chore: improve performance

Author: darkskygit

Cargo.lock

@@ -45,6 +45,7 @@ pdf-extract            = "0.8.2"
 rand                   = "0.9"
 rayon                  = "1.10"
 readability            = { version = "0.3.0", default-features = false }
+regex                  = "1"
 rubato                 = "0.16"
 screencapturekit       = "0.3"
 serde                  = "1"

Cargo.toml

@@ -13,6 +13,7 @@ file-format   = { workspace = true }
 napi          = { workspace = true, features = ["async"] }
 napi-derive   = { workspace = true }
 rand          = { workspace = true }
+regex         = { workspace = true }
 sha3          = { workspace = true }
 tiktoken-rs   = { workspace = true }
 v_htmlescape  = { workspace = true }

packages/backend/native/Cargo.toml

@@ -4,22 +4,37 @@ use napi::{
   bindgen_prelude::{AsyncTask, Buffer},
   Env, JsObject, Result, Task,
 };
+use regex::Regex;
 
 pub struct Document {
   inner: Doc,
+  re: Regex,
 }
 
 impl Document {
   fn name(&self) -> String {
     self.inner.name.clone()
   }
 
+  fn clean_input(&self, input: &str) -> String {
+    self
+      .re
+      .replace_all(input, "")
+      .replace("\n", " ")
+      .replace("  ", " ")
+      .replace("\x00", "")
+      .trim()
+      .to_string()
+  }
+
   fn chunks(&self, env: Env) -> Result<JsObject> {
     let mut array = env.create_array_with_length(self.inner.chunks.len())?;
     for (i, chunk) in self.inner.chunks.iter().enumerate() {
+      let content = self.clean_input(&chunk.content);
+
       let mut obj = env.create_object()?;
       obj.set_named_property("index", i as i64)?;
-      obj.set_named_property("content", chunk.content.clone())?;
+      obj.set_named_property("content", content)?;
       array.set_element(i as u32, obj)?;
     }
     Ok(array)
@@ -45,7 +60,8 @@ impl Task for AsyncParseDocResponse {
 
   fn compute(&mut self) -> Result<Self::Output> {
     let doc = Doc::new(&self.file_path, &self.doc).map_err(|e| anyhow!(e))?;
-    Ok(Document { inner: doc })
+    let re = Regex::new(r"(Figure|Table)\s+\d+\.").map_err(|e| anyhow!(e))?;
+    Ok(Document { inner: doc, re })
   }
 
   fn resolve(&mut self, env: Env, doc: Document) -> Result<Self::JsValue> {

packages/backend/native/src/doc_loader.rs

@@ -1,5 +1,6 @@
 import { randomUUID } from 'node:crypto';
 
+import { Package } from '@affine-tools/utils/workspace';
 import type { TestFn } from 'ava';
 import ava from 'ava';
 import Sinon from 'sinon';
@@ -755,7 +756,7 @@ test('should be able to manage context', async t => {
 
   const fs = await import('node:fs');
   const buffer = fs.readFileSync(
-    new URL('../../../../common/native/fixtures/sample.pdf', import.meta.url)
+    new Package('@affine/native').join('fixtures/sample.pdf').toFileUrl()
   );
 
   {

packages/backend/server/src/__tests__/copilot.e2e.ts

@@ -1,5 +1,6 @@
 import { randomUUID } from 'node:crypto';
 
+import { Package } from '@affine-tools/utils/workspace';
 import type { TestFn } from 'ava';
 import ava from 'ava';
 import Sinon from 'sinon';
@@ -1330,7 +1331,7 @@ test('should be able to manage context', async t => {
 
   const fs = await import('node:fs');
   const buffer = fs.readFileSync(
-    new URL('../../../../common/native/fixtures/sample.pdf', import.meta.url)
+    new Package('@affine/native').join('fixtures/sample.pdf').toFileUrl()
   );
 
   {

packages/backend/server/src/__tests__/copilot.spec.ts

@@ -10,7 +10,6 @@ import {
   Config,
   EventBus,
   JobQueue,
-  metrics,
   OnEvent,
   OnJob,
 } from '../../../base';
@@ -161,9 +160,6 @@ export class CopilotContextDocJob implements OnModuleInit {
         chunkSize: total,
       });
     } catch (e: any) {
-      metrics.doc
-        .counter('auto_embed_pending_files_error')
-        .add(1, { contextId, fileId });
       this.logger.error(
         `Failed to embed pending file: ${contextId}::${fileId}`,
         e
@@ -200,9 +196,6 @@ export class CopilotContextDocJob implements OnModuleInit {
         }
       }
     } catch (e: any) {
-      metrics.doc
-        .counter('auto_embed_pending_docs_error')
-        .add(1, { workspaceId });
       this.logger.error(
         `Failed to embed pending doc: ${workspaceId}::${docId}`,
         e

packages/backend/server/src/plugins/copilot/context/job.ts

@@ -122,10 +122,10 @@ export abstract class EmbeddingClient {
         });
       }
       const input = doc.chunks.toSorted((a, b) => a.index - b.index);
-      // chunk input into 2048 every array
+      // chunk input into 32 every array
       const chunks: Chunk[][] = [];
-      for (let i = 0; i < input.length; i += 2048) {
-        chunks.push(input.slice(i, i + 2048));
+      for (let i = 0; i < input.length; i += 32) {
+        chunks.push(input.slice(i, i + 32));
       }
       return chunks;
     }