feat(core): new worker workspace engine (#9257)

packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts

@@ -25,7 +25,7 @@ const logger = new DebugLogger('crawler');
 const WORKSPACE_DOCS_INDEXER_VERSION_KEY = 'docs-indexer-version';
 
 interface IndexerJobPayload {
-  storageDocId: string;
+  docId: string;
 }
 
 export class DocsIndexer extends Entity {
@@ -81,26 +81,31 @@ export class DocsIndexer extends Entity {
   }
 
   setupListener() {
-    this.disposables.push(
-      this.workspaceEngine.doc.storage.eventBus.on(event => {
-        if (WorkspaceDBService.isDBDocId(event.docId)) {
-          // skip db doc
-          return;
-        }
-        if (event.clientId === this.workspaceEngine.doc.clientId) {
-          this.jobQueue
-            .enqueue([
-              {
-                batchKey: event.docId,
-                payload: { storageDocId: event.docId },
-              },
-            ])
-            .catch(err => {
-              console.error('Error enqueueing job', err);
-            });
-        }
+    this.workspaceEngine.doc.storage.connection
+      .waitForConnected()
+      .then(() => {
+        this.disposables.push(
+          this.workspaceEngine.doc.storage.subscribeDocUpdate(updated => {
+            if (WorkspaceDBService.isDBDocId(updated.docId)) {
+              // skip db doc
+              return;
+            }
+            this.jobQueue
+              .enqueue([
+                {
+                  batchKey: updated.docId,
+                  payload: { docId: updated.docId },
+                },
+              ])
+              .catch(err => {
+                console.error('Error enqueueing job', err);
+              });
+          })
+        );
       })
-    );
+      .catch(err => {
+        console.error('Error waiting for doc storage connection', err);
+      });
   }
 
   async execJob(jobs: Job<IndexerJobPayload>[], signal: AbortSignal) {
@@ -119,20 +124,19 @@ export class DocsIndexer extends Entity {
     const isUpgrade = dbVersion < DocsIndexer.INDEXER_VERSION;
 
     // jobs should have the same storage docId, so we just pick the first one
-    const storageDocId = jobs[0].payload.storageDocId;
+    const docId = jobs[0].payload.docId;
 
     const worker = await this.ensureWorker(signal);
 
     const startTime = performance.now();
-    logger.debug('Start crawling job for storageDocId:', storageDocId);
+    logger.debug('Start crawling job for docId:', docId);
 
     let workerOutput;
 
-    if (storageDocId === this.workspaceId) {
-      const rootDocBuffer =
-        await this.workspaceEngine.doc.storage.loadDocFromLocal(
-          this.workspaceId
-        );
+    if (docId === this.workspaceId) {
+      const rootDocBuffer = (
+        await this.workspaceEngine.doc.storage.getDoc(this.workspaceId)
+      )?.bin;
       if (!rootDocBuffer) {
         return;
       }
@@ -147,15 +151,13 @@ export class DocsIndexer extends Entity {
         rootDocId: this.workspaceId,
       });
     } else {
-      const rootDocBuffer =
-        await this.workspaceEngine.doc.storage.loadDocFromLocal(
-          this.workspaceId
-        );
+      const rootDocBuffer = (
+        await this.workspaceEngine.doc.storage.getDoc(this.workspaceId)
+      )?.bin;
 
       const docBuffer =
-        (await this.workspaceEngine.doc.storage.loadDocFromLocal(
-          storageDocId
-        )) ?? new Uint8Array(0);
+        (await this.workspaceEngine.doc.storage.getDoc(docId))?.bin ??
+        new Uint8Array(0);
 
       if (!rootDocBuffer) {
         return;
@@ -164,7 +166,7 @@ export class DocsIndexer extends Entity {
       workerOutput = await worker.run({
         type: 'doc',
         docBuffer,
-        storageDocId,
+        docId,
         rootDocBuffer,
         rootDocId: this.workspaceId,
       });
@@ -231,9 +233,9 @@ export class DocsIndexer extends Entity {
 
     if (workerOutput.reindexDoc) {
       await this.jobQueue.enqueue(
-        workerOutput.reindexDoc.map(({ storageDocId }) => ({
-          batchKey: storageDocId,
-          payload: { storageDocId },
+        workerOutput.reindexDoc.map(({ docId }) => ({
+          batchKey: docId,
+          payload: { docId },
         }))
       );
     }
@@ -244,11 +246,7 @@ export class DocsIndexer extends Entity {
 
     const duration = performance.now() - startTime;
     logger.debug(
-      'Finish crawling job for storageDocId:' +
-        storageDocId +
-        ' in ' +
-        duration +
-        'ms '
+      'Finish crawling job for docId:' + docId + ' in ' + duration + 'ms '
     );
   }
 
@@ -259,7 +257,7 @@ export class DocsIndexer extends Entity {
       .enqueue([
         {
           batchKey: this.workspaceId,
-          payload: { storageDocId: this.workspaceId },
+          payload: { docId: this.workspaceId },
         },
       ])
       .catch(err => {

packages/frontend/core/src/modules/docs-search/worker/in-worker.ts

@@ -1,4 +1,3 @@
-import { getElectronAPIs } from '@affine/electron-api/web-worker';
 import type {
   AttachmentBlockModel,
   BookmarkBlockModel,
@@ -50,11 +49,6 @@ const LRU_CACHE_SIZE = 5;
 // lru cache for ydoc instances, last used at the end of the array
 const lruCache = [] as { doc: YDoc; hash: string }[];
 
-const electronAPIs = BUILD_CONFIG.isElectron ? getElectronAPIs() : null;
-
-// @ts-expect-error test
-globalThis.__electronAPIs = electronAPIs;
-
 async function digest(data: Uint8Array) {
   if (
     globalThis.crypto &&
@@ -478,7 +472,7 @@ function unindentMarkdown(markdown: string) {
 
 async function crawlingDocData({
   docBuffer,
-  storageDocId,
+  docId,
   rootDocBuffer,
   rootDocId,
 }: WorkerInput & { type: 'doc' }): Promise<WorkerOutput> {
@@ -489,18 +483,6 @@ async function crawlingDocData({
 
   const yRootDoc = await getOrCreateCachedYDoc(rootDocBuffer);
 
-  let docId = null;
-  for (const [id, subdoc] of yRootDoc.getMap('spaces')) {
-    if (subdoc instanceof YDoc && storageDocId === subdoc.guid) {
-      docId = id;
-      break;
-    }
-  }
-
-  if (docId === null) {
-    return {};
-  }
-
   let docExists: boolean | null = null;
 
   (

packages/frontend/core/src/modules/docs-search/worker/out-worker.ts

@@ -1,5 +1,4 @@
 import { DebugLogger } from '@affine/debug';
-import { connectWebWorker } from '@affine/electron-api/web-worker';
 import { MANUALLY_STOP, throwIfAborted } from '@toeverything/infra';
 
 import type {
@@ -13,7 +12,6 @@ const logger = new DebugLogger('affine:indexer-worker');
 
 export async function createWorker(abort: AbortSignal) {
   let worker: Worker | null = null;
-  let electronApiCleanup: (() => void) | null = null;
   while (throwIfAborted(abort)) {
     try {
       worker = await new Promise<Worker>((resolve, reject) => {
@@ -32,10 +30,6 @@ export async function createWorker(abort: AbortSignal) {
         });
         worker.postMessage({ type: 'init', msgId: 0 } as WorkerIngoingMessage);
 
-        if (BUILD_CONFIG.isElectron) {
-          electronApiCleanup = connectWebWorker(worker);
-        }
-
         setTimeout(() => {
           reject('timeout');
         }, 1000 * 30 /* 30 sec */);
@@ -104,7 +98,6 @@ export async function createWorker(abort: AbortSignal) {
     dispose: () => {
       terminateAbort.abort(MANUALLY_STOP);
       worker.terminate();
-      electronApiCleanup?.();
     },
   };
 }

packages/frontend/core/src/modules/docs-search/worker/types.ts

@@ -36,14 +36,14 @@ export type WorkerInput =
     }
   | {
       type: 'doc';
-      storageDocId: string;
+      docId: string;
       rootDocId: string;
       rootDocBuffer: Uint8Array;
       docBuffer: Uint8Array;
     };
 
 export interface WorkerOutput {
-  reindexDoc?: { docId: string; storageDocId: string }[];
+  reindexDoc?: { docId: string }[];
   addedDoc?: {
     id: string;
     blocks: Document<BlockIndexSchema>[];