fix(infra): memory leak (#9013)

This commit is contained in:
EYHN
2024-12-16 16:55:49 +00:00
parent 55f1cc4b1e
commit b36b398957
30 changed files with 399 additions and 135 deletions

View File

@@ -36,7 +36,7 @@ export class DocsIndexer extends Entity {
/**
* increase this number to re-index all docs
*/
static INDEXER_VERSION = 10;
static INDEXER_VERSION = 11;
private readonly jobQueue: JobQueue<IndexerJobPayload> =
new IndexedDBJobQueue<IndexerJobPayload>(
@@ -85,24 +85,26 @@ export class DocsIndexer extends Entity {
}
setupListener() {
this.workspaceEngine.doc.storage.eventBus.on(event => {
if (WorkspaceDBService.isDBDocId(event.docId)) {
// skip db doc
return;
}
if (event.clientId === this.workspaceEngine.doc.clientId) {
this.jobQueue
.enqueue([
{
batchKey: event.docId,
payload: { storageDocId: event.docId },
},
])
.catch(err => {
console.error('Error enqueueing job', err);
});
}
});
this.disposables.push(
this.workspaceEngine.doc.storage.eventBus.on(event => {
if (WorkspaceDBService.isDBDocId(event.docId)) {
// skip db doc
return;
}
if (event.clientId === this.workspaceEngine.doc.clientId) {
this.jobQueue
.enqueue([
{
batchKey: event.docId,
payload: { storageDocId: event.docId },
},
])
.catch(err => {
console.error('Error enqueueing job', err);
});
}
})
);
}
async execJob(jobs: Job<IndexerJobPayload>[], signal: AbortSignal) {
@@ -298,6 +300,8 @@ export class DocsIndexer extends Entity {
}
override dispose(): void {
super.dispose();
this.runner.stop();
this.worker?.dispose();
}
}

View File

@@ -1,6 +1,7 @@
import { defineSchema } from '@toeverything/infra';
export const docIndexSchema = defineSchema({
docId: 'String',
title: 'FullText',
// summary of the doc, used for preview
summary: { type: 'String', index: false },

View File

@@ -632,10 +632,31 @@ export class DocsSearchService extends Service {
);
}
async getDocTitle(docId: string) {
const doc = await this.indexer.docIndex.get(docId);
const title = doc?.get('title');
return typeof title === 'string' ? title : title?.[0];
watchDocSummary(docId: string) {
return this.indexer.docIndex
.search$(
{
type: 'match',
field: 'docId',
match: docId,
},
{
fields: ['summary'],
pagination: {
limit: 1,
},
}
)
.pipe(
map(({ nodes }) => {
const node = nodes.at(0);
return (
(typeof node?.fields.summary === 'string'
? node?.fields.summary
: node?.fields.summary[0]) ?? null
);
})
);
}
override dispose(): void {

View File

@@ -105,6 +105,11 @@ const bookmarkFlavours = new Set([
'affine:embed-loom',
]);
const markdownPreviewDocCollection = new DocCollection({
id: 'indexer',
schema: blocksuiteSchema,
});
function generateMarkdownPreviewBuilder(
yRootDoc: YDoc,
workspaceId: string,
@@ -164,10 +169,7 @@ function generateMarkdownPreviewBuilder(
const markdownAdapter = new MarkdownAdapter(
new Job({
collection: new DocCollection({
id: 'indexer',
schema: blocksuiteSchema,
}),
collection: markdownPreviewDocCollection,
middlewares: [docLinkBaseURLMiddleware, titleMiddleware],
})
);
@@ -875,6 +877,7 @@ async function crawlingDocData({
{
id: docId,
doc: Document.from<DocIndexSchema>(docId, {
docId,
title: docTitle,
summary,
}),

View File

@@ -95,8 +95,8 @@ export async function createWorker(abort: AbortSignal) {
});
},
dispose: () => {
worker.terminate();
terminateAbort.abort(MANUALLY_STOP);
worker.terminate();
},
};
}