luulam/AFFiNE
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(server): get full content for doc embedding (#11138)

Browse Source
This commit is contained in:
darkskygit
2025-03-25 01:10:43 +00:00
parent c1b3e25fc7
commit bf5d8b1211
3 changed files with 18 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
packages/backend/server/src/core/doc/reader.ts
View File

@@ -42,10 +42,10 @@ export abstract class DocReader {
protected readonly blobStorage: WorkspaceBlobStorage protected readonly blobStorage: WorkspaceBlobStorage
) {} ) {}
parseDocContent(bin: Uint8Array) { parseDocContent(bin: Uint8Array, maxSummaryLength = 150) {
const doc = new YDoc(); const doc = new YDoc();
applyUpdate(doc, bin); applyUpdate(doc, bin);
return parsePageDoc(doc); return parsePageDoc(doc, { maxSummaryLength });
} }
parseWorkspaceContent(bin: Uint8Array) { parseWorkspaceContent(bin: Uint8Array) {
@@ -85,6 +85,13 @@ export abstract class DocReader {
return content; return content;
} }
async getFullDocContent(
workspaceId: string,
docId: string
): Promise<PageDocContent | null> {
return await this.getDocContentWithoutCache(workspaceId, docId, true);
}
/** /**
* Get workspace content, try to read from database first. * Get workspace content, try to read from database first.
* If not exists, read from `getWorkspaceContentWithoutCache()` and save it back to database. * If not exists, read from `getWorkspaceContentWithoutCache()` and save it back to database.
@@ -131,7 +138,8 @@ export abstract class DocReader {
protected abstract getDocContentWithoutCache( protected abstract getDocContentWithoutCache(
workspaceId: string, workspaceId: string,
guid: string guid: string,
fullContent?: boolean
): Promise<PageDocContent | null>; ): Promise<PageDocContent | null>;
protected abstract getWorkspaceContentWithoutCache( protected abstract getWorkspaceContentWithoutCache(
@@ -180,13 +188,14 @@ export class DatabaseDocReader extends DocReader {
protected override async getDocContentWithoutCache( protected override async getDocContentWithoutCache(
workspaceId: string, workspaceId: string,
guid: string guid: string,
fullContent?: boolean
): Promise<PageDocContent | null> { ): Promise<PageDocContent | null> {
const docRecord = await this.workspace.getDoc(workspaceId, guid); const docRecord = await this.workspace.getDoc(workspaceId, guid);
if (!docRecord) { if (!docRecord) {
return null; return null;
} }
return this.parseDocContent(docRecord.bin); return this.parseDocContent(docRecord.bin, fullContent ? -1 : 150);
} }
protected override async getWorkspaceContentWithoutCache( protected override async getWorkspaceContentWithoutCache(

4
packages/backend/server/src/core/utils/blocksuite.ts
View File

@@ -115,7 +115,9 @@ export function parsePageDoc(
continue; continue;
} }
if (summaryLenNeeded > 0) { if (summaryLenNeeded === -1) {
content.summary += text.toString();
} else if (summaryLenNeeded > 0) {
content.summary += text.toString(); content.summary += text.toString();
summaryLenNeeded -= text.length; summaryLenNeeded -= text.length;
} else { } else {

2
packages/backend/server/src/plugins/copilot/context/job.ts
View File

@@ -159,7 +159,7 @@ export class CopilotContextDocJob implements OnModuleInit {
if (!this.supportEmbedding) return; if (!this.supportEmbedding) return;
try { try {
const content = await this.doc.getDocContent(workspaceId, docId); const content = await this.doc.getFullDocContent(workspaceId, docId);
if (content) { if (content) {
// no need to check if embeddings is empty, will throw internally // no need to check if embeddings is empty, will throw internally
const embeddings = await this.embeddingClient.getFileEmbeddings( const embeddings = await this.embeddingClient.getFileEmbeddings(