diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts index 6a5c9880c..f12f39a2a 100644 --- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts @@ -369,7 +369,7 @@ The term **“CRDT”** was first introduced by Marc Shapiro, Nuno Preguiça, Ca .map(c => JSON.parse(c.citationJson).type) .filter(type => ['attachment', 'doc'].includes(type)).length === 0, - 'should not have citation' + `should not have citation: ${JSON.stringify(c, null, 2)}` ); }); }, diff --git a/packages/backend/server/src/plugins/copilot/embedding/client.ts b/packages/backend/server/src/plugins/copilot/embedding/client.ts index cdc345e2f..1ca8bd082 100644 --- a/packages/backend/server/src/plugins/copilot/embedding/client.ts +++ b/packages/backend/server/src/plugins/copilot/embedding/client.ts @@ -112,11 +112,14 @@ class ProductionEmbeddingClient extends EmbeddingClient { ); try { - return ranks.map((score, chunk) => ({ - chunk, - targetId: this.getTargetId(embeddings[chunk]), - score, - })); + return ranks.map((score, i) => { + const chunk = embeddings[i]; + return { + chunk: chunk.chunk, + targetId: this.getTargetId(chunk), + score: Math.max(score, 1 - (chunk.distance || -Infinity)), + }; + }); } catch (error) { this.logger.error('Failed to parse rerank results', error); // silent error, will fallback to default sorting in parent method @@ -148,7 +151,7 @@ class ProductionEmbeddingClient extends EmbeddingClient { const chunks = sortedEmbeddings.reduce( (acc, e) => { - const targetId = 'docId' in e ? e.docId : 'fileId' in e ? e.fileId : ''; + const targetId = this.getTargetId(e); const key = `${targetId}:${e.chunk}`; acc[key] = e; return acc; @@ -179,7 +182,10 @@ class ProductionEmbeddingClient extends EmbeddingClient { .filter(Boolean); this.logger.verbose( - `ReRank completed: ${highConfidenceChunks.length} high-confidence results found` + `ReRank completed: ${highConfidenceChunks.length} high-confidence results found, total ${sortedEmbeddings.length} embeddings`, + highConfidenceChunks.length !== sortedEmbeddings.length + ? JSON.stringify(ranks) + : undefined ); return highConfidenceChunks.slice(0, topK); } catch (error) { diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index ff2b4b667..2e5d8cf04 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -338,7 +338,7 @@ Convert a multi-speaker audio recording into a structured JSON format by transcr { name: 'Rerank results', action: 'Rerank results', - model: 'gpt-4.1-mini', + model: 'gpt-4.1', messages: [ { role: 'system', @@ -1677,7 +1677,7 @@ This sentence contains information from the first source[^1]. This sentence refe Before starting Tool calling, you need to follow: - DO NOT explain what operation you will perform. - DO NOT embed a tool call mid-sentence. -- When searching for unknown information or keyword, prioritize searching the user's workspace. +- When searching for unknown information, personal information or keyword, prioritize searching the user's workspace rather than the web. - Depending on the complexity of the question and the information returned by the search tools, you can call different tools multiple times to search. diff --git a/packages/backend/server/src/plugins/copilot/prompt/service.ts b/packages/backend/server/src/plugins/copilot/prompt/service.ts index c4d19bf5a..fd8d84ac8 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/service.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/service.ts @@ -53,8 +53,11 @@ export class PromptService implements OnApplicationBootstrap { * @returns prompt messages */ async get(name: string): Promise { - const cached = this.cache.get(name); - if (cached) return cached; + // skip cache in dev mode to ensure the latest prompt is always fetched + if (!env.dev) { + const cached = this.cache.get(name); + if (cached) return cached; + } const prompt = await this.db.aiPrompt.findUnique({ where: { diff --git a/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts b/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts index 7e2cb554f..e2912a7cf 100644 --- a/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts +++ b/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts @@ -62,6 +62,7 @@ export abstract class AnthropicProvider extends CopilotProvider { try { metrics.ai.counter('chat_text_calls').add(1, { model: model.id }); + const [system, msgs] = await chatToGPTMessage(messages, true, true); const modelInstance = this.instance(model.id); diff --git a/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts b/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts index 982265343..20dd0c29f 100644 --- a/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts +++ b/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts @@ -88,6 +88,12 @@ export abstract class GeminiProvider extends CopilotProvider { system, messages: msgs, abortSignal: options.signal, + providerOptions: { + google: this.getGeminiOptions(options, model.id), + }, + tools: await this.getTools(options, model.id), + maxSteps: this.MAX_STEPS, + experimental_continueSteps: true, }); if (!text) throw new Error('Failed to generate text'); @@ -254,16 +260,16 @@ export abstract class GeminiProvider extends CopilotProvider { ) { const [system, msgs] = await chatToGPTMessage(messages); const { fullStream } = streamText({ - model: this.instance(model.id, { - useSearchGrounding: this.useSearchGrounding(options), - }), + model: this.instance(model.id), system, messages: msgs, abortSignal: options.signal, - maxSteps: this.MAX_STEPS, providerOptions: { google: this.getGeminiOptions(options, model.id), }, + tools: await this.getTools(options, model.id), + maxSteps: this.MAX_STEPS, + experimental_continueSteps: true, }); return fullStream; } @@ -282,8 +288,4 @@ export abstract class GeminiProvider extends CopilotProvider { private isReasoningModel(model: string) { return model.startsWith('gemini-2.5'); } - - private useSearchGrounding(options: CopilotChatOptions) { - return options?.tools?.includes('webSearch'); - } } diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index f7d8d9c8b..8c2ffea05 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -274,9 +274,11 @@ export class OpenAIProvider extends CopilotProvider { override getProviderSpecificTools( toolName: CopilotChatTools, model: string - ): [string, Tool] | undefined { + ): [string, Tool?] | undefined { if (toolName === 'webSearch' && !this.isReasoningModel(model)) { return ['web_search_preview', openai.tools.webSearchPreview()]; + } else if (toolName === 'docEdit') { + return ['doc_edit', undefined]; } return; } diff --git a/packages/backend/server/src/plugins/copilot/providers/provider.ts b/packages/backend/server/src/plugins/copilot/providers/provider.ts index 54605efaa..ebe2f8073 100644 --- a/packages/backend/server/src/plugins/copilot/providers/provider.ts +++ b/packages/backend/server/src/plugins/copilot/providers/provider.ts @@ -126,7 +126,7 @@ export abstract class CopilotProvider { protected getProviderSpecificTools( _toolName: CopilotChatTools, _model: string - ): [string, Tool] | undefined { + ): [string, Tool?] | undefined { return; } @@ -143,7 +143,10 @@ export abstract class CopilotProvider { for (const tool of options.tools) { const toolDef = this.getProviderSpecificTools(tool, model); if (toolDef) { - tools[toolDef[0]] = toolDef[1]; + // allow provider prevent tool creation + if (toolDef[1]) { + tools[toolDef[0]] = toolDef[1]; + } continue; } switch (tool) { diff --git a/packages/backend/server/src/plugins/copilot/tools/doc-semantic-search.ts b/packages/backend/server/src/plugins/copilot/tools/doc-semantic-search.ts index 299b4dd2d..20bcbd6c8 100644 --- a/packages/backend/server/src/plugins/copilot/tools/doc-semantic-search.ts +++ b/packages/backend/server/src/plugins/copilot/tools/doc-semantic-search.ts @@ -58,7 +58,7 @@ export const createDocSemanticSearchTool = ( ) => { return tool({ description: - 'Retrieve conceptually related passages by performing vector-based semantic similarity search across embedded documents; use this tool only when exact keyword search fails or the user explicitly needs meaning-level matches (e.g., paraphrases, synonyms, broader concepts).', + 'Retrieve conceptually related passages by performing vector-based semantic similarity search across embedded documents; use this tool only when exact keyword search fails or the user explicitly needs meaning-level matches (e.g., paraphrases, synonyms, broader concepts, recent documents).', parameters: z.object({ query: z .string() diff --git a/tests/affine-cloud-copilot/e2e/ai-action/image-processing.spec.ts b/tests/affine-cloud-copilot/e2e/ai-action/image-processing.spec.ts index a286d509a..5d57d59ed 100644 --- a/tests/affine-cloud-copilot/e2e/ai-action/image-processing.spec.ts +++ b/tests/affine-cloud-copilot/e2e/ai-action/image-processing.spec.ts @@ -36,6 +36,8 @@ test.describe('AIAction/ImageProcessing', () => { await expect(answer.getByTestId('ai-answer-image')).toBeVisible(); const insert = answer.getByTestId('answer-insert-below'); await insert.click(); + await page.reload(); + await utils.chatPanel.waitForHistory(page, [ { role: 'action', diff --git a/tests/affine-cloud-copilot/e2e/basic/chat.spec.ts b/tests/affine-cloud-copilot/e2e/basic/chat.spec.ts index c416c3ccb..d78519c8a 100644 --- a/tests/affine-cloud-copilot/e2e/basic/chat.spec.ts +++ b/tests/affine-cloud-copilot/e2e/basic/chat.spec.ts @@ -45,28 +45,30 @@ test.describe('AIBasic/Chat', () => { // Type and send a message await utils.chatPanel.makeChat( page, - 'Introduce AFFiNE to me. Answer in 50 words.' + 'Introduce AFFiNE to me. Answer in 500 words.' ); - // AI is loading - await utils.chatPanel.waitForHistory(page, [ - { - role: 'user', - content: 'Introduce AFFiNE to me. Answer in 50 words.', - }, - { - role: 'assistant', - status: 'loading', - }, - ]); + if (!(await page.getByTestId('ai-loading').isVisible())) { + // AI is loading + await utils.chatPanel.waitForHistory(page, [ + { + role: 'user', + content: 'Introduce AFFiNE to me. Answer in 500 words.', + }, + { + role: 'assistant', + status: 'loading', + }, + ]); - await expect(page.getByTestId('ai-loading')).toBeVisible(); + await expect(page.getByTestId('ai-loading')).toBeVisible(); + } // AI Generating await utils.chatPanel.waitForHistory(page, [ { role: 'user', - content: 'Introduce AFFiNE to me. Answer in 50 words.', + content: 'Introduce AFFiNE to me. Answer in 500 words.', }, { role: 'assistant', @@ -79,7 +81,7 @@ test.describe('AIBasic/Chat', () => { await utils.chatPanel.waitForHistory(page, [ { role: 'user', - content: 'Introduce AFFiNE to me. Answer in 50 words.', + content: 'Introduce AFFiNE to me. Answer in 500 words.', }, { role: 'assistant', @@ -94,14 +96,14 @@ test.describe('AIBasic/Chat', () => { }) => { await utils.chatPanel.makeChat( page, - 'Introduce AFFiNE to me. Answer in 50 words.' + 'Introduce AFFiNE to me. Answer in 5000 words.' ); // AI Generating await utils.chatPanel.waitForHistory(page, [ { role: 'user', - content: 'Introduce AFFiNE to me. Answer in 50 words.', + content: 'Introduce AFFiNE to me. Answer in 5000 words.', }, { role: 'assistant', @@ -113,7 +115,7 @@ test.describe('AIBasic/Chat', () => { await utils.chatPanel.waitForHistory(page, [ { role: 'user', - content: 'Introduce AFFiNE to me. Answer in 50 words.', + content: 'Introduce AFFiNE to me. Answer in 5000 words.', }, { role: 'assistant', @@ -183,13 +185,14 @@ test.describe('AIBasic/Chat', () => { // Type and send a message await utils.chatPanel.makeChat( page, - 'Hello, write a poem about the moon. Answer in 50 words.' + 'Hello, give a introduction about the moon. Answer in 500 words.' ); await utils.chatPanel.waitForHistory(page, [ { role: 'user', - content: 'Hello, write a poem about the moon. Answer in 50 words.', + content: + 'Hello, give a introduction about the moon. Answer in 500 words.', }, { role: 'assistant', diff --git a/tests/affine-cloud-copilot/e2e/chat-with/attachments.spec.ts b/tests/affine-cloud-copilot/e2e/chat-with/attachments.spec.ts index 8f85ee033..bc8dc7803 100644 --- a/tests/affine-cloud-copilot/e2e/chat-with/attachments.spec.ts +++ b/tests/affine-cloud-copilot/e2e/chat-with/attachments.spec.ts @@ -2,9 +2,14 @@ import { expect } from '@playwright/test'; import { test } from '../base/base-test'; +test.describe.configure({ mode: 'serial' }); + test.describe('AIChatWith/Attachments', () => { test.beforeEach(async ({ loggedInPage: page, utils }) => { - await utils.testUtils.setupTestEnvironment(page); + await utils.testUtils.setupTestEnvironment( + page, + 'claude-sonnet-4@20250514' + ); await utils.chatPanel.openChatPanel(page); }); @@ -48,8 +53,10 @@ test.describe('AIChatWith/Attachments', () => { loggedInPage: page, utils, }) => { - const textContent1 = 'AttachmentEEee is a cute cat'; - const textContent2 = 'AttachmentFFff is a cute dog'; + const randomStr1 = Math.random().toString(36).substring(2, 6); + const randomStr2 = Math.random().toString(36).substring(2, 6); + const textContent1 = `Attachment${randomStr1} is a cute cat`; + const textContent2 = `Attachment${randomStr2} is a cute dog`; const buffer1 = Buffer.from(textContent1); const buffer2 = Buffer.from(textContent2); @@ -67,13 +74,13 @@ test.describe('AIChatWith/Attachments', () => { buffer: buffer2, }, ], - 'What is AttachmentEEee? What is AttachmentFFff?' + `What is Attachment${randomStr1}? What is Attachment${randomStr2}?` ); await utils.chatPanel.waitForHistory(page, [ { role: 'user', - content: 'What is AttachmentEEee? What is AttachmentFFff?', + content: `What is Attachment${randomStr1}? What is Attachment${randomStr2}?`, }, { role: 'assistant', @@ -84,8 +91,8 @@ test.describe('AIChatWith/Attachments', () => { await expect(async () => { const { content, message } = await utils.chatPanel.getLatestAssistantMessage(page); - expect(content).toMatch(/AttachmentEEee/); - expect(content).toMatch(/AttachmentFFff/); + expect(content).toMatch(new RegExp(`Attachment${randomStr1}`)); + expect(content).toMatch(new RegExp(`Attachment${randomStr2}`)); expect(await message.locator('affine-footnote-node').count()).toBe(2); }).toPass({ timeout: 20000 }); }); diff --git a/tests/affine-cloud-copilot/e2e/chat-with/collections.spec.ts b/tests/affine-cloud-copilot/e2e/chat-with/collections.spec.ts index 6035d0a16..08cd61f60 100644 --- a/tests/affine-cloud-copilot/e2e/chat-with/collections.spec.ts +++ b/tests/affine-cloud-copilot/e2e/chat-with/collections.spec.ts @@ -6,7 +6,10 @@ test.describe.configure({ mode: 'serial' }); test.describe('AIChatWith/Collections', () => { test.beforeEach(async ({ loggedInPage: page, utils }) => { - await utils.testUtils.setupTestEnvironment(page); + await utils.testUtils.setupTestEnvironment( + page, + 'claude-sonnet-4@20250514' + ); await utils.chatPanel.openChatPanel(page); await utils.editor.clearAllCollections(page); diff --git a/tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts b/tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts index 080556cf4..b24f1522d 100644 --- a/tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts +++ b/tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts @@ -1,12 +1,16 @@ import { createLocalWorkspace } from '@affine-test/kit/utils/workspace'; -import { faker } from '@faker-js/faker'; import { expect } from '@playwright/test'; import { test } from '../base/base-test'; +test.describe.configure({ mode: 'serial' }); + test.describe('AISettings/Embedding', () => { test.beforeEach(async ({ loggedInPage: page, utils }) => { - await utils.testUtils.setupTestEnvironment(page); + await utils.testUtils.setupTestEnvironment( + page, + 'claude-sonnet-4@20250514' + ); await utils.chatPanel.openChatPanel(page); }); @@ -246,7 +250,7 @@ test.describe('AISettings/Embedding', () => { await createLocalWorkspace({ name: 'test' }, page, false, 'affine-cloud'); await utils.settings.openSettingsPanel(page); await utils.settings.enableWorkspaceEmbedding(page); - const person = faker.person.fullName(); + const person = 'test123'; const hobby1 = Buffer.from(`${person} love climbing`); const hobby2 = Buffer.from(`${person} love skating`); diff --git a/tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts b/tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts index 0cb4eab45..058acfb77 100644 --- a/tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts +++ b/tests/affine-cloud-copilot/e2e/utils/chat-panel-utils.ts @@ -201,7 +201,7 @@ export class ChatPanelUtils { public static async chatWithDoc(page: Page, docName: string) { const withButton = page.getByTestId('chat-panel-with-button'); await withButton.hover(); - await withButton.click(); + await withButton.click({ delay: 200 }); const withMenu = page.getByTestId('ai-add-popover'); await withMenu.waitFor({ state: 'visible' }); await withMenu.getByText(docName).click(); @@ -221,7 +221,7 @@ export class ChatPanelUtils { const fileChooserPromise = page.waitForEvent('filechooser'); const withButton = page.getByTestId('chat-panel-with-button'); await withButton.hover(); - await withButton.click(); + await withButton.click({ delay: 200 }); const withMenu = page.getByTestId('ai-add-popover'); await withMenu.waitFor({ state: 'visible' }); await withMenu.getByTestId('ai-chat-with-files').click(); @@ -282,7 +282,7 @@ export class ChatPanelUtils { for (const tag of tags) { const withButton = page.getByTestId('chat-panel-with-button'); await withButton.hover(); - await withButton.click(); + await withButton.click({ delay: 200 }); const withMenu = page.getByTestId('ai-add-popover'); await withMenu.waitFor({ state: 'visible' }); await withMenu.getByTestId('ai-chat-with-tags').click(); @@ -299,7 +299,7 @@ export class ChatPanelUtils { for (const collection of collections) { const withButton = page.getByTestId('chat-panel-with-button'); await withButton.hover(); - await withButton.click(); + await withButton.click({ delay: 200 }); const withMenu = page.getByTestId('ai-add-popover'); await withMenu.waitFor({ state: 'visible' }); await withMenu.getByTestId('ai-chat-with-collections').click(); diff --git a/tests/affine-cloud-copilot/e2e/utils/test-utils.ts b/tests/affine-cloud-copilot/e2e/utils/test-utils.ts index 474845055..4a363985a 100644 --- a/tests/affine-cloud-copilot/e2e/utils/test-utils.ts +++ b/tests/affine-cloud-copilot/e2e/utils/test-utils.ts @@ -1,5 +1,8 @@ import { skipOnboarding } from '@affine-test/kit/playwright'; -import { createRandomAIUser } from '@affine-test/kit/utils/cloud'; +import { + createRandomAIUser, + switchDefaultChatModel, +} from '@affine-test/kit/utils/cloud'; import { openHomePage, setCoreUrl } from '@affine-test/kit/utils/load-page'; import { clickNewPageButton, @@ -58,7 +61,12 @@ export class TestUtils { await waitForEditorLoad(page); } - public async setupTestEnvironment(page: Page) { + public async setupTestEnvironment( + page: Page, + defaultModel = 'gemini-2.5-flash' + ) { + await switchDefaultChatModel(defaultModel); + await skipOnboarding(page.context()); await openHomePage(page); await this.createNewPage(page); diff --git a/tests/kit/src/utils/cloud.ts b/tests/kit/src/utils/cloud.ts index 230d443ed..4ef207dcf 100644 --- a/tests/kit/src/utils/cloud.ts +++ b/tests/kit/src/utils/cloud.ts @@ -152,6 +152,22 @@ export async function createRandomUser(): Promise<{ } as any; } +export async function switchDefaultChatModel(model: string) { + await runPrisma(async client => { + const promptId = await client.aiPrompt + .findFirst({ + where: { name: 'Chat With AFFiNE AI' }, + select: { id: true }, + }) + .then(f => f!.id); + + await client.aiPrompt.update({ + where: { id: promptId }, + data: { model }, + }); + }); +} + export async function createRandomAIUser(): Promise<{ name: string; email: string;