test(core): embedding settings (#11554)

### TL;DR

tests: workspace embedding e2e

> CLOSE BS-3052

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->

## Summary by CodeRabbit

- **New Features**
  - Introduced comprehensive end-to-end tests for workspace embedding settings, including toggling embedding, uploading and managing attachments, pagination, and ignoring documents.
  - Added utilities for automated interaction with the settings panel and document creation in tests.

- **Tests**
  - Implemented detailed scenarios to verify workspace embedding functionality and user interactions within the settings panel.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

tests/affine-cloud-copilot/e2e/base/base-test.ts

@@ -4,6 +4,7 @@ import type { Page } from '@playwright/test';
 
 import { ChatPanelUtils } from '../utils/chat-panel-utils';
 import { EditorUtils } from '../utils/editor-utils';
+import { SettingsPanelUtils } from '../utils/settings-panel-utils';
 import { TestUtils } from '../utils/test-utils';
 
 interface TestUtilsFixtures {
@@ -11,6 +12,7 @@ interface TestUtilsFixtures {
     testUtils: TestUtils;
     chatPanel: typeof ChatPanelUtils;
     editor: typeof EditorUtils;
+    settings: typeof SettingsPanelUtils;
   };
   loggedInPage: Page;
 }
@@ -22,6 +24,7 @@ export const test = base.extend<TestUtilsFixtures>({
       testUtils,
       chatPanel: ChatPanelUtils,
       editor: EditorUtils,
+      settings: SettingsPanelUtils,
     });
   },
   loggedInPage: async ({ browser }, use) => {

tests/affine-cloud-copilot/e2e/settings/embedding.spec.ts

@@ -0,0 +1,302 @@
+import { expect } from '@playwright/test';
+
+import { test } from '../base/base-test';
+
+test.describe.configure({ mode: 'serial' });
+
+test.describe.skip('AISettings/Embedding', () => {
+  test.beforeEach(async ({ loggedInPage: page, utils }) => {
+    await utils.testUtils.setupTestEnvironment(page);
+    await utils.chatPanel.openChatPanel(page);
+    await utils.settings.openSettingsPanel(page);
+  });
+
+  test.afterEach(async ({ loggedInPage: page, utils }) => {
+    await utils.settings.openSettingsPanel(page);
+    await utils.settings.clearAllIgnoredDocs(page);
+    await utils.settings.removeAllAttachments(page);
+    await utils.settings.closeSettingsPanel(page);
+  });
+
+  test('should show workspace embedding enabled status', async ({
+    loggedInPage: page,
+    utils,
+  }) => {
+    await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, true);
+  });
+
+  test('should support disable workspace embedding', async ({
+    loggedInPage: page,
+    utils,
+  }) => {
+    await utils.settings.enableWorkspaceEmbedding(page);
+    await utils.settings.disableWorkspaceEmbedding(page);
+    await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, false);
+  });
+
+  test('should support enable workspace embedding', async ({
+    loggedInPage: page,
+    utils,
+  }) => {
+    await utils.settings.disableWorkspaceEmbedding(page);
+    await utils.settings.enableWorkspaceEmbedding(page);
+    await utils.settings.waitForWorkspaceEmbeddingSwitchToBe(page, true);
+  });
+
+  test('should allow manual attachment upload for embedding', async ({
+    loggedInPage: page,
+    utils,
+  }) => {
+    await utils.settings.enableWorkspaceEmbedding(page);
+    const textContent1 = 'WorkspaceEBEEE is a cute cat';
+    const textContent2 = 'WorkspaceEBFFF is a cute dog';
+    const buffer1 = Buffer.from(textContent1);
+    const buffer2 = Buffer.from(textContent2);
+    const attachments = [
+      {
+        name: 'document1.txt',
+        mimeType: 'text/plain',
+        buffer: buffer1,
+      },
+      {
+        name: 'document2.txt',
+        mimeType: 'text/plain',
+        buffer: buffer2,
+      },
+    ];
+    await utils.settings.uploadWorkspaceEmbedding(page, attachments);
+
+    const attachmentList = await page.getByTestId(
+      'workspace-embedding-setting-attachment-list'
+    );
+    await expect(
+      attachmentList.getByTestId('workspace-embedding-setting-attachment-item')
+    ).toHaveCount(2);
+
+    await utils.settings.closeSettingsPanel(page);
+
+    await page.waitForTimeout(5000); // wait for the embedding to be ready
+
+    await utils.chatPanel.makeChat(
+      page,
+      'What is WorkspaceEBEEE? What is WorkspaceEBFFF?'
+    );
+
+    await utils.chatPanel.waitForHistory(page, [
+      {
+        role: 'user',
+        content: 'What is WorkspaceEBEEE? What is WorkspaceEBFFF?',
+      },
+      {
+        role: 'assistant',
+        status: 'success',
+      },
+    ]);
+
+    await expect(async () => {
+      const { content, message } =
+        await utils.chatPanel.getLatestAssistantMessage(page);
+      expect(content).toMatch(/WorkspaceEBEEE.*cat/);
+      expect(content).toMatch(/WorkspaceEBFFF.*dog/);
+      expect(await message.locator('affine-footnote-node').count()).toBe(2);
+    }).toPass({ timeout: 20000 });
+  });
+
+  test('should support hybrid search for both globally uploaded attachments and those uploaded in the current session', async ({
+    loggedInPage: page,
+    utils,
+  }) => {
+    await utils.settings.enableWorkspaceEmbedding(page);
+    const hobby1 = Buffer.from('Jerry-Affine love climbing');
+    const hobby2 = Buffer.from('Jerry-Affine love skating');
+    const attachments = [
+      {
+        name: 'jerry-affine-hobby.txt',
+        mimeType: 'text/plain',
+        buffer: hobby1,
+      },
+    ];
+    await utils.settings.uploadWorkspaceEmbedding(page, attachments);
+
+    const attachmentList = await page.getByTestId(
+      'workspace-embedding-setting-attachment-list'
+    );
+    await expect(
+      attachmentList.getByTestId('workspace-embedding-setting-attachment-item')
+    ).toHaveCount(1);
+
+    await utils.settings.closeSettingsPanel(page);
+
+    await page.waitForTimeout(5000); // wait for the embedding to be ready
+
+    await utils.chatPanel.chatWithAttachments(
+      page,
+      [
+        {
+          name: 'jerry-affine-hobby2.txt',
+          mimeType: 'text/plain',
+          buffer: hobby2,
+        },
+      ],
+      'What is Jerry-Affine hobby?'
+    );
+
+    await utils.chatPanel.waitForHistory(page, [
+      {
+        role: 'user',
+        content: 'What is Jerry-Affine hobby?',
+      },
+      {
+        role: 'assistant',
+        status: 'success',
+      },
+    ]);
+
+    await expect(async () => {
+      const { content, message } =
+        await utils.chatPanel.getLatestAssistantMessage(page);
+      expect(content).toMatch(/climbing/i);
+      expect(content).toMatch(/skating/i);
+      expect(await message.locator('affine-footnote-node').count()).toBe(2);
+    }).toPass({ timeout: 20000 });
+  });
+
+  test('should support attachments pagination', async ({
+    loggedInPage: page,
+    utils,
+  }) => {
+    await utils.settings.enableWorkspaceEmbedding(page);
+    const attachments = Array.from({ length: 11 }, (_, i) => ({
+      name: `document${i + 1}.txt`,
+      mimeType: 'text/plain',
+      buffer: Buffer.from('attachment content'),
+    }));
+
+    await utils.settings.uploadWorkspaceEmbedding(page, attachments);
+
+    const attachmentList = await page.getByTestId(
+      'workspace-embedding-setting-attachment-list'
+    );
+
+    await expect(
+      attachmentList.getByTestId('workspace-embedding-setting-attachment-item')
+    ).toHaveCount(10);
+    const pagination = await attachmentList.getByRole('navigation');
+    const currentPage = await pagination.locator('li.active');
+    await expect(currentPage).toHaveText('1');
+
+    const page2 = await pagination.locator('li').nth(2);
+    await page2.click();
+
+    await expect(
+      attachmentList.getByTestId('workspace-embedding-setting-attachment-item')
+    ).toHaveCount(1);
+    await expect(
+      attachmentList
+        .getByTestId('workspace-embedding-setting-attachment-item')
+        .first()
+    ).toHaveText('document1.txt');
+  });
+
+  test('should support remove attachment', async ({
+    loggedInPage: page,
+    utils,
+  }) => {
+    await utils.settings.enableWorkspaceEmbedding(page);
+    const textContent = 'WorkspaceEBEEE is a cute cat';
+    const attachments = [
+      {
+        name: 'document1.txt',
+        mimeType: 'text/plain',
+        buffer: Buffer.from(textContent),
+      },
+    ];
+    await utils.settings.uploadWorkspaceEmbedding(page, attachments);
+
+    const attachmentList = await page.getByTestId(
+      'workspace-embedding-setting-attachment-list'
+    );
+    await expect(
+      attachmentList.getByTestId('workspace-embedding-setting-attachment-item')
+    ).toHaveCount(1);
+    await utils.settings.removeAttachment(page, 'document1.txt');
+  });
+
+  // FIXME: wait for indexer
+  test.skip('should support ignore docs for embedding', async ({
+    loggedInPage: page,
+    utils,
+  }) => {
+    await utils.settings.enableWorkspaceEmbedding(page);
+    await utils.settings.closeSettingsPanel(page);
+    await utils.editor.createDoc(
+      page,
+      'WBIgnoreDoc1',
+      'WBIgnoreEEE is a cute cat'
+    );
+    await utils.editor.createDoc(
+      page,
+      'WBIgnoreDoc2',
+      'WBIgnoreFFF is a cute dog'
+    );
+
+    await page.waitForTimeout(5000); // wait for the embedding to be ready
+
+    await utils.chatPanel.makeChat(
+      page,
+      'What is WBIgnoreEEE? What is WBIgnoreFFF?If you dont know, just say "I dont know"'
+    );
+
+    await utils.chatPanel.waitForHistory(page, [
+      {
+        role: 'user',
+        content:
+          'What is WBIgnoreEEE? What is WBIgnoreFFF?If you dont know, just say "I dont know"',
+      },
+      {
+        role: 'assistant',
+        status: 'success',
+      },
+    ]);
+
+    await expect(async () => {
+      const { content, message } =
+        await utils.chatPanel.getLatestAssistantMessage(page);
+      expect(content).toMatch(/WBIgnoreEEE.*cat/);
+      expect(content).toMatch(/WBIgnoreFFF.*dog/);
+      expect(await message.locator('affine-footnote-node').count()).toBe(2);
+    }).toPass({ timeout: 20000 });
+
+    // Ignore docs
+    await utils.settings.openSettingsPanel(page);
+    await utils.settings.ignoreDocForEmbedding(page, 'WBIgnoreDoc1');
+    await utils.settings.ignoreDocForEmbedding(page, 'WBIgnoreDoc2');
+
+    await utils.settings.closeSettingsPanel(page);
+
+    // Clear history
+    await utils.chatPanel.clearChat(page);
+
+    // Ignored docs should not be used for embedding
+    await utils.chatPanel.makeChat(
+      page,
+      'What is WBIgnoreEEE? What is WBIgnoreFFF?If you dont know, just say "I dont know"'
+    );
+
+    await utils.chatPanel.waitForHistory(page, [
+      {
+        role: 'user',
+        content: 'What is WBIgnoreEEE? What is WBIgnoreFFF?',
+      },
+      {
+        role: 'assistant',
+        status: 'success',
+      },
+    ]);
+
+    await expect(async () => {
+      const { content } = await utils.chatPanel.getLatestAssistantMessage(page);
+      expect(content).toMatch(/I dont know/i);
+    }).toPass({ timeout: 20000 });
+  });
+});

tests/affine-cloud-copilot/e2e/utils/editor-utils.ts

@@ -6,7 +6,10 @@ import {
   pressEscape,
   selectAllByKeyboard,
 } from '@affine-test/kit/utils/keyboard';
-import { getBlockSuiteEditorTitle } from '@affine-test/kit/utils/page-logic';
+import {
+  clickNewPageButton,
+  getBlockSuiteEditorTitle,
+} from '@affine-test/kit/utils/page-logic';
 import type { EdgelessRootBlockComponent } from '@blocksuite/affine/blocks/root';
 import type {
   MindmapElementModel,
@@ -288,6 +291,19 @@ export class EditorUtils {
     await page.waitForTimeout(100);
   }
 
+  public static async createDoc(page: Page, title: string, docContent: string) {
+    await clickNewPageButton(page);
+    await page.keyboard.insertText(title);
+    await this.focusToEditor(page);
+    const texts = docContent.split('\n');
+    for (const [index, line] of texts.entries()) {
+      await page.keyboard.insertText(line);
+      if (index !== texts.length - 1) {
+        await page.keyboard.press('Enter');
+      }
+    }
+  }
+
   public static async createCollectionAndDoc(
     page: Page,
     collectionName: string,

tests/affine-cloud-copilot/e2e/utils/settings-panel-utils.ts

@@ -0,0 +1,175 @@
+import { expect, type Page } from '@playwright/test';
+
+const WORKSPACE_EMBEDDING_SWITCH_TEST_ID = 'workspace-embedding-setting-switch';
+
+export class SettingsPanelUtils {
+  public static async openSettingsPanel(page: Page) {
+    if (
+      await page.getByTestId('workspace-setting:indexer-embedding').isHidden()
+    ) {
+      await page.getByTestId('slider-bar-workspace-setting-button').click();
+      await page.getByTestId('workspace-setting:indexer-embedding').click();
+      await page.getByTestId('workspace-embedding-setting-wrapper').waitFor({
+        state: 'visible',
+      });
+    }
+  }
+
+  public static async closeSettingsPanel(page: Page) {
+    if (
+      await page.getByTestId('workspace-embedding-setting-wrapper').isVisible()
+    ) {
+      await page.getByTestId('modal-close-button').click();
+      await page.getByTestId('workspace-embedding-setting-wrapper').waitFor({
+        state: 'hidden',
+      });
+    }
+  }
+
+  public static async isWorkspaceEmbeddingEnabled(page: Page) {
+    const input = await page
+      .getByTestId(WORKSPACE_EMBEDDING_SWITCH_TEST_ID)
+      .locator('input');
+    return (await input.getAttribute('value')) === 'on';
+  }
+
+  public static async waitForWorkspaceEmbeddingSwitchToBe(
+    page: Page,
+    enabled: boolean
+  ) {
+    const input = await page
+      .getByTestId(WORKSPACE_EMBEDDING_SWITCH_TEST_ID)
+      .locator('input');
+    await expect(input).toHaveAttribute('value', enabled ? 'on' : 'off');
+  }
+
+  public static async toggleWorkspaceEmbedding(page: Page) {
+    const input = await page.getByTestId(WORKSPACE_EMBEDDING_SWITCH_TEST_ID);
+    await input.click();
+  }
+
+  public static async enableWorkspaceEmbedding(page: Page) {
+    const enabled = await this.isWorkspaceEmbeddingEnabled(page);
+    if (!enabled) {
+      await this.toggleWorkspaceEmbedding(page);
+    }
+    await this.waitForWorkspaceEmbeddingSwitchToBe(page, true);
+  }
+
+  public static async disableWorkspaceEmbedding(page: Page) {
+    const enabled = await this.isWorkspaceEmbeddingEnabled(page);
+    if (enabled) {
+      await this.toggleWorkspaceEmbedding(page);
+    }
+    await this.waitForWorkspaceEmbeddingSwitchToBe(page, false);
+  }
+
+  public static async uploadWorkspaceEmbedding(
+    page: Page,
+    attachments: { name: string; mimeType: string; buffer: Buffer }[]
+  ) {
+    await page.evaluate(() => {
+      delete window.showOpenFilePicker;
+    });
+
+    for (const attachment of attachments) {
+      const fileChooserPromise = page.waitForEvent('filechooser');
+      await page
+        .getByTestId('workspace-embedding-setting-upload-button')
+        .click();
+
+      const fileChooser = await fileChooserPromise;
+      await fileChooser.setFiles(attachment);
+    }
+  }
+
+  public static async removeAllAttachments(page: Page) {
+    const itemId = 'workspace-embedding-setting-attachment-item';
+    let count = await page.getByTestId(itemId).count();
+
+    while (count > 0) {
+      const attachmentItem = await page.getByTestId(itemId).first();
+      await attachmentItem
+        .getByTestId('workspace-embedding-setting-attachment-delete-button')
+        .click();
+      await page.getByTestId('confirm-modal-confirm').click();
+      await page.waitForTimeout(1000);
+      count = await page.getByTestId(itemId).count();
+    }
+  }
+
+  public static async removeAttachment(page: Page, attachment: string) {
+    const attachmentItem = await page
+      .getByTestId('workspace-embedding-setting-attachment-item')
+      .filter({ hasText: attachment });
+    await attachmentItem
+      .getByTestId('workspace-embedding-setting-attachment-delete-button')
+      .click();
+    await page.getByTestId('confirm-modal-confirm').click();
+    await page
+      .getByTestId('workspace-embedding-setting-attachment-item')
+      .filter({ hasText: attachment })
+      .waitFor({
+        state: 'hidden',
+      });
+  }
+
+  public static async ignoreDocForEmbedding(page: Page, doc: string) {
+    // Open Dos Searcher
+    const ignoreDocsButton = await page.getByTestId(
+      'workspace-embedding-setting-ignore-docs-button'
+    );
+    await ignoreDocsButton.click();
+    // Search and select the doc
+    const searcher = await page.getByTestId('doc-selector-layout');
+    const searchInput = await page.getByTestId('doc-selector-search-input');
+
+    await searchInput.focus();
+    await page.keyboard.insertText(doc);
+
+    const pageListItem = searcher.getByTestId('page-list-item');
+    await expect(pageListItem).toHaveCount(1);
+    const pageListItemTitle = pageListItem.getByTestId(
+      'page-list-item-title-text'
+    );
+    await expect(pageListItemTitle).toHaveText(doc);
+
+    await pageListItem.getByTestId('affine-checkbox').check();
+
+    await searcher.getByTestId('doc-selector-confirm-button').click();
+
+    const ignoredDocs = await page.getByTestId(
+      'workspace-embedding-setting-ignore-docs-list'
+    );
+    await expect(
+      ignoredDocs
+        .getByTestId('workspace-embedding-setting-ignore-docs-list-item')
+        .filter({ hasText: doc })
+    ).toBeVisible();
+  }
+
+  public static async clearAllIgnoredDocs(page: Page) {
+    const ignoredDocs = await page.getByTestId('ignore-doc-title').all();
+    for (const ignoredDoc of ignoredDocs) {
+      const doc = await ignoredDoc.innerText();
+      // Open Dos Searcher
+      const ignoreDocsButton = await page.getByTestId(
+        'workspace-embedding-setting-ignore-docs-button'
+      );
+      await ignoreDocsButton.click();
+      // Search and select the doc
+      const searcher = await page.getByTestId('doc-selector-layout');
+      const searchInput = await page.getByTestId('doc-selector-search-input');
+
+      await searchInput.focus();
+      await page.keyboard.insertText(doc);
+
+      const pageListItem = searcher.getByTestId('page-list-item');
+      await expect(pageListItem).toHaveCount(1);
+
+      await pageListItem.getByTestId('affine-checkbox').uncheck();
+
+      await searcher.getByTestId('doc-selector-confirm-button').click();
+    }
+  }
+}