From d048ac6c91c4fbf40a436ee5a9772a0fb737ad3a Mon Sep 17 00:00:00 2001 From: akumatus Date: Sat, 18 Jan 2025 08:35:19 +0000 Subject: [PATCH] feat(core): support chat panel chips and suggest current doc for embedding (#9747) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support issue [BS-2347](https://linear.app/affine-design/issue/BS-2347). ## What Changed? - Add chat panel components - `chat-panel-chips` - `chat-panel-doc-chip` - `chat-panel-file-chip` - `chat-panel-chip` - Add `chips` and `docs` field in `ChatContextValue` - Add `extractMarkdownFromDoc` function to extract markdown content of a doc - Add e2e test Click a candidate card to add it into AI chat context:
🎥 Video uploaded on Graphite:
--- .../presets/ai/chat-panel/chat-config.ts | 17 +++ .../presets/ai/chat-panel/chat-context.ts | 46 +++++++ .../presets/ai/chat-panel/chat-panel-chips.ts | 59 ++++++++ .../presets/ai/chat-panel/chat-panel-input.ts | 13 +- .../presets/ai/chat-panel/components/chip.ts | 106 +++++++++++++++ .../ai/chat-panel/components/doc-chip.ts | 127 ++++++++++++++++++ .../ai/chat-panel/components/file-chip.ts | 31 +++++ .../presets/ai/chat-panel/components/utils.ts | 61 +++++++++ .../blocksuite/presets/ai/chat-panel/index.ts | 31 ++++- .../blocksuite/presets/ai/utils/extract.ts | 89 +++++++++--- .../presets/ai/utils/selection-utils.ts | 2 +- .../core/src/blocksuite/presets/effects.ts | 8 ++ .../pages/workspace/detail-page/tabs/chat.tsx | 21 ++- .../affine-cloud-copilot/e2e/copilot.spec.ts | 80 ++++++++++- 14 files changed, 656 insertions(+), 35 deletions(-) create mode 100644 packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-config.ts create mode 100644 packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts create mode 100644 packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/chip.ts create mode 100644 packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/doc-chip.ts create mode 100644 packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts create mode 100644 packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/utils.ts diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-config.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-config.ts new file mode 100644 index 000000000..4882e5ce0 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-config.ts @@ -0,0 +1,17 @@ +import type { Store } from '@blocksuite/store'; +import type { Signal } from '@preact/signals-core'; + +export interface AINetworkSearchConfig { + visible: Signal; + enabled: Signal; + setEnabled: (state: boolean) => void; +} + +export interface DocDisplayConfig { + getIcon: (docId: string) => any; + getTitle: (docId: string) => { + signal: Signal; + cleanup: () => void; + }; + getDoc: (docId: string) => Store | null; +} diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts index 41b49c506..c2c2eb126 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts @@ -24,13 +24,28 @@ export type ChatStatus = | 'idle' | 'transmitting'; +export interface DocContext { + docId: string; + plaintext?: string; + markdown?: string; + images?: File[]; +} + export type ChatContextValue = { + // history messages of the chat items: ChatItem[]; status: ChatStatus; error: AIError | null; + // plain-text of the selected content quote: string; + // markdown of the selected content markdown: string; + // images of the selected content or user uploaded images: File[]; + // chips of workspace doc or user uploaded file + chips: ChatChip[]; + // content of selected workspace doc + docs: DocContext[]; abortController: AbortController | null; chatSessionId: string | null; }; @@ -40,3 +55,34 @@ export type ChatBlockMessage = ChatMessage & { userName?: string; avatarUrl?: string; }; + +export type ChipState = + | 'candidate' + | 'uploading' + | 'embedding' + | 'success' + | 'failed'; + +export interface BaseChip { + /** + * candidate: the chip is a candidate for the chat + * uploading: the chip is uploading + * embedding: the chip is embedding + * success: the chip is successfully embedded + * failed: the chip is failed to embed + */ + state: ChipState; + tooltip?: string; +} + +export interface DocChip extends BaseChip { + docId: string; +} + +export interface FileChip extends BaseChip { + fileName: string; + fileId: string; + fileType: string; +} + +export type ChatChip = DocChip | FileChip; diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts new file mode 100644 index 000000000..966ce99f3 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts @@ -0,0 +1,59 @@ +import { + type EditorHost, + ShadowlessElement, +} from '@blocksuite/affine/block-std'; +import { WithDisposable } from '@blocksuite/affine/global/utils'; +import { css, html } from 'lit'; +import { property } from 'lit/decorators.js'; +import { repeat } from 'lit/directives/repeat.js'; + +import type { DocDisplayConfig } from './chat-config'; +import type { ChatContextValue } from './chat-context'; +import { getChipKey, isDocChip, isFileChip } from './components/utils'; + +export class ChatPanelChips extends WithDisposable(ShadowlessElement) { + static override styles = css` + .chip-list { + display: flex; + flex-wrap: wrap; + } + `; + + @property({ attribute: false }) + accessor host!: EditorHost; + + @property({ attribute: false }) + accessor chatContextValue!: ChatContextValue; + + @property({ attribute: false }) + accessor updateContext!: (context: Partial) => void; + + @property({ attribute: false }) + accessor docDisplayConfig!: DocDisplayConfig; + + override render() { + return html`
+ ${repeat( + this.chatContextValue.chips, + chip => getChipKey(chip), + chip => { + if (isDocChip(chip)) { + return html``; + } + if (isFileChip(chip)) { + return html``; + } + return null; + } + )} +
`; + } +} diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts index 38c46bd84..53b3399e1 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts @@ -8,7 +8,6 @@ import { } from '@blocksuite/affine/global/utils'; import { unsafeCSSVarV2 } from '@blocksuite/affine-shared/theme'; import { ImageIcon, PublishIcon } from '@blocksuite/icons/lit'; -import type { Signal } from '@preact/signals-core'; import { css, html, LitElement, nothing } from 'lit'; import { property, query, state } from 'lit/decorators.js'; import { repeat } from 'lit/directives/repeat.js'; @@ -22,6 +21,7 @@ import { import { AIProvider } from '../provider'; import { reportResponse } from '../utils/action-reporter'; import { readBlobAsURL } from '../utils/image'; +import type { AINetworkSearchConfig } from './chat-config'; import type { ChatContextValue, ChatMessage } from './chat-context'; const MaximumImageCount = 32; @@ -31,12 +31,6 @@ function getFirstTwoLines(text: string) { return lines.slice(0, 2); } -export interface AINetworkSearchConfig { - visible: Signal; - enabled: Signal; - setEnabled: (state: boolean) => void; -} - export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) { static override styles = css` .chat-panel-input { @@ -510,7 +504,7 @@ export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) { }; send = async (text: string) => { - const { status, markdown } = this.chatContextValue; + const { status, markdown, docs } = this.chatContextValue; if (status === 'loading' || status === 'transmitting') return; const { images } = this.chatContextValue; @@ -531,7 +525,8 @@ export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) { images?.map(image => readBlobAsURL(image)) ); - const content = (markdown ? `${markdown}\n` : '') + text; + const refDocs = docs.map(doc => doc.markdown).join('\n'); + const content = (markdown ? `${markdown}\n` : '') + `${refDocs}\n` + text; this.updateContext({ items: [ diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/chip.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/chip.ts new file mode 100644 index 000000000..d4f21d81d --- /dev/null +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/chip.ts @@ -0,0 +1,106 @@ +import { ShadowlessElement } from '@blocksuite/affine/block-std'; +import { SignalWatcher, WithDisposable } from '@blocksuite/affine/global/utils'; +import { CloseIcon } from '@blocksuite/icons/lit'; +import { css, html, type TemplateResult } from 'lit'; +import { property } from 'lit/decorators.js'; + +import type { ChipState } from '../chat-context'; + +export class ChatPanelChip extends SignalWatcher( + WithDisposable(ShadowlessElement) +) { + static override styles = css` + .chip-card { + display: flex; + align-items: center; + justify-content: center; + padding: 4px; + margin: 4px; + border-radius: 4px; + border: 0.5px solid var(--affine-border-color); + background: var(--affine-background-primary-color); + } + .chip-card[data-state='candidate'] { + border-width: 0.5px; + border-style: dashed; + background: var(--affine-background-secondary-color); + } + .chip-card[data-state='failed'] { + color: var(--affine-error-color); + background: var(--affine-background-error-color); + } + .chip-card[data-state='failed'] svg { + color: var(--affine-error-color); + } + .chip-card svg { + width: 16px; + height: 16px; + color: var(--affine-v2-icon-primary); + } + .chip-card-title { + display: inline-block; + margin: 0 4px; + font-size: 12px; + min-width: 16px; + max-width: 124px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + cursor: pointer; + } + .chip-card-close { + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + border-radius: 4px; + } + .chip-card-close:hover { + background: var(--affine-hover-color); + } + `; + + @property({ attribute: false }) + accessor state!: ChipState; + + @property({ attribute: false }) + accessor name!: string; + + @property({ attribute: false }) + accessor tooltip!: string; + + @property({ attribute: false }) + accessor icon!: TemplateResult<1>; + + @property({ attribute: false }) + accessor closeable: boolean = false; + + @property({ attribute: false }) + accessor onChipDelete: () => void = () => {}; + + @property({ attribute: false }) + accessor onChipClick: () => void = () => {}; + + override render() { + return html` +
+ ${this.icon} + + ${this.tooltip} + ${this.name} + + ${this.closeable + ? html` +
+ ${CloseIcon()} +
+ ` + : ''} +
+ `; + } +} diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/doc-chip.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/doc-chip.ts new file mode 100644 index 000000000..e56934af2 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/doc-chip.ts @@ -0,0 +1,127 @@ +import { + type EditorHost, + ShadowlessElement, +} from '@blocksuite/affine/block-std'; +import { SignalWatcher, WithDisposable } from '@blocksuite/affine/global/utils'; +import { Signal } from '@preact/signals-core'; +import { html } from 'lit'; +import { property } from 'lit/decorators.js'; + +import { extractMarkdownFromDoc } from '../../utils/extract'; +import type { DocDisplayConfig } from '../chat-config'; +import type { ChatContextValue, DocChip } from '../chat-context'; +import { getChipIcon, getChipTooltip, isDocChip } from './utils'; + +export class ChatPanelDocChip extends SignalWatcher( + WithDisposable(ShadowlessElement) +) { + @property({ attribute: false }) + accessor chip!: DocChip; + + @property({ attribute: false }) + accessor docDisplayConfig!: DocDisplayConfig; + + @property({ attribute: false }) + accessor host!: EditorHost; + + @property({ attribute: false }) + accessor chatContextValue!: ChatContextValue; + + @property({ attribute: false }) + accessor updateContext!: (context: Partial) => void; + + private name = new Signal(''); + + private cleanup?: () => any; + + override connectedCallback() { + super.connectedCallback(); + const { signal, cleanup } = this.docDisplayConfig.getTitle(this.chip.docId); + this.name = signal; + this.cleanup = cleanup; + } + + override disconnectedCallback() { + super.disconnectedCallback(); + this.cleanup?.(); + } + + private readonly onChipClick = () => { + if (this.chip.state === 'candidate') { + const doc = this.docDisplayConfig.getDoc(this.chip.docId); + if (!doc) { + return; + } + this.updateChipContext({ + state: 'embedding', + }); + extractMarkdownFromDoc(doc, this.host.std.provider) + .then(result => { + this.updateChipContext({ + state: 'success', + }); + this.updateContext({ + docs: [...this.chatContextValue.docs, result], + }); + }) + .catch(e => { + this.updateChipContext({ + state: 'failed', + tooltip: e.message, + }); + }); + } + }; + + private updateChipContext(options: Partial) { + const index = this.chatContextValue.chips.findIndex(item => { + return isDocChip(item) && item.docId === this.chip.docId; + }); + const nextChip: DocChip = { + ...this.chip, + ...options, + }; + this.updateContext({ + chips: [ + ...this.chatContextValue.chips.slice(0, index), + nextChip, + ...this.chatContextValue.chips.slice(index + 1), + ], + }); + } + + private readonly onChipDelete = () => { + if (this.chip.state === 'success') { + this.updateContext({ + docs: this.chatContextValue.docs.filter( + doc => doc.docId !== this.chip.docId + ), + }); + } + + this.updateContext({ + chips: this.chatContextValue.chips.filter( + chip => isDocChip(chip) && chip.docId !== this.chip.docId + ), + }); + }; + + override render() { + const { state, docId } = this.chip; + const isLoading = state === 'embedding' || state === 'uploading'; + const getIcon = this.docDisplayConfig.getIcon(docId); + const docIcon = typeof getIcon === 'function' ? getIcon() : getIcon; + const icon = getChipIcon(state, docIcon); + const tooltip = getChipTooltip(state, this.name.value, this.chip.tooltip); + + return html``; + } +} diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts new file mode 100644 index 000000000..789750790 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts @@ -0,0 +1,31 @@ +import { ShadowlessElement } from '@blocksuite/affine/block-std'; +import { SignalWatcher, WithDisposable } from '@blocksuite/affine/global/utils'; +import { getAttachmentFileIcons } from '@blocksuite/affine-components/icons'; +import { html } from 'lit'; +import { property } from 'lit/decorators.js'; + +import type { FileChip } from '../chat-context'; +import { getChipIcon, getChipTooltip } from './utils'; + +export class ChatPanelFileChip extends SignalWatcher( + WithDisposable(ShadowlessElement) +) { + @property({ attribute: false }) + accessor chip!: FileChip; + + override render() { + const { state, fileName, fileType } = this.chip; + const isLoading = state === 'embedding' || state === 'uploading'; + const tooltip = getChipTooltip(state, fileName, this.chip.tooltip); + const fileIcon = getAttachmentFileIcons(fileType); + const icon = getChipIcon(state, fileIcon); + + return html``; + } +} diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/utils.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/utils.ts new file mode 100644 index 000000000..9bf9e99dd --- /dev/null +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/utils.ts @@ -0,0 +1,61 @@ +import { WarningIcon } from '@blocksuite/icons/lit'; +import { type TemplateResult } from 'lit'; + +import { LoadingIcon } from '../../../blocks/_common/icon'; +import type { ChatChip, ChipState, DocChip, FileChip } from '../chat-context'; + +export function getChipTooltip( + state: ChipState, + title: string, + tooltip?: string +) { + if (tooltip) { + return tooltip; + } + if (state === 'candidate') { + return 'Click to add doc'; + } + if (state === 'embedding') { + return 'Embedding...'; + } + if (state === 'uploading') { + return 'Uploading...'; + } + if (state === 'failed') { + return 'Failed to embed'; + } + return title; +} + +export function getChipIcon( + state: ChipState, + icon: TemplateResult<1> +): TemplateResult<1> { + const isLoading = state === 'embedding' || state === 'uploading'; + const isFailed = state === 'failed'; + if (isFailed) { + return WarningIcon(); + } + if (isLoading) { + return LoadingIcon; + } + return icon; +} + +export function isDocChip(chip: ChatChip): chip is DocChip { + return 'docId' in chip; +} + +export function isFileChip(chip: ChatChip): chip is FileChip { + return 'fileId' in chip; +} + +export function getChipKey(chip: ChatChip) { + if (isDocChip(chip)) { + return chip.docId; + } + if (isFileChip(chip)) { + return chip.fileId; + } + return null; +} diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts index ba273ce5b..94a82bca3 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts @@ -17,8 +17,13 @@ import { getSelectedImagesAsBlobs, getSelectedTextContent, } from '../utils/selection-utils'; -import type { ChatAction, ChatContextValue, ChatItem } from './chat-context'; -import type { AINetworkSearchConfig } from './chat-panel-input'; +import type { AINetworkSearchConfig, DocDisplayConfig } from './chat-config'; +import type { + ChatAction, + ChatContextValue, + ChatItem, + DocChip, +} from './chat-context'; import type { ChatPanelMessages } from './chat-panel-messages'; export class ChatPanel extends WithDisposable(ShadowlessElement) { @@ -125,6 +130,13 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) { AIProvider.LAST_ROOT_SESSION_ID = history.sessionId; } + const { chips } = this.chatContextValue; + const defaultChip: DocChip = { + docId: this.doc.id, + state: 'candidate', + }; + const nextChips = + items.length === 0 && chips.length === 0 ? [defaultChip] : chips; this.chatContextValue = { ...this.chatContextValue, items: items.sort((a, b) => { @@ -132,6 +144,7 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) { new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime() ); }), + chips: nextChips, }; this.isLoading = false; @@ -148,6 +161,9 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) { @property({ attribute: false }) accessor networkSearchConfig!: AINetworkSearchConfig; + @property({ attribute: false }) + accessor docDisplayConfig!: DocDisplayConfig; + @state() accessor isLoading = false; @@ -157,6 +173,8 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) { images: [], abortController: null, items: [], + chips: [], + docs: [], status: 'idle', error: null, markdown: '', @@ -198,6 +216,9 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) { if (_changedProperties.has('doc')) { requestAnimationFrame(() => { this.chatContextValue.chatSessionId = null; + // TODO get from CopilotContext + this.chatContextValue.chips = []; + this.chatContextValue.docs = []; this._resetItems(); }); } @@ -281,6 +302,12 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) { .host=${this.host} .isLoading=${this.isLoading} > + | null> { if (!isInsideEdgelessEditor(host)) return null; @@ -113,21 +120,10 @@ async function extractEdgelessAll( }; } -async function extractPageAll( +export async function extractPageAll( host: EditorHost ): Promise | null> { - const notes = host.doc - .getBlocksByFlavour('affine:note') - .filter( - note => - (note.model as NoteBlockModel).displayMode !== - NoteDisplayMode.EdgelessOnly - ) - .map(note => note.model as NoteBlockModel); - const blockModels = notes.reduce((acc, note) => { - acc.push(...note.children); - return acc; - }, [] as BlockModel[]); + const blockModels = getNoteBlockModels(host.doc); const text = await getTextContentFromBlockModels( host, blockModels, @@ -156,3 +152,64 @@ async function extractPageAll( images, }; } + +export async function extractMarkdownFromDoc( + doc: Store, + provider: ServiceProvider +): Promise { + const transformer = await getTransformer(doc); + const adapter = new MarkdownAdapter(transformer, provider); + const blockModels = getNoteBlockModels(doc); + const textModels = blockModels.filter( + model => + !BlocksUtils.matchFlavours(model, ['affine:image', 'affine:database']) + ); + const drafts = textModels.map(toDraftModel); + drafts.forEach(draft => traverse(draft, drafts)); + const slice = Slice.fromModels(doc, drafts); + + const snapshot = transformer.sliceToSnapshot(slice); + if (!snapshot) { + throw new Error('Failed to extract markdown, snapshot is undefined'); + } + const content = await adapter.fromSliceSnapshot({ + snapshot, + assets: transformer.assetsManager, + }); + return { + docId: doc.id, + markdown: content.file, + }; +} + +function getNoteBlockModels(doc: Store) { + const notes = doc + .getBlocksByFlavour('affine:note') + .filter( + note => + (note.model as NoteBlockModel).displayMode !== + NoteDisplayMode.EdgelessOnly + ) + .map(note => note.model as NoteBlockModel); + const blockModels = notes.reduce((acc, note) => { + acc.push(...note.children); + return acc; + }, [] as BlockModel[]); + return blockModels; +} + +async function getTransformer(doc: Store) { + return new Transformer({ + schema: doc.workspace.schema, + blobCRUD: doc.workspace.blobSync, + docCRUD: { + create: (id: string) => doc.workspace.createDoc({ id }), + get: (id: string) => doc.workspace.getDoc(id), + delete: (id: string) => doc.workspace.removeDoc(id), + }, + middlewares: [ + titleMiddleware(doc.workspace.meta.docMetas), + embedSyncedDocMiddleware('content'), + ], + }); +} diff --git a/packages/frontend/core/src/blocksuite/presets/ai/utils/selection-utils.ts b/packages/frontend/core/src/blocksuite/presets/ai/utils/selection-utils.ts index 660991d95..8a6c156b9 100644 --- a/packages/frontend/core/src/blocksuite/presets/ai/utils/selection-utils.ts +++ b/packages/frontend/core/src/blocksuite/presets/ai/utils/selection-utils.ts @@ -105,7 +105,7 @@ export function getSelectedModels(editorHost: EditorHost) { return selectedModels; } -function traverse(model: DraftModel, drafts: DraftModel[]) { +export function traverse(model: DraftModel, drafts: DraftModel[]) { const isDatabase = model.flavour === 'affine:database'; const children = isDatabase ? model.children diff --git a/packages/frontend/core/src/blocksuite/presets/effects.ts b/packages/frontend/core/src/blocksuite/presets/effects.ts index 4b35112ef..fa8f6d1de 100644 --- a/packages/frontend/core/src/blocksuite/presets/effects.ts +++ b/packages/frontend/core/src/blocksuite/presets/effects.ts @@ -15,8 +15,12 @@ import { ActionMindmap } from './ai/chat-panel/actions/mindmap'; import { ActionSlides } from './ai/chat-panel/actions/slides'; import { ActionText } from './ai/chat-panel/actions/text'; import { AILoading } from './ai/chat-panel/ai-loading'; +import { ChatPanelChips } from './ai/chat-panel/chat-panel-chips'; import { ChatPanelInput } from './ai/chat-panel/chat-panel-input'; import { ChatPanelMessages } from './ai/chat-panel/chat-panel-messages'; +import { ChatPanelChip } from './ai/chat-panel/components/chip'; +import { ChatPanelDocChip } from './ai/chat-panel/components/doc-chip'; +import { ChatPanelFileChip } from './ai/chat-panel/components/file-chip'; import { AIErrorWrapper } from './ai/messages/error'; import { AISlidesRenderer } from './ai/messages/slides-renderer'; import { AIAnswerWrapper } from './ai/messages/wrapper'; @@ -57,6 +61,10 @@ export function registerBlocksuitePresetsCustomComponents() { customElements.define('chat-panel-input', ChatPanelInput); customElements.define('chat-panel-messages', ChatPanelMessages); customElements.define('chat-panel', ChatPanel); + customElements.define('chat-panel-chips', ChatPanelChips); + customElements.define('chat-panel-doc-chip', ChatPanelDocChip); + customElements.define('chat-panel-file-chip', ChatPanelFileChip); + customElements.define('chat-panel-chip', ChatPanelChip); customElements.define('ai-error-wrapper', AIErrorWrapper); customElements.define('ai-slides-renderer', AISlidesRenderer); customElements.define('ai-answer-wrapper', AIAnswerWrapper); diff --git a/packages/frontend/core/src/desktop/pages/workspace/detail-page/tabs/chat.tsx b/packages/frontend/core/src/desktop/pages/workspace/detail-page/tabs/chat.tsx index 63344a978..d0a70520e 100644 --- a/packages/frontend/core/src/desktop/pages/workspace/detail-page/tabs/chat.tsx +++ b/packages/frontend/core/src/desktop/pages/workspace/detail-page/tabs/chat.tsx @@ -1,10 +1,13 @@ import { ChatPanel } from '@affine/core/blocksuite/presets/ai'; import { AINetworkSearchService } from '@affine/core/modules/ai-button/services/network-search'; +import { DocDisplayMetaService } from '@affine/core/modules/doc-display-meta'; +import { WorkspaceService } from '@affine/core/modules/workspace'; import { DocModeProvider, RefNodeSlotsProvider, } from '@blocksuite/affine/blocks'; import type { AffineEditorContainer } from '@blocksuite/affine/presets'; +import { createSignalFromObservable } from '@blocksuite/affine-shared/utils'; import { useFramework } from '@toeverything/infra'; import { forwardRef, useEffect, useRef } from 'react'; @@ -49,12 +52,26 @@ export const EditorChatPanel = forwardRef(function EditorChatPanel( chatPanelRef.current.doc = editor.doc; containerRef.current?.append(chatPanelRef.current); const searchService = framework.get(AINetworkSearchService); - const networkSearchConfig = { + const docDisplayMetaService = framework.get(DocDisplayMetaService); + const workspaceService = framework.get(WorkspaceService); + chatPanelRef.current.networkSearchConfig = { visible: searchService.visible, enabled: searchService.enabled, setEnabled: searchService.setEnabled, }; - chatPanelRef.current.networkSearchConfig = networkSearchConfig; + chatPanelRef.current.docDisplayConfig = { + getIcon: (docId: string) => { + return docDisplayMetaService.icon$(docId, { type: 'lit' }).value; + }, + getTitle: (docId: string) => { + const title$ = docDisplayMetaService.title$(docId); + return createSignalFromObservable(title$, ''); + }, + getDoc: (docId: string) => { + const doc = workspaceService.workspace.docCollection.getDoc(docId); + return doc; + }, + }; } else { chatPanelRef.current.host = editor.host; chatPanelRef.current.doc = editor.doc; diff --git a/tests/affine-cloud-copilot/e2e/copilot.spec.ts b/tests/affine-cloud-copilot/e2e/copilot.spec.ts index 599aac392..55ee19c2b 100644 --- a/tests/affine-cloud-copilot/e2e/copilot.spec.ts +++ b/tests/affine-cloud-copilot/e2e/copilot.spec.ts @@ -86,6 +86,13 @@ const typeChat = async (page: Page, content: string) => { await page.keyboard.type(content); }; +const typeChatSequentially = async (page: Page, content: string) => { + const input = await page.locator('chat-panel-input textarea').nth(0); + await input.pressSequentially(content, { + delay: 50, + }); +}; + const makeChat = async (page: Page, content: string) => { await openChat(page); await typeChat(page, content); @@ -210,8 +217,8 @@ test.describe('chat panel', () => { await createLocalWorkspace({ name: 'test' }, page); await clickNewPageButton(page); const sendButton = await page.getByTestId('chat-panel-send'); - // oxlint-disable-next-line unicorn/prefer-dom-node-dataset await openChat(page); + // oxlint-disable-next-line unicorn/prefer-dom-node-dataset expect(await sendButton.getAttribute('aria-disabled')).toBe('true'); await typeChat(page, 'hello'); // oxlint-disable-next-line unicorn/prefer-dom-node-dataset @@ -399,11 +406,12 @@ test.describe('chat panel', () => { await page.getByTestId('modal-close-button').click(); await openChat(page); await page.getByTestId('chat-network-search').click(); - await makeChat(page, 'hello'); + await typeChatSequentially(page, 'What is the weather in Shanghai today?'); + await page.keyboard.press('Enter'); let history = await collectChat(page); expect(history[0]).toEqual({ name: 'You', - content: 'hello', + content: 'What is the weather in Shanghai today?', }); expect(history[1].name).toBe('AFFiNE AI'); expect( @@ -413,11 +421,12 @@ test.describe('chat panel', () => { await clearChat(page); expect((await collectChat(page)).length).toBe(0); await page.getByTestId('chat-network-search').click(); - await makeChat(page, 'hello'); + await typeChatSequentially(page, 'What is the weather in Shanghai today?'); + await page.keyboard.press('Enter'); history = await collectChat(page); expect(history[0]).toEqual({ name: 'You', - content: 'hello', + content: 'What is the weather in Shanghai today?', }); expect(history[1].name).toBe('AFFiNE AI'); expect(await page.locator('chat-panel affine-link').count()).toBe(0); @@ -704,3 +713,64 @@ test.describe('chat with block', () => { }); }); }); + +test.describe('chat with doc', () => { + let user: { + email: string; + password: string; + }; + + test.beforeEach(async ({ page }) => { + user = await getUser(); + await loginUser(page, user); + }); + + test('can chat with current doc', async ({ page }) => { + await page.reload(); + await clickSideBarAllPageButton(page); + await page.waitForTimeout(200); + await createLocalWorkspace({ name: 'test' }, page); + await clickNewPageButton(page); + + await openChat(page); + const chipTitle = await page.getByTestId('chat-panel-chip-title'); + expect(await chipTitle.textContent()).toBe('Untitled'); + + const editorTitle = await page.locator('doc-title .inline-editor').nth(0); + await editorTitle.pressSequentially('AFFiNE AI', { + delay: 50, + }); + await page.keyboard.press('Enter', { delay: 50 }); + // Wait for the editor to be ready and focused + await page.waitForSelector('page-editor affine-paragraph .inline-editor'); + const richText = await page + .locator('page-editor affine-paragraph .inline-editor') + .nth(0); + await richText.click(); // Ensure proper focus + await page.keyboard.type( + 'AFFiNE AI is an assistant with the ability to create well-structured outlines for any given content.', + { + delay: 50, + } + ); + + expect(await chipTitle.textContent()).toBe('AFFiNE AI'); + const chip = await page.getByTestId('chat-panel-chip'); + // oxlint-disable-next-line unicorn/prefer-dom-node-dataset + expect(await chip.getAttribute('data-state')).toBe('candidate'); + await chip.click(); + // oxlint-disable-next-line unicorn/prefer-dom-node-dataset + expect(await chip.getAttribute('data-state')).toBe('success'); + + await makeChat(page, 'summarize'); + const history = await collectChat(page); + expect(history[0]).toEqual({ + name: 'You', + content: + 'AFFiNE AI is an assistant with the ability to create well-structured outlines for any given content.\nsummarize', + }); + expect(history[1].name).toBe('AFFiNE AI'); + await clearChat(page); + expect((await collectChat(page)).length).toBe(0); + }); +});