feat(core): continue with AI (#7253)

Closes: [AFF-1251](https://linear.app/affine-design/issue/AFF-1251/continue-with-ai-的-action-的入口区分-open-with-ai) [BS-515](https://linear.app/affine-design/issue/BS-515/添加白板选区的-continue-with-ai-支持) [AFF-1256](https://linear.app/affine-design/issue/AFF-1256/continue-with-ai-当只有不支持的-block-时，需要特殊处理) [AF-919](https://linear.app/affine-design/issue/AF-919/内容为空时，不需要显示-start-with-this-doc)

* add the selected element to the candidate card list
* select the candidate card to hide the candidate card list and add quote into input
* close quote to show the candidate card list
* show only the three newest candidates
* **`Start with this doc`**: Currently only determines if a document has content after the first load

https://github.com/toeverything/AFFiNE/assets/27926/d19c8ab6-37eb-495f-9c38-e579b2f57000

https://github.com/toeverything/AFFiNE/assets/27926/3ba654c3-6af4-4662-a641-17cfe2ed5ff7

packages/frontend/core/src/blocksuite/presets/ai/_common/config.ts

@@ -36,6 +36,7 @@ import {
   AISearchIcon,
   AIStarIconWithAnimation,
   ChatWithAIIcon,
+  CommentIcon,
   ExplainIcon,
   ImproveWritingIcon,
   LanguageIcon,
@@ -395,6 +396,15 @@ const GenerateWithAIGroup: AIItemGroupConfig = {
 const OthersAIGroup: AIItemGroupConfig = {
   name: 'Others',
   items: [
+    {
+      name: 'Continue with AI',
+      icon: CommentIcon,
+      handler: host => {
+        const panel = getAIPanel(host);
+        AIProvider.slots.requestContinueWithAIInChat.emit({ host });
+        panel.hide();
+      },
+    },
     {
       name: 'Open AI Chat',
       icon: ChatWithAIIcon,

packages/frontend/core/src/blocksuite/presets/ai/_common/icons.ts

@@ -1061,3 +1061,17 @@ export const MoreIcon = html`<svg
     </clipPath>
   </defs>
 </svg> `;
+
+export const CommentIcon = html`<svg
+  width="20"
+  height="20"
+  viewBox="0 0 20 20"
+  fill="currentColor"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <path
+    fill-rule="evenodd"
+    clip-rule="evenodd"
+    d="M10.4167 3.125C6.84987 3.125 3.95837 6.01649 3.95837 9.58333C3.95837 10.5626 4.1759 11.4893 4.56469 12.3193C4.62513 12.4484 4.64581 12.5938 4.62202 12.7365L4.09372 15.9063L7.26351 15.378C7.40628 15.3542 7.55167 15.3749 7.68071 15.4354C8.51071 15.8241 9.43744 16.0417 10.4167 16.0417C13.9835 16.0417 16.875 13.1502 16.875 9.58333C16.875 6.01649 13.9835 3.125 10.4167 3.125ZM2.70837 9.58333C2.70837 5.32614 6.15951 1.875 10.4167 1.875C14.6739 1.875 18.125 5.32614 18.125 9.58333C18.125 13.8405 14.6739 17.2917 10.4167 17.2917C9.31104 17.2917 8.25828 17.0585 7.30619 16.6382L3.5512 17.264C3.07179 17.3439 2.65615 16.9283 2.73606 16.4488L3.36189 12.6939C2.94154 11.7418 2.70837 10.689 2.70837 9.58333Z"
+  />
+</svg>`;

packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-cards.ts

@@ -1,12 +1,10 @@
-import type { BaseSelection, EditorHost } from '@blocksuite/block-std';
+import type { EditorHost } from '@blocksuite/block-std';
 import { WithDisposable } from '@blocksuite/block-std';
 import {
-  type CopilotSelectionController,
   type ImageBlockModel,
   type NoteBlockModel,
   NoteDisplayMode,
 } from '@blocksuite/blocks';
-import { debounce } from '@blocksuite/global/utils';
 import type { BlockModel } from '@blocksuite/store';
 import { css, html, LitElement, nothing, type PropertyValues } from 'lit';
 import { customElement, property, state } from 'lit/decorators.js';
@@ -18,8 +16,8 @@ import {
   DocIcon,
   SmallImageIcon,
 } from '../_common/icons';
+import { AIProvider } from '../provider';
 import {
-  getEdgelessRootFromEditor,
   getSelectedImagesAsBlobs,
   getSelectedTextContent,
   getTextContentFromBlockModels,
@@ -54,15 +52,68 @@ const cardsStyles = css`
   }
 `;
 
-const ChatCardsConfig = [
+enum CardType {
-  {
+  Text,
-    name: 'current-selection',
+  Image,
-    render: (text?: string, _?: File, __?: string) => {
+  Block,
-      if (!text) return nothing;
+  Doc,
+}
 
+type CardBase = {
+  id: number;
+};
+
+type CardText = CardBase & {
+  type: CardType.Text;
+  text: string;
+  markdown: string;
+};
+
+type CardImage = CardBase & {
+  type: CardType.Image;
+  image: File;
+  caption?: string;
+};
+
+type CardBlock = CardBase & {
+  type: CardType.Block | CardType.Doc;
+  text?: string;
+  markdown?: string;
+  images?: File[];
+};
+
+type Card = CardText | CardImage | CardBlock;
+
+const MAX_CARDS = 3;
+
+@customElement('chat-cards')
+export class ChatCards extends WithDisposable(LitElement) {
+  static override styles = css`
+    :host {
+      display: flex;
+      flex-direction: column;
+      gap: 12px;
+    }
+
+    ${cardsStyles}
+  `;
+
+  @property({ attribute: false })
+  accessor host!: EditorHost;
+
+  @property({ attribute: false })
+  accessor updateContext!: (context: Partial<ChatContextValue>) => void;
+
+  @state()
+  accessor cards: Card[] = [];
+
+  private _selectedCardId: number = 0;
+
+  static renderText({ text }: CardText) {
     const lines = text.split('\n');
 
-      return html`<div class="card-wrapper">
+    return html`
+      <div class="card-wrapper">
         <div class="card-title">
           ${CurrentSelectionIcon}
           <div>Start with current selection</div>
@@ -71,8 +122,8 @@ const ChatCardsConfig = [
           ${repeat(
             lines.slice(0, 2),
             line => line,
-            line => {
+            line => html`
-              return html`<div
+              <div
                 style=${styleMap({
                   overflow: 'hidden',
                   textOverflow: 'ellipsis',
@@ -80,34 +131,17 @@ const ChatCardsConfig = [
                 })}
               >
                 ${line}
-              </div>`;
+              </div>
-            }
+            `
           )}
         </div>
-      </div> `;
+      </div>
-    },
+    `;
-    handler: (
-      updateContext: (context: Partial<ChatContextValue>) => void,
-      text: string,
-      markdown: string,
-      images?: File[]
-    ) => {
-      const value: Partial<ChatContextValue> = {
-        quote: text,
-        markdown: markdown,
-      };
-      if (images) {
-        value.images = images;
   }
-      updateContext(value);
-    },
-  },
-  {
-    name: 'image',
-    render: (_?: string, image?: File, caption?: string) => {
-      if (!image) return nothing;
 
-      return html`<div
+  static renderImage({ caption, image }: CardImage) {
+    return html`
+      <div
         class="card-wrapper"
         style=${styleMap({
           display: 'flex',
@@ -134,24 +168,11 @@ const ChatCardsConfig = [
           })}
           src="${URL.createObjectURL(image)}"
         />
-      </div>`;
+      </div>
-    },
+    `;
-    handler: (
-      updateContext: (context: Partial<ChatContextValue>) => void,
-      _: string,
-      __: string,
-      images?: File[]
-    ) => {
-      const value: Partial<ChatContextValue> = {};
-      if (images) {
-        value.images = images;
   }
-      updateContext(value);
+
-    },
+  static renderDoc(_: CardBlock) {
-  },
-  {
-    name: 'doc',
-    render: () => {
     return html`
       <div class="card-wrapper">
         <div class="card-title">
@@ -161,115 +182,140 @@ const ChatCardsConfig = [
         <div class="second-text">you've chosen within the doc</div>
       </div>
     `;
-    },
-    handler: (
-      updateContext: (context: Partial<ChatContextValue>) => void,
-      text: string,
-      markdown: string,
-      images?: File[]
-    ) => {
-      const value: Partial<ChatContextValue> = {
-        quote: text,
-        markdown: markdown,
-      };
-      if (images) {
-        value.images = images;
   }
-      updateContext(value);
-    },
-  },
-];
 
-@customElement('chat-cards')
+  private _renderCard(card: Card) {
-export class ChatCards extends WithDisposable(LitElement) {
+    if (card.type === CardType.Text) {
-  static override styles = css`
+      return ChatCards.renderText(card);
-    ${cardsStyles}
-    .cards-container {
-      display: flex;
-      flex-direction: column;
-      gap: 12px;
     }
-  `;
 
-  @property({ attribute: false })
+    if (card.type === CardType.Image) {
-  accessor host!: EditorHost;
+      return ChatCards.renderImage(card);
+    }
 
-  @property({ attribute: false })
+    if (card.type === CardType.Doc) {
-  accessor chatContextValue!: ChatContextValue;
+      return ChatCards.renderDoc(card);
+    }
 
-  @property({ attribute: false })
+    return nothing;
-  accessor updateContext!: (context: Partial<ChatContextValue>) => void;
+  }
 
-  @property({ attribute: false })
+  private _updateCards(card: Card) {
-  accessor selectionValue: BaseSelection[] = [];
+    this.cards.unshift(card);
 
-  @state()
+    if (this.cards.length > MAX_CARDS) {
-  accessor text: string = '';
+      this.cards.pop();
+    }
 
-  @state()
+    this.requestUpdate();
-  accessor markdown: string = '';
+  }
 
-  @state()
+  private async _handleDocSelection(card: CardBlock) {
-  accessor images: File[] = [];
+    const { text, markdown, images } = await this._extractAll();
 
-  @state()
+    card.text = text;
-  accessor caption: string = '';
+    card.markdown = markdown;
+    card.images = images;
+  }
 
-  private _onEdgelessCopilotAreaUpdated() {
+  private async _handleClick(card: Card) {
-    if (!this.host.closest('edgeless-editor')) return;
+    AIProvider.slots.toggleChatCards.emit({ visible: false });
-    const edgeless = getEdgelessRootFromEditor(this.host);
 
-    const copilotSelectionTool = edgeless.tools.controllers
+    this._selectedCardId = card.id;
-      .copilot as CopilotSelectionController;
 
-    this._disposables.add(
+    switch (card.type) {
-      copilotSelectionTool.draggingAreaUpdated.on(
+      case CardType.Text: {
-        debounce(() => {
+        this.updateContext({
-          selectedToCanvas(this.host)
+          quote: card.text,
-            .then(canvas => {
+          markdown: card.markdown,
-              canvas?.toBlob(blob => {
-                if (!blob) return;
-                const file = new File([blob], 'selected.png');
-                this.images = [file];
         });
-            })
+        break;
-            .catch(console.error);
+      }
-        }, 300)
+      case CardType.Image: {
-      )
+        this.updateContext({
-    );
+          images: [card.image],
+        });
+        break;
+      }
+      case CardType.Doc: {
+        await this._handleDocSelection(card);
+        this.updateContext({
+          quote: card.text,
+          markdown: card.markdown,
+          images: card.images,
+        });
+        break;
+      }
+    }
   }
 
-  private async _updateState() {
+  private async _extract() {
-    if (
+    const text = await getSelectedTextContent(this.host, 'plain-text');
-      this.selectionValue.some(
+    const images = await getSelectedImagesAsBlobs(this.host);
-        selection => selection.is('text') || selection.is('image')
+    const hasText = text.length > 0;
-      )
+    const hasImages = images.length > 0;
-    )
+
+    if (hasText && !hasImages) {
+      const markdown = await getSelectedTextContent(this.host, 'markdown');
+
+      this._updateCards({
+        id: Date.now(),
+        type: CardType.Text,
+        text,
+        markdown,
+      });
+
       return;
-    this.text = await getSelectedTextContent(this.host, 'plain-text');
+    }
-    this.markdown = await getSelectedTextContent(this.host, 'markdown');
+
-    this.images = await getSelectedImagesAsBlobs(this.host);
+    if (!hasText && hasImages && images.length === 1) {
       const [_, data] = this.host.command
         .chain()
-      .tryAll(chain => [
+        .tryAll(chain => [chain.getImageSelections()])
-        chain.getTextSelection(),
-        chain.getBlockSelections(),
-        chain.getImageSelections(),
-      ])
         .getSelectedBlocks({
           types: ['image'],
         })
         .run();
-    if (data.currentBlockSelections?.[0]) {
+      let caption = '';
-      this.caption =
+
+      if (data.currentImageSelections?.[0]) {
+        caption =
           (
-          this.host.doc.getBlock(data.currentBlockSelections[0].blockId)
+            this.host.doc.getBlock(data.currentImageSelections[0].blockId)
               ?.model as ImageBlockModel
           ).caption ?? '';
       }
+
+      this._updateCards({
+        id: Date.now(),
+        type: CardType.Image,
+        image: images[0],
+        caption,
+      });
+
+      return;
+    }
   }
 
-  private async _handleDocSelection() {
+  private async _extractOnEdgeless() {
+    if (!this.host.closest('edgeless-editor')) return;
+
+    const canvas = await selectedToCanvas(this.host);
+    if (!canvas) return;
+
+    const blob: Blob | null = await new Promise(resolve =>
+      canvas.toBlob(resolve)
+    );
+    if (!blob) return;
+
+    this._updateCards({
+      id: Date.now(),
+      type: CardType.Image,
+      image: new File([blob], 'selected.png'),
+    });
+  }
+
+  private async _extractAll() {
     const notes = this.host.doc
       .getBlocksByFlavour('affine:note')
       .filter(
@@ -305,50 +351,68 @@ export class ChatCards extends WithDisposable(LitElement) {
       }) ?? []
     );
     const images = blobs.filter((blob): blob is File => !!blob);
-    this.text = text;
+
-    this.markdown = markdown;
+    return {
-    this.images = images;
+      text,
+      markdown,
+      images,
+    };
   }
 
-  protected override async updated(_changedProperties: PropertyValues) {
+  protected override async updated(changedProperties: PropertyValues) {
-    if (_changedProperties.has('selectionValue')) {
+    if (changedProperties.has('host')) {
-      await this._updateState();
+      const { text, images } = await this._extractAll();
+      const hasText = text.length > 0;
+      const hasImages = images.length > 0;
+
+      // Currently only supports checking on first load
+      if (
+        (hasText || hasImages) &&
+        !this.cards.some(card => card.type === CardType.Doc)
+      ) {
+        this._updateCards({
+          id: Date.now(),
+          type: CardType.Doc,
+        });
+      }
+    }
   }
 
-    if (_changedProperties.has('host')) {
+  override async connectedCallback() {
-      this._onEdgelessCopilotAreaUpdated();
+    super.connectedCallback();
+
+    this._disposables.add(
+      AIProvider.slots.requestContinueWithAIInChat.on(async ({ mode }) => {
+        if (mode === 'edgeless') {
+          await this._extractOnEdgeless();
+        } else {
+          await this._extract();
         }
+      })
+    );
+
+    this._disposables.add(
+      AIProvider.slots.toggleChatCards.on(({ visible, ok }) => {
+        if (visible && ok && this._selectedCardId > 0) {
+          this.cards = this.cards.filter(
+            card => card.id !== this._selectedCardId
+          );
+          this._selectedCardId = 0;
+        }
+      })
+    );
   }
 
   protected override render() {
-    return html`<div class="cards-container">
+    return repeat(
-      ${repeat(
+      this.cards,
-        ChatCardsConfig,
+      card => card.id,
-        card => card.name,
+      card => html`
-        card => {
+        <div @click=${() => this._handleClick(card)}>
-          if (
+          ${this._renderCard(card)}
-            card.render(this.text, this.images[0], this.caption) !== nothing
+        </div>
-          ) {
+      `
-            return html`<div
-              @click=${async () => {
-                if (card.name === 'doc') {
-                  await this._handleDocSelection();
-                }
-                card.handler(
-                  this.updateContext,
-                  this.text,
-                  this.markdown,
-                  this.images
     );
-              }}
-            >
-              ${card.render(this.text, this.images[0], this.caption)}
-            </div> `;
-          }
-          return nothing;
-        }
-      )}
-    </div>`;
   }
 }
 

packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts

@@ -268,6 +268,8 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
         <div
           class="close-wrapper"
           @click=${() => {
+            AIProvider.slots.toggleChatCards.emit({ visible: true });
+
             if (this.curIndex >= 0 && this.curIndex < images.length) {
               const newImages = [...images];
               newImages.splice(this.curIndex, 1);
@@ -315,6 +317,7 @@ export class ChatPanelInput extends WithDisposable(LitElement) {
               <div
                 class="chat-quote-close"
                 @click=${() => {
+                  AIProvider.slots.toggleChatCards.emit({ visible: true });
                   this.updateContext({ quote: '', markdown: '' });
                 }}
               >

packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-messages.ts

@@ -27,6 +27,7 @@ import {
 } from '@blocksuite/blocks';
 import { css, html, nothing, type PropertyValues } from 'lit';
 import { customElement, property, query, state } from 'lit/decorators.js';
+import { cache } from 'lit/directives/cache.js';
 import { repeat } from 'lit/directives/repeat.js';
 import { styleMap } from 'lit/directives/style-map.js';
 
@@ -144,7 +145,8 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
     }
   `;
 
-  private _selectionValue: BaseSelection[] = [];
+  @state()
+  accessor _selectionValue: BaseSelection[] = [];
 
   @state()
   accessor showDownIndicator = false;
@@ -167,14 +169,16 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
   @query('.chat-panel-messages')
   accessor messagesContainer!: HTMLDivElement;
 
-  protected override updated(_changedProperties: PropertyValues) {
+  @state()
-    if (_changedProperties.has('host')) {
+  accessor showChatCards = true;
+
+  protected override updated(changedProperties: PropertyValues) {
+    if (changedProperties.has('host')) {
       const { disposables } = this;
 
       disposables.add(
         this.host.selection.slots.changed.on(() => {
           this._selectionValue = this.host.selection.value;
-          this.requestUpdate();
         })
       );
       const { docModeService } = this.host.spec.getService('affine:page');
@@ -226,12 +230,16 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
                     : 'What can I help you with?'}
                 </div>
               </div>
+              ${cache(
+                this.showChatCards
+                  ? html`
                       <chat-cards
-                .chatContextValue=${this.chatContextValue}
                         .updateContext=${this.updateContext}
                         .host=${this.host}
-                .selectionValue=${this._selectionValue}
+                      ></chat-cards>
-              ></chat-cards> `
+                    `
+                  : nothing
+              )}`
           : repeat(filteredItems, (item, index) => {
               const isLast = index === filteredItems.length - 1;
               return html`<div class="message">
@@ -265,6 +273,12 @@ export class ChatPanelMessages extends WithDisposable(ShadowlessElement) {
         }
       })
     );
+
+    this.disposables.add(
+      AIProvider.slots.toggleChatCards.on(({ visible }) => {
+        this.showChatCards = visible;
+      })
+    );
   }
 
   renderError() {

packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts

@@ -190,6 +190,7 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
 
     AIProvider.slots.actions.on(({ action, event }) => {
       const { status } = this.chatContextValue;
+
       if (
         action !== 'chat' &&
         event === 'finished' &&
@@ -197,6 +198,13 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
       ) {
         this._resetItems();
       }
+
+      if (action === 'chat' && event === 'finished') {
+        AIProvider.slots.toggleChatCards.emit({
+          visible: true,
+          ok: status === 'success',
+        });
+      }
     });
 
     AIProvider.slots.userInfo.on(userInfo => {

packages/frontend/core/src/blocksuite/presets/ai/entries/edgeless/actions-config.ts

@@ -19,6 +19,7 @@ import {
   AIPresentationIconWithAnimation,
   AISearchIcon,
   ChatWithAIIcon,
+  CommentIcon,
   ExplainIcon,
   ImproveWritingIcon,
   LanguageIcon,
@@ -98,6 +99,19 @@ export const imageProcessingSubItem = imageProcessingTypes.map(type => {
 const othersGroup: AIItemGroupConfig = {
   name: 'others',
   items: [
+    {
+      name: 'Continue with AI',
+      icon: CommentIcon,
+      showWhen: () => true,
+      handler: host => {
+        const panel = getAIPanel(host);
+        AIProvider.slots.requestContinueWithAIInChat.emit({
+          host,
+          mode: 'edgeless',
+        });
+        panel.hide();
+      },
+    },
     {
       name: 'Open AI Chat',
       icon: ChatWithAIIcon,

packages/frontend/core/src/blocksuite/presets/ai/provider.ts

@@ -81,6 +81,10 @@ export class AIProvider {
     // use case: when user selects "continue in chat" in an ask ai result panel
     // do we need to pass the context to the chat panel?
     requestContinueInChat: new Slot<{ host: EditorHost; show: boolean }>(),
+    requestContinueWithAIInChat: new Slot<{
+      host: EditorHost;
+      mode?: 'page' | 'edgeless';
+    }>(),
     requestLogin: new Slot<{ host: EditorHost }>(),
     requestUpgradePlan: new Slot<{ host: EditorHost }>(),
     // when an action is requested to run in edgeless mode (show a toast in affine)
@@ -94,6 +98,10 @@ export class AIProvider {
     // downstream can emit this slot to notify ai presets that user info has been updated
     userInfo: new Slot<AIUserInfo | null>(),
     // add more if needed
+    toggleChatCards: new Slot<{
+      visible: boolean;
+      ok?: boolean;
+    }>(),
   };
 
   // track the history of triggered actions (in memory only)