diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index 175895cd8..825cf5dbd 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -27,23 +27,24 @@ export interface Chunk { content: string } -export declare function fromModelName(modelName: string): Tokenizer | null - -export declare function getMime(input: Uint8Array): string - -export declare function htmlSanitize(input: string): string - /** * Converts markdown content to AFFiNE-compatible y-octo document binary. * * # Arguments + * * `title` - The document title * * `markdown` - The markdown content to convert * * `doc_id` - The document ID to use for the y-octo doc * * # Returns * A Buffer containing the y-octo document update binary */ -export declare function markdownToDocBinary(markdown: string, docId: string): Buffer +export declare function createDocWithMarkdown(title: string, markdown: string, docId: string): Buffer + +export declare function fromModelName(modelName: string): Tokenizer | null + +export declare function getMime(input: Uint8Array): string + +export declare function htmlSanitize(input: string): string /** * Merge updates in form like `Y.applyUpdate(doc, update)` way and return the @@ -103,9 +104,38 @@ export declare function parseWorkspaceDoc(docBin: Buffer): NativeWorkspaceDocCon export declare function readAllDocIdsFromRootDoc(docBin: Buffer, includeTrash?: boolean | undefined | null): Array +/** + * Updates or creates the docProperties record for a document. + * + * # Arguments + * * `existing_binary` - The current docProperties document binary + * * `properties_doc_id` - The docProperties document ID + * (db$${workspaceId}$docProperties) + * * `target_doc_id` - The document ID to update in docProperties + * * `created_by` - Optional creator user ID + * * `updated_by` - Optional updater user ID + * + * # Returns + * A Buffer containing only the delta (changes) as a y-octo update binary + */ +export declare function updateDocProperties(existingBinary: Buffer, propertiesDocId: string, targetDocId: string, createdBy?: string | undefined | null, updatedBy?: string | undefined | null): Buffer + +/** + * Updates a document's title without touching content blocks. + * + * # Arguments + * * `existing_binary` - The current document binary + * * `title` - The new title + * * `doc_id` - The document ID + * + * # Returns + * A Buffer containing only the delta (changes) as a y-octo update binary + */ +export declare function updateDocTitle(existingBinary: Buffer, title: string, docId: string): Buffer + /** * Updates an existing document with new markdown content. - * Uses structural and text-level diffing to apply minimal changes. + * Uses structural diffing to apply block-level replacements for changes. * * # Arguments * * `existing_binary` - The current document binary @@ -117,4 +147,17 @@ export declare function readAllDocIdsFromRootDoc(docBin: Buffer, includeTrash?: */ export declare function updateDocWithMarkdown(existingBinary: Buffer, newMarkdown: string, docId: string): Buffer +/** + * Updates a document title in the workspace root doc's meta.pages array. + * + * # Arguments + * * `root_doc_bin` - The current root doc binary (workspaceId doc) + * * `doc_id` - The document ID to update + * * `title` - The new title for the document + * + * # Returns + * A Buffer containing the y-octo update binary to apply to the root doc + */ +export declare function updateRootDocMetaTitle(rootDocBin: Buffer, docId: string, title: string): Buffer + export declare function verifyChallengeResponse(response: string, bits: number, resource: string): Promise diff --git a/packages/backend/native/src/doc.rs b/packages/backend/native/src/doc.rs index 550150d54..642300f0d 100644 --- a/packages/backend/native/src/doc.rs +++ b/packages/backend/native/src/doc.rs @@ -136,20 +136,21 @@ pub fn read_all_doc_ids_from_root_doc(doc_bin: Buffer, include_trash: Option Result { - let result = - doc_parser::markdown_to_ydoc(&markdown, &doc_id).map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?; +pub fn create_doc_with_markdown(title: String, markdown: String, doc_id: String) -> Result { + let result = doc_parser::build_full_doc(&title, &markdown, &doc_id) + .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?; Ok(Buffer::from(result)) } /// Updates an existing document with new markdown content. -/// Uses structural and text-level diffing to apply minimal changes. +/// Uses structural diffing to apply block-level replacements for changes. /// /// # Arguments /// * `existing_binary` - The current document binary @@ -160,11 +161,58 @@ pub fn markdown_to_doc_binary(markdown: String, doc_id: String) -> Result Result { - let result = doc_parser::update_ydoc(&existing_binary, &new_markdown, &doc_id) + let result = doc_parser::update_doc(&existing_binary, &new_markdown, &doc_id) .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?; Ok(Buffer::from(result)) } +/// Updates a document's title without touching content blocks. +/// +/// # Arguments +/// * `existing_binary` - The current document binary +/// * `title` - The new title +/// * `doc_id` - The document ID +/// +/// # Returns +/// A Buffer containing only the delta (changes) as a y-octo update binary +#[napi] +pub fn update_doc_title(existing_binary: Buffer, title: String, doc_id: String) -> Result { + let result = doc_parser::update_doc_title(&existing_binary, &doc_id, &title) + .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?; + Ok(Buffer::from(result)) +} + +/// Updates or creates the docProperties record for a document. +/// +/// # Arguments +/// * `existing_binary` - The current docProperties document binary +/// * `properties_doc_id` - The docProperties document ID +/// (db$${workspaceId}$docProperties) +/// * `target_doc_id` - The document ID to update in docProperties +/// * `created_by` - Optional creator user ID +/// * `updated_by` - Optional updater user ID +/// +/// # Returns +/// A Buffer containing only the delta (changes) as a y-octo update binary +#[napi] +pub fn update_doc_properties( + existing_binary: Buffer, + properties_doc_id: String, + target_doc_id: String, + created_by: Option, + updated_by: Option, +) -> Result { + let result = doc_parser::update_doc_properties( + &existing_binary, + &properties_doc_id, + &target_doc_id, + created_by.as_deref(), + updated_by.as_deref(), + ) + .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?; + Ok(Buffer::from(result)) +} + /// Adds a document ID to the workspace root doc's meta.pages array. /// This registers the document in the workspace so it appears in the UI. /// @@ -181,3 +229,19 @@ pub fn add_doc_to_root_doc(root_doc_bin: Buffer, doc_id: String, title: Option Result { + let result = doc_parser::update_root_doc_meta_title(&root_doc_bin, &doc_id, &title) + .map_err(|e| Error::new(Status::GenericFailure, e.to_string()))?; + Ok(Buffer::from(result)) +} diff --git a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md index fc81da2a2..f05073bd1 100644 --- a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md +++ b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.md @@ -70,6 +70,33 @@ Generated by [AVA](https://avajs.dev). ␊ ␊ ␊ + ␊ + [](Bookmark,https://affine.pro/)␊ + ␊ + ␊ + [](Bookmark,https://www.youtube.com/@affinepro)␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ `, title: 'Write, Draw, Plan all at Once.', } diff --git a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap index bc9899e48..7bd07afac 100644 Binary files a/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap and b/packages/backend/server/src/__tests__/e2e/doc-service/__snapshots__/controller.spec.ts.snap differ diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md index 9e86c8604..a2672336e 100644 --- a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md +++ b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.md @@ -70,6 +70,33 @@ Generated by [AVA](https://avajs.dev). ␊ ␊ ␊ + ␊ + [](Bookmark,https://affine.pro/)␊ + ␊ + ␊ + [](Bookmark,https://www.youtube.com/@affinepro)␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ `, title: 'Write, Draw, Plan all at Once.', } diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap index 70dca3ed8..c2e0c3a45 100644 Binary files a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap and b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-database.spec.ts.snap differ diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md index 73ecc0f05..32460b778 100644 --- a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md +++ b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.md @@ -70,6 +70,33 @@ Generated by [AVA](https://avajs.dev). ␊ ␊ ␊ + ␊ + [](Bookmark,https://affine.pro/)␊ + ␊ + ␊ + [](Bookmark,https://www.youtube.com/@affinepro)␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ `, title: 'Write, Draw, Plan all at Once.', } diff --git a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap index 70dca3ed8..c2e0c3a45 100644 Binary files a/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap and b/packages/backend/server/src/core/doc/__tests__/__snapshots__/reader-from-rpc.spec.ts.snap differ diff --git a/packages/backend/server/src/core/doc/writer.ts b/packages/backend/server/src/core/doc/writer.ts index ecfcf04f0..c4017489d 100644 --- a/packages/backend/server/src/core/doc/writer.ts +++ b/packages/backend/server/src/core/doc/writer.ts @@ -1,10 +1,14 @@ import { Injectable, Logger, NotFoundException } from '@nestjs/common'; import { nanoid } from 'nanoid'; +import { EventBus } from '../../base'; import { addDocToRootDoc, - markdownToDocBinary, + createDocWithMarkdown, + updateDocProperties, + updateDocTitle, updateDocWithMarkdown, + updateRootDocMetaTitle, } from '../../native'; import { PgWorkspaceDocStorageAdapter } from './adapters/workspace'; @@ -16,22 +20,40 @@ export interface UpdateDocResult { success: boolean; } +declare global { + interface Events { + 'doc.updates.pushed': { + spaceType: 'workspace' | 'userspace'; + spaceId: string; + docId: string; + updates: Uint8Array[]; + timestamp: number; + editor?: string; + }; + } +} + @Injectable() export class DocWriter { private readonly logger = new Logger(DocWriter.name); - constructor(private readonly storage: PgWorkspaceDocStorageAdapter) {} + constructor( + private readonly storage: PgWorkspaceDocStorageAdapter, + private readonly event: EventBus + ) {} /** * Creates a new document from markdown content. * * @param workspaceId - The workspace ID - * @param markdown - The markdown content + * @param title - The document title + * @param markdown - The markdown content (body only) * @param editorId - Optional editor ID for tracking * @returns The created document ID */ async createDoc( workspaceId: string, + title: string, markdown: string, editorId?: string ): Promise { @@ -58,24 +80,50 @@ export class DocWriter { `Creating doc ${docId} in workspace ${workspaceId} from markdown` ); - // Convert markdown to y-octo binary - const binary = markdownToDocBinary(markdown, docId); - - // Extract title from markdown (first H1 heading) - const titleMatch = markdown.match(/^#\s+(.+?)(?:\s*#+)?\s*$/m); - const title = titleMatch ? titleMatch[1].trim() : undefined; + // Convert markdown to y-octo binary using the provided title + const binary = createDocWithMarkdown(title, markdown, docId); // Prepare root doc update to register the new document const rootDocUpdate = addDocToRootDoc(rootDocBin, docId, title); // Push both updates together - root doc first, then the new doc - await this.storage.pushDocUpdates( + const rootTimestamp = await this.storage.pushDocUpdates( workspaceId, workspaceId, [rootDocUpdate], editorId ); - await this.storage.pushDocUpdates(workspaceId, docId, [binary], editorId); + this.emitDocUpdatesPushed({ + spaceId: workspaceId, + docId: workspaceId, + updates: [rootDocUpdate], + timestamp: rootTimestamp, + editor: editorId, + }); + + const docTimestamp = await this.storage.pushDocUpdates( + workspaceId, + docId, + [binary], + editorId + ); + this.emitDocUpdatesPushed({ + spaceId: workspaceId, + docId, + updates: [binary], + timestamp: docTimestamp, + editor: editorId, + }); + + await this.updateDocProperties( + workspaceId, + docId, + { + createdBy: editorId, + updatedBy: editorId, + }, + editorId + ); this.logger.debug( `Created and registered doc ${docId} in workspace ${workspaceId}` @@ -88,8 +136,10 @@ export class DocWriter { * Updates an existing document with new markdown content. * * Uses structural diffing to compute minimal changes between the existing - * document and new markdown, then applies only the delta. This preserves - * document history and enables proper CRDT merging with concurrent edits. + * document and new markdown, then applies block-level replacements for + * changed blocks. This preserves document history and enables proper CRDT + * merging with concurrent edits. + * Note: this does not update the document title. * * @param workspaceId - The workspace ID * @param docId - The document ID to update @@ -124,8 +174,194 @@ export class DocWriter { const delta = updateDocWithMarkdown(existingBinary, markdown, docId); // Push only the delta changes - await this.storage.pushDocUpdates(workspaceId, docId, [delta], editorId); + const timestamp = await this.storage.pushDocUpdates( + workspaceId, + docId, + [delta], + editorId + ); + this.emitDocUpdatesPushed({ + spaceId: workspaceId, + docId, + updates: [delta], + timestamp, + editor: editorId, + }); + + await this.updateDocProperties( + workspaceId, + docId, + { updatedBy: editorId }, + editorId + ); return { success: true }; } + + /** + * Updates document metadata (currently title only). + * + * @param workspaceId - The workspace ID + * @param docId - The document ID to update + * @param meta - Metadata updates + * @param editorId - Optional editor ID for tracking + */ + async updateDocMeta( + workspaceId: string, + docId: string, + meta: { title?: string }, + editorId?: string + ): Promise { + if (meta.title === undefined) { + throw new Error('No metadata provided'); + } + + this.logger.debug(`Updating doc meta ${docId} in workspace ${workspaceId}`); + + const existingDoc = await this.storage.getDoc(workspaceId, docId); + if (!existingDoc?.bin) { + throw new NotFoundException(`Document ${docId} not found`); + } + + const rootDoc = await this.storage.getDoc(workspaceId, workspaceId); + if (!rootDoc?.bin) { + throw new NotFoundException( + `Workspace ${workspaceId} not found or has no root document` + ); + } + + const existingBinary = Buffer.isBuffer(existingDoc.bin) + ? existingDoc.bin + : Buffer.from( + existingDoc.bin.buffer, + existingDoc.bin.byteOffset, + existingDoc.bin.byteLength + ); + const rootDocBin = Buffer.isBuffer(rootDoc.bin) + ? rootDoc.bin + : Buffer.from( + rootDoc.bin.buffer, + rootDoc.bin.byteOffset, + rootDoc.bin.byteLength + ); + + const titleUpdate = updateDocTitle(existingBinary, meta.title, docId); + const rootMetaUpdate = updateRootDocMetaTitle( + rootDocBin, + docId, + meta.title + ); + + const rootTimestamp = await this.storage.pushDocUpdates( + workspaceId, + workspaceId, + [rootMetaUpdate], + editorId + ); + this.emitDocUpdatesPushed({ + spaceId: workspaceId, + docId: workspaceId, + updates: [rootMetaUpdate], + timestamp: rootTimestamp, + editor: editorId, + }); + + const docTimestamp = await this.storage.pushDocUpdates( + workspaceId, + docId, + [titleUpdate], + editorId + ); + this.emitDocUpdatesPushed({ + spaceId: workspaceId, + docId, + updates: [titleUpdate], + timestamp: docTimestamp, + editor: editorId, + }); + + await this.updateDocProperties( + workspaceId, + docId, + { updatedBy: editorId }, + editorId + ); + + return { success: true }; + } + + private emitDocUpdatesPushed(payload: { + spaceId: string; + docId: string; + updates: Uint8Array[]; + timestamp: number; + editor?: string; + }) { + this.event.emit('doc.updates.pushed', { + spaceType: 'workspace', + spaceId: payload.spaceId, + docId: payload.docId, + updates: payload.updates, + timestamp: payload.timestamp, + editor: payload.editor, + }); + } + + private async updateDocProperties( + workspaceId: string, + docId: string, + props: { createdBy?: string; updatedBy?: string }, + editorId?: string + ) { + if (!editorId) { + return; + } + if ( + workspaceId === docId || + docId.startsWith('db$') || + docId.startsWith('userdata$') + ) { + return; + } + if (!props.createdBy && !props.updatedBy) { + return; + } + + const propertiesDocId = `db$${workspaceId}$docProperties`; + const existingDoc = await this.storage.getDoc(workspaceId, propertiesDocId); + const existingBinary = existingDoc?.bin + ? Buffer.isBuffer(existingDoc.bin) + ? existingDoc.bin + : Buffer.from( + existingDoc.bin.buffer, + existingDoc.bin.byteOffset, + existingDoc.bin.byteLength + ) + : Buffer.alloc(0); + + const update = updateDocProperties( + existingBinary, + propertiesDocId, + docId, + props.createdBy, + props.updatedBy + ); + if (this.storage.isEmptyBin(update)) { + return; + } + + const timestamp = await this.storage.pushDocUpdates( + workspaceId, + propertiesDocId, + [update], + editorId + ); + this.emitDocUpdatesPushed({ + spaceId: workspaceId, + docId: propertiesDocId, + updates: [update], + timestamp, + editor: editorId, + }); + } } diff --git a/packages/backend/server/src/core/sync/gateway.ts b/packages/backend/server/src/core/sync/gateway.ts index 9c15517c8..d789ac58e 100644 --- a/packages/backend/server/src/core/sync/gateway.ts +++ b/packages/backend/server/src/core/sync/gateway.ts @@ -6,9 +6,10 @@ import { OnGatewayDisconnect, SubscribeMessage as RawSubscribeMessage, WebSocketGateway, + WebSocketServer, } from '@nestjs/websockets'; import { ClsInterceptor } from 'nestjs-cls'; -import { Socket } from 'socket.io'; +import { type Server, Socket } from 'socket.io'; import { CallMetric, @@ -18,6 +19,7 @@ import { GatewayErrorWrapper, metrics, NotInSpace, + OnEvent, SpaceAccessDenied, } from '../../base'; import { Models } from '../../models'; @@ -141,6 +143,9 @@ export class SpaceSyncGateway { protected logger = new Logger(SpaceSyncGateway.name); + @WebSocketServer() + private readonly server!: Server; + private connectionCount = 0; constructor( @@ -166,6 +171,46 @@ export class SpaceSyncGateway metrics.socketio.gauge('connections').record(this.connectionCount); } + @OnEvent('doc.updates.pushed') + onDocUpdatesPushed({ + spaceType, + spaceId, + docId, + updates, + timestamp, + editor, + }: Events['doc.updates.pushed']) { + if (!this.server || updates.length === 0) { + return; + } + + const encodedUpdates = updates.map(update => + Buffer.from(update).toString('base64') + ); + + this.server + .to(Room(spaceId, 'sync-019')) + .emit('space:broadcast-doc-updates', { + spaceType, + spaceId, + docId, + updates: encodedUpdates, + timestamp, + }); + + const room = `${spaceType}:${Room(spaceId)}`; + encodedUpdates.forEach(update => { + this.server.to(room).emit('space:broadcast-doc-update', { + spaceType, + spaceId, + docId, + update, + timestamp, + editor, + }); + }); + } + selectAdapter(client: Socket, spaceType: SpaceType): SyncSocketAdapter { let adapters: Record = (client as any) .affineSyncAdapters; diff --git a/packages/backend/server/src/core/telemetry/service.ts b/packages/backend/server/src/core/telemetry/service.ts index 918ca963b..886cbf6c9 100644 --- a/packages/backend/server/src/core/telemetry/service.ts +++ b/packages/backend/server/src/core/telemetry/service.ts @@ -133,14 +133,24 @@ export class TelemetryService { }; } catch (error) { const err = error as Error; - this.logger.error('Telemetry forwarding failed', err); - return { - ok: false, - error: { - name: err?.name ?? 'TelemetryForwardingError', - message: err?.message ?? 'Telemetry forwarding failed', - }, - }; + if (env.dev) { + this.logger.error('Telemetry forwarding failed', err); + return { + ok: false, + error: { + name: err?.name ?? 'TelemetryForwardingError', + message: err?.message ?? 'Telemetry forwarding failed', + }, + }; + } else { + return { + ok: false, + error: { + name: 'TelemetryForwardingError', + message: 'Telemetry forwarding failed', + }, + }; + } } } diff --git a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md index ea7ffb3ff..998a1ed4e 100644 --- a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md +++ b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.md @@ -1440,6 +1440,33 @@ Generated by [AVA](https://avajs.dev). ␊ ␊ ␊ + ␊ + [](Bookmark,https://affine.pro/)␊ + ␊ + ␊ + [](Bookmark,https://www.youtube.com/@affinepro)␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ `, title: 'Write, Draw, Plan all at Once.', } @@ -1449,55 +1476,74 @@ Generated by [AVA](https://avajs.dev). > Snapshot 1 { - markdown: `␊ + markdown: `␊ AFFiNE is an open source all in one workspace, an operating system for all the building blocks of your team wiki, knowledge management and digital assets and a better alternative to Notion and Miro.␊ ␊ + ␊ ␊ ␊ + ␊ # You own your data, with no compromises␊ ␊ + ␊ ## Local-first & Real-time collaborative␊ ␊ + ␊ We love the idea proposed by Ink & Switch in the famous article about you owning your data, despite the cloud. Furthermore, AFFiNE is the first all-in-one workspace that keeps your data ownership with no compromises on real-time collaboration and editing experience.␊ ␊ + ␊ AFFiNE is a local-first application upon CRDTs with real-time collaboration support. Your data is always stored locally while multiple nodes remain synced in real-time.␊ ␊ + ␊ ␊ ␊ - ␊ + ␊ ### Blocks that assemble your next docs, tasks kanban or whiteboard␊ ␊ - ␊ + ␊ There is a large overlap of their atomic "building blocks" between these apps. They are neither open source nor have a plugin system like VS Code for contributors to customize. We want to have something that contains all the features we love and goes one step further.␊ ␊ + ␊ We are building AFFiNE to be a fundamental open source platform that contains all the building blocks for docs, task management and visual collaboration, hoping you can shape your next workflow with us that can make your life better and also connect others, too.␊ ␊ + ␊ If you want to learn more about the product design of AFFiNE, here goes the concepts:␊ ␊ + ␊ To Shape, not to adapt. AFFiNE is built for individuals & teams who care about their data, who refuse vendor lock-in, and who want to have control over their essential tools.␊ ␊ - ␊ + ␊ ## A true canvas for blocks in any form␊ ␊ + ␊ [Many editor apps](http://notion.so) claimed to be a canvas for productivity. Since _the Mother of All Demos,_ Douglas Engelbart, a creative and programable digital workspace has been a pursuit and an ultimate mission for generations of tool makers.␊ ␊ + ␊ ␊ ␊ + ␊ "We shape our tools and thereafter our tools shape us”. A lot of pioneers have inspired us a long the way, e.g.:␊ ␊ + ␊ * Quip & Notion with their great concept of "everything is a block"␊ + ␊ * Trello with their Kanban␊ + ␊ * Airtable & Miro with their no-code programable datasheets␊ + ␊ * Miro & Whimiscal with their edgeless visual whiteboard␊ + ␊ * Remnote & Capacities with their object-based tag system␊ - ␊ + ␊ For more details, please refer to our [RoadMap](https://docs.affine.pro/docs/core-concepts/roadmap)␊ ␊ + ␊ ## Self Host␊ ␊ + ␊ Self host AFFiNE␊ ␊ - ␊ + ␊ ␊ ### Learning From␊ ||Title|Tag|␊ @@ -1510,14 +1556,47 @@ Generated by [AVA](https://avajs.dev). |Miro & Whimiscal with their edgeless visual whiteboard|Miro & Whimiscal with their edgeless visual whiteboard|Reference|␊ |Remnote & Capacities with their object-based tag system|Remnote & Capacities with their object-based tag system||␊ ␊ - ␊ + ␊ ## Affine Development␊ ␊ + ␊ For developer or installation guides, please go to [AFFiNE Development](https://docs.affine.pro/docs/development/quick-start)␊ ␊ + ␊ ␊ ␊ - ␊ + ␊ + ␊ + [](Bookmark,https://affine.pro/)␊ + ␊ + ␊ + ␊ + [](Bookmark,https://www.youtube.com/@affinepro)␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ + ␊ `, title: 'Write, Draw, Plan all at Once.', } diff --git a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap index fa26beded..9833dcdc0 100644 Binary files a/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap and b/packages/backend/server/src/core/utils/__tests__/__snapshots__/blocksute.spec.ts.snap differ diff --git a/packages/backend/server/src/native.ts b/packages/backend/server/src/native.ts index 8d37adcd9..7563eb7a5 100644 --- a/packages/backend/server/src/native.ts +++ b/packages/backend/server/src/native.ts @@ -51,6 +51,9 @@ export const AFFINE_PRO_LICENSE_AES_KEY = serverNativeModule.AFFINE_PRO_LICENSE_AES_KEY; // MCP write tools exports -export const markdownToDocBinary = serverNativeModule.markdownToDocBinary; +export const createDocWithMarkdown = serverNativeModule.createDocWithMarkdown; export const updateDocWithMarkdown = serverNativeModule.updateDocWithMarkdown; export const addDocToRootDoc = serverNativeModule.addDocToRootDoc; +export const updateDocTitle = serverNativeModule.updateDocTitle; +export const updateDocProperties = serverNativeModule.updateDocProperties; +export const updateRootDocMetaTitle = serverNativeModule.updateRootDocMetaTitle; diff --git a/packages/backend/server/src/plugins/copilot/mcp/provider.ts b/packages/backend/server/src/plugins/copilot/mcp/provider.ts index c23f4986b..536e77cdb 100644 --- a/packages/backend/server/src/plugins/copilot/mcp/provider.ts +++ b/packages/backend/server/src/plugins/copilot/mcp/provider.ts @@ -166,146 +166,217 @@ export class WorkspaceMcpProvider { } ); - // Write tools - create and update documents - server.registerTool( - 'create_document', - { - title: 'Create Document', - description: - 'Create a new document in the workspace with the given title and markdown content. Returns the ID of the created document.', - inputSchema: z.object({ - title: z.string().min(1).describe('The title of the new document'), - content: z - .string() - .describe( - 'The markdown content for the document body (should NOT include a title H1 - the title parameter will be used)' - ), - }), - }, - async ({ title, content }) => { - try { - // Check if user can create docs in this workspace - await this.ac + if (env.dev || env.namespaces.canary) { + // Write tools - create and update documents + server.registerTool( + 'create_document', + { + title: 'Create Document', + description: + 'Create a new document in the workspace with the given title and markdown content. Returns the ID of the created document. This tool not support insert or update database block and image yet.', + inputSchema: z.object({ + title: z.string().min(1).describe('The title of the new document'), + content: z + .string() + .describe('The markdown content for the document body'), + }), + }, + async ({ title, content }) => { + try { + // Check if user can create docs in this workspace + await this.ac + .user(userId) + .workspace(workspaceId) + .assert('Workspace.CreateDoc'); + + // Sanitize title by removing newlines and trimming + const sanitizedTitle = title.replace(/[\r\n]+/g, ' ').trim(); + if (!sanitizedTitle) { + throw new Error('Title cannot be empty'); + } + + // Strip any leading H1 from content to prevent duplicates + // Per CommonMark spec, ATX headings allow only 0-3 spaces before the # + // Handles: "# Title", " # Title", "# Title #" + const strippedContent = content.replace( + /^[ \t]{0,3}#\s+[^\n]*#*\s*\n*/, + '' + ); + + // Create the document + const result = await this.writer.createDoc( + workspaceId, + sanitizedTitle, + strippedContent, + userId + ); + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + docId: result.docId, + message: `Document "${title}" created successfully`, + }), + }, + ], + } as const; + } catch (error) { + return { + isError: true, + content: [ + { + type: 'text', + text: `Failed to create document: ${error instanceof Error ? error.message : 'Unknown error'}`, + }, + ], + }; + } + } + ); + + server.registerTool( + 'update_document', + { + title: 'Update Document', + description: + 'Update an existing document with new markdown content (body only). Uses structural diffing to apply minimal changes, preserving document history and enabling real-time collaboration. This does NOT update the document title. This tool not support insert or update database block and image yet.', + inputSchema: z.object({ + docId: z.string().describe('The ID of the document to update'), + content: z + .string() + .describe( + 'The complete new markdown content for the document body (do NOT include a title H1)' + ), + }), + }, + async ({ docId, content }) => { + const notFoundError: CallToolResult = { + isError: true, + content: [ + { + type: 'text', + text: `Doc with id ${docId} not found.`, + }, + ], + }; + + // Use can() instead of assert() to avoid leaking doc existence info + const accessible = await this.ac .user(userId) .workspace(workspaceId) - .assert('Workspace.CreateDoc'); + .doc(docId) + .can('Doc.Update'); - // Combine title and content into markdown - // Sanitize title by removing newlines and trimming - const sanitizedTitle = title.replace(/[\r\n]+/g, ' ').trim(); - if (!sanitizedTitle) { - throw new Error('Title cannot be empty'); + if (!accessible) { + return notFoundError; } - // Strip any leading H1 from content to prevent duplicates - // Per CommonMark spec, ATX headings allow only 0-3 spaces before the # - // Handles: "# Title", " # Title", "# Title #" - const strippedContent = content.replace( - /^[ \t]{0,3}#\s+[^\n]*#*\s*\n*/, - '' - ); + try { + // Update the document + await this.writer.updateDoc(workspaceId, docId, content, userId); - const markdown = `# ${sanitizedTitle}\n\n${strippedContent}`; + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + docId, + message: `Document updated successfully`, + }), + }, + ], + } as const; + } catch (error) { + return { + isError: true, + content: [ + { + type: 'text', + text: `Failed to update document: ${error instanceof Error ? error.message : 'Unknown error'}`, + }, + ], + }; + } + } + ); - // Create the document - const result = await this.writer.createDoc( - workspaceId, - markdown, - userId - ); - - return { - content: [ - { - type: 'text', - text: JSON.stringify({ - success: true, - docId: result.docId, - message: `Document "${title}" created successfully`, - }), - }, - ], - } as const; - } catch (error) { - return { + server.registerTool( + 'update_document_meta', + { + title: 'Update Document Metadata', + description: 'Update document metadata (currently title only).', + inputSchema: z.object({ + docId: z.string().describe('The ID of the document to update'), + title: z.string().min(1).describe('The new document title'), + }), + }, + async ({ docId, title }) => { + const notFoundError: CallToolResult = { isError: true, content: [ { type: 'text', - text: `Failed to create document: ${error instanceof Error ? error.message : 'Unknown error'}`, + text: `Doc with id ${docId} not found.`, }, ], }; - } - } - ); - server.registerTool( - 'update_document', - { - title: 'Update Document', - description: - 'Update an existing document with new markdown content. Uses structural diffing to apply minimal changes, preserving document history and enabling real-time collaboration.', - inputSchema: z.object({ - docId: z.string().describe('The ID of the document to update'), - content: z - .string() - .describe( - 'The complete new markdown content for the document (including title as H1)' - ), - }), - }, - async ({ docId, content }) => { - const notFoundError: CallToolResult = { - isError: true, - content: [ - { - type: 'text', - text: `Doc with id ${docId} not found.`, - }, - ], - }; + // Use can() instead of assert() to avoid leaking doc existence info + const accessible = await this.ac + .user(userId) + .workspace(workspaceId) + .doc(docId) + .can('Doc.Update'); - // Use can() instead of assert() to avoid leaking doc existence info - const accessible = await this.ac - .user(userId) - .workspace(workspaceId) - .doc(docId) - .can('Doc.Update'); + if (!accessible) { + return notFoundError; + } - if (!accessible) { - return notFoundError; - } + try { + const sanitizedTitle = title.replace(/[\r\n]+/g, ' ').trim(); + if (!sanitizedTitle) { + throw new Error('Title cannot be empty'); + } - try { - // Update the document - await this.writer.updateDoc(workspaceId, docId, content, userId); - - return { - content: [ + await this.writer.updateDocMeta( + workspaceId, + docId, { - type: 'text', - text: JSON.stringify({ - success: true, - docId, - message: `Document updated successfully`, - }), + title: sanitizedTitle, }, - ], - } as const; - } catch (error) { - return { - isError: true, - content: [ - { - type: 'text', - text: `Failed to update document: ${error instanceof Error ? error.message : 'Unknown error'}`, - }, - ], - }; + userId + ); + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + docId, + message: `Document title updated successfully`, + }), + }, + ], + } as const; + } catch (error) { + return { + isError: true, + content: [ + { + type: 'text', + text: `Failed to update document metadata: ${error instanceof Error ? error.message : 'Unknown error'}`, + }, + ], + }; + } } - } - ); + ); + } return server; } diff --git a/packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts b/packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts index 76a541162..a2ceefd2b 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/chat-prompt.ts @@ -127,6 +127,7 @@ export class ChatPrompt { selectedMarkdown, selectedSnapshot, html, + currentDocId, } = params; return { 'affine::date': new Date().toLocaleDateString(), @@ -135,6 +136,8 @@ export class ChatPrompt { 'affine::hasDocsRef': Array.isArray(docs) && docs.length > 0, 'affine::hasFilesRef': Array.isArray(files) && files.length > 0, 'affine::hasSelected': !!selectedMarkdown || !!selectedSnapshot || !!html, + 'affine::hasCurrentDoc': + typeof currentDocId === 'string' && currentDocId.trim().length > 0, }; } diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index db9f98c69..38216e135 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -1950,6 +1950,13 @@ User's preferred language is {{affine::language}}. User's timezone is {{affine::timezone}}. +{{#affine::hasCurrentDoc}} + +The user is chatting within the current document: {{currentDocId}}. +If the user's request relates to this document, call the doc_read tool with docId {{currentDocId}} to read it before answering. + +{{/affine::hasCurrentDoc}} + - If documents are provided, analyze all documents based on the user's query - Identify key information relevant to the user's specific request @@ -2086,7 +2093,10 @@ Below is the user's query. Please respond in the user's preferred language witho config: { tools: [ 'docRead', - 'sectionEdit', + 'docCreate', + 'docUpdate', + 'docUpdateMeta', + // 'sectionEdit', 'docKeywordSearch', 'docSemanticSearch', 'webSearch', diff --git a/packages/backend/server/src/plugins/copilot/providers/provider.ts b/packages/backend/server/src/plugins/copilot/providers/provider.ts index f18035448..a7dc7d289 100644 --- a/packages/backend/server/src/plugins/copilot/providers/provider.ts +++ b/packages/backend/server/src/plugins/copilot/providers/provider.ts @@ -9,7 +9,7 @@ import { CopilotProviderNotSupported, OnEvent, } from '../../../base'; -import { DocReader } from '../../../core/doc'; +import { DocReader, DocWriter } from '../../../core/doc'; import { AccessController } from '../../../core/permission'; import { Models } from '../../../models'; import { IndexerService } from '../../indexer'; @@ -19,16 +19,22 @@ import { buildBlobContentGetter, buildContentGetter, buildDocContentGetter, + buildDocCreateHandler, buildDocKeywordSearchGetter, buildDocSearchGetter, + buildDocUpdateHandler, + buildDocUpdateMetaHandler, createBlobReadTool, createCodeArtifactTool, createConversationSummaryTool, createDocComposeTool, + createDocCreateTool, createDocEditTool, createDocKeywordSearchTool, createDocReadTool, createDocSemanticSearchTool, + createDocUpdateMetaTool, + createDocUpdateTool, createExaCrawlTool, createExaSearchTool, createSectionEditTool, @@ -163,6 +169,7 @@ export abstract class CopilotProvider { strict: false, }); const docReader = this.moduleRef.get(DocReader, { strict: false }); + const docWriter = this.moduleRef.get(DocWriter, { strict: false }); const models = this.moduleRef.get(Models, { strict: false }); const prompt = this.moduleRef.get(PromptService, { strict: false, @@ -177,6 +184,12 @@ export abstract class CopilotProvider { } continue; } + if ( + !(env.dev || env.namespaces.canary) && + ['docCreate', 'docUpdate', 'docUpdateMeta'].includes(tool) + ) { + continue; + } switch (tool) { case 'blobRead': { const docContext = options.session @@ -244,6 +257,27 @@ export abstract class CopilotProvider { tools.doc_read = createDocReadTool(getDoc.bind(null, options)); break; } + case 'docCreate': { + const createDoc = buildDocCreateHandler(ac, docWriter); + tools.doc_create = createDocCreateTool( + createDoc.bind(null, options) + ); + break; + } + case 'docUpdate': { + const updateDoc = buildDocUpdateHandler(ac, docWriter); + tools.doc_update = createDocUpdateTool( + updateDoc.bind(null, options) + ); + break; + } + case 'docUpdateMeta': { + const updateDocMeta = buildDocUpdateMetaHandler(ac, docWriter); + tools.doc_update_meta = createDocUpdateMetaTool( + updateDocMeta.bind(null, options) + ); + break; + } case 'webSearch': { tools.web_search_exa = createExaSearchTool(this.AFFiNEConfig); tools.web_crawl_exa = createExaCrawlTool(this.AFFiNEConfig); diff --git a/packages/backend/server/src/plugins/copilot/providers/types.ts b/packages/backend/server/src/plugins/copilot/providers/types.ts index e568be80e..2a06e832f 100644 --- a/packages/backend/server/src/plugins/copilot/providers/types.ts +++ b/packages/backend/server/src/plugins/copilot/providers/types.ts @@ -66,6 +66,9 @@ export const PromptToolsSchema = z 'docEdit', // work with indexer 'docRead', + 'docCreate', + 'docUpdate', + 'docUpdateMeta', 'docKeywordSearch', // work with embeddings 'docSemanticSearch', diff --git a/packages/backend/server/src/plugins/copilot/tools/doc-write.ts b/packages/backend/server/src/plugins/copilot/tools/doc-write.ts new file mode 100644 index 000000000..73b03fd84 --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/tools/doc-write.ts @@ -0,0 +1,207 @@ +import { Logger } from '@nestjs/common'; +import { tool } from 'ai'; +import { z } from 'zod'; + +import { DocWriter } from '../../../core/doc'; +import { AccessController } from '../../../core/permission'; +import type { CopilotChatOptions } from '../providers'; +import { toolError } from './error'; + +const logger = new Logger('DocWriteTool'); + +const stripLeadingH1 = (content: string) => + content.replace(/^[ \t]{0,3}#\s+[^\n]*#*\s*\n*/, ''); + +const sanitizeTitle = (title: string) => title.replace(/[\r\n]+/g, ' ').trim(); + +export const buildDocCreateHandler = ( + ac: AccessController, + writer: DocWriter +) => { + return async ( + options: CopilotChatOptions, + title: string, + content: string + ) => { + if (!options?.user || !options.workspace) { + return toolError( + 'Doc Create Failed', + 'Missing user or workspace context' + ); + } + + await ac + .user(options.user) + .workspace(options.workspace) + .assert('Workspace.CreateDoc'); + + const sanitizedTitle = sanitizeTitle(title); + if (!sanitizedTitle) { + return toolError('Doc Create Failed', 'Title cannot be empty'); + } + + const strippedContent = stripLeadingH1(content); + const result = await writer.createDoc( + options.workspace, + sanitizedTitle, + strippedContent, + options.user + ); + + return { + success: true, + docId: result.docId, + message: `Document "${sanitizedTitle}" created successfully`, + }; + }; +}; + +export const buildDocUpdateHandler = ( + ac: AccessController, + writer: DocWriter +) => { + return async ( + options: CopilotChatOptions, + docId: string, + content: string + ) => { + const notFound = toolError( + 'Doc Update Failed', + `Doc with id ${docId} not found.` + ); + + if (!options?.user || !options.workspace) { + return notFound; + } + + const canAccess = await ac + .user(options.user) + .workspace(options.workspace) + .doc(docId) + .can('Doc.Update'); + + if (!canAccess) { + return notFound; + } + + await writer.updateDoc(options.workspace, docId, content, options.user); + + return { + success: true, + docId, + message: 'Document updated successfully', + }; + }; +}; + +export const buildDocUpdateMetaHandler = ( + ac: AccessController, + writer: DocWriter +) => { + return async (options: CopilotChatOptions, docId: string, title: string) => { + const notFound = toolError( + 'Doc Meta Update Failed', + `Doc with id ${docId} not found.` + ); + + if (!options?.user || !options.workspace) { + return notFound; + } + + const canAccess = await ac + .user(options.user) + .workspace(options.workspace) + .doc(docId) + .can('Doc.Update'); + + if (!canAccess) { + return notFound; + } + + const sanitizedTitle = sanitizeTitle(title); + if (!sanitizedTitle) { + return toolError('Doc Meta Update Failed', 'Title cannot be empty'); + } + + await writer.updateDocMeta( + options.workspace, + docId, + { title: sanitizedTitle }, + options.user + ); + + return { + success: true, + docId, + message: 'Document title updated successfully', + }; + }; +}; + +export const createDocCreateTool = ( + createDoc: (title: string, content: string) => Promise +) => { + return tool({ + description: + 'Create a new document in the workspace with the given title and markdown content. Returns the ID of the created document. This tool not support insert or update database block and image yet.', + inputSchema: z.object({ + title: z.string().min(1).describe('The title of the new document'), + content: z + .string() + .describe('The markdown content for the document body'), + }), + execute: async ({ title, content }) => { + try { + return await createDoc(title, content); + } catch (err: any) { + logger.error(`Failed to create document: ${title}`, err); + return toolError('Doc Create Failed', err.message); + } + }, + }); +}; + +export const createDocUpdateTool = ( + updateDoc: (docId: string, content: string) => Promise +) => { + return tool({ + description: + 'Update an existing document with new markdown content (body only). Uses structural diffing to apply minimal changes. This does NOT update the document title. This tool not support insert or update database block and image yet.', + inputSchema: z.object({ + doc_id: z.string().describe('The ID of the document to update'), + content: z + .string() + .describe( + 'The complete new markdown content for the document body (do NOT include a title H1)' + ), + }), + execute: async ({ doc_id, content }) => { + try { + return await updateDoc(doc_id, content); + } catch (err: any) { + logger.error(`Failed to update document: ${doc_id}`, err); + return toolError('Doc Update Failed', err.message); + } + }, + }); +}; + +export const createDocUpdateMetaTool = ( + updateDocMeta: (docId: string, title: string) => Promise +) => { + return tool({ + description: 'Update document metadata (currently title only).', + inputSchema: z.object({ + doc_id: z.string().describe('The ID of the document to update'), + title: z.string().min(1).describe('The new document title'), + }), + execute: async ({ doc_id, title }) => { + try { + return await updateDocMeta(doc_id, title); + } catch (err: any) { + logger.error(`Failed to update document meta: ${doc_id}`, err); + return toolError('Doc Meta Update Failed', err.message); + } + }, + }); +}; diff --git a/packages/backend/server/src/plugins/copilot/tools/index.ts b/packages/backend/server/src/plugins/copilot/tools/index.ts index 7dee9b136..7cc7e2594 100644 --- a/packages/backend/server/src/plugins/copilot/tools/index.ts +++ b/packages/backend/server/src/plugins/copilot/tools/index.ts @@ -8,6 +8,11 @@ import { createDocEditTool } from './doc-edit'; import { createDocKeywordSearchTool } from './doc-keyword-search'; import { createDocReadTool } from './doc-read'; import { createDocSemanticSearchTool } from './doc-semantic-search'; +import { + createDocCreateTool, + createDocUpdateMetaTool, + createDocUpdateTool, +} from './doc-write'; import { createExaCrawlTool } from './exa-crawl'; import { createExaSearchTool } from './exa-search'; import { createSectionEditTool } from './section-edit'; @@ -20,6 +25,9 @@ export interface CustomAITools extends ToolSet { doc_semantic_search: ReturnType; doc_keyword_search: ReturnType; doc_read: ReturnType; + doc_create: ReturnType; + doc_update: ReturnType; + doc_update_meta: ReturnType; doc_compose: ReturnType; section_edit: ReturnType; web_search_exa: ReturnType; @@ -34,6 +42,7 @@ export * from './doc-edit'; export * from './doc-keyword-search'; export * from './doc-read'; export * from './doc-semantic-search'; +export * from './doc-write'; export * from './error'; export * from './exa-crawl'; export * from './exa-search'; diff --git a/packages/common/native/src/doc_parser/affine.rs b/packages/common/native/src/doc_parser/affine.rs deleted file mode 100644 index f4efe3628..000000000 --- a/packages/common/native/src/doc_parser/affine.rs +++ /dev/null @@ -1,1283 +0,0 @@ -use std::collections::HashSet; - -use serde::{Deserialize, Serialize}; -use serde_json::{Map as JsonMap, Value as JsonValue}; -use thiserror::Error; -use y_octo::{Any, DocOptions, JwstCodecError, Map, Value}; - -use super::{ - blocksuite::{ - DocContext, collect_child_ids, get_block_id, get_flavour, get_list_depth, get_string, nearest_by_flavour, - }, - delta_markdown::{ - DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references, - extract_inline_references_from_value, text_to_inline_markdown, text_to_markdown, - }, - value::{any_as_string, any_truthy, build_reference_payload, params_value_to_json, value_to_string}, -}; - -const SUMMARY_LIMIT: usize = 1000; -const PAGE_FLAVOUR: &str = "affine:page"; -const NOTE_FLAVOUR: &str = "affine:note"; - -const BOOKMARK_FLAVOURS: [&str; 5] = [ - "affine:bookmark", - "affine:embed-youtube", - "affine:embed-figma", - "affine:embed-github", - "affine:embed-loom", -]; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BlockInfo { - pub block_id: String, - pub flavour: String, - pub content: Option>, - pub blob: Option>, - pub ref_doc_id: Option>, - pub ref_info: Option>, - pub parent_flavour: Option, - pub parent_block_id: Option, - pub additional: Option, -} - -impl BlockInfo { - fn base( - block_id: &str, - flavour: &str, - parent_flavour: Option<&String>, - parent_block_id: Option<&String>, - additional: Option, - ) -> Self { - Self { - block_id: block_id.to_string(), - flavour: flavour.to_string(), - content: None, - blob: None, - ref_doc_id: None, - ref_info: None, - parent_flavour: parent_flavour.cloned(), - parent_block_id: parent_block_id.cloned(), - additional, - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CrawlResult { - pub blocks: Vec, - pub title: String, - pub summary: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PageDocContent { - pub title: String, - pub summary: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WorkspaceDocContent { - pub name: String, - #[serde(rename = "avatarKey")] - pub avatar_key: String, -} - -#[derive(Error, Debug, Serialize, Deserialize)] -pub enum ParseError { - #[error("doc_not_found")] - DocNotFound, - #[error("invalid_binary")] - InvalidBinary, - #[error("sqlite_error: {0}")] - SqliteError(String), - #[error("parser_error: {0}")] - ParserError(String), - #[error("unknown: {0}")] - Unknown(String), -} - -impl From for ParseError { - fn from(value: JwstCodecError) -> Self { - Self::ParserError(value.to_string()) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MarkdownResult { - pub title: String, - pub markdown: String, -} - -pub fn parse_workspace_doc(doc_bin: Vec) -> Result, ParseError> { - if doc_bin.is_empty() || doc_bin == [0, 0] { - return Err(ParseError::InvalidBinary); - } - - let mut doc = DocOptions::new().build(); - doc - .apply_update_from_binary_v1(&doc_bin) - .map_err(|_| ParseError::InvalidBinary)?; - - let meta = match doc.get_map("meta") { - Ok(meta) => meta, - Err(_) => return Ok(None), - }; - - let name = get_string(&meta, "name").unwrap_or_default(); - let avatar_key = get_string(&meta, "avatar").unwrap_or_default(); - - Ok(Some(WorkspaceDocContent { name, avatar_key })) -} - -pub fn parse_page_doc( - doc_bin: Vec, - max_summary_length: Option, -) -> Result, ParseError> { - if doc_bin.is_empty() || doc_bin == [0, 0] { - return Err(ParseError::InvalidBinary); - } - - let mut doc = DocOptions::new().build(); - doc - .apply_update_from_binary_v1(&doc_bin) - .map_err(|_| ParseError::InvalidBinary)?; - - let blocks_map = match doc.get_map("blocks") { - Ok(map) => map, - Err(_) => return Ok(None), - }; - - if blocks_map.is_empty() { - return Ok(None); - } - - let Some(context) = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) else { - return Ok(None); - }; - - let mut stack = vec![context.root_block_id.clone()]; - let mut content = PageDocContent { - title: context - .block_pool - .get(&context.root_block_id) - .and_then(|block| get_string(block, "prop:title")) - .unwrap_or_default(), - summary: String::new(), - }; - - let mut summary_remaining = max_summary_length.unwrap_or(150); - - while let Some(block_id) = stack.pop() { - let Some(block) = context.block_pool.get(&block_id) else { - break; - }; - - let Some(flavour) = get_flavour(block) else { - continue; - }; - - match flavour.as_str() { - "affine:page" | "affine:note" => { - push_children(&mut stack, block); - } - "affine:attachment" | "affine:transcription" | "affine:callout" => { - if summary_remaining == -1 { - push_children(&mut stack, block); - } - } - "affine:database" => { - if summary_remaining == -1 { - append_database_summary(&mut content.summary, block, &context); - } - } - "affine:table" => { - if summary_remaining == -1 { - let contents = gather_table_contents(block); - if !contents.is_empty() { - content.summary.push_str(&contents.join("|")); - } - } - } - "affine:paragraph" | "affine:list" | "affine:code" => { - push_children(&mut stack, block); - if let Some((text, len)) = text_content_for_summary(block, "prop:text") { - if summary_remaining == -1 { - content.summary.push_str(&text); - } else if summary_remaining > 0 { - content.summary.push_str(&text); - summary_remaining -= len as isize; - } - } - } - _ => {} - } - } - - Ok(Some(content)) -} - -pub fn parse_doc_to_markdown( - doc_bin: Vec, - doc_id: String, - ai_editable: bool, - doc_url_prefix: Option, -) -> Result { - if doc_bin.is_empty() || doc_bin == [0, 0] { - return Err(ParseError::InvalidBinary); - } - - let mut doc = DocOptions::new().with_guid(doc_id.clone()).build(); - doc - .apply_update_from_binary_v1(&doc_bin) - .map_err(|_| ParseError::InvalidBinary)?; - - let blocks_map = doc.get_map("blocks")?; - if blocks_map.is_empty() { - return Ok(MarkdownResult { - title: "".into(), - markdown: "".into(), - }); - } - - let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) - .ok_or_else(|| ParseError::ParserError("root block not found".into()))?; - let root_block_id = context.root_block_id.clone(); - let mut walker = context.walker(); - let mut doc_title = String::from("Untitled"); - let mut markdown = String::new(); - let md_options = DeltaToMdOptions::new(doc_url_prefix); - - while let Some((parent_block_id, block_id)) = walker.next() { - let block = match context.block_pool.get(&block_id) { - Some(block) => block, - None => continue, - }; - - let flavour = match get_flavour(block) { - Some(flavour) => flavour, - None => continue, - }; - - let parent_id = context.parent_lookup.get(&block_id); - let parent_flavour = parent_id - .and_then(|id| context.block_pool.get(id)) - .and_then(get_flavour); - - if parent_flavour.as_deref() == Some("affine:database") { - continue; - } - - // enqueue children first to keep traversal order similar to JS implementation - walker.enqueue_children(&block_id, block); - - if flavour == PAGE_FLAVOUR { - let title = get_string(block, "prop:title").unwrap_or_default(); - doc_title = title.clone(); - continue; - } - - if flavour == "affine:database" { - let title = get_string(block, "prop:title").unwrap_or_default(); - markdown.push_str(&format!("\n### {title}\n")); - - if let Some(table) = build_database_table(block, &context, &md_options) { - let escape_table = |s: &str| s.replace('|', "\\|").replace('\n', "
"); - let mut table_md = String::new(); - - table_md.push('|'); - for column in &table.columns { - table_md.push_str(&escape_table(column.name.as_deref().unwrap_or_default())); - table_md.push('|'); - } - table_md.push('\n'); - - table_md.push('|'); - for _ in &table.columns { - table_md.push_str("---|"); - } - table_md.push('\n'); - - for row in table.rows.into_iter() { - table_md.push('|'); - for cell_text in row.into_iter() { - table_md.push_str(&escape_table(&cell_text)); - table_md.push('|'); - } - table_md.push('\n'); - } - append_table_block(&mut markdown, &table_md); - } - continue; - } - - if flavour == "affine:table" { - let contents = gather_table_contents(block); - let table = contents.join("|"); - append_table_block(&mut markdown, &table); - continue; - } - - if ai_editable && parent_block_id.as_ref() == Some(&root_block_id) { - markdown.push_str(&format!("\n")); - } - - if flavour == "affine:paragraph" { - let type_ = get_string(block, "prop:type").unwrap_or_default(); - let prefix = paragraph_prefix(type_.as_str()); - if let Some(text_md) = text_to_markdown(block, "prop:text", &md_options) { - append_paragraph(&mut markdown, prefix, &text_md); - } else if let Some((text, _)) = text_content(block, "prop:text") { - append_paragraph(&mut markdown, prefix, &text); - } - continue; - } - - if flavour == "affine:list" { - let type_ = get_string(block, "prop:type").unwrap_or_default(); - let checked = block - .get("prop:checked") - .and_then(|value| value.to_any()) - .as_ref() - .map(any_truthy) - .unwrap_or(false); - let prefix = list_prefix(type_.as_str(), checked); - let depth = get_list_depth(&block_id, &context.parent_lookup, &context.block_pool); - let indent = list_indent(depth); - if let Some(text_md) = text_to_markdown(block, "prop:text", &md_options) { - append_list_item(&mut markdown, &indent, prefix, &text_md); - } else if let Some((text, _)) = text_content(block, "prop:text") { - append_list_item(&mut markdown, &indent, prefix, &text); - } - continue; - } - - if flavour == "affine:code" { - if let Some((text, _)) = text_content(block, "prop:text") { - let lang = get_string(block, "prop:language").unwrap_or_default(); - append_code_block(&mut markdown, &lang, &text); - } - continue; - } - } - - Ok(MarkdownResult { - title: doc_title, - markdown, - }) -} - -pub fn parse_doc_from_binary(doc_bin: Vec, doc_id: String) -> Result { - if doc_bin.is_empty() || doc_bin == [0, 0] { - return Err(ParseError::InvalidBinary); - } - - let mut doc = DocOptions::new().with_guid(doc_id.clone()).build(); - doc - .apply_update_from_binary_v1(&doc_bin) - .map_err(|_| ParseError::InvalidBinary)?; - - let blocks_map = doc.get_map("blocks")?; - if blocks_map.is_empty() { - return Err(ParseError::ParserError("blocks map is empty".into())); - } - - let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) - .ok_or_else(|| ParseError::ParserError("root block not found".into()))?; - let mut walker = context.walker(); - let mut blocks: Vec = Vec::with_capacity(context.block_pool.len()); - let mut doc_title = String::new(); - let mut summary = String::new(); - let mut summary_remaining = SUMMARY_LIMIT as isize; - - while let Some((parent_block_id, block_id)) = walker.next() { - let block = match context.block_pool.get(&block_id) { - Some(block) => block, - None => continue, - }; - - let flavour = match get_flavour(block) { - Some(flavour) => flavour, - None => continue, - }; - - let parent_block = parent_block_id.as_ref().and_then(|id| context.block_pool.get(id)); - let parent_flavour = parent_block.and_then(get_flavour); - - let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &context.parent_lookup, &context.block_pool); - let note_block_id = note_block.as_ref().and_then(get_block_id); - let display_mode = determine_display_mode(note_block.as_ref()); - - // enqueue children first to keep traversal order similar to JS implementation - walker.enqueue_children(&block_id, block); - - let build_block = |database_name: Option<&String>| { - BlockInfo::base( - &block_id, - &flavour, - parent_flavour.as_ref(), - parent_block_id.as_ref(), - compose_additional(&display_mode, note_block_id.as_ref(), database_name), - ) - }; - - if flavour == PAGE_FLAVOUR { - let title = get_string(block, "prop:title").unwrap_or_default(); - doc_title = title.clone(); - let mut info = build_block(None); - info.content = Some(vec![title]); - blocks.push(info); - continue; - } - - if matches!(flavour.as_str(), "affine:paragraph" | "affine:list" | "affine:code") { - if let Some(text) = block.get("prop:text").and_then(|value| value.to_text()) { - let database_name = if flavour == "affine:paragraph" && parent_flavour.as_deref() == Some("affine:database") { - parent_block.and_then(|map| get_string(map, "prop:title")) - } else { - None - }; - - let content = text.to_string(); - let text_len = text.len() as usize; - let refs = extract_inline_references(&text.to_delta()); - - let mut info = build_block(database_name.as_ref()); - info.content = Some(vec![content.clone()]); - if !refs.is_empty() { - info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect()); - info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect()); - } - blocks.push(info); - append_summary(&mut summary, &mut summary_remaining, text_len, &content); - } - continue; - } - - if matches!(flavour.as_str(), "affine:embed-linked-doc" | "affine:embed-synced-doc") { - if let Some(page_id) = get_string(block, "prop:pageId") { - let mut info = build_block(None); - let payload = embed_ref_payload(block, &page_id); - apply_doc_ref(&mut info, page_id, payload); - blocks.push(info); - } - continue; - } - - if flavour == "affine:attachment" { - if let Some(blob_id) = get_string(block, "prop:sourceId") { - let mut info = build_block(None); - let name = get_string(block, "prop:name").unwrap_or_default(); - apply_blob_info(&mut info, blob_id, name); - blocks.push(info); - } - continue; - } - - if flavour == "affine:image" { - if let Some(blob_id) = get_string(block, "prop:sourceId") { - let mut info = build_block(None); - let caption = get_string(block, "prop:caption").unwrap_or_default(); - apply_blob_info(&mut info, blob_id, caption); - blocks.push(info); - } - continue; - } - - if flavour == "affine:surface" { - let texts = gather_surface_texts(block); - let mut info = build_block(None); - info.content = Some(texts); - blocks.push(info); - continue; - } - - if flavour == "affine:database" { - let (texts, database_name) = gather_database_texts(block); - let mut info = BlockInfo::base( - &block_id, - &flavour, - parent_flavour.as_ref(), - parent_block_id.as_ref(), - compose_additional(&display_mode, note_block_id.as_ref(), database_name.as_ref()), - ); - info.content = Some(texts); - let refs = collect_database_cell_references(block); - if !refs.is_empty() { - info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect()); - info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect()); - } - blocks.push(info); - continue; - } - - if flavour == "affine:latex" { - if let Some(content) = get_string(block, "prop:latex") { - let mut info = build_block(None); - info.content = Some(vec![content]); - blocks.push(info); - } - continue; - } - - if flavour == "affine:table" { - let contents = gather_table_contents(block); - let mut info = build_block(None); - info.content = Some(contents); - blocks.push(info); - continue; - } - - if BOOKMARK_FLAVOURS.contains(&flavour.as_str()) { - blocks.push(build_block(None)); - } - } - - if doc_title.is_empty() { - doc_title = "Untitled".into(); - } - - Ok(CrawlResult { - blocks, - title: doc_title, - summary, - }) -} - -pub fn get_doc_ids_from_binary(doc_bin: Vec, include_trash: bool) -> Result, ParseError> { - if doc_bin.is_empty() || doc_bin == [0, 0] { - return Err(ParseError::InvalidBinary); - } - - let mut doc = DocOptions::new().build(); - doc - .apply_update_from_binary_v1(&doc_bin) - .map_err(|_| ParseError::InvalidBinary)?; - - let meta = doc.get_map("meta")?; - let pages = match meta.get("pages").and_then(|v| v.to_array()) { - Some(arr) => arr, - None => return Ok(vec![]), - }; - - let mut doc_ids = Vec::new(); - for page_val in pages.iter() { - if let Some(page) = page_val.to_map() { - let id = get_string(&page, "id"); - if let Some(id) = id { - let trash = page - .get("trash") - .and_then(|v| match v.to_any() { - Some(Any::True) => Some(true), - Some(Any::False) => Some(false), - _ => None, - }) - .unwrap_or(false); - - if include_trash || !trash { - doc_ids.push(id); - } - } - } - } - - Ok(doc_ids) -} - -/// Adds a document ID to the root doc's meta.pages array. -/// Returns a binary update that can be applied to the root doc. -/// -/// # Arguments -/// * `root_doc_bin` - The current root doc binary -/// * `doc_id` - The document ID to add -/// * `title` - Optional title for the document -/// -/// # Returns -/// A Vec containing the y-octo update binary to add the doc -pub fn add_doc_to_root_doc(root_doc_bin: Vec, doc_id: &str, title: Option<&str>) -> Result, ParseError> { - // Handle empty or minimal root doc - create a new one - let doc = if root_doc_bin.is_empty() || root_doc_bin == [0, 0] { - DocOptions::new().build() - } else { - let mut doc = DocOptions::new().build(); - doc - .apply_update_from_binary_v1(&root_doc_bin) - .map_err(|_| ParseError::InvalidBinary)?; - doc - }; - - // Capture state before modifications to encode only the delta - let state_before = doc.get_state_vector(); - - // Get or create the meta map - let mut meta = doc.get_or_create_map("meta")?; - - // Get existing pages array or create new one - let pages_exists = meta.get("pages").and_then(|v| v.to_array()).is_some(); - - if pages_exists { - // Get the existing array and add to it - let mut pages = meta.get("pages").and_then(|v| v.to_array()).unwrap(); - - // Check if doc already exists - let doc_exists = pages.iter().any(|page_val| { - page_val - .to_map() - .and_then(|page| get_string(&page, "id")) - .map(|id| id == doc_id) - .unwrap_or(false) - }); - - if !doc_exists { - // Create a new page entry - let page_map = doc.create_map().map_err(|e| ParseError::ParserError(e.to_string()))?; - - // Insert into pages array first, then populate - let idx = pages.len(); - pages - .insert(idx, page_map) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - // Now get the inserted map and populate it - if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) { - inserted_page - .insert("id".to_string(), Any::String(doc_id.to_string())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - if let Some(t) = title { - inserted_page - .insert("title".to_string(), Any::String(t.to_string())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - - // Set createDate to current timestamp - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0); - inserted_page - .insert("createDate".to_string(), Any::BigInt64(timestamp)) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - } - } else { - // Create new pages array with this doc - let page_entry = vec![Any::Object( - [ - ("id".to_string(), Any::String(doc_id.to_string())), - ("title".to_string(), Any::String(title.unwrap_or("").to_string())), - ( - "createDate".to_string(), - Any::BigInt64( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as i64) - .unwrap_or(0), - ), - ), - ] - .into_iter() - .collect(), - )]; - - meta - .insert("pages".to_string(), Any::Array(page_entry)) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - - // Encode only the changes (delta) since state_before - doc - .encode_state_as_update_v1(&state_before) - .map_err(|e| ParseError::ParserError(e.to_string())) -} - -fn paragraph_prefix(type_: &str) -> &'static str { - match type_ { - "h1" => "# ", - "h2" => "## ", - "h3" => "### ", - "h4" => "#### ", - "h5" => "##### ", - "h6" => "###### ", - "quote" => "> ", - _ => "", - } -} - -fn list_prefix(type_: &str, checked: bool) -> &'static str { - match type_ { - "bulleted" => "* ", - "todo" => { - if checked { - "- [x] " - } else { - "- [ ] " - } - } - _ => "1. ", - } -} - -fn list_indent(depth: usize) -> String { - " ".repeat(depth) -} - -fn append_paragraph(markdown: &mut String, prefix: &str, text: &str) { - markdown.push_str(prefix); - markdown.push_str(text); - if !text.ends_with('\n') { - markdown.push('\n'); - } - markdown.push('\n'); -} - -fn append_list_item(markdown: &mut String, indent: &str, prefix: &str, text: &str) { - markdown.push_str(indent); - markdown.push_str(prefix); - markdown.push_str(text); - if !text.ends_with('\n') { - markdown.push('\n'); - } -} - -fn append_code_block(markdown: &mut String, lang: &str, text: &str) { - markdown.push_str("```"); - markdown.push_str(lang); - markdown.push('\n'); - markdown.push_str(text); - markdown.push_str("\n```\n\n"); -} - -fn append_table_block(markdown: &mut String, table: &str) { - if table.is_empty() { - markdown.push('\n'); - return; - } - markdown.push_str(table); - if !table.ends_with('\n') { - markdown.push('\n'); - } - markdown.push('\n'); -} - -fn text_content(block: &Map, key: &str) -> Option<(String, usize)> { - block.get(key).and_then(|value| { - value.to_text().map(|text| { - let content = text.to_string(); - let len = text.len() as usize; - (content, len) - }) - }) -} - -fn determine_display_mode(note_block: Option<&Map>) -> String { - match note_block.and_then(|block| get_string(block, "prop:displayMode")) { - Some(mode) if mode == "both" => "page".into(), - Some(mode) => mode, - None => "edgeless".into(), - } -} - -fn compose_additional( - display_mode: &str, - note_block_id: Option<&String>, - database_name: Option<&String>, -) -> Option { - let mut payload = JsonMap::new(); - payload.insert("displayMode".into(), JsonValue::String(display_mode.to_string())); - if let Some(note_id) = note_block_id { - payload.insert("noteBlockId".into(), JsonValue::String(note_id.clone())); - } - if let Some(name) = database_name { - payload.insert("databaseName".into(), JsonValue::String(name.clone())); - } - Some(JsonValue::Object(payload).to_string()) -} - -fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) { - info.blob = Some(vec![blob_id]); - info.content = Some(vec![content]); -} - -fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option) { - info.ref_doc_id = Some(vec![page_id]); - if let Some(payload) = payload { - info.ref_info = Some(vec![payload]); - } -} - -fn embed_ref_payload(block: &Map, page_id: &str) -> Option { - let params = block.get("prop:params").as_ref().and_then(params_value_to_json); - Some(build_reference_payload(page_id, params)) -} - -fn gather_surface_texts(block: &Map) -> Vec { - let mut texts = Vec::new(); - let elements = match block.get("prop:elements").and_then(|value| value.to_map()) { - Some(map) => map, - None => return texts, - }; - - if elements - .get("type") - .and_then(|value| value_to_string(&value)) - .as_deref() - != Some("$blocksuite:internal:native$") - { - return texts; - } - - if let Some(value_map) = elements.get("value").and_then(|value| value.to_map()) { - for value in value_map.values() { - if let Some(element) = value.to_map() - && let Some(text) = element.get("text").and_then(|value| value.to_text()) - { - texts.push(text.to_string()); - } - } - } - - texts.sort(); - texts -} - -fn gather_database_texts(block: &Map) -> (Vec, Option) { - let mut texts = Vec::new(); - let database_title = get_string(block, "prop:title"); - if let Some(title) = &database_title { - texts.push(title.clone()); - } - - if let Some(columns) = parse_database_columns(block) { - for column in columns.iter() { - if let Some(name) = column.name.as_ref() { - texts.push(name.clone()); - } - for option in column.options.iter() { - if let Some(value) = option.value.as_ref() { - texts.push(value.clone()); - } - } - } - } - - (texts, database_title) -} - -fn collect_database_cell_references(block: &Map) -> Vec { - let cells_map = match block.get("prop:cells").and_then(|value| value.to_map()) { - Some(map) => map, - None => return Vec::new(), - }; - - let mut refs = Vec::new(); - let mut seen: HashSet<(String, String)> = HashSet::new(); - - for row in cells_map.values() { - let Some(row_map) = row.to_map() else { - continue; - }; - for cell in row_map.values() { - let Some(cell_map) = cell.to_map() else { - continue; - }; - let Some(value) = cell_map.get("value") else { - continue; - }; - for reference in extract_inline_references_from_value(&value) { - let key = (reference.doc_id.clone(), reference.payload.clone()); - if seen.insert(key) { - refs.push(reference); - } - } - } - } - - refs -} - -fn gather_table_contents(block: &Map) -> Vec { - let mut contents = Vec::new(); - for key in block.keys() { - if key.starts_with("prop:cells.") - && key.ends_with(".text") - && let Some(value) = block.get(key).and_then(|value| value_to_string(&value)) - && !value.is_empty() - { - contents.push(value); - } - } - contents -} - -struct DatabaseTable { - columns: Vec, - rows: Vec>, -} - -fn build_database_table(block: &Map, context: &DocContext, md_options: &DeltaToMdOptions) -> Option { - let columns = parse_database_columns(block)?; - let cells_map = block.get("prop:cells").and_then(|v| v.to_map())?; - let child_ids = collect_child_ids(block); - - let mut rows = Vec::new(); - for child_id in child_ids { - let row_cells = cells_map.get(&child_id).and_then(|v| v.to_map()); - let mut row = Vec::new(); - - for column in columns.iter() { - let mut cell_text = String::new(); - if column.col_type == "title" { - if let Some(child_block) = context.block_pool.get(&child_id) { - if let Some(text_md) = text_to_inline_markdown(child_block, "prop:text", md_options) { - cell_text = text_md; - } else if let Some((text, _)) = text_content(child_block, "prop:text") { - cell_text = text; - } else if let Some((text, _)) = text_content_for_summary(child_block, "prop:text") { - cell_text = text; - } - } - } else if let Some(row_cells) = &row_cells - && let Some(cell_val) = row_cells.get(&column.id).and_then(|v| v.to_map()) - && let Some(value) = cell_val.get("value") - { - if let Some(text_md) = delta_value_to_inline_markdown(&value, md_options) { - cell_text = text_md; - } else { - cell_text = format_cell_value(&value, column); - } - } - - row.push(cell_text); - } - rows.push(row); - } - - Some(DatabaseTable { columns, rows }) -} - -fn append_database_summary(summary: &mut String, block: &Map, context: &DocContext) { - let md_options = DeltaToMdOptions::new(None); - let Some(table) = build_database_table(block, context, &md_options) else { - return; - }; - - if let Some(title) = get_string(block, "prop:title") - && !title.is_empty() - { - summary.push_str(&title); - summary.push('|'); - } - - for column in table.columns.iter() { - if let Some(name) = column.name.as_ref() - && !name.is_empty() - { - summary.push_str(name); - summary.push('|'); - } - for option in column.options.iter() { - if let Some(value) = option.value.as_ref() - && !value.is_empty() - { - summary.push_str(value); - summary.push('|'); - } - } - } - - for row in table.rows.iter() { - for cell_text in row.iter() { - if !cell_text.is_empty() { - summary.push_str(cell_text); - summary.push('|'); - } - } - } -} - -fn push_children(queue: &mut Vec, block: &Map) { - let mut child_ids = collect_child_ids(block); - for child_id in child_ids.drain(..).rev() { - queue.push(child_id); - } -} - -fn text_content_for_summary(block: &Map, key: &str) -> Option<(String, usize)> { - if let Some((text, len)) = text_content(block, key) { - return Some((text, len)); - } - - block.get(key).and_then(|value| { - value_to_string(&value).map(|text| { - let len = text.chars().count(); - (text, len) - }) - }) -} - -struct DatabaseOption { - id: Option, - value: Option, - color: Option, -} - -struct DatabaseColumn { - id: String, - name: Option, - col_type: String, - options: Vec, -} - -fn parse_database_columns(block: &Map) -> Option> { - let columns = block.get("prop:columns").and_then(|value| value.to_array())?; - let mut parsed = Vec::new(); - for column_value in columns.iter() { - if let Some(column) = column_value.to_map() { - let id = get_string(&column, "id").unwrap_or_default(); - let name = get_string(&column, "name"); - let col_type = get_string(&column, "type").unwrap_or_default(); - let options = parse_database_options(&column); - parsed.push(DatabaseColumn { - id, - name, - col_type, - options, - }); - } - } - Some(parsed) -} - -fn parse_database_options(column: &Map) -> Vec { - let Some(data) = column.get("data").and_then(|value| value.to_map()) else { - return Vec::new(); - }; - let Some(options) = data.get("options").and_then(|value| value.to_array()) else { - return Vec::new(); - }; - - let mut parsed = Vec::new(); - for option_value in options.iter() { - if let Some(option) = option_value.to_map() { - parsed.push(DatabaseOption { - id: get_string(&option, "id"), - value: get_string(&option, "value"), - color: get_string(&option, "color"), - }); - } - } - parsed -} - -fn format_option_tag(option: &DatabaseOption) -> String { - let id = option.id.as_deref().unwrap_or_default(); - let value = option.value.as_deref().unwrap_or_default(); - let color = option.color.as_deref().unwrap_or_default(); - - format!("{value}") -} - -fn format_cell_value(value: &Value, column: &DatabaseColumn) -> String { - match column.col_type.as_str() { - "select" => { - let id = match value { - Value::Any(any) => any_as_string(any).map(str::to_string), - Value::Text(text) => Some(text.to_string()), - _ => None, - }; - if let Some(id) = id { - for option in column.options.iter() { - if option.id.as_deref() == Some(id.as_str()) { - return format_option_tag(option); - } - } - } - String::new() - } - "multi-select" => { - let ids: Vec = match value { - Value::Any(Any::Array(ids)) => ids.iter().filter_map(any_as_string).map(str::to_string).collect(), - Value::Array(array) => array.iter().filter_map(|id_val| value_to_string(&id_val)).collect(), - _ => Vec::new(), - }; - - if ids.is_empty() { - return String::new(); - } - - let mut selected = Vec::new(); - for id in ids.iter() { - for option in column.options.iter() { - if option.id.as_deref() == Some(id.as_str()) { - selected.push(format_option_tag(option)); - } - } - } - selected.join("") - } - _ => value_to_string(value).unwrap_or_default(), - } -} - -fn append_summary(summary: &mut String, remaining: &mut isize, text_len: usize, text: &str) { - if *remaining > 0 { - summary.push_str(text); - *remaining -= text_len as isize; - } -} - -#[cfg(test)] -mod tests { - use serde_json::json; - use y_octo::{AHashMap, Any, TextAttributes, TextDeltaOp, TextInsert, Value}; - - use super::*; - - #[test] - fn test_parse_doc_from_binary() { - let json = include_bytes!("../../fixtures/demo.ydoc.json"); - let doc_bin = include_bytes!("../../fixtures/demo.ydoc").to_vec(); - let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string(); - - let result = parse_doc_from_binary(doc_bin, doc_id).unwrap(); - let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict) - .numeric_mode(assert_json_diff::NumericMode::AssumeFloat); - assert_json_diff::assert_json_matches!( - serde_json::from_slice::(json).unwrap(), - serde_json::json!(result), - config - ); - } - - #[test] - fn test_database_cell_references() { - let doc_id = "doc-with-db".to_string(); - let doc = DocOptions::new().with_guid(doc_id.clone()).build(); - let mut blocks = doc.get_or_create_map("blocks").unwrap(); - - let mut page = doc.create_map().unwrap(); - page.insert("sys:id".into(), "page").unwrap(); - page.insert("sys:flavour".into(), "affine:page").unwrap(); - let mut page_children = doc.create_array().unwrap(); - page_children.push("note").unwrap(); - page.insert("sys:children".into(), Value::Array(page_children)).unwrap(); - let mut page_title = doc.create_text().unwrap(); - page_title.insert(0, "Page").unwrap(); - page.insert("prop:title".into(), Value::Text(page_title)).unwrap(); - blocks.insert("page".into(), Value::Map(page)).unwrap(); - - let mut note = doc.create_map().unwrap(); - note.insert("sys:id".into(), "note").unwrap(); - note.insert("sys:flavour".into(), "affine:note").unwrap(); - let mut note_children = doc.create_array().unwrap(); - note_children.push("db").unwrap(); - note.insert("sys:children".into(), Value::Array(note_children)).unwrap(); - note.insert("prop:displayMode".into(), "page").unwrap(); - blocks.insert("note".into(), Value::Map(note)).unwrap(); - - let mut db = doc.create_map().unwrap(); - db.insert("sys:id".into(), "db").unwrap(); - db.insert("sys:flavour".into(), "affine:database").unwrap(); - db.insert("sys:children".into(), Value::Array(doc.create_array().unwrap())) - .unwrap(); - let mut db_title = doc.create_text().unwrap(); - db_title.insert(0, "Database").unwrap(); - db.insert("prop:title".into(), Value::Text(db_title)).unwrap(); - - let mut columns = doc.create_array().unwrap(); - let mut column = doc.create_map().unwrap(); - column.insert("id".into(), "col1").unwrap(); - column.insert("name".into(), "Text").unwrap(); - column.insert("type".into(), "rich-text").unwrap(); - column - .insert("data".into(), Value::Map(doc.create_map().unwrap())) - .unwrap(); - columns.push(Value::Map(column)).unwrap(); - db.insert("prop:columns".into(), Value::Array(columns)).unwrap(); - - let mut cell_text = doc.create_text().unwrap(); - let mut reference = AHashMap::default(); - reference.insert("pageId".into(), Any::String("target-doc".into())); - let mut params = AHashMap::default(); - params.insert("mode".into(), Any::String("page".into())); - reference.insert("params".into(), Any::Object(params)); - let mut attrs = TextAttributes::new(); - attrs.insert("reference".into(), Any::Object(reference)); - cell_text - .apply_delta(&[ - TextDeltaOp::Insert { - insert: TextInsert::Text("See ".into()), - format: None, - }, - TextDeltaOp::Insert { - insert: TextInsert::Text("Target".into()), - format: Some(attrs), - }, - ]) - .unwrap(); - - let mut cell = doc.create_map().unwrap(); - cell.insert("columnId".into(), "col1").unwrap(); - cell.insert("value".into(), Value::Text(cell_text)).unwrap(); - let mut row = doc.create_map().unwrap(); - row.insert("col1".into(), Value::Map(cell)).unwrap(); - let mut cells = doc.create_map().unwrap(); - cells.insert("row1".into(), Value::Map(row)).unwrap(); - db.insert("prop:cells".into(), Value::Map(cells)).unwrap(); - - blocks.insert("db".into(), Value::Map(db)).unwrap(); - - let doc_bin = doc.encode_update_v1().unwrap(); - let result = parse_doc_from_binary(doc_bin, doc_id).unwrap(); - let db_block = result.blocks.iter().find(|block| block.block_id == "db").unwrap(); - assert_eq!(db_block.ref_doc_id, Some(vec!["target-doc".to_string()])); - assert_eq!( - db_block.ref_info, - Some(vec![build_reference_payload( - "target-doc", - Some(json!({"mode": "page"})) - )]) - ); - } - - #[test] - fn test_paragraph_newlines() { - let mut markdown = String::new(); - append_paragraph(&mut markdown, "# ", "Title\n"); - assert_eq!(markdown, "# Title\n\n"); - - markdown.clear(); - append_paragraph(&mut markdown, "", "Plain"); - assert_eq!(markdown, "Plain\n\n"); - } - - #[test] - fn test_list_newlines() { - let mut markdown = String::new(); - append_list_item(&mut markdown, " ", "* ", "Item\n"); - assert_eq!(markdown, " * Item\n"); - - markdown.clear(); - append_list_item(&mut markdown, "", "- [ ] ", "Task"); - assert_eq!(markdown, "- [ ] Task\n"); - } - - #[test] - fn test_code_block_newlines() { - let mut markdown = String::new(); - append_code_block(&mut markdown, "rs", "fn main() {}"); - assert_eq!(markdown, "```rs\nfn main() {}\n```\n\n"); - } - - #[test] - fn test_table_newlines() { - let mut markdown = String::new(); - append_table_block(&mut markdown, "|a|b|\n|---|---|\n|1|2|\n"); - assert_eq!(markdown, "|a|b|\n|---|---|\n|1|2|\n\n"); - - markdown.clear(); - append_table_block(&mut markdown, "|a|b|"); - assert_eq!(markdown, "|a|b|\n\n"); - } -} diff --git a/packages/common/native/src/doc_parser/block_spec.rs b/packages/common/native/src/doc_parser/block_spec.rs new file mode 100644 index 000000000..e0195bd68 --- /dev/null +++ b/packages/common/native/src/doc_parser/block_spec.rs @@ -0,0 +1,601 @@ +use y_octo::{Any, Map, TextAttributes, TextDeltaOp, TextInsert}; + +use super::{ + ParseError, + blocksuite::get_string, + schema::{ + PROP_CAPTION, PROP_CHECKED, PROP_COLUMN_ID_SUFFIX, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER, + PROP_ORDER_SUFFIX, PROP_ROW_ID_SUFFIX, PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL, + PROP_VIDEO_ID, PROP_WIDTH, SYS_FLAVOUR, table_cell_text_key, + }, + table::{MarkdownTableOptions, render_markdown_table}, + value::{value_to_f64, value_to_string}, +}; + +/// Block flavours used in AFFiNE documents. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlockFlavour { + Paragraph, + List, + Code, + Divider, + Image, + Table, + Bookmark, + EmbedYoutube, + EmbedIframe, + Callout, +} + +impl BlockFlavour { + pub fn as_str(&self) -> &'static str { + match self { + BlockFlavour::Paragraph => "affine:paragraph", + BlockFlavour::List => "affine:list", + BlockFlavour::Code => "affine:code", + BlockFlavour::Divider => "affine:divider", + BlockFlavour::Image => "affine:image", + BlockFlavour::Table => "affine:table", + BlockFlavour::Bookmark => "affine:bookmark", + BlockFlavour::EmbedYoutube => "affine:embed-youtube", + BlockFlavour::EmbedIframe => "affine:embed-iframe", + BlockFlavour::Callout => "affine:callout", + } + } + + pub fn from_str(value: &str) -> Option { + match value { + "affine:paragraph" => Some(BlockFlavour::Paragraph), + "affine:list" => Some(BlockFlavour::List), + "affine:code" => Some(BlockFlavour::Code), + "affine:divider" => Some(BlockFlavour::Divider), + "affine:image" => Some(BlockFlavour::Image), + "affine:table" => Some(BlockFlavour::Table), + "affine:bookmark" => Some(BlockFlavour::Bookmark), + "affine:embed-youtube" => Some(BlockFlavour::EmbedYoutube), + "affine:embed-iframe" => Some(BlockFlavour::EmbedIframe), + "affine:callout" => Some(BlockFlavour::Callout), + _ => None, + } + } +} + +/// Block types for paragraphs and lists. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BlockType { + // Paragraph types + Text, + H1, + H2, + H3, + H4, + H5, + H6, + Quote, + // List types + Bulleted, + Numbered, + Todo, + // Preserve unknown types when loading from ydoc. + Unknown(String), +} + +impl BlockType { + pub fn as_str(&self) -> &str { + match self { + BlockType::Text => "text", + BlockType::H1 => "h1", + BlockType::H2 => "h2", + BlockType::H3 => "h3", + BlockType::H4 => "h4", + BlockType::H5 => "h5", + BlockType::H6 => "h6", + BlockType::Quote => "quote", + BlockType::Bulleted => "bulleted", + BlockType::Numbered => "numbered", + BlockType::Todo => "todo", + BlockType::Unknown(value) => value.as_str(), + } + } + + pub fn from_str(value: &str) -> Option { + match value { + "text" => Some(BlockType::Text), + "h1" => Some(BlockType::H1), + "h2" => Some(BlockType::H2), + "h3" => Some(BlockType::H3), + "h4" => Some(BlockType::H4), + "h5" => Some(BlockType::H5), + "h6" => Some(BlockType::H6), + "quote" => Some(BlockType::Quote), + "bulleted" => Some(BlockType::Bulleted), + "numbered" => Some(BlockType::Numbered), + "todo" => Some(BlockType::Todo), + _ => None, + } + } + + pub fn from_str_lossy(value: String) -> Self { + Self::from_str(&value).unwrap_or(BlockType::Unknown(value)) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(super) struct ImageSpec { + pub(super) source_id: String, + pub(super) caption: Option, + pub(super) width: Option, + pub(super) height: Option, +} + +impl ImageSpec { + pub(super) fn render_markdown(&self) -> String { + let blob_url = format!("blob://{}", self.source_id); + let caption = self.caption.as_deref().unwrap_or(""); + + if self.width.is_some() || self.height.is_some() || !caption.is_empty() { + let width_text = self + .width + .map(|value| value.to_string()) + .unwrap_or_else(|| "auto".into()); + let height_text = self + .height + .map(|value| value.to_string()) + .unwrap_or_else(|| "auto".into()); + return format!( + "\n\n" + ); + } + + let alt = if caption.is_empty() { + self.source_id.as_str() + } else { + caption + }; + format!("\n![{alt}]({blob_url})\n\n") + } + + pub(super) fn from_block_map(block: &Map) -> Self { + let source_id = get_string(block, PROP_SOURCE_ID).unwrap_or_default(); + let caption = get_string(block, PROP_CAPTION); + let width = block.get(PROP_WIDTH).and_then(value_to_f64); + let height = block.get(PROP_HEIGHT).and_then(value_to_f64); + ImageSpec { + source_id, + caption, + width, + height, + } + } + + pub(super) fn normalize_source(src: &str) -> Result { + let trimmed = src.trim(); + if trimmed.is_empty() { + return Err(ParseError::ParserError("invalid_image_source".into())); + } + if let Some(rest) = trimmed.strip_prefix("blob://") { + if rest.is_empty() { + return Err(ParseError::ParserError("invalid_image_source".into())); + } + return Ok(rest.to_string()); + } + if !trimmed.contains('/') && !trimmed.contains(':') { + return Ok(trimmed.to_string()); + } + if let Some(pos) = trimmed.rfind("/blobs/") { + let id = &trimmed[pos + "/blobs/".len()..]; + let id = id.split(['?', '#']).next().unwrap_or(""); + if !id.is_empty() { + return Ok(id.to_string()); + } + } + Err(ParseError::ParserError("unsupported_image_source".into())) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(super) struct TableSpec { + pub(super) rows: Vec>, +} + +impl TableSpec { + pub(super) fn from_block_map(block: &Map) -> Self { + let mut row_entries: Vec<(String, String)> = Vec::new(); + let mut column_entries: Vec<(String, String)> = Vec::new(); + + for key in block.keys() { + if key.starts_with(PROP_ROWS_PREFIX) + && key.ends_with(PROP_ROW_ID_SUFFIX) + && let Some(row_id) = block.get(key).and_then(|value| value_to_string(&value)) + { + let base = key.trim_end_matches(PROP_ROW_ID_SUFFIX); + let order_key = format!("{base}{PROP_ORDER_SUFFIX}"); + let order = block + .get(&order_key) + .and_then(|value| value_to_string(&value)) + .unwrap_or_default(); + row_entries.push((order, row_id)); + } + if key.starts_with(PROP_COLUMNS_PREFIX) + && key.ends_with(PROP_COLUMN_ID_SUFFIX) + && let Some(column_id) = block.get(key).and_then(|value| value_to_string(&value)) + { + let base = key.trim_end_matches(PROP_COLUMN_ID_SUFFIX); + let order_key = format!("{base}{PROP_ORDER_SUFFIX}"); + let order = block + .get(&order_key) + .and_then(|value| value_to_string(&value)) + .unwrap_or_default(); + column_entries.push((order, column_id)); + } + } + + row_entries.sort_by(|a, b| a.0.cmp(&b.0)); + column_entries.sort_by(|a, b| a.0.cmp(&b.0)); + + let mut rows = Vec::new(); + for (_, row_id) in row_entries { + let mut row = Vec::new(); + for (_, column_id) in &column_entries { + let cell_key = table_cell_text_key(&row_id, column_id); + let cell_text = block + .get(&cell_key) + .and_then(|value| value_to_string(&value)) + .unwrap_or_default(); + row.push(cell_text); + } + rows.push(row); + } + + Self { rows } + } + + pub(super) fn render_markdown(&self) -> Option { + let options = MarkdownTableOptions::new(false, "
", true); + render_markdown_table(&self.rows, options) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub(super) struct BookmarkSpec { + pub(super) url: String, + pub(super) caption: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub(super) struct EmbedYoutubeSpec { + pub(super) video_id: String, +} + +#[derive(Debug, Clone, PartialEq)] +pub(super) struct EmbedIframeSpec { + pub(super) url: String, +} + +#[derive(Debug, Clone)] +pub(super) struct BlockSpec { + pub(super) flavour: BlockFlavour, + pub(super) block_type: Option, + pub(super) text: Vec, + pub(super) checked: Option, + pub(super) language: Option, + pub(super) order: Option, + pub(super) image: Option, + pub(super) table: Option, + pub(super) bookmark: Option, + pub(super) embed_youtube: Option, + pub(super) embed_iframe: Option, +} + +impl BlockSpec { + pub(super) fn is_exact(&self, other: &BlockSpec) -> bool { + self.flavour == other.flavour + && self.block_type == other.block_type + && self.checked == other.checked + && self.language == other.language + && self.image == other.image + && self.table == other.table + && self.bookmark == other.bookmark + && self.embed_youtube == other.embed_youtube + && self.embed_iframe == other.embed_iframe + && text_delta_eq(&self.text, &other.text) + } + + pub(super) fn is_similar(&self, other: &BlockSpec) -> bool { + self.flavour == other.flavour && self.block_type == other.block_type + } + + pub(super) fn block_type_str(&self) -> Option<&str> { + self.block_type.as_ref().map(BlockType::as_str) + } + + pub(super) fn from_block_map(block: &Map) -> Result { + let flavour_value = get_string(block, SYS_FLAVOUR).unwrap_or_default(); + let Some(flavour) = BlockFlavour::from_str(&flavour_value) else { + return Err(ParseError::ParserError(format!( + "unsupported block flavour: {flavour_value}" + ))); + }; + Ok(Self::from_block_map_with_flavour(block, flavour)) + } + + pub(super) fn from_block_map_with_flavour(block: &Map, flavour: BlockFlavour) -> Self { + if flavour == BlockFlavour::Image { + return BlockSpec { + flavour, + block_type: None, + text: Vec::new(), + checked: None, + language: None, + order: None, + image: Some(ImageSpec::from_block_map(block)), + table: None, + bookmark: None, + embed_youtube: None, + embed_iframe: None, + }; + } + + if flavour == BlockFlavour::Table { + return BlockSpec { + flavour, + block_type: None, + text: Vec::new(), + checked: None, + language: None, + order: None, + image: None, + table: Some(TableSpec::from_block_map(block)), + bookmark: None, + embed_youtube: None, + embed_iframe: None, + }; + } + + if flavour == BlockFlavour::Bookmark { + return BlockSpec { + flavour, + block_type: None, + text: Vec::new(), + checked: None, + language: None, + order: None, + image: None, + table: None, + bookmark: Some(BookmarkSpec { + url: get_string(block, PROP_URL).unwrap_or_default(), + caption: get_string(block, PROP_CAPTION), + }), + embed_youtube: None, + embed_iframe: None, + }; + } + + if flavour == BlockFlavour::EmbedYoutube { + return BlockSpec { + flavour, + block_type: None, + text: Vec::new(), + checked: None, + language: None, + order: None, + image: None, + table: None, + bookmark: None, + embed_youtube: Some(EmbedYoutubeSpec { + video_id: get_string(block, PROP_VIDEO_ID).unwrap_or_default(), + }), + embed_iframe: None, + }; + } + + if flavour == BlockFlavour::EmbedIframe { + return BlockSpec { + flavour, + block_type: None, + text: Vec::new(), + checked: None, + language: None, + order: None, + image: None, + table: None, + bookmark: None, + embed_youtube: None, + embed_iframe: Some(EmbedIframeSpec { + url: get_string(block, PROP_URL).unwrap_or_default(), + }), + }; + } + + let block_type = match get_string(block, PROP_TYPE) { + Some(value) => Some(BlockType::from_str_lossy(value)), + None => match flavour { + BlockFlavour::Paragraph => Some(BlockType::Text), + BlockFlavour::List => Some(BlockType::Bulleted), + _ => None, + }, + }; + + let text = block + .get(PROP_TEXT) + .and_then(|v| v.to_text()) + .map(|text| text.to_delta()) + .unwrap_or_default(); + + let checked = block.get(PROP_CHECKED).and_then(|v| v.to_any()).and_then(|a| match a { + Any::True => Some(true), + Any::False => Some(false), + _ => None, + }); + let language = get_string(block, PROP_LANGUAGE); + let order = block.get(PROP_ORDER).and_then(|v| v.to_any()).and_then(|a| match a { + Any::Integer(value) => Some(value as i64), + Any::BigInt64(value) => Some(value), + Any::Float32(value) => Some(value.0 as i64), + Any::Float64(value) => Some(value.0 as i64), + _ => None, + }); + + BlockSpec { + flavour, + block_type, + text, + checked, + language, + order, + image: None, + table: None, + bookmark: None, + embed_youtube: None, + embed_iframe: None, + } + } +} + +#[derive(Debug, Clone)] +pub(super) struct BlockNode { + pub(super) spec: BlockSpec, + pub(super) children: Vec, +} + +pub(super) trait TreeNode { + fn children(&self) -> &[Self] + where + Self: Sized; +} + +impl TreeNode for BlockNode { + fn children(&self) -> &[BlockNode] { + &self.children + } +} + +pub(super) fn count_tree_nodes(nodes: &[T]) -> usize { + nodes.iter().map(|node| 1 + count_tree_nodes(node.children())).sum() +} + +pub(super) fn text_delta_eq(a: &[TextDeltaOp], b: &[TextDeltaOp]) -> bool { + if a.len() != b.len() { + return false; + } + + for (left, right) in a.iter().zip(b.iter()) { + match (left, right) { + ( + TextDeltaOp::Insert { + insert: TextInsert::Text(text_a), + format: format_a, + }, + TextDeltaOp::Insert { + insert: TextInsert::Text(text_b), + format: format_b, + }, + ) => { + if text_a != text_b { + return false; + } + if !attrs_eq(format_a.as_ref(), format_b.as_ref()) { + return false; + } + } + _ => return false, + } + } + + true +} + +fn attrs_eq(a: Option<&TextAttributes>, b: Option<&TextAttributes>) -> bool { + match (a, b) { + (None, None) => true, + (Some(left), Some(right)) => { + if left.len() != right.len() { + return false; + } + left.iter().all(|(key, value)| right.get(key) == Some(value)) + } + _ => false, + } +} + +#[cfg(test)] +mod tests { + use y_octo::DocOptions; + + use super::{super::write::builder::text_ops_from_plain, *}; + use crate::doc_parser::build_full_doc; + + fn spec_from_markdown(markdown: &str, doc_id: &str, flavour: &str) -> BlockSpec { + let bin = build_full_doc("Title", markdown, doc_id).expect("create doc"); + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc.apply_update_from_binary_v1(&bin).expect("apply update"); + let blocks_map = doc.get_map("blocks").expect("blocks map"); + + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() + && get_string(&block_map, SYS_FLAVOUR).as_deref() == Some(flavour) + { + return BlockSpec::from_block_map(&block_map).expect("spec"); + } + } + + panic!("block not found: {flavour}"); + } + + #[test] + fn test_from_block_map_paragraph() { + let spec = spec_from_markdown("Plain paragraph.", "block-spec-paragraph", "affine:paragraph"); + assert_eq!(spec.flavour, BlockFlavour::Paragraph); + assert_eq!(spec.block_type, Some(BlockType::Text)); + assert_eq!(spec.text, text_ops_from_plain("Plain paragraph.")); + } + + #[test] + fn test_from_block_map_list_checked() { + let spec = spec_from_markdown("- [x] Done", "block-spec-list", "affine:list"); + assert_eq!(spec.flavour, BlockFlavour::List); + assert_eq!(spec.block_type, Some(BlockType::Todo)); + assert_eq!(spec.checked, Some(true)); + assert_eq!(spec.text, text_ops_from_plain("Done")); + } + + #[test] + fn test_from_block_map_image() { + let spec = spec_from_markdown("![Alt](blob://image-id)", "block-spec-image", "affine:image"); + assert_eq!(spec.flavour, BlockFlavour::Image); + let image = spec.image.expect("image spec"); + assert_eq!(image.source_id, "image-id"); + assert_eq!(image.caption.as_deref(), Some("Alt")); + assert_eq!(image.width, None); + assert_eq!(image.height, None); + } + + #[test] + fn test_from_block_map_table() { + let spec = spec_from_markdown( + "| A | B |\n| --- | --- |\n| 1 | 2 |", + "block-spec-table", + "affine:table", + ); + assert_eq!(spec.flavour, BlockFlavour::Table); + let table = spec.table.expect("table spec"); + assert_eq!( + table.rows, + vec![ + vec!["A".to_string(), "B".to_string()], + vec!["1".to_string(), "2".to_string()] + ] + ); + } + + #[test] + fn test_from_block_map_embed_iframe() { + let spec = spec_from_markdown( + r#""#, + "block-spec-embed-iframe", + "affine:embed-iframe", + ); + assert_eq!(spec.flavour, BlockFlavour::EmbedIframe); + assert_eq!(spec.embed_iframe.as_ref().unwrap().url, "https://example.com/embed"); + } +} diff --git a/packages/common/native/src/doc_parser/blocksuite.rs b/packages/common/native/src/doc_parser/blocksuite.rs index a4f15a1a7..ad266b0b1 100644 --- a/packages/common/native/src/doc_parser/blocksuite.rs +++ b/packages/common/native/src/doc_parser/blocksuite.rs @@ -2,7 +2,10 @@ use std::collections::{HashMap, HashSet}; use y_octo::Map; -use super::value::value_to_string; +use super::{ + schema::{SYS_CHILDREN, SYS_FLAVOUR, SYS_ID}, + value::value_to_string, +}; pub(super) struct BlockIndex { pub(super) block_pool: HashMap, @@ -96,7 +99,7 @@ pub(super) fn find_block_id_by_flavour(block_pool: &HashMap, flavou pub(super) fn collect_child_ids(block: &Map) -> Vec { block - .get("sys:children") + .get(SYS_CHILDREN) .and_then(|value| value.to_array()) .map(|array| { array @@ -108,11 +111,11 @@ pub(super) fn collect_child_ids(block: &Map) -> Vec { } pub(super) fn get_block_id(block: &Map) -> Option { - get_string(block, "sys:id") + get_string(block, SYS_ID) } pub(super) fn get_flavour(block: &Map) -> Option { - get_string(block, "sys:flavour") + get_string(block, SYS_FLAVOUR) } pub(super) fn get_string(block: &Map, key: &str) -> Option { @@ -157,3 +160,17 @@ pub(super) fn nearest_by_flavour( } None } + +pub(super) fn find_child_id_by_flavour( + parent: &Map, + block_pool: &HashMap, + flavour: &str, +) -> Option { + collect_child_ids(parent).into_iter().find(|id| { + block_pool + .get(id) + .and_then(get_flavour) + .as_deref() + .is_some_and(|value| value == flavour) + }) +} diff --git a/packages/common/native/src/doc_parser/doc_loader.rs b/packages/common/native/src/doc_parser/doc_loader.rs new file mode 100644 index 000000000..10e2f210d --- /dev/null +++ b/packages/common/native/src/doc_parser/doc_loader.rs @@ -0,0 +1,39 @@ +use y_octo::{Doc, DocOptions}; + +use super::ParseError; + +pub(super) fn is_empty_doc(binary: &[u8]) -> bool { + binary.is_empty() || binary == [0, 0] +} + +pub(super) fn load_doc(binary: &[u8], doc_id: Option<&str>) -> Result { + if is_empty_doc(binary) { + return Err(ParseError::InvalidBinary); + } + + let mut doc = build_doc(doc_id); + doc + .apply_update_from_binary_v1(binary) + .map_err(|_| ParseError::InvalidBinary)?; + Ok(doc) +} + +pub(super) fn load_doc_or_new(binary: &[u8]) -> Result { + if is_empty_doc(binary) { + return Ok(DocOptions::new().build()); + } + + let mut doc = DocOptions::new().build(); + doc + .apply_update_from_binary_v1(binary) + .map_err(|_| ParseError::InvalidBinary)?; + Ok(doc) +} + +fn build_doc(doc_id: Option<&str>) -> Doc { + let options = DocOptions::new(); + match doc_id { + Some(doc_id) => options.with_guid(doc_id.to_string()).build(), + None => options.build(), + } +} diff --git a/packages/common/native/src/doc_parser/error.rs b/packages/common/native/src/doc_parser/error.rs new file mode 100644 index 000000000..cf9aed437 --- /dev/null +++ b/packages/common/native/src/doc_parser/error.rs @@ -0,0 +1,38 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use y_octo::JwstCodecError; + +#[derive(Error, Debug, Serialize, Deserialize)] +pub enum ParseError { + #[error("doc_not_found")] + DocNotFound, + #[error("invalid_binary")] + InvalidBinary, + #[error("sqlite_error: {0}")] + SqliteError(String), + #[error("parser_error: {0}")] + ParserError(String), + #[error("unknown: {0}")] + Unknown(String), +} + +impl From for ParseError { + fn from(value: JwstCodecError) -> Self { + if matches!( + value, + JwstCodecError::DamagedDocumentJson + | JwstCodecError::IncompleteDocument(_) + | JwstCodecError::InvalidWriteBuffer(_) + | JwstCodecError::UpdateInvalid(_) + | JwstCodecError::StructClockInvalid { expect: _, actually: _ } + | JwstCodecError::StructSequenceInvalid { client_id: _, clock: _ } + | JwstCodecError::StructSequenceNotExists(_) + | JwstCodecError::RootStructNotFound(_) + | JwstCodecError::ParentNotFound + | JwstCodecError::IndexOutOfBound(_) + ) { + return ParseError::InvalidBinary; + } + Self::ParserError(value.to_string()) + } +} diff --git a/packages/common/native/src/doc_parser/delta_markdown.rs b/packages/common/native/src/doc_parser/markdown/delta.rs similarity index 86% rename from packages/common/native/src/doc_parser/delta_markdown.rs rename to packages/common/native/src/doc_parser/markdown/delta.rs index 9f011ba3b..29a1d96f0 100644 --- a/packages/common/native/src/doc_parser/delta_markdown.rs +++ b/packages/common/native/src/doc_parser/markdown/delta.rs @@ -6,8 +6,11 @@ use std::{ use y_octo::{AHashMap, Any, Map, Text, TextAttributes, TextDeltaOp, TextInsert, Value}; -use super::value::{ - any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any, +use super::{ + super::value::{ + any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json, value_to_any, + }, + inline::InlineStyle, }; #[derive(Debug, Clone)] @@ -20,18 +23,18 @@ struct InlineReference { } #[derive(Debug, Clone)] -pub(super) struct InlineReferencePayload { - pub(super) doc_id: String, - pub(super) payload: String, +pub(crate) struct InlineReferencePayload { + pub(crate) doc_id: String, + pub(crate) payload: String, } #[derive(Debug, Clone)] -pub(super) struct DeltaToMdOptions { +pub(crate) struct DeltaToMdOptions { doc_url_prefix: Option, } impl DeltaToMdOptions { - pub(super) fn new(doc_url_prefix: Option) -> Self { + pub(crate) fn new(doc_url_prefix: Option) -> Self { Self { doc_url_prefix } } @@ -54,21 +57,32 @@ impl DeltaToMdOptions { } } -pub(super) fn text_to_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option { - block - .get(key) - .and_then(|value| value.to_text()) - .map(|text| delta_to_markdown(&text, options)) -} - -pub(super) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option { +pub(crate) fn text_to_inline_markdown(block: &Map, key: &str, options: &DeltaToMdOptions) -> Option { block .get(key) .and_then(|value| value.to_text()) .map(|text| delta_to_inline_markdown(&text, options)) } -pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec { +pub(crate) fn delta_ops_to_markdown(ops: &[TextDeltaOp], options: &DeltaToMdOptions) -> String { + delta_to_markdown_with_options(ops, options, true) +} + +pub(crate) fn delta_ops_to_plain_text(ops: &[TextDeltaOp]) -> String { + let mut out = String::new(); + for op in ops { + if let TextDeltaOp::Insert { + insert: TextInsert::Text(text), + .. + } = op + { + out.push_str(text); + } + } + out +} + +pub(crate) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec { let mut refs = Vec::new(); let mut seen: HashSet<(String, String)> = HashSet::new(); @@ -80,7 +94,7 @@ pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec continue, }; - let reference = match attrs.get("reference").and_then(parse_inline_reference) { + let reference = match attrs.get(InlineStyle::Reference.key()).and_then(parse_inline_reference) { Some(reference) => reference, None => continue, }; @@ -102,7 +116,7 @@ pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec Vec { +pub(crate) fn extract_inline_references_from_value(value: &Value) -> Vec { if let Some(text) = value.to_text() { return extract_inline_references(&text.to_delta()); } @@ -123,7 +137,11 @@ fn extract_inline_references_from_any(value: &Any) -> Vec = HashSet::new(); for op in ops { - let reference = match op.attributes.get("reference").and_then(parse_inline_reference) { + let reference = match op + .attributes + .get(InlineStyle::Reference.key()) + .and_then(parse_inline_reference) + { Some(reference) => reference, None => continue, }; @@ -178,10 +196,6 @@ fn inline_reference_payload(reference: &InlineReference) -> Option { Some(build_reference_payload(&reference.page_id, params)) } -fn delta_to_markdown(text: &Text, options: &DeltaToMdOptions) -> String { - delta_to_markdown_with_options(&text.to_delta(), options, true) -} - fn delta_to_inline_markdown(text: &Text, options: &DeltaToMdOptions) -> String { delta_to_markdown_with_options(&text.to_delta(), options, false) } @@ -274,7 +288,7 @@ fn delta_any_to_inline_markdown(value: &Any, options: &DeltaToMdOptions) -> Opti delta_ops_from_any(value).map(|ops| delta_ops_to_markdown_with_options(&ops, options, false)) } -pub(super) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option { +pub(crate) fn delta_value_to_inline_markdown(value: &Value, options: &DeltaToMdOptions) -> Option { if let Some(text) = value.to_text() { return Some(delta_to_inline_markdown(&text, options)); } @@ -538,19 +552,29 @@ fn apply_inline_attributes( } fn inline_node_for_attr(attr: &str, attrs: &TextAttributes, options: &DeltaToMdOptions) -> Option>> { - match attr { - "italic" => Some(Node::new_inline("_", "_")), - "bold" => Some(Node::new_inline("**", "**")), - "link" => attrs + let style = InlineStyle::from_key(attr)?; + if let Some(delimiter) = style.delimiter() { + return Some(Node::new_inline(delimiter.open, delimiter.close)); + } + + match style { + InlineStyle::Underline => Some(Node::new_inline("", "")), + InlineStyle::Color => { + let color = attrs.get(attr).and_then(any_as_string)?.trim(); + if color.is_empty() { + None + } else { + Some(Node::new_inline(&format!(""), "")) + } + } + InlineStyle::Link => attrs .get(attr) .and_then(any_as_string) .map(|url| Node::new_inline("[", &format!("]({url})"))), - "reference" => attrs.get(attr).and_then(parse_inline_reference).map(|reference| { + InlineStyle::Reference => attrs.get(attr).and_then(parse_inline_reference).map(|reference| { let (title, link) = options.build_reference_link(&reference); Node::new_inline("[", &format!("{title}]({link})")) }), - "strike" => Some(Node::new_inline("~~", "~~")), - "code" => Some(Node::new_inline("`", "`")), _ => None, } } @@ -560,7 +584,7 @@ fn has_block_level_attribute(attrs: &TextAttributes) -> bool { } fn is_inline_attribute(attr: &str) -> bool { - matches!(attr, "italic" | "bold" | "link" | "reference" | "strike" | "code") + InlineStyle::from_key(attr).is_some() } fn encode_link(link: &str) -> String { @@ -741,13 +765,17 @@ fn new_line( #[cfg(test)] mod tests { use serde_json::Value; + use y_octo::{Any, TextAttributes, TextDeltaOp, TextInsert}; use super::*; #[test] fn test_delta_to_inline_markdown_link() { let mut attrs = TextAttributes::new(); - attrs.insert("link".into(), Any::String("https://example.com".into())); + attrs.insert( + InlineStyle::Link.key().into(), + Any::String("https://example.com".into()), + ); let delta = vec![TextDeltaOp::Insert { insert: TextInsert::Text("AFFiNE".into()), @@ -767,7 +795,7 @@ mod tests { ref_map.insert("type".into(), Any::String("LinkedPage".into())); let mut attrs = TextAttributes::new(); - attrs.insert("reference".into(), Any::Object(ref_map)); + attrs.insert(InlineStyle::Reference.key().into(), Any::Object(ref_map)); let delta = vec![TextDeltaOp::Insert { insert: TextInsert::Text("Doc Title".into()), @@ -781,4 +809,34 @@ mod tests { let payload: Value = serde_json::from_str(&refs[0].payload).unwrap(); assert_eq!(payload, serde_json::json!({ "docId": "doc123" })); } + + #[test] + fn test_delta_to_inline_markdown_underline() { + let mut attrs = TextAttributes::new(); + attrs.insert(InlineStyle::Underline.key().into(), Any::True); + + let delta = vec![TextDeltaOp::Insert { + insert: TextInsert::Text("Under".into()), + format: Some(attrs), + }]; + + let options = DeltaToMdOptions::new(None); + let rendered = delta_to_markdown_with_options(&delta, &options, false); + assert_eq!(rendered, "Under"); + } + + #[test] + fn test_delta_to_inline_markdown_color() { + let mut attrs = TextAttributes::new(); + attrs.insert(InlineStyle::Color.key().into(), Any::String("red".into())); + + let delta = vec![TextDeltaOp::Insert { + insert: TextInsert::Text("Red".into()), + format: Some(attrs), + }]; + + let options = DeltaToMdOptions::new(None); + let rendered = delta_to_markdown_with_options(&delta, &options, false); + assert_eq!(rendered, "Red"); + } } diff --git a/packages/common/native/src/doc_parser/markdown/inline.rs b/packages/common/native/src/doc_parser/markdown/inline.rs new file mode 100644 index 000000000..aa8d9b31d --- /dev/null +++ b/packages/common/native/src/doc_parser/markdown/inline.rs @@ -0,0 +1,71 @@ +const INLINE_ATTR_BOLD: &str = "bold"; +const INLINE_ATTR_ITALIC: &str = "italic"; +const INLINE_ATTR_UNDERLINE: &str = "underline"; +const INLINE_ATTR_STRIKE: &str = "strike"; +const INLINE_ATTR_CODE: &str = "code"; +const INLINE_ATTR_LINK: &str = "link"; +const INLINE_ATTR_REFERENCE: &str = "reference"; +const INLINE_ATTR_COLOR: &str = "color"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum InlineStyle { + Bold, + Italic, + Underline, + Strike, + Code, + Link, + Reference, + Color, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct InlineDelimiter { + pub(super) open: &'static str, + pub(super) close: &'static str, +} + +impl InlineStyle { + pub(super) fn key(self) -> &'static str { + match self { + InlineStyle::Bold => INLINE_ATTR_BOLD, + InlineStyle::Italic => INLINE_ATTR_ITALIC, + InlineStyle::Underline => INLINE_ATTR_UNDERLINE, + InlineStyle::Strike => INLINE_ATTR_STRIKE, + InlineStyle::Code => INLINE_ATTR_CODE, + InlineStyle::Link => INLINE_ATTR_LINK, + InlineStyle::Reference => INLINE_ATTR_REFERENCE, + InlineStyle::Color => INLINE_ATTR_COLOR, + } + } + + pub(super) fn from_key(key: &str) -> Option { + match key { + INLINE_ATTR_BOLD => Some(InlineStyle::Bold), + INLINE_ATTR_ITALIC => Some(InlineStyle::Italic), + INLINE_ATTR_UNDERLINE => Some(InlineStyle::Underline), + INLINE_ATTR_STRIKE => Some(InlineStyle::Strike), + INLINE_ATTR_CODE => Some(InlineStyle::Code), + INLINE_ATTR_LINK => Some(InlineStyle::Link), + INLINE_ATTR_REFERENCE => Some(InlineStyle::Reference), + INLINE_ATTR_COLOR => Some(InlineStyle::Color), + _ => None, + } + } + + pub(super) fn delimiter(self) -> Option { + match self { + InlineStyle::Italic => Some(InlineDelimiter { open: "_", close: "_" }), + InlineStyle::Bold => Some(InlineDelimiter { + open: "**", + close: "**", + }), + InlineStyle::Strike => Some(InlineDelimiter { + open: "~~", + close: "~~", + }), + InlineStyle::Code => Some(InlineDelimiter { open: "`", close: "`" }), + InlineStyle::Link | InlineStyle::Reference | InlineStyle::Underline | InlineStyle::Color => None, + } + } +} diff --git a/packages/common/native/src/doc_parser/markdown/mod.rs b/packages/common/native/src/doc_parser/markdown/mod.rs new file mode 100644 index 000000000..0a036bc7c --- /dev/null +++ b/packages/common/native/src/doc_parser/markdown/mod.rs @@ -0,0 +1,13 @@ +mod delta; +mod inline; +mod parser; +mod render; + +pub(crate) use delta::{ + DeltaToMdOptions, InlineReferencePayload, delta_value_to_inline_markdown, extract_inline_references, + extract_inline_references_from_value, text_to_inline_markdown, +}; +#[cfg(test)] +pub(crate) use parser::MAX_MARKDOWN_CHARS; +pub(crate) use parser::{MAX_BLOCKS, parse_markdown_blocks}; +pub(crate) use render::{MarkdownRenderer, MarkdownWriter}; diff --git a/packages/common/native/src/doc_parser/markdown/parser.rs b/packages/common/native/src/doc_parser/markdown/parser.rs new file mode 100644 index 000000000..ea7194d99 --- /dev/null +++ b/packages/common/native/src/doc_parser/markdown/parser.rs @@ -0,0 +1,1725 @@ +//! Shared markdown utilities for the doc_parser module + +use std::collections::HashMap; + +use pulldown_cmark::{CodeBlockKind, Event, HeadingLevel, Options, Parser, Tag, TagEnd}; +use y_octo::{Any, TextAttributes, TextDeltaOp, TextInsert}; + +use super::{ + super::{ + ParseError, + block_spec::{ + BlockFlavour, BlockNode, BlockSpec, BlockType, BookmarkSpec, EmbedIframeSpec, EmbedYoutubeSpec, ImageSpec, + TableSpec, count_tree_nodes, + }, + }, + inline::InlineStyle, +}; + +const DEFAULT_CODE_LANG: &str = "plain text"; +pub(crate) const MAX_MARKDOWN_CHARS: usize = 200_000; +pub(crate) const MAX_BLOCKS: usize = 2_000; + +fn markdown_options() -> Options { + Options::ENABLE_STRIKETHROUGH + | Options::ENABLE_TABLES + | Options::ENABLE_TASKLISTS + | Options::ENABLE_HEADING_ATTRIBUTES +} + +impl BlockType { + pub fn from_heading_level(level: HeadingLevel) -> Self { + match level { + HeadingLevel::H1 => BlockType::H1, + HeadingLevel::H2 => BlockType::H2, + HeadingLevel::H3 => BlockType::H3, + HeadingLevel::H4 => BlockType::H4, + HeadingLevel::H5 => BlockType::H5, + HeadingLevel::H6 => BlockType::H6, + } + } +} + +#[derive(Debug, Clone)] +pub(super) struct MarkdownDocument { + pub blocks: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct InlineAttr { + style: InlineStyle, + value: Option, +} + +impl InlineAttr { + fn new(style: InlineStyle) -> Self { + Self { style, value: None } + } + + fn color(value: String) -> Self { + Self { + style: InlineStyle::Color, + value: Some(value), + } + } + + fn link(url: String) -> Self { + Self { + style: InlineStyle::Link, + value: Some(url), + } + } + + fn key(&self) -> &'static str { + self.style.key() + } +} + +#[derive(Debug, Default)] +struct InlineState { + stack: Vec, +} + +impl InlineState { + fn push(&mut self, attr: InlineAttr) { + self.stack.push(attr); + } + + fn pop(&mut self, attr: InlineAttr) { + if let Some(pos) = self.stack.iter().rposition(|item| item.key() == attr.key()) { + self.stack.remove(pos); + } + } + + fn attrs(&self) -> Option { + if self.stack.is_empty() { + return None; + } + + let mut attrs = TextAttributes::new(); + for attr in &self.stack { + match attr.style { + InlineStyle::Link => { + if let Some(url) = attr.value.as_ref() { + attrs.insert(attr.key().into(), Any::String(url.clone())); + } + } + InlineStyle::Color => { + if let Some(color) = attr.value.as_ref() { + attrs.insert(attr.key().into(), Any::String(color.clone())); + } + } + _ => { + attrs.insert(attr.key().into(), Any::True); + } + } + } + + Some(attrs) + } + + fn attrs_with(&self, extra: InlineAttr) -> Option { + let mut attrs = self.attrs().unwrap_or_default(); + let key = extra.key(); + match extra.style { + InlineStyle::Link => { + if let Some(url) = extra.value.as_ref() { + attrs.insert(key.into(), Any::String(url.clone())); + } + } + InlineStyle::Color => { + if let Some(color) = extra.value.as_ref() { + attrs.insert(key.into(), Any::String(color.clone())); + } + } + _ => { + attrs.insert(key.into(), Any::True); + } + } + Some(attrs) + } +} + +#[derive(Debug)] +struct BlockDraft { + flavour: BlockFlavour, + block_type: Option, + checked: Option, + language: Option, + order: Option, + text: Vec, + children: Vec, +} + +impl BlockDraft { + fn new(flavour: BlockFlavour, block_type: Option) -> Self { + let block_type = if flavour == BlockFlavour::Paragraph { + block_type.or(Some(BlockType::Text)) + } else { + block_type + }; + + Self { + flavour, + block_type, + checked: None, + language: None, + order: None, + text: Vec::new(), + children: Vec::new(), + } + } + + fn push_text(&mut self, text: &str, attrs: Option) { + if text.is_empty() { + return; + } + + self.text.push(TextDeltaOp::Insert { + insert: TextInsert::Text(text.to_string()), + format: attrs, + }); + } + + fn is_empty(&self) -> bool { + self.text.is_empty() && self.children.is_empty() + } + + fn finish(self) -> BlockNode { + BlockNode { + spec: BlockSpec { + flavour: self.flavour, + block_type: self.block_type, + text: self.text, + checked: self.checked, + language: self.language, + order: self.order, + image: None, + table: None, + bookmark: None, + embed_youtube: None, + embed_iframe: None, + }, + children: self.children, + } + } +} + +#[derive(Debug)] +struct ListContext { + ordered: bool, + next_index: i64, +} + +#[derive(Debug)] +struct ImageDraft { + source: String, + caption: String, + width: Option, + height: Option, +} + +impl ImageDraft { + fn finish(self) -> Result { + let caption = if self.caption.trim().is_empty() { + None + } else { + Some(self.caption) + }; + let source_id = ImageSpec::normalize_source(&self.source)?; + Ok(ImageSpec { + source_id, + caption, + width: self.width, + height: self.height, + }) + } +} + +#[derive(Debug, Default)] +struct TableState { + rows: Vec>, + current_row: Vec, + current_cell: String, + pending_link: Option, + pending_image: Option, + row_in_progress: bool, + cell_in_progress: bool, + in_head: bool, +} + +impl TableState { + fn start_row(&mut self) { + self.current_row.clear(); + self.row_in_progress = true; + } + + fn finish_row(&mut self) { + if self.row_in_progress { + self.rows.push(std::mem::take(&mut self.current_row)); + } + self.row_in_progress = false; + } + + fn start_cell(&mut self) { + self.current_cell.clear(); + self.cell_in_progress = true; + } + + fn finish_cell(&mut self) { + if self.cell_in_progress { + let cell = self.current_cell.trim().to_string(); + self.current_row.push(cell); + } + self.current_cell.clear(); + self.cell_in_progress = false; + } + + fn push_text(&mut self, text: &str) { + self.current_cell.push_str(text); + } + + fn push_marker(&mut self, marker: &str) { + self.current_cell.push_str(marker); + } +} + +pub(crate) fn parse_markdown_blocks(markdown: &str) -> Result, ParseError> { + let normalized = normalize_markdown(markdown); + if normalized.len() > MAX_MARKDOWN_CHARS { + return Err(ParseError::ParserError("markdown_too_large".into())); + } + + validate_markdown_inner(&normalized)?; + let parsed = parse_markdown_inner(&normalized)?; + if count_tree_nodes(&parsed.blocks) > MAX_BLOCKS { + return Err(ParseError::ParserError("block_count_too_large".into())); + } + Ok(parsed.blocks) +} + +/// Parses markdown content into blocks suitable for building a ydoc. +/// +/// The first H1 can be skipped to act as the document title. +fn parse_markdown_inner(markdown: &str) -> Result { + let parser = Parser::new_ext(markdown, markdown_options()); + + let mut blocks: Vec = Vec::new(); + + let mut inline = InlineState::default(); + let mut list_stack: Vec = Vec::new(); + let mut list_items: Vec = Vec::new(); + let mut active: Option = None; + let mut in_blockquote = false; + let mut pending_image: Option = None; + let mut pending_bookmark: Option = None; + let mut table_state: Option = None; + let mut span_stack: Vec = Vec::new(); + + for event in parser { + let mut table_completed: Option>> = None; + let mut table_handled = false; + if let Some(state) = table_state.as_mut() { + match &event { + Event::Start(Tag::TableHead) => { + state.in_head = true; + table_handled = true; + } + Event::End(TagEnd::TableHead) => { + if state.cell_in_progress { + state.finish_cell(); + } + state.finish_row(); + state.in_head = false; + table_handled = true; + } + Event::Start(Tag::TableRow) => { + state.start_row(); + table_handled = true; + } + Event::End(TagEnd::TableRow) => { + if state.cell_in_progress { + state.finish_cell(); + } + state.finish_row(); + table_handled = true; + } + Event::Start(Tag::TableCell) => { + if state.in_head && !state.row_in_progress { + state.start_row(); + } + state.start_cell(); + table_handled = true; + } + Event::End(TagEnd::TableCell) => { + state.finish_cell(); + table_handled = true; + } + Event::Start(Tag::Image { dest_url, .. }) => { + state.pending_image = Some(ImageDraft { + source: dest_url.to_string(), + caption: String::new(), + width: None, + height: None, + }); + table_handled = true; + } + Event::End(TagEnd::Image) => { + if let Some(image) = state.pending_image.take() { + let alt = image.caption.trim(); + let src = image.source; + let fragment = if alt.is_empty() { + format!("![]({src})") + } else { + format!("![{alt}]({src})") + }; + state.push_text(&fragment); + } + table_handled = true; + } + Event::Text(text) => { + if let Some(image) = state.pending_image.as_mut() { + image.caption.push_str(text); + } else { + state.push_text(text); + } + table_handled = true; + } + Event::Code(code) => { + let fragment = format!("`{code}`"); + state.push_text(&fragment); + table_handled = true; + } + Event::SoftBreak | Event::HardBreak => { + state.push_text(" "); + table_handled = true; + } + Event::Start(Tag::Strong) => { + state.push_marker("**"); + table_handled = true; + } + Event::End(TagEnd::Strong) => { + state.push_marker("**"); + table_handled = true; + } + Event::Start(Tag::Emphasis) => { + state.push_marker("_"); + table_handled = true; + } + Event::End(TagEnd::Emphasis) => { + state.push_marker("_"); + table_handled = true; + } + Event::Start(Tag::Strikethrough) => { + state.push_marker("~~"); + table_handled = true; + } + Event::End(TagEnd::Strikethrough) => { + state.push_marker("~~"); + table_handled = true; + } + Event::Start(Tag::Link { dest_url, .. }) => { + state.push_marker("["); + state.pending_link = Some(dest_url.to_string()); + table_handled = true; + } + Event::End(TagEnd::Link) => { + if let Some(url) = state.pending_link.take() { + state.push_marker(&format!("]({url})")); + } + table_handled = true; + } + Event::Html(html) | Event::InlineHtml(html) => { + if let Some(text) = extract_wrapped_html_text(html) { + state.push_text(&text); + } else if is_html_line_break(html) { + state.push_text("\n"); + } else if let Some(tag) = parse_html_tag(html) + && matches!(tag.name.as_str(), "u" | "span") + { + // Ignore inline formatting tags inside table cells. + } else if !html.trim().is_empty() { + state.push_text(html); + } + table_handled = true; + } + Event::End(TagEnd::Table) => { + if state.cell_in_progress { + state.finish_cell(); + } + state.finish_row(); + table_completed = Some(std::mem::take(&mut state.rows)); + table_handled = true; + } + _ => {} + } + } + if let Some(rows) = table_completed { + table_state = None; + attach_block(table_block(rows), &mut list_items, &mut blocks); + continue; + } + if table_handled { + continue; + } + if matches!(event, Event::Start(Tag::Table(_))) { + if let Some(block) = active.take() { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + table_state = Some(TableState::default()); + continue; + } + + match event { + Event::Start(Tag::Heading { level, .. }) => { + active = Some(BlockDraft::new( + BlockFlavour::Paragraph, + Some(BlockType::from_heading_level(level)), + )); + } + Event::End(TagEnd::Heading(_)) => { + if let Some(block) = active.take() { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + } + Event::Start(Tag::Paragraph) => { + if in_blockquote { + if active.is_none() { + active = Some(BlockDraft::new(BlockFlavour::Paragraph, Some(BlockType::Quote))); + } + } else if list_items.is_empty() { + active = Some(BlockDraft::new(BlockFlavour::Paragraph, Some(BlockType::Text))); + } + } + Event::End(TagEnd::Paragraph) => { + if let Some(url) = pending_bookmark.take() + && active.as_ref().is_some_and(|block| block.is_empty()) + { + active = None; + attach_block(bookmark_block(url), &mut list_items, &mut blocks); + continue; + } + if in_blockquote { + if let Some(block) = active.as_mut() { + block.push_text("\n", None); + } + } else if let Some(block) = active.take() { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + } + Event::Start(Tag::BlockQuote(_)) => { + in_blockquote = true; + } + Event::End(TagEnd::BlockQuote(_)) => { + in_blockquote = false; + if let Some(block) = active.take() { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + } + Event::Start(Tag::List(start_num)) => { + let start = start_num.unwrap_or(1) as i64; + list_stack.push(ListContext { + ordered: start_num.is_some(), + next_index: start, + }); + } + Event::End(TagEnd::List(_)) => { + list_stack.pop(); + } + Event::Start(Tag::Item) => { + let Some(context) = list_stack.last_mut() else { + continue; + }; + let order = if context.ordered { + let order = context.next_index; + context.next_index += 1; + Some(order) + } else { + None + }; + + let block_type = if context.ordered { + BlockType::Numbered + } else { + BlockType::Bulleted + }; + + let mut draft = BlockDraft::new(BlockFlavour::List, Some(block_type)); + draft.checked = None; + draft.order = order; + list_items.push(draft); + } + Event::End(TagEnd::Item) => { + if let Some(block) = list_items.pop() { + let finished = block.finish(); + if let Some(parent) = list_items.last_mut() { + parent.children.push(finished); + } else { + blocks.push(finished); + } + } + } + Event::TaskListMarker(checked) => { + if let Some(item) = list_items.last_mut() { + item.checked = Some(checked); + item.block_type = Some(BlockType::Todo); + item.order = None; + } + } + Event::Start(Tag::CodeBlock(kind)) => { + let mut draft = BlockDraft::new(BlockFlavour::Code, None); + match kind { + CodeBlockKind::Fenced(lang) => { + if !lang.is_empty() { + draft.language = Some(lang.to_string()); + } else { + draft.language = Some(DEFAULT_CODE_LANG.to_string()); + } + } + CodeBlockKind::Indented => { + draft.language = Some(DEFAULT_CODE_LANG.to_string()); + } + } + active = Some(draft); + } + Event::End(TagEnd::CodeBlock) => { + if let Some(block) = active.take() { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + } + Event::Start(Tag::Image { dest_url, .. }) => { + if let Some(block) = active.take() + && !block.is_empty() + { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + pending_image = Some(ImageDraft { + source: dest_url.to_string(), + caption: String::new(), + width: None, + height: None, + }); + } + Event::End(TagEnd::Image) => { + if let Some(image) = pending_image.take() { + let image = image.finish()?; + attach_block(image_block(image), &mut list_items, &mut blocks); + } + } + Event::Text(text) => { + if pending_bookmark.is_some() && !text.trim().is_empty() { + pending_bookmark = None; + } + if let Some(image) = pending_image.as_mut() { + image.caption.push_str(&text); + } else if let Some(block) = active.as_mut() { + let attrs = inline.attrs(); + block.push_text(&text, attrs); + } else if let Some(item) = list_items.last_mut() { + let attrs = inline.attrs(); + item.push_text(&text, attrs); + } + } + Event::Html(html) | Event::InlineHtml(html) => { + if is_ai_editable_comment(&html) { + continue; + } + if let Some((text, attrs)) = parse_wrapped_inline_html(&html) { + if let Some(image) = pending_image.as_mut() { + image.caption.push_str(&text); + } else if let Some(block) = active.as_mut() { + block.push_text(&text, attrs); + } else if let Some(item) = list_items.last_mut() { + item.push_text(&text, attrs); + } + continue; + } + if handle_inline_html_tag(&html, &mut inline, &mut span_stack) { + continue; + } + if let Some(image) = parse_img_tag(&html) { + if let Some(block) = active.take() + && !block.is_empty() + { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + let image = image.finish()?; + attach_block(image_block(image), &mut list_items, &mut blocks); + } else if let Some(embed) = parse_iframe_tag(&html) { + if let Some(block) = active.take() + && !block.is_empty() + { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + match embed { + IframeEmbed::Youtube(video_id) => { + attach_block(embed_youtube_block(video_id), &mut list_items, &mut blocks); + } + IframeEmbed::Iframe(url) => { + attach_block(embed_iframe_block(url), &mut list_items, &mut blocks); + } + } + } else if is_html_line_break(&html) { + if let Some(image) = pending_image.as_mut() { + image.caption.push(' '); + } else if let Some(block) = active.as_mut() { + let attrs = inline.attrs(); + block.push_text("\n", attrs); + } else if let Some(item) = list_items.last_mut() { + let attrs = inline.attrs(); + item.push_text("\n", attrs); + } + } else if let Some(block) = active.as_mut() { + let attrs = inline.attrs(); + block.push_text(&html, attrs); + } else if let Some(item) = list_items.last_mut() { + let attrs = inline.attrs(); + item.push_text(&html, attrs); + } else if !html.trim().is_empty() { + let mut draft = BlockDraft::new(BlockFlavour::Code, None); + draft.language = Some("html".to_string()); + draft.push_text(&html, None); + attach_block(draft.finish(), &mut list_items, &mut blocks); + } + } + Event::Code(code) => { + if pending_bookmark.is_some() && !code.trim().is_empty() { + pending_bookmark = None; + } + if let Some(image) = pending_image.as_mut() { + image.caption.push_str(&code); + } else { + let attrs = inline.attrs_with(InlineAttr::new(InlineStyle::Code)); + if let Some(block) = active.as_mut() { + block.push_text(&code, attrs); + } else if let Some(item) = list_items.last_mut() { + item.push_text(&code, attrs); + } + } + } + Event::SoftBreak | Event::HardBreak => { + if pending_bookmark.is_some() { + pending_bookmark = None; + } + if let Some(image) = pending_image.as_mut() { + image.caption.push(' '); + } else { + let break_text = if matches!(active.as_ref().map(|b| b.flavour), Some(BlockFlavour::Code)) { + "\n" + } else { + " " + }; + if let Some(block) = active.as_mut() { + let attrs = inline.attrs(); + block.push_text(break_text, attrs); + } else if let Some(item) = list_items.last_mut() { + let attrs = inline.attrs(); + item.push_text(break_text, attrs); + } + } + } + Event::Rule => { + let divider = BlockDraft::new(BlockFlavour::Divider, None).finish(); + attach_block(divider, &mut list_items, &mut blocks); + } + Event::Start(Tag::Strong) => inline.push(InlineAttr::new(InlineStyle::Bold)), + Event::End(TagEnd::Strong) => inline.pop(InlineAttr::new(InlineStyle::Bold)), + Event::Start(Tag::Emphasis) => inline.push(InlineAttr::new(InlineStyle::Italic)), + Event::End(TagEnd::Emphasis) => inline.pop(InlineAttr::new(InlineStyle::Italic)), + Event::Start(Tag::Strikethrough) => inline.push(InlineAttr::new(InlineStyle::Strike)), + Event::End(TagEnd::Strikethrough) => inline.pop(InlineAttr::new(InlineStyle::Strike)), + Event::Start(Tag::Link { dest_url, .. }) => { + if let Some(url) = parse_bookmark_url(&dest_url) + && active + .as_ref() + .is_some_and(|block| block.flavour == BlockFlavour::Paragraph && block.is_empty()) + && list_items.is_empty() + { + pending_bookmark = Some(url); + } else { + inline.push(InlineAttr::link(dest_url.to_string())); + } + } + Event::End(TagEnd::Link) => { + if pending_bookmark.is_none() { + inline.pop(InlineAttr::new(InlineStyle::Link)); + } + } + _ => {} + } + } + + if let Some(block) = active.take() { + attach_block(block.finish(), &mut list_items, &mut blocks); + } + if let Some(image) = pending_image.take() { + let image = image.finish()?; + attach_block(image_block(image), &mut list_items, &mut blocks); + } + if let Some(mut state) = table_state.take() { + state.finish_row(); + if !state.rows.is_empty() { + attach_block(table_block(state.rows), &mut list_items, &mut blocks); + } + } + + Ok(MarkdownDocument { blocks }) +} + +fn validate_markdown_inner(markdown: &str) -> Result<(), ParseError> { + let parser = Parser::new_ext(markdown, markdown_options()); + + for event in parser { + match event { + Event::Start(tag) => ensure_supported_tag(&tag)?, + Event::Html(html) | Event::InlineHtml(html) => { + if is_ai_editable_comment(&html) { + continue; + } + if parse_img_tag(&html).is_some() { + continue; + } + if parse_iframe_tag(&html).is_some() { + continue; + } + if is_html_line_break(&html) { + continue; + } + if is_supported_inline_html(&html) { + continue; + } + return Err(ParseError::ParserError("unsupported_markdown:html".into())); + } + Event::FootnoteReference(_) => { + return Err(ParseError::ParserError("unsupported_markdown:footnote".into())); + } + Event::InlineMath(_) | Event::DisplayMath(_) => { + return Err(ParseError::ParserError("unsupported_markdown:math".into())); + } + _ => {} + } + } + + Ok(()) +} + +fn ensure_supported_tag(tag: &Tag) -> Result<(), ParseError> { + match tag { + Tag::Paragraph + | Tag::Heading { .. } + | Tag::BlockQuote(_) + | Tag::CodeBlock(_) + | Tag::List(_) + | Tag::Item + | Tag::Emphasis + | Tag::Strong + | Tag::Strikethrough + | Tag::Link { .. } + | Tag::Image { .. } + | Tag::Table(_) + | Tag::TableHead + | Tag::TableRow + | Tag::TableCell => Ok(()), + Tag::HtmlBlock => Ok(()), + Tag::FootnoteDefinition(_) => Err(ParseError::ParserError("unsupported_markdown:footnote".into())), + Tag::DefinitionList | Tag::DefinitionListTitle | Tag::DefinitionListDefinition => { + Err(ParseError::ParserError("unsupported_markdown:definition_list".into())) + } + Tag::Superscript => Err(ParseError::ParserError("unsupported_markdown:superscript".into())), + Tag::Subscript => Err(ParseError::ParserError("unsupported_markdown:subscript".into())), + Tag::MetadataBlock(_) => Err(ParseError::ParserError("unsupported_markdown:metadata".into())), + } +} + +fn parse_img_tag(html: &str) -> Option { + let tag = html.trim(); + if !tag.starts_with("().ok()); + let height = attrs.get("height").and_then(|value| value.parse::().ok()); + Some(ImageDraft { + source, + caption: caption.unwrap_or_default(), + width, + height, + }) +} + +enum IframeEmbed { + Youtube(String), + Iframe(String), +} + +fn parse_iframe_tag(html: &str) -> Option { + let tag = html.trim(); + if !tag.to_ascii_lowercase().starts_with(" Option { + let src = src.trim(); + let prefix = "https://www.youtube.com/embed/"; + if !src.starts_with(prefix) { + return None; + } + let id = &src[prefix.len()..]; + let id = id.split(['?', '#', '/']).next().unwrap_or(""); + if id.is_empty() { None } else { Some(id.to_string()) } +} + +const AFFINE_DOMAINS: [&str; 6] = [ + "affine.pro", + "app.affine.pro", + "insider.affine.pro", + "affine.fail", + "toeverything.app", + "apple.getaffineapp.com", +]; + +fn is_valid_generic_embed_url(url: &str) -> bool { + let Some(host) = parse_https_host(url) else { + return false; + }; + !AFFINE_DOMAINS + .iter() + .any(|domain| host == *domain || host.ends_with(&format!(".{domain}"))) +} + +fn parse_https_host(url: &str) -> Option { + let trimmed = url.trim(); + if trimmed.is_empty() { + return None; + } + let lower = trimmed.to_ascii_lowercase(); + let (scheme, rest) = lower.split_once("://")?; + if scheme != "https" { + return None; + } + let host_port = rest.split(&['/', '?', '#'][..]).next().unwrap_or(""); + if host_port.is_empty() { + return None; + } + let host = host_port.split('@').next_back().unwrap_or(""); + let host = host.split(':').next().unwrap_or(""); + if host.is_empty() { + return None; + } + Some(host.to_string()) +} + +fn is_ai_editable_comment(html: &str) -> bool { + let trimmed = html.trim(); + if !trimmed.starts_with("") { + return false; + } + let body = trimmed.trim_start_matches("").trim(); + body.contains("block_id=") && body.contains("flavour=") +} + +fn is_html_line_break(html: &str) -> bool { + let trimmed = html.trim(); + if !trimmed.starts_with('<') || !trimmed.ends_with('>') { + return false; + } + let inner = trimmed.trim_start_matches('<').trim_end_matches('>').trim(); + let inner = inner.trim_end_matches('/').trim(); + inner.eq_ignore_ascii_case("br") +} + +fn parse_bookmark_url(dest_url: &str) -> Option { + let (prefix, url) = dest_url.split_once(',')?; + if !prefix.trim().eq_ignore_ascii_case("bookmark") { + return None; + } + let url = url.trim(); + if url.is_empty() { None } else { Some(url.to_string()) } +} + +fn normalize_markdown(markdown: &str) -> String { + if !markdown.contains('<') { + return markdown.to_string(); + } + normalize_html_lists(markdown) +} + +#[derive(Debug, Clone, Copy)] +enum ListKind { + Unordered, + Ordered, +} + +#[derive(Debug, Clone, Copy)] +struct ListState { + kind: ListKind, + counter: usize, +} + +fn normalize_html_lists(markdown: &str) -> String { + if !markdown.contains(" = None; + let mut list_stack: Vec = Vec::new(); + + for chunk in markdown.split_inclusive('\n') { + let line = chunk.strip_suffix('\n').unwrap_or(chunk); + let newline = if chunk.ends_with('\n') { "\n" } else { "" }; + let trimmed = line.trim_start(); + + if let Some(marker) = fence_marker_start(trimmed) { + if !in_fence { + in_fence = true; + fence_marker = Some(marker.to_string()); + } else if fence_marker.as_deref() == Some(marker) { + in_fence = false; + fence_marker = None; + } + out.push_str(line); + out.push_str(newline); + continue; + } + + if in_fence { + out.push_str(line); + out.push_str(newline); + continue; + } + + let normalized_line = normalize_html_lists_line(line, &mut list_stack); + out.push_str(&normalized_line); + out.push_str(newline); + } + + out +} + +fn fence_marker_start(line: &str) -> Option<&'static str> { + if line.starts_with("```") { + Some("```") + } else if line.starts_with("~~~") { + Some("~~~") + } else { + None + } +} + +fn normalize_html_lists_line(line: &str, list_stack: &mut Vec) -> String { + let mut out = String::with_capacity(line.len()); + let bytes = line.as_bytes(); + let mut i = 0; + + while i < line.len() { + if bytes[i] == b'<' + && let Some(rel_end) = line[i..].find('>') + { + let end = i + rel_end; + let tag = &line[i..=end]; + if let Some(tag_info) = parse_html_tag(tag) { + match tag_info.name.as_str() { + "ul" => { + if tag_info.closing { + list_stack.pop(); + } else if !tag_info.self_closing { + list_stack.push(ListState { + kind: ListKind::Unordered, + counter: 0, + }); + } + } + "ol" => { + if tag_info.closing { + list_stack.pop(); + } else if !tag_info.self_closing { + list_stack.push(ListState { + kind: ListKind::Ordered, + counter: 0, + }); + } + } + "li" => { + if tag_info.closing { + if !out.ends_with('\n') { + out.push('\n'); + } + } else { + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + let depth = list_stack.len().saturating_sub(1); + if depth > 0 { + out.push_str(&" ".repeat(depth)); + } + let prefix = match list_stack.last_mut() { + Some(state) => match state.kind { + ListKind::Unordered => "- ".to_string(), + ListKind::Ordered => { + state.counter += 1; + format!("{}. ", state.counter) + } + }, + None => "- ".to_string(), + }; + out.push_str(&prefix); + } + } + _ => out.push_str(tag), + } + i = end + 1; + continue; + } + } + + let ch = line[i..].chars().next().unwrap(); + out.push(ch); + i += ch.len_utf8(); + } + + out +} + +#[derive(Debug, Clone)] +struct HtmlTag { + name: String, + closing: bool, + self_closing: bool, + attrs: HashMap, +} + +fn parse_html_tag(html: &str) -> Option { + let trimmed = html.trim(); + if !trimmed.starts_with('<') || !trimmed.ends_with('>') { + return None; + } + if trimmed.starts_with("\n")); + } + markdown.push_str(&block_markdown); + } + + Ok(MarkdownResult { + title: doc_title, + markdown, + }) +} + +pub fn parse_doc_from_binary(doc_bin: Vec, doc_id: String) -> Result { + let doc = load_doc(&doc_bin, Some(doc_id.as_str()))?; + + let blocks_map = doc.get_map("blocks")?; + if blocks_map.is_empty() { + return Err(ParseError::ParserError("blocks map is empty".into())); + } + + let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR) + .ok_or_else(|| ParseError::ParserError("root block not found".into()))?; + let mut walker = context.walker(); + let mut blocks: Vec = Vec::with_capacity(context.block_pool.len()); + let mut doc_title = String::new(); + let mut summary = SummaryBuilder::new(SUMMARY_LIMIT as isize); + + while let Some((parent_block_id, block_id)) = walker.next() { + let block = match context.block_pool.get(&block_id) { + Some(block) => block, + None => continue, + }; + + let flavour = match get_flavour(block) { + Some(flavour) => flavour, + None => continue, + }; + + let parent_block = parent_block_id.as_ref().and_then(|id| context.block_pool.get(id)); + let parent_flavour = parent_block.and_then(get_flavour); + + let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &context.parent_lookup, &context.block_pool); + let note_block_id = note_block.as_ref().and_then(get_block_id); + let display_mode = determine_display_mode(note_block.as_ref()); + + // enqueue children first to keep traversal order similar to JS implementation + walker.enqueue_children(&block_id, block); + + let build_block = |database_name: Option<&String>| { + BlockInfo::base( + &block_id, + &flavour, + parent_flavour.as_ref(), + parent_block_id.as_ref(), + compose_additional(&display_mode, note_block_id.as_ref(), database_name), + ) + }; + + if flavour == PAGE_FLAVOUR { + let title = get_string(block, "prop:title").unwrap_or_default(); + doc_title = title.clone(); + let mut info = build_block(None); + info.content = Some(vec![title]); + blocks.push(info); + continue; + } + + if matches!(flavour.as_str(), "affine:paragraph" | "affine:list" | "affine:code") { + if let Some(text) = block.get("prop:text").and_then(|value| value.to_text()) { + let database_name = if flavour == "affine:paragraph" && parent_flavour.as_deref() == Some("affine:database") { + parent_block.and_then(|map| get_string(map, "prop:title")) + } else { + None + }; + + let content = text.to_string(); + let text_len = text.len() as usize; + let refs = extract_inline_references(&text.to_delta()); + + let mut info = build_block(database_name.as_ref()); + info.content = Some(vec![content.clone()]); + if !refs.is_empty() { + info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect()); + info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect()); + } + blocks.push(info); + summary.push_text(&content, text_len); + } + continue; + } + + if matches!(flavour.as_str(), "affine:embed-linked-doc" | "affine:embed-synced-doc") { + if let Some(page_id) = get_string(block, "prop:pageId") { + let mut info = build_block(None); + let payload = embed_ref_payload(block, &page_id); + apply_doc_ref(&mut info, page_id, payload); + blocks.push(info); + } + continue; + } + + if flavour == "affine:attachment" { + if let Some(blob_id) = get_string(block, "prop:sourceId") { + let mut info = build_block(None); + let name = get_string(block, "prop:name").unwrap_or_default(); + apply_blob_info(&mut info, blob_id, name); + blocks.push(info); + } + continue; + } + + if flavour == "affine:image" { + let image = ImageSpec::from_block_map(block); + if !image.source_id.is_empty() { + let mut info = build_block(None); + let caption = image.caption.unwrap_or_default(); + apply_blob_info(&mut info, image.source_id, caption); + blocks.push(info); + } + continue; + } + + if flavour == "affine:surface" { + let texts = gather_surface_texts(block); + let mut info = build_block(None); + info.content = Some(texts); + blocks.push(info); + continue; + } + + if flavour == "affine:database" { + let (texts, database_name) = gather_database_texts(block); + let mut info = BlockInfo::base( + &block_id, + &flavour, + parent_flavour.as_ref(), + parent_block_id.as_ref(), + compose_additional(&display_mode, note_block_id.as_ref(), database_name.as_ref()), + ); + info.content = Some(texts); + let refs = collect_database_cell_references(block); + if !refs.is_empty() { + info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect()); + info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect()); + } + blocks.push(info); + continue; + } + + if flavour == "affine:latex" { + if let Some(content) = get_string(block, "prop:latex") { + let mut info = build_block(None); + info.content = Some(vec![content]); + blocks.push(info); + } + continue; + } + + if flavour == "affine:table" { + let contents = table_cell_texts(block); + let mut info = build_block(None); + info.content = Some(contents); + blocks.push(info); + continue; + } + + if BOOKMARK_FLAVOURS.contains(&flavour.as_str()) { + blocks.push(build_block(None)); + } + } + + if doc_title.is_empty() { + doc_title = DEFAULT_PAGE_TITLE.into(); + } + + Ok(CrawlResult { + blocks, + title: doc_title, + summary: summary.into_string(), + }) +} + +pub fn get_doc_ids_from_binary(doc_bin: Vec, include_trash: bool) -> Result, ParseError> { + let doc = load_doc(&doc_bin, None)?; + + let mut doc_ids = Vec::new(); + let meta = doc.get_map("meta")?; + let pages_value = meta.get("pages"); + if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) { + for page_val in pages.iter() { + if let Some(page) = page_val.to_map() { + let id = get_string(&page, "id"); + if let Some(id) = id { + let trash = page + .get("trash") + .and_then(|v| match v.to_any() { + Some(Any::True) => Some(true), + Some(Any::False) => Some(false), + _ => None, + }) + .unwrap_or(false); + + if include_trash || !trash { + doc_ids.push(id); + } + } + } + } + return Ok(doc_ids); + } + + if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) { + for entry in entries { + let Any::Object(map) = entry else { + continue; + }; + let id = map.get("id").and_then(any_as_string).map(str::to_string); + if let Some(id) = id { + let trash = map.get("trash").map(any_truthy).unwrap_or(false); + if include_trash || !trash { + doc_ids.push(id); + } + } + } + } + + Ok(doc_ids) +} + +fn block_level(block_id: &str, root_id: &str, parent_lookup: &HashMap) -> usize { + let mut level = 0; + let mut cursor = block_id; + while let Some(parent) = parent_lookup.get(cursor) { + level += 1; + if parent == root_id { + break; + } + cursor = parent; + } + level +} + +pub(super) fn text_content(block: &Map, key: &str) -> Option<(String, usize)> { + block.get(key).and_then(|value| { + value.to_text().map(|text| { + let content = text.to_string(); + let len = text.len() as usize; + (content, len) + }) + }) +} + +fn determine_display_mode(note_block: Option<&Map>) -> String { + match note_block.and_then(|block| get_string(block, "prop:displayMode")) { + Some(mode) if mode == "both" => "page".into(), + Some(mode) => mode, + None => "edgeless".into(), + } +} + +fn compose_additional( + display_mode: &str, + note_block_id: Option<&String>, + database_name: Option<&String>, +) -> Option { + let mut payload = JsonMap::new(); + payload.insert("displayMode".into(), JsonValue::String(display_mode.to_string())); + if let Some(note_id) = note_block_id { + payload.insert("noteBlockId".into(), JsonValue::String(note_id.clone())); + } + if let Some(name) = database_name { + payload.insert("databaseName".into(), JsonValue::String(name.clone())); + } + Some(JsonValue::Object(payload).to_string()) +} + +fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) { + info.blob = Some(vec![blob_id]); + info.content = Some(vec![content]); +} + +fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option) { + info.ref_doc_id = Some(vec![page_id]); + if let Some(payload) = payload { + info.ref_info = Some(vec![payload]); + } +} + +fn embed_ref_payload(block: &Map, page_id: &str) -> Option { + let params = block.get("prop:params").as_ref().and_then(params_value_to_json); + Some(build_reference_payload(page_id, params)) +} + +fn gather_surface_texts(block: &Map) -> Vec { + let mut texts = Vec::new(); + let elements = match block.get("prop:elements").and_then(|value| value.to_map()) { + Some(map) => map, + None => return texts, + }; + + if elements + .get("type") + .and_then(|value| value_to_string(&value)) + .as_deref() + != Some("$blocksuite:internal:native$") + { + return texts; + } + + if let Some(value_map) = elements.get("value").and_then(|value| value.to_map()) { + for value in value_map.values() { + if let Some(element) = value.to_map() + && let Some(text) = element.get("text").and_then(|value| value.to_text()) + { + texts.push(text.to_string()); + } + } + } + + texts.sort(); + texts +} + +fn table_cell_texts(block: &Map) -> Vec { + let mut contents = Vec::new(); + for key in block.keys() { + if key.starts_with("prop:cells.") + && key.ends_with(".text") + && let Some(value) = block.get(key).and_then(|value| value_to_string(&value)) + && !value.is_empty() + { + contents.push(value); + } + } + contents +} + +pub(super) fn text_content_for_summary(block: &Map, key: &str) -> Option<(String, usize)> { + if let Some((text, len)) = text_content(block, key) { + return Some((text, len)); + } + + block.get(key).and_then(|value| { + value_to_string(&value).map(|text| { + let len = text.chars().count(); + (text, len) + }) + }) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use y_octo::{AHashMap, Any, DocOptions, TextAttributes, TextDeltaOp, TextInsert, Value}; + + use super::*; + use crate::doc_parser::build_full_doc; + + #[test] + fn test_parse_doc_from_binary() { + let json = include_bytes!("../../../fixtures/demo.ydoc.json"); + let doc_bin = include_bytes!("../../../fixtures/demo.ydoc").to_vec(); + let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string(); + + let result = parse_doc_from_binary(doc_bin, doc_id).unwrap(); + let config = assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict) + .numeric_mode(assert_json_diff::NumericMode::AssumeFloat); + assert_json_diff::assert_json_matches!( + serde_json::from_slice::(json).unwrap(), + serde_json::json!(result), + config + ); + } + + #[test] + fn test_database_cell_references() { + let doc_id = "doc-with-db".to_string(); + let doc = DocOptions::new().with_guid(doc_id.clone()).build(); + let mut blocks = doc.get_or_create_map("blocks").unwrap(); + + let mut page = doc.create_map().unwrap(); + page.insert("sys:id".into(), "page").unwrap(); + page.insert("sys:flavour".into(), "affine:page").unwrap(); + let mut page_children = doc.create_array().unwrap(); + page_children.push("note").unwrap(); + page.insert("sys:children".into(), Value::Array(page_children)).unwrap(); + let mut page_title = doc.create_text().unwrap(); + page_title.insert(0, "Page").unwrap(); + page.insert("prop:title".into(), Value::Text(page_title)).unwrap(); + blocks.insert("page".into(), Value::Map(page)).unwrap(); + + let mut note = doc.create_map().unwrap(); + note.insert("sys:id".into(), "note").unwrap(); + note.insert("sys:flavour".into(), "affine:note").unwrap(); + let mut note_children = doc.create_array().unwrap(); + note_children.push("db").unwrap(); + note.insert("sys:children".into(), Value::Array(note_children)).unwrap(); + note.insert("prop:displayMode".into(), "page").unwrap(); + blocks.insert("note".into(), Value::Map(note)).unwrap(); + + let mut db = doc.create_map().unwrap(); + db.insert("sys:id".into(), "db").unwrap(); + db.insert("sys:flavour".into(), "affine:database").unwrap(); + db.insert("sys:children".into(), Value::Array(doc.create_array().unwrap())) + .unwrap(); + let mut db_title = doc.create_text().unwrap(); + db_title.insert(0, "Database").unwrap(); + db.insert("prop:title".into(), Value::Text(db_title)).unwrap(); + + let mut columns = doc.create_array().unwrap(); + let mut column = doc.create_map().unwrap(); + column.insert("id".into(), "col1").unwrap(); + column.insert("name".into(), "Text").unwrap(); + column.insert("type".into(), "rich-text").unwrap(); + column + .insert("data".into(), Value::Map(doc.create_map().unwrap())) + .unwrap(); + columns.push(Value::Map(column)).unwrap(); + db.insert("prop:columns".into(), Value::Array(columns)).unwrap(); + + let mut cell_text = doc.create_text().unwrap(); + let mut reference = AHashMap::default(); + reference.insert("pageId".into(), Any::String("target-doc".into())); + let mut params = AHashMap::default(); + params.insert("mode".into(), Any::String("page".into())); + reference.insert("params".into(), Any::Object(params)); + let mut attrs = TextAttributes::new(); + attrs.insert("reference".into(), Any::Object(reference)); + cell_text + .apply_delta(&[ + TextDeltaOp::Insert { + insert: TextInsert::Text("See ".into()), + format: None, + }, + TextDeltaOp::Insert { + insert: TextInsert::Text("Target".into()), + format: Some(attrs), + }, + ]) + .unwrap(); + + let mut cell = doc.create_map().unwrap(); + cell.insert("columnId".into(), "col1").unwrap(); + cell.insert("value".into(), Value::Text(cell_text)).unwrap(); + let mut row = doc.create_map().unwrap(); + row.insert("col1".into(), Value::Map(cell)).unwrap(); + let mut cells = doc.create_map().unwrap(); + cells.insert("row1".into(), Value::Map(row)).unwrap(); + db.insert("prop:cells".into(), Value::Map(cells)).unwrap(); + + blocks.insert("db".into(), Value::Map(db)).unwrap(); + + let doc_bin = doc.encode_update_v1().unwrap(); + let result = parse_doc_from_binary(doc_bin, doc_id).unwrap(); + let db_block = result.blocks.iter().find(|block| block.block_id == "db").unwrap(); + assert_eq!(db_block.ref_doc_id, Some(vec!["target-doc".to_string()])); + assert_eq!( + db_block.ref_info, + Some(vec![build_reference_payload( + "target-doc", + Some(json!({"mode": "page"})) + )]) + ); + } + + #[test] + fn test_parse_doc_to_markdown_ai_editable_image_table() { + let doc_id = "ai-editable-doc"; + let markdown = "![Alt](blob://image-id)\n\n| A | B |\n| --- | --- |\n| 1 | 2 |"; + let doc_bin = build_full_doc("Title", markdown, doc_id).expect("create doc"); + + let result = parse_doc_to_markdown(doc_bin, doc_id.to_string(), true, None).expect("parse doc"); + let md = result.markdown; + + assert!(md.contains("flavour=affine:image")); + assert!(md.contains("blob://image-id")); + assert!(md.contains("|A|B|")); + assert!(md.contains("|---|---|")); + } +} diff --git a/packages/common/native/src/doc_parser/roundtrip_tests.rs b/packages/common/native/src/doc_parser/roundtrip_tests.rs new file mode 100644 index 000000000..0418507f5 --- /dev/null +++ b/packages/common/native/src/doc_parser/roundtrip_tests.rs @@ -0,0 +1,56 @@ +use super::{build_full_doc, parse_doc_to_markdown}; + +fn assert_markdown_roundtrip(markdown: &str, expected: &str) { + let doc_id = "roundtrip-doc"; + let title = "Roundtrip Title"; + let bin = build_full_doc(title, markdown, doc_id).expect("create doc"); + let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc"); + assert_eq!(result.title, title); + assert_eq!(result.markdown, expected); +} + +#[test] +fn test_roundtrip_inline_styles() { + let markdown = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com)."; + let expected = "Inline **bold** _italic_ ~~strike~~ `code` [Link](https://example.com).\n\n"; + assert_markdown_roundtrip(markdown, expected); +} + +#[test] +fn test_roundtrip_list_items() { + let markdown = "- Item 1\n- Item 2\n- [ ] Task\n- [x] Done"; + let expected = "* Item 1\n* Item 2\n- [ ] Task\n- [x] Done\n"; + assert_markdown_roundtrip(markdown, expected); +} + +#[test] +fn test_roundtrip_code_block() { + let markdown = "```rust\nfn main() {}\n```"; + let expected = "```rust\nfn main() {}\n\n```\n\n"; + assert_markdown_roundtrip(markdown, expected); +} + +#[test] +fn test_roundtrip_code_block_indentation() { + let markdown = "```python\n def indented():\n return \"ok\"\n```"; + let doc_id = "roundtrip-indent"; + let title = "Roundtrip Title"; + let bin = build_full_doc(title, markdown, doc_id).expect("create doc"); + let result = parse_doc_to_markdown(bin, doc_id.to_string(), false, None).expect("parse doc"); + assert!(result.markdown.contains("\n def indented():")); + assert!(result.markdown.contains("\n return \"ok\"")); +} + +#[test] +fn test_roundtrip_table() { + let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |"; + let expected = "|A|B|\n|---|---|\n|1|2|\n\n"; + assert_markdown_roundtrip(markdown, expected); +} + +#[test] +fn test_roundtrip_image_with_caption() { + let markdown = "![Alt](blob://image-id)"; + let expected = "\n\n"; + assert_markdown_roundtrip(markdown, expected); +} diff --git a/packages/common/native/src/doc_parser/schema.rs b/packages/common/native/src/doc_parser/schema.rs new file mode 100644 index 000000000..f14b4fd99 --- /dev/null +++ b/packages/common/native/src/doc_parser/schema.rs @@ -0,0 +1,57 @@ +pub(super) const PAGE_FLAVOUR: &str = "affine:page"; +pub(super) const NOTE_FLAVOUR: &str = "affine:note"; +pub(super) const SURFACE_FLAVOUR: &str = "affine:surface"; + +pub(super) const SYS_ID: &str = "sys:id"; +pub(super) const SYS_FLAVOUR: &str = "sys:flavour"; +pub(super) const SYS_VERSION: &str = "sys:version"; +pub(super) const SYS_CHILDREN: &str = "sys:children"; + +pub(super) const PROP_TITLE: &str = "prop:title"; +pub(super) const PROP_TEXT: &str = "prop:text"; +pub(super) const PROP_TYPE: &str = "prop:type"; +pub(super) const PROP_CHECKED: &str = "prop:checked"; +pub(super) const PROP_LANGUAGE: &str = "prop:language"; +pub(super) const PROP_ORDER: &str = "prop:order"; + +pub(super) const PROP_ELEMENTS: &str = "prop:elements"; +pub(super) const PROP_BACKGROUND: &str = "prop:background"; +pub(super) const PROP_XYWH: &str = "prop:xywh"; +pub(super) const PROP_INDEX: &str = "prop:index"; +pub(super) const PROP_HIDDEN: &str = "prop:hidden"; +pub(super) const PROP_DISPLAY_MODE: &str = "prop:displayMode"; + +pub(super) const PROP_SOURCE_ID: &str = "prop:sourceId"; +pub(super) const PROP_CAPTION: &str = "prop:caption"; +pub(super) const PROP_WIDTH: &str = "prop:width"; +pub(super) const PROP_HEIGHT: &str = "prop:height"; +pub(super) const PROP_URL: &str = "prop:url"; +pub(super) const PROP_VIDEO_ID: &str = "prop:videoId"; + +pub(super) const PROP_ROWS_PREFIX: &str = "prop:rows."; +pub(super) const PROP_COLUMNS_PREFIX: &str = "prop:columns."; +pub(super) const PROP_CELLS_PREFIX: &str = "prop:cells."; +pub(super) const PROP_ROW_ID_SUFFIX: &str = ".rowId"; +pub(super) const PROP_COLUMN_ID_SUFFIX: &str = ".columnId"; +pub(super) const PROP_ORDER_SUFFIX: &str = ".order"; +pub(super) const PROP_TEXT_SUFFIX: &str = ".text"; + +pub(super) fn table_row_id_key(row_id: &str) -> String { + format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ROW_ID_SUFFIX}") +} + +pub(super) fn table_row_order_key(row_id: &str) -> String { + format!("{PROP_ROWS_PREFIX}{row_id}{PROP_ORDER_SUFFIX}") +} + +pub(super) fn table_column_id_key(column_id: &str) -> String { + format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_COLUMN_ID_SUFFIX}") +} + +pub(super) fn table_column_order_key(column_id: &str) -> String { + format!("{PROP_COLUMNS_PREFIX}{column_id}{PROP_ORDER_SUFFIX}") +} + +pub(super) fn table_cell_text_key(row_id: &str, column_id: &str) -> String { + format!("{PROP_CELLS_PREFIX}{row_id}:{column_id}{PROP_TEXT_SUFFIX}") +} diff --git a/packages/common/native/src/doc_parser/table.rs b/packages/common/native/src/doc_parser/table.rs new file mode 100644 index 000000000..6b8241426 --- /dev/null +++ b/packages/common/native/src/doc_parser/table.rs @@ -0,0 +1,71 @@ +#[derive(Clone, Copy)] +pub(super) struct MarkdownTableOptions { + pub(super) escape_pipes: bool, + pub(super) newline_replacement: &'static str, + pub(super) trim: bool, +} + +impl MarkdownTableOptions { + pub(super) const fn new(escape_pipes: bool, newline_replacement: &'static str, trim: bool) -> Self { + Self { + escape_pipes, + newline_replacement, + trim, + } + } +} + +pub(super) fn render_markdown_table(rows: &[Vec], options: MarkdownTableOptions) -> Option { + let (header, body) = rows.split_first()?; + let header_line = format_table_row(header, options); + let separator_line = format_table_row(&vec!["---".to_string(); header.len()], options); + let mut lines = vec![header_line, separator_line]; + for row in body { + lines.push(format_table_row(row, options)); + } + Some(lines.join("\n")) +} + +fn format_table_row(row: &[String], options: MarkdownTableOptions) -> String { + let cells = row + .iter() + .map(|cell| format_table_cell(cell, options)) + .collect::>(); + format!("|{}|", cells.join("|")) +} + +fn format_table_cell(cell: &str, options: MarkdownTableOptions) -> String { + let mut value = if options.trim { + cell.trim().to_string() + } else { + cell.to_string() + }; + + if options.escape_pipes { + value = value.replace('|', "\\|"); + } + if !options.newline_replacement.is_empty() { + value = collapse_newlines(&value, options.newline_replacement); + } + value +} + +fn collapse_newlines(value: &str, replacement: &str) -> String { + if replacement.is_empty() { + return value.to_string(); + } + let mut out = String::with_capacity(value.len()); + let mut in_newline = false; + for ch in value.chars() { + if ch == '\n' { + if !in_newline { + out.push_str(replacement); + in_newline = true; + } + } else { + in_newline = false; + out.push(ch); + } + } + out +} diff --git a/packages/common/native/src/doc_parser/update_ydoc.rs b/packages/common/native/src/doc_parser/update_ydoc.rs deleted file mode 100644 index b345574e1..000000000 --- a/packages/common/native/src/doc_parser/update_ydoc.rs +++ /dev/null @@ -1,1102 +0,0 @@ -//! Update YDoc module -//! -//! Provides functionality to update existing AFFiNE documents by applying -//! surgical y-octo operations based on content differences. - -use std::collections::HashMap; - -use y_octo::{Any, Doc, DocOptions, Map}; - -use super::{ - affine::ParseError, - blocksuite::{collect_child_ids, get_string}, - markdown_utils::{BlockFlavour, ParsedBlock, extract_title, parse_markdown_blocks}, -}; - -const PAGE_FLAVOUR: &str = "affine:page"; -const NOTE_FLAVOUR: &str = "affine:note"; - -/// Represents a content block for diffing purposes -#[derive(Debug, Clone, PartialEq)] -pub struct ContentBlock { - pub flavour: String, - pub block_type: Option, // h1, h2, text, bulleted, numbered, todo, etc. - pub content: String, - pub checked: Option, // For todo items - pub language: Option, // For code blocks -} - -impl ContentBlock { - /// Check if two blocks are similar enough to be considered "the same" for - /// diffing - fn is_similar(&self, other: &ContentBlock) -> bool { - self.flavour == other.flavour && self.block_type == other.block_type - } -} - -/// Converts a ParsedBlock from the shared parser into a ContentBlock -impl From for ContentBlock { - fn from(parsed: ParsedBlock) -> Self { - // Default paragraph type to "text" to match existing documents - let block_type = if parsed.flavour == BlockFlavour::Paragraph && parsed.block_type.is_none() { - Some("text".to_string()) - } else { - parsed.block_type.map(|bt| bt.as_str().to_string()) - }; - - ContentBlock { - flavour: parsed.flavour.as_str().to_string(), - block_type, - content: parsed.content, - checked: parsed.checked, - language: parsed.language, - } - } -} - -/// Represents the existing document structure -struct ExistingDoc { - doc: Doc, - page_id: String, - note_id: String, - content_block_ids: Vec, - content_blocks: Vec<(String, ContentBlock)>, // (id, block) -} - -/// Represents a diff operation -#[derive(Debug)] -enum DiffOp { - Keep(usize), // old_idx - block unchanged - Delete(usize), // old_idx - block removed - Insert(usize), // new_idx - block added - Update(usize, usize), // (old_idx, new_idx) - block content changed -} - -/// Updates an existing document with new markdown content. -/// -/// This function performs structural diffing between the existing document -/// and the new markdown content, then applies minimal y-octo operations -/// to update only what changed. This enables proper CRDT merging with -/// concurrent edits from other clients. -/// -/// If the existing document is empty or invalid, falls back to creating -/// a new document from the markdown using `markdown_to_ydoc`. -/// -/// # Arguments -/// * `existing_binary` - The current document binary -/// * `new_markdown` - The new markdown content -/// * `doc_id` - The document ID -/// -/// # Returns -/// A binary vector representing only the delta (changes) to apply -pub fn update_ydoc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> Result, ParseError> { - // Load and parse the existing document - // If the document is empty or invalid, fall back to creating a new one - let mut existing = match load_existing_doc(existing_binary, doc_id) { - Ok(doc) => doc, - Err(ParseError::InvalidBinary) | Err(ParseError::ParserError(_)) => { - // Empty or invalid document - create from scratch - return super::markdown_to_ydoc::markdown_to_ydoc(new_markdown, doc_id); - } - Err(e) => return Err(e), - }; - - // Parse new markdown into content blocks - let new_blocks = parse_markdown_to_content_blocks(new_markdown)?; - - // Compute diff between old and new blocks - let diff_ops = compute_diff(&existing.content_blocks, &new_blocks); - - // Capture state before modifications to encode only the delta - let state_before = existing.doc.get_state_vector(); - - // Update the title if changed - let new_title = extract_title(new_markdown); - update_title(&mut existing, &new_title)?; - - // Apply diff operations to update the document structure - apply_diff(&mut existing, &new_blocks, &diff_ops)?; - - // Encode only the changes (delta) since state_before - existing - .doc - .encode_state_as_update_v1(&state_before) - .map_err(|e| ParseError::ParserError(e.to_string())) -} - -/// Loads an existing document and extracts its structure -fn load_existing_doc(binary: &[u8], doc_id: &str) -> Result { - // Check for empty or minimal empty Y-Doc binary - // [0, 0] represents an empty Y-Doc update (0 structs, 0 deletes) - a convention - // used throughout the AFFiNE codebase for uninitialized/empty documents - if binary.is_empty() || binary == [0, 0] { - return Err(ParseError::InvalidBinary); - } - - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(binary) - .map_err(|_| ParseError::InvalidBinary)?; - - let blocks_map = doc.get_map("blocks")?; - if blocks_map.is_empty() { - return Err(ParseError::ParserError("blocks map is empty".into())); - } - - // Build block index - let mut block_pool: HashMap = HashMap::new(); - for (_, value) in blocks_map.iter() { - if let Some(block_map) = value.to_map() - && let Some(block_id) = get_string(&block_map, "sys:id") - { - block_pool.insert(block_id, block_map); - } - } - - // Find page block - let page_id = block_pool - .iter() - .find_map(|(id, block)| { - get_string(block, "sys:flavour") - .filter(|f| f == PAGE_FLAVOUR) - .map(|_| id.clone()) - }) - .ok_or_else(|| ParseError::ParserError("page block not found".into()))?; - - // Find note block (child of page) - let page_block = block_pool - .get(&page_id) - .ok_or_else(|| ParseError::ParserError("page block not found".into()))?; - let note_id = collect_child_ids(page_block) - .into_iter() - .find(|id| block_pool.get(id).and_then(|b| get_string(b, "sys:flavour")).as_deref() == Some(NOTE_FLAVOUR)) - .ok_or_else(|| ParseError::ParserError("note block not found".into()))?; - - // Get content block IDs (children of note) - let note_block = block_pool - .get(¬e_id) - .ok_or_else(|| ParseError::ParserError("note block not found".into()))?; - let raw_content_block_ids = collect_child_ids(note_block); - - // Extract content blocks with their data, filtering to only existing blocks - // This ensures content_block_ids and content_blocks stay in sync - let mut content_blocks = Vec::new(); - let mut content_block_ids = Vec::new(); - for block_id in raw_content_block_ids { - if let Some(block) = block_pool.get(&block_id) { - let content_block = extract_content_block(block); - content_blocks.push((block_id.clone(), content_block)); - content_block_ids.push(block_id); - } - } - - Ok(ExistingDoc { - doc, - page_id, - note_id, - content_block_ids, - content_blocks, - }) -} - -/// Extracts content block data from a y-octo Map -fn extract_content_block(block: &Map) -> ContentBlock { - let flavour = get_string(block, "sys:flavour").unwrap_or_default(); - let block_type = get_string(block, "prop:type"); - // Use get_string which handles both Y.Text and Any::String via value_to_string - let content = get_string(block, "prop:text").unwrap_or_default(); - let checked = block - .get("prop:checked") - .and_then(|v| v.to_any()) - .and_then(|a| match a { - Any::True => Some(true), - Any::False => Some(false), - _ => None, - }); - let language = get_string(block, "prop:language"); - - ContentBlock { - flavour, - block_type, - content, - checked, - language, - } -} - -/// Parses markdown into content blocks for diffing. -/// -/// Uses the shared `parse_markdown_blocks` function and converts to -/// `ContentBlock`. -fn parse_markdown_to_content_blocks(markdown: &str) -> Result, ParseError> { - let parsed_blocks = parse_markdown_blocks(markdown, true); - Ok(parsed_blocks.into_iter().map(ContentBlock::from).collect()) -} - -/// Updates the document title if it has changed -fn update_title(existing: &mut ExistingDoc, new_title: &str) -> Result<(), ParseError> { - let blocks_map = existing - .doc - .get_map("blocks") - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - if let Some(mut page_block) = blocks_map.get(&existing.page_id).and_then(|v| v.to_map()) { - let current_title = get_string(&page_block, "prop:title").unwrap_or_default(); - if current_title != new_title { - page_block - .insert("prop:title".to_string(), Any::String(new_title.to_string())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - } - - Ok(()) -} - -/// Computes the diff between old and new blocks using weighted LCS algorithm. -/// Uses a two-tier matching: exact matches (same type + content) get priority, -/// then similar matches (same type, different content) for update operations. -fn compute_diff(old_blocks: &[(String, ContentBlock)], new_blocks: &[ContentBlock]) -> Vec { - let old_len = old_blocks.len(); - let new_len = new_blocks.len(); - - if old_len == 0 { - // All inserts - return (0..new_len).map(DiffOp::Insert).collect(); - } - if new_len == 0 { - // All deletes - return (0..old_len).map(DiffOp::Delete).collect(); - } - - // Build weighted LCS table using exact content match - // This ensures identical blocks are matched together - let mut lcs = vec![vec![0usize; new_len + 1]; old_len + 1]; - - for i in 1..=old_len { - for j in 1..=new_len { - let old_block = &old_blocks[i - 1].1; - let new_block = &new_blocks[j - 1]; - - // Only count as match if blocks are identical (same type AND content) - if old_block.flavour == new_block.flavour - && old_block.block_type == new_block.block_type - && old_block.content == new_block.content - && old_block.checked == new_block.checked - && old_block.language == new_block.language - { - lcs[i][j] = lcs[i - 1][j - 1] + 1; - } else { - lcs[i][j] = std::cmp::max(lcs[i - 1][j], lcs[i][j - 1]); - } - } - } - - // Backtrack to find the diff - let mut ops = Vec::new(); - let mut i = old_len; - let mut j = new_len; - - while i > 0 || j > 0 { - if i > 0 && j > 0 { - let old_block = &old_blocks[i - 1].1; - let new_block = &new_blocks[j - 1]; - - let is_exact_match = old_block.flavour == new_block.flavour - && old_block.block_type == new_block.block_type - && old_block.content == new_block.content - && old_block.checked == new_block.checked - && old_block.language == new_block.language; - - if is_exact_match { - // Exact match - Keep - ops.push(DiffOp::Keep(i - 1)); - i -= 1; - j -= 1; - } else if old_block.is_similar(new_block) - && lcs[i - 1][j - 1] >= lcs[i - 1][j] - && lcs[i - 1][j - 1] >= lcs[i][j - 1] - { - // Similar block (same type, different content) - Update if it doesn't hurt LCS - ops.push(DiffOp::Update(i - 1, j - 1)); - i -= 1; - j -= 1; - } else if lcs[i][j - 1] >= lcs[i - 1][j] { - ops.push(DiffOp::Insert(j - 1)); - j -= 1; - } else { - ops.push(DiffOp::Delete(i - 1)); - i -= 1; - } - } else if j > 0 { - ops.push(DiffOp::Insert(j - 1)); - j -= 1; - } else { - ops.push(DiffOp::Delete(i - 1)); - i -= 1; - } - } - - // Reverse to get operations in order - ops.reverse(); - ops -} - -/// Applies diff operations to update the document -fn apply_diff(existing: &mut ExistingDoc, new_blocks: &[ContentBlock], diff_ops: &[DiffOp]) -> Result<(), ParseError> { - let mut blocks_map = existing - .doc - .get_map("blocks") - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - // Track new children for the note block - let mut new_children: Vec = Vec::new(); - - // Track which old blocks to delete - let mut blocks_to_delete: Vec = Vec::new(); - - for op in diff_ops { - match op { - DiffOp::Keep(old_idx) => { - // Keep the existing block - let block_id = &existing.content_block_ids[*old_idx]; - new_children.push(block_id.clone()); - } - DiffOp::Delete(old_idx) => { - // Mark block for deletion - let block_id = &existing.content_block_ids[*old_idx]; - blocks_to_delete.push(block_id.clone()); - } - DiffOp::Insert(new_idx) => { - // Create a new block - let new_block = &new_blocks[*new_idx]; - let block_id = create_new_block(&mut blocks_map, &existing.doc, new_block)?; - new_children.push(block_id); - } - DiffOp::Update(old_idx, new_idx) => { - // Update existing block content - let block_id = &existing.content_block_ids[*old_idx]; - let new_block = &new_blocks[*new_idx]; - update_block_content(&mut existing.doc, &mut blocks_map, block_id, new_block)?; - new_children.push(block_id.clone()); - } - } - } - - // Delete removed blocks from blocks map - for block_id in blocks_to_delete { - blocks_map.remove(&block_id); - } - - // Update note block's children only if they changed - // First check if they're different - let note_block = blocks_map - .get(&existing.note_id) - .and_then(|v| v.to_map()) - .ok_or_else(|| ParseError::ParserError("Note block not found".into()))?; - - let current_children: Vec = note_block - .get("sys:children") - .and_then(|v| v.to_array()) - .map(|arr| { - arr - .iter() - .filter_map(|v| { - v.to_any().and_then(|a| match a { - Any::String(s) => Some(s.clone()), - _ => None, - }) - }) - .collect() - }) - .unwrap_or_default(); - - if current_children != new_children { - update_note_children(&mut blocks_map, &existing.note_id, new_children)?; - } - - Ok(()) -} - -// ============================================================================ -// Two-Phase Insertion Helpers -// ============================================================================ -// -// IMPORTANT: These helpers implement the two-phase insertion pattern required -// for YJS compatibility. When creating nested CRDT types (Text, Array, Map), -// we must: -// 1. Insert the empty container into the parent FIRST (gets clock value) -// 2. Then retrieve and populate it (content gets later clock values) -// -// This ensures parent items always have earlier clocks than children, -// avoiding "forward parent references" that YJS cannot handle. - -/// Creates an empty Text, inserts it into the parent map, then returns it for -/// population. -fn insert_and_get_text(doc: &Doc, parent_map: &mut Map, key: &str) -> Result { - let text = doc.create_text().map_err(|e| ParseError::ParserError(e.to_string()))?; - parent_map - .insert(key.to_string(), text) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - parent_map - .get(key) - .and_then(|v| v.to_text()) - .ok_or_else(|| ParseError::ParserError("Failed to retrieve inserted text".into())) -} - -/// Creates a new block in the blocks map -/// -/// IMPORTANT: Uses two-phase approach for YJS compatibility: -/// 1. Insert empty map into blocks_map first (gets clock value) -/// 2. Then populate the map with properties (gets later clock values) -/// -/// This ensures parent items have earlier clocks than children. -/// -/// Uses Any types (Any::Array, Any::String) for children and text to avoid -/// the "get back" pattern which can hang in release builds. -fn create_new_block(blocks_map: &mut Map, doc: &Doc, block: &ContentBlock) -> Result { - let block_id = nanoid::nanoid!(); - - // Step 1: Create and insert empty map into blocks_map - let empty_map = doc.create_map().map_err(|e| ParseError::ParserError(e.to_string()))?; - blocks_map - .insert(block_id.clone(), empty_map) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - // Step 2: Retrieve the inserted map - let mut block_map = blocks_map - .get(&block_id) - .and_then(|v| v.to_map()) - .ok_or_else(|| ParseError::ParserError("Failed to get inserted block map".into()))?; - - // Step 3: Insert primitive values - block_map - .insert("sys:id".to_string(), Any::String(block_id.clone())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - block_map - .insert("sys:flavour".to_string(), Any::String(block.flavour.clone())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - if let Some(ref block_type) = block.block_type { - block_map - .insert("prop:type".to_string(), Any::String(block_type.clone())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - - if let Some(checked) = block.checked { - block_map - .insert("prop:checked".to_string(), if checked { Any::True } else { Any::False }) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - - if let Some(ref language) = block.language { - block_map - .insert("prop:language".to_string(), Any::String(language.clone())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - - // Step 4: Use Any::Array for children (avoids "get back" pattern) - block_map - .insert("sys:children".to_string(), Any::Array(vec![])) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - // Step 5: Use Any::String for text content (avoids "get back" pattern) - if !block.content.is_empty() { - block_map - .insert("prop:text".to_string(), Any::String(block.content.clone())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - - Ok(block_id) -} - -/// Updates an existing block's content using text-level diff -fn update_block_content( - doc: &mut Doc, - blocks_map: &mut Map, - block_id: &str, - new_block: &ContentBlock, -) -> Result<(), ParseError> { - let mut block = blocks_map - .get(block_id) - .and_then(|v| v.to_map()) - .ok_or_else(|| ParseError::ParserError(format!("Block {} not found", block_id)))?; - - // Update text content using text-level diff - if let Some(mut text) = block.get("prop:text").and_then(|v| v.to_text()) { - let old_content = text.to_string(); - apply_text_diff(&mut text, &old_content, &new_block.content)?; - } else if !new_block.content.is_empty() { - // Block didn't have text before, but now it does (e.g., divider becoming - // paragraph) Use two-phase helper to avoid forward parent references - let mut text = insert_and_get_text(doc, &mut block, "prop:text")?; - text - .insert(0, &new_block.content) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - - // Update checked state - set if present, clear if stale - match new_block.checked { - Some(checked) => { - block - .insert("prop:checked".to_string(), if checked { Any::True } else { Any::False }) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - None => { - // Clear stale checked state if block had it but shouldn't anymore - if block.get("prop:checked").is_some() { - block.remove("prop:checked"); - } - } - } - - // Update language - set if present, clear if stale - match &new_block.language { - Some(language) => { - block - .insert("prop:language".to_string(), Any::String(language.clone())) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - } - None => { - // Clear stale language if block had it but shouldn't anymore - if block.get("prop:language").is_some() { - block.remove("prop:language"); - } - } - } - - Ok(()) -} - -/// Applies a text-level diff to a YText field -fn apply_text_diff(text: &mut y_octo::Text, old_content: &str, new_content: &str) -> Result<(), ParseError> { - // Use greedy diff algorithm for character-level changes - let old_chars: Vec = old_content.chars().collect(); - let new_chars: Vec = new_content.chars().collect(); - - let ops = compute_text_diff(&old_chars, &new_chars); - - // Apply operations in order, adjusting positions based on accumulated offset - // IMPORTANT: y_octo uses UTF-16 code units for positions, not char indices - let mut offset = 0i64; - for op in ops { - match op { - TextDiffOp::Delete { start_utf16, len_utf16 } => { - let raw_pos = start_utf16 as i64 + offset; - // Fail fast if position goes negative - indicates a bug in diff computation - if raw_pos < 0 { - return Err(ParseError::ParserError(format!( - "Invalid delete position: start_utf16={}, offset={}, raw_pos={}", - start_utf16, offset, raw_pos - ))); - } - let adjusted_start = raw_pos as u64; - text - .remove(adjusted_start, len_utf16 as u64) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - offset -= len_utf16 as i64; - } - TextDiffOp::Insert { - pos_utf16, - text: insert_text, - } => { - let raw_pos = pos_utf16 as i64 + offset; - // Fail fast if position goes negative - indicates a bug in diff computation - if raw_pos < 0 { - return Err(ParseError::ParserError(format!( - "Invalid insert position: pos_utf16={}, offset={}, raw_pos={}", - pos_utf16, offset, raw_pos - ))); - } - let adjusted_pos = raw_pos as u64; - let utf16_len: usize = insert_text.chars().map(|c| c.len_utf16()).sum(); - text - .insert(adjusted_pos, &insert_text) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - offset += utf16_len as i64; - } - } - } - - Ok(()) -} - -#[derive(Debug)] -enum TextDiffOp { - /// Delete operation with UTF-16 code unit positions - Delete { start_utf16: usize, len_utf16: usize }, - /// Insert operation with UTF-16 code unit position and text to insert - Insert { pos_utf16: usize, text: String }, -} - -/// Computes character-level diff between two strings using greedy matching. -/// Returns operations with UTF-16 code unit positions (required by y_octo). -fn compute_text_diff(old: &[char], new: &[char]) -> Vec { - // Find common prefix - let mut prefix_len = 0; - while prefix_len < old.len() && prefix_len < new.len() && old[prefix_len] == new[prefix_len] { - prefix_len += 1; - } - - // Find common suffix (from the non-prefix parts) - let old_remaining = &old[prefix_len..]; - let new_remaining = &new[prefix_len..]; - - let mut suffix_len = 0; - while suffix_len < old_remaining.len() - && suffix_len < new_remaining.len() - && old_remaining[old_remaining.len() - 1 - suffix_len] == new_remaining[new_remaining.len() - 1 - suffix_len] - { - suffix_len += 1; - } - - // The middle parts that differ - let old_mid_len = old_remaining.len() - suffix_len; - let new_mid_start = prefix_len; - let new_mid_len = new_remaining.len() - suffix_len; - - #[derive(Debug, Clone)] - enum Edit { - Keep(char), // Keep this char from old - Delete(char), // Delete this char from old - Insert(char), // Insert this char (from new) - } - - let mut edits = Vec::new(); - - // Keep prefix (store the actual chars for UTF-16 length calculation) - for &c in old.iter().take(prefix_len) { - edits.push(Edit::Keep(c)); - } - - // Delete middle of old - for &c in old.iter().skip(prefix_len).take(old_mid_len) { - edits.push(Edit::Delete(c)); - } - - // Insert middle of new - for &c in new.iter().skip(new_mid_start).take(new_mid_len) { - edits.push(Edit::Insert(c)); - } - - // Keep suffix (store the actual chars for UTF-16 length calculation) - for &c in old.iter().skip(prefix_len + old_mid_len).take(suffix_len) { - edits.push(Edit::Keep(c)); - } - - // Convert edits to operations, tracking position in UTF-16 code units - let mut ops = Vec::new(); - let mut old_pos_utf16 = 0usize; - - // Pending delete - let mut del_start_utf16: Option = None; - let mut del_len_utf16 = 0usize; - - // Pending insert - let mut ins_pos_utf16: Option = None; - let mut ins_text = String::new(); - - for edit in edits { - match edit { - Edit::Keep(c) => { - // Flush pending operations - if let Some(start) = del_start_utf16.take() { - ops.push(TextDiffOp::Delete { - start_utf16: start, - len_utf16: del_len_utf16, - }); - del_len_utf16 = 0; - } - if let Some(pos) = ins_pos_utf16.take() { - ops.push(TextDiffOp::Insert { - pos_utf16: pos, - text: std::mem::take(&mut ins_text), - }); - } - old_pos_utf16 += c.len_utf16(); - } - Edit::Delete(c) => { - // Flush pending inserts first - if let Some(pos) = ins_pos_utf16.take() { - ops.push(TextDiffOp::Insert { - pos_utf16: pos, - text: std::mem::take(&mut ins_text), - }); - } - if del_start_utf16.is_none() { - del_start_utf16 = Some(old_pos_utf16); - } - del_len_utf16 += c.len_utf16(); - old_pos_utf16 += c.len_utf16(); - } - Edit::Insert(c) => { - // Flush pending deletes first - if let Some(start) = del_start_utf16.take() { - ops.push(TextDiffOp::Delete { - start_utf16: start, - len_utf16: del_len_utf16, - }); - del_len_utf16 = 0; - } - if ins_pos_utf16.is_none() { - ins_pos_utf16 = Some(old_pos_utf16); - } - ins_text.push(c); - } - } - } - - // Flush remaining operations - if let Some(start) = del_start_utf16 { - ops.push(TextDiffOp::Delete { - start_utf16: start, - len_utf16: del_len_utf16, - }); - } - if let Some(pos) = ins_pos_utf16 { - ops.push(TextDiffOp::Insert { - pos_utf16: pos, - text: ins_text, - }); - } - - ops -} - -/// Updates the note block's children array, only if the children have changed -fn update_note_children(blocks_map: &mut Map, note_id: &str, new_children: Vec) -> Result<(), ParseError> { - let mut note_block = blocks_map - .get(note_id) - .and_then(|v| v.to_map()) - .ok_or_else(|| ParseError::ParserError("Note block not found".into()))?; - - // Replace children atomically with Any::Array (single CRDT operation) - // This is cleaner than clearing element-by-element and re-inserting - let children_any: Vec = new_children.into_iter().map(Any::String).collect(); - note_block - .insert("sys:children".to_string(), Any::Array(children_any)) - .map_err(|e| ParseError::ParserError(e.to_string()))?; - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_markdown_to_content_blocks() { - let markdown = "# Title\n\nParagraph one.\n\nParagraph two."; - let blocks = parse_markdown_to_content_blocks(markdown).unwrap(); - - assert_eq!(blocks.len(), 2); - assert_eq!(blocks[0].flavour, BlockFlavour::Paragraph.as_str()); - assert_eq!(blocks[0].content, "Paragraph one."); - assert_eq!(blocks[1].content, "Paragraph two."); - } - - #[test] - fn test_compute_text_diff_simple() { - let old: Vec = "hello".chars().collect(); - let new: Vec = "hello world".chars().collect(); - let ops = compute_text_diff(&old, &new); - - // Should have one insert operation at UTF-16 position 5 - assert!(!ops.is_empty()); - match &ops[0] { - TextDiffOp::Insert { pos_utf16, text } => { - assert_eq!(*pos_utf16, 5); // "hello" is 5 UTF-16 code units - assert_eq!(text, " world"); - } - _ => panic!("Expected Insert operation"), - } - } - - #[test] - fn test_compute_text_diff_emoji() { - // Test with emoji (outside BMP, uses 2 UTF-16 code units per char) - let old: Vec = "a😀b".chars().collect(); - let new: Vec = "a😀c".chars().collect(); - let ops = compute_text_diff(&old, &new); - - // Should delete 'b' at UTF-16 position 3 (1 for 'a', 2 for emoji) - // Insert position is 4 (after 'b'), but offset adjustment in apply_text_diff - // accounts for the delete (-1), resulting in actual insert at position 3 - assert_eq!(ops.len(), 2); - match &ops[0] { - TextDiffOp::Delete { start_utf16, len_utf16 } => { - assert_eq!(*start_utf16, 3); // 'a'=1, '😀'=2, total=3 - assert_eq!(*len_utf16, 1); // 'b' is 1 UTF-16 code unit - } - _ => panic!("Expected Delete operation"), - } - match &ops[1] { - TextDiffOp::Insert { pos_utf16, text } => { - // Position recorded as 4 (after processing delete in old string) - // Offset adjustment will bring this to 3 when applied - assert_eq!(*pos_utf16, 4); - assert_eq!(text, "c"); - } - _ => panic!("Expected Insert operation"), - } - } - - #[test] - fn test_compute_text_diff_replace() { - let old: Vec = "abc".chars().collect(); - let new: Vec = "axc".chars().collect(); - let ops = compute_text_diff(&old, &new); - - // Should have delete 'b' and insert 'x' - assert_eq!(ops.len(), 2); - } - - #[test] - fn test_content_block_similarity() { - let paragraph_flavour = BlockFlavour::Paragraph.as_str(); - let b1 = ContentBlock { - flavour: paragraph_flavour.to_string(), - block_type: Some("h1".to_string()), - content: "Hello".to_string(), - checked: None, - language: None, - }; - let b2 = ContentBlock { - flavour: paragraph_flavour.to_string(), - block_type: Some("h1".to_string()), - content: "World".to_string(), - checked: None, - language: None, - }; - let b3 = ContentBlock { - flavour: paragraph_flavour.to_string(), - block_type: Some("h2".to_string()), - content: "Hello".to_string(), - checked: None, - language: None, - }; - - assert!(b1.is_similar(&b2)); // Same type, different content - assert!(!b1.is_similar(&b3)); // Different type - } - - #[test] - fn test_extract_title() { - assert_eq!(extract_title("# My Title\n\nContent"), "My Title"); - assert_eq!(extract_title("No heading"), "Untitled"); - assert_eq!(extract_title("## Secondary\n\nContent"), "Untitled"); - assert_eq!(extract_title("# **Bold** Title"), "Bold Title"); - } - - #[test] - fn test_update_ydoc_roundtrip() { - use crate::doc_parser::markdown_to_ydoc; - - // Create initial document - let initial_md = "# Test Document\n\nFirst paragraph.\n\nSecond paragraph."; - let doc_id = "update-test"; - - let initial_bin = markdown_to_ydoc(initial_md, doc_id).expect("Should create initial doc"); - - // Update with new content - let updated_md = "# Test Document\n\nFirst paragraph.\n\nModified second paragraph.\n\nNew third paragraph."; - - let delta = update_ydoc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); - - // Delta should not be empty (changes were made) - assert!(!delta.is_empty(), "Delta should contain changes"); - - // Apply delta to original and verify structure - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - doc.apply_update_from_binary_v1(&delta).expect("Should apply delta"); - - // Verify the document has the expected structure - let blocks_map = doc.get_map("blocks").expect("Should have blocks"); - assert!(!blocks_map.is_empty(), "Blocks should not be empty"); - } - - #[test] - fn test_update_ydoc_title_change() { - use crate::doc_parser::markdown_to_ydoc; - - let initial_md = "# Original Title\n\nContent here."; - let doc_id = "title-test"; - - let initial_bin = markdown_to_ydoc(initial_md, doc_id).expect("Should create initial doc"); - - let updated_md = "# New Title\n\nContent here."; - let delta = update_ydoc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); - - // Apply and verify - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - doc.apply_update_from_binary_v1(&delta).expect("Should apply delta"); - - let blocks_map = doc.get_map("blocks").expect("Should have blocks"); - assert!(!blocks_map.is_empty()); - } - - #[test] - fn test_update_ydoc_no_changes() { - use crate::doc_parser::markdown_to_ydoc; - - let markdown = "# Same Title\n\nSame content."; - let doc_id = "no-change-test"; - - let initial_bin = markdown_to_ydoc(markdown, doc_id).expect("Should create initial doc"); - - // Update with identical content - let delta = update_ydoc(&initial_bin, markdown, doc_id).expect("Should compute delta"); - - // Applying the delta should not fail - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - doc - .apply_update_from_binary_v1(&delta) - .expect("Should apply delta even with no changes"); - - // Document should still be valid - let blocks_map = doc.get_map("blocks").expect("Should have blocks"); - assert!(!blocks_map.is_empty()); - } - - #[test] - fn test_update_ydoc_add_block() { - use crate::doc_parser::markdown_to_ydoc; - - // Create initial document with one paragraph - let initial_md = "# Add Block Test\n\nOriginal paragraph."; - let doc_id = "add-block-test"; - - let initial_bin = markdown_to_ydoc(initial_md, doc_id).expect("Should create initial doc"); - let initial_size = initial_bin.len(); - - // Add a new paragraph - let updated_md = "# Add Block Test\n\nOriginal paragraph.\n\nNew paragraph added."; - let delta = update_ydoc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); - - // Delta should be smaller than a full document (indicates true delta encoding) - // Note: For small changes, delta might not always be smaller due to overhead - assert!(!delta.is_empty(), "Delta should contain changes"); - - // Apply delta and verify - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - doc - .apply_update_from_binary_v1(&delta) - .expect("Should apply delta with new block"); - - // Verify block count increased - let blocks_map = doc.get_map("blocks").expect("Should have blocks"); - let block_count = blocks_map.len(); - // Should have: page + note + 2 content blocks = 4 blocks - assert!(block_count >= 4, "Should have at least 4 blocks, got {}", block_count); - - println!( - "Add block test: initial={} bytes, delta={} bytes, blocks={}", - initial_size, - delta.len(), - block_count - ); - } - - #[test] - fn test_update_ydoc_delete_block() { - use crate::doc_parser::markdown_to_ydoc; - - // Create initial document with two paragraphs - let initial_md = "# Delete Block Test\n\nFirst paragraph.\n\nSecond paragraph to delete."; - let doc_id = "delete-block-test"; - - let initial_bin = markdown_to_ydoc(initial_md, doc_id).expect("Should create initial doc"); - - // Remove the second paragraph - let updated_md = "# Delete Block Test\n\nFirst paragraph."; - let delta = update_ydoc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); - - assert!(!delta.is_empty(), "Delta should contain changes"); - - // Apply delta and verify - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&initial_bin) - .expect("Should apply initial"); - doc - .apply_update_from_binary_v1(&delta) - .expect("Should apply delta with block deletion"); - - // Verify document still valid - let blocks_map = doc.get_map("blocks").expect("Should have blocks"); - assert!(!blocks_map.is_empty(), "Blocks should not be empty after deletion"); - } - - #[test] - fn test_update_ydoc_concurrent_merge_simulation() { - use crate::doc_parser::markdown_to_ydoc; - - // This test simulates concurrent editing by creating two different updates - // from the same base document and merging them. - let base_md = "# Concurrent Test\n\nBase paragraph."; - let doc_id = "concurrent-test"; - - let base_bin = markdown_to_ydoc(base_md, doc_id).expect("Should create base doc"); - - // Client A modifies the paragraph - let client_a_md = "# Concurrent Test\n\nModified by client A."; - let delta_a = update_ydoc(&base_bin, client_a_md, doc_id).expect("Delta A"); - - // Client B adds a new paragraph (from same base) - let client_b_md = "# Concurrent Test\n\nBase paragraph.\n\nAdded by client B."; - let delta_b = update_ydoc(&base_bin, client_b_md, doc_id).expect("Delta B"); - - // Apply both deltas to base document - let mut final_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - final_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base"); - final_doc.apply_update_from_binary_v1(&delta_a).expect("Apply delta A"); - final_doc.apply_update_from_binary_v1(&delta_b).expect("Apply delta B"); - - // Document should be valid with merged changes - let blocks_map = final_doc.get_map("blocks").expect("Should have blocks"); - assert!(!blocks_map.is_empty(), "Merged document should have blocks"); - - // Should have more blocks than just page + note + 1 paragraph - // The merge should result in at least 4 blocks (page, note, modified para, new - // para) - let block_count = blocks_map.len(); - println!("Concurrent merge test: final block count = {}", block_count); - assert!(block_count >= 4, "Should have merged blocks, got {}", block_count); - } - - #[test] - fn test_update_ydoc_empty_binary_fallback() { - // Test that update_ydoc falls back to markdown_to_ydoc for empty binaries - let markdown = "# New Document\n\nCreated from empty binary."; - let doc_id = "empty-fallback-test"; - - // Empty binary should trigger fallback - let result = update_ydoc(&[], markdown, doc_id).expect("Should create from empty"); - assert!(!result.is_empty(), "Result should not be empty"); - - // [0, 0] minimal empty binary should also trigger fallback - let result = update_ydoc(&[0, 0], markdown, doc_id).expect("Should create from minimal empty"); - assert!(!result.is_empty(), "Result should not be empty"); - - // Verify the result is a valid document - let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); - doc - .apply_update_from_binary_v1(&result) - .expect("Should apply created doc"); - - let blocks_map = doc.get_map("blocks").expect("Should have blocks"); - assert!(!blocks_map.is_empty(), "Document created from empty should have blocks"); - } -} diff --git a/packages/common/native/src/doc_parser/value.rs b/packages/common/native/src/doc_parser/value.rs index 7bf238305..8526f56ac 100644 --- a/packages/common/native/src/doc_parser/value.rs +++ b/packages/common/native/src/doc_parser/value.rs @@ -81,6 +81,16 @@ pub(super) fn value_to_any(value: &Value) -> Option { None } +pub(super) fn value_to_f64(value: Value) -> Option { + value.to_any().and_then(|any| match any { + Any::Integer(v) => Some(v as f64), + Any::BigInt64(v) => Some(v as f64), + Any::Float32(v) => Some(v.0 as f64), + Any::Float64(v) => Some(v.0), + _ => None, + }) +} + pub(super) fn any_to_string(any: &Any) -> Option { match any { Any::String(value) => Some(value.to_string()), diff --git a/packages/common/native/src/doc_parser/write/builder.rs b/packages/common/native/src/doc_parser/write/builder.rs new file mode 100644 index 000000000..37b962cea --- /dev/null +++ b/packages/common/native/src/doc_parser/write/builder.rs @@ -0,0 +1,426 @@ +use y_octo::{TextDeltaOp, TextInsert}; + +use super::{ + super::schema::{ + PROP_CAPTION, PROP_CELLS_PREFIX, PROP_CHECKED, PROP_COLUMNS_PREFIX, PROP_HEIGHT, PROP_LANGUAGE, PROP_ORDER, + PROP_ROWS_PREFIX, PROP_SOURCE_ID, PROP_TEXT, PROP_TYPE, PROP_URL, PROP_VIDEO_ID, PROP_WIDTH, SYS_CHILDREN, + SYS_FLAVOUR, SYS_ID, SYS_VERSION, table_cell_text_key, table_column_id_key, table_column_order_key, + table_row_id_key, table_row_order_key, + }, + *, +}; + +pub(super) const BOXED_NATIVE_TYPE: &str = "$blocksuite:internal:native$"; +pub(super) const NOTE_BG_LIGHT: &str = "#ffffff"; +pub(super) const NOTE_BG_DARK: &str = "#252525"; +const TABLE_ORDER_WIDTH: usize = 6; + +pub(super) fn block_version(flavour: &str) -> i32 { + match flavour { + "affine:page" => 2, + "affine:surface" => 5, + "affine:note" => 1, + "affine:paragraph" => 1, + "affine:list" => 1, + "affine:code" => 1, + "affine:divider" => 1, + "affine:image" => 1, + "affine:table" => 1, + "affine:bookmark" => 1, + "affine:embed-youtube" => 1, + "affine:embed-iframe" => 1, + "affine:callout" => 1, + _ => 1, + } +} + +pub(super) struct TextBlockProps<'a> { + pub block_type: Option<&'a str>, + pub checked: Option, + pub language: Option<&'a str>, + pub order: Option, + pub text: &'a [TextDeltaOp], +} + +pub(super) struct ImageBlockProps<'a> { + pub source_id: &'a str, + pub caption: Option<&'a str>, + pub width: Option, + pub height: Option, +} + +pub(super) struct BookmarkBlockProps<'a> { + pub url: &'a str, + pub caption: Option<&'a str>, +} + +pub(super) struct EmbedYoutubeBlockProps<'a> { + pub video_id: &'a str, +} + +pub(super) struct EmbedIframeBlockProps<'a> { + pub url: &'a str, +} + +pub(super) fn insert_text(doc: &Doc, block: &mut Map, key: &str, ops: &[TextDeltaOp]) -> Result<(), ParseError> { + let mut text = doc.create_text()?; + // Attach first so updates encode parent types before their contents. + block.insert(key.to_string(), Value::Text(text.clone()))?; + if !ops.is_empty() { + text.apply_delta(ops)?; + } + Ok(()) +} + +pub(crate) fn text_ops_from_plain(text: &str) -> Vec { + if text.is_empty() { + Vec::new() + } else { + vec![TextDeltaOp::Insert { + insert: TextInsert::Text(text.to_string()), + format: None, + }] + } +} + +pub(super) fn insert_children(doc: &Doc, block: &mut Map, children: &[String]) -> Result<(), ParseError> { + let mut array = doc.create_array()?; + // Attach first so updates encode parent types before their contents. + block.insert(SYS_CHILDREN.to_string(), Value::Array(array.clone()))?; + for child_id in children { + array.push(child_id.to_string())?; + } + Ok(()) +} + +pub(super) fn insert_block_map(doc: &Doc, blocks_map: &mut Map, block_id: &str) -> Result { + let empty_map = doc.create_map()?; + blocks_map.insert(block_id.to_string(), Value::Map(empty_map))?; + + blocks_map + .get(block_id) + .and_then(|value| value.to_map()) + .ok_or_else(|| ParseError::ParserError("Failed to retrieve inserted block map".into())) +} + +pub(super) fn insert_sys_fields(block: &mut Map, block_id: &str, flavour: &str) -> Result<(), ParseError> { + block.insert(SYS_ID.to_string(), Any::String(block_id.to_string()))?; + block.insert(SYS_FLAVOUR.to_string(), Any::String(flavour.to_string()))?; + block.insert(SYS_VERSION.to_string(), Any::Integer(block_version(flavour)))?; + Ok(()) +} + +pub(super) fn apply_text_block_props( + doc: &Doc, + block: &mut Map, + props: &TextBlockProps<'_>, + preserve_text: bool, + clear_missing: bool, +) -> Result<(), ParseError> { + match props.block_type { + Some(block_type) => { + block.insert(PROP_TYPE.to_string(), Any::String(block_type.to_string()))?; + } + None => { + if clear_missing && block.get(PROP_TYPE).is_some() { + block.remove(PROP_TYPE); + } + } + } + + if !preserve_text && !props.text.is_empty() { + insert_text(doc, block, PROP_TEXT, props.text)?; + } else if !preserve_text && clear_missing && block.get(PROP_TEXT).is_some() { + block.remove(PROP_TEXT); + } + + match props.checked { + Some(checked) => { + block.insert(PROP_CHECKED.to_string(), if checked { Any::True } else { Any::False })?; + } + None => { + if clear_missing && block.get(PROP_CHECKED).is_some() { + block.remove(PROP_CHECKED); + } + } + } + + match props.language { + Some(language) => { + block.insert(PROP_LANGUAGE.to_string(), Any::String(language.to_string()))?; + } + None => { + if clear_missing && block.get(PROP_LANGUAGE).is_some() { + block.remove(PROP_LANGUAGE); + } + } + } + + match props.order { + Some(order) => { + block.insert(PROP_ORDER.to_string(), Any::Float64((order as f64).into()))?; + } + None => { + if clear_missing && block.get(PROP_ORDER).is_some() { + block.remove(PROP_ORDER); + } + } + } + + Ok(()) +} + +pub(super) fn apply_image_block_props( + block: &mut Map, + props: &ImageBlockProps<'_>, + clear_missing: bool, +) -> Result<(), ParseError> { + block.insert(PROP_SOURCE_ID.to_string(), Any::String(props.source_id.to_string()))?; + + match props.caption { + Some(caption) => { + block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?; + } + None => { + if clear_missing && block.get(PROP_CAPTION).is_some() { + block.remove(PROP_CAPTION); + } + } + } + + match props.width { + Some(width) => { + block.insert(PROP_WIDTH.to_string(), Any::Float64(width.into()))?; + } + None => { + if clear_missing && block.get(PROP_WIDTH).is_some() { + block.remove(PROP_WIDTH); + } + } + } + + match props.height { + Some(height) => { + block.insert(PROP_HEIGHT.to_string(), Any::Float64(height.into()))?; + } + None => { + if clear_missing && block.get(PROP_HEIGHT).is_some() { + block.remove(PROP_HEIGHT); + } + } + } + + Ok(()) +} + +pub(super) fn apply_bookmark_block_props( + block: &mut Map, + props: &BookmarkBlockProps<'_>, + clear_missing: bool, +) -> Result<(), ParseError> { + block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?; + + match props.caption { + Some(caption) => { + block.insert(PROP_CAPTION.to_string(), Any::String(caption.to_string()))?; + } + None => { + if clear_missing && block.get(PROP_CAPTION).is_some() { + block.remove(PROP_CAPTION); + } + } + } + + Ok(()) +} + +pub(super) fn apply_embed_youtube_block_props( + block: &mut Map, + props: &EmbedYoutubeBlockProps<'_>, +) -> Result<(), ParseError> { + block.insert(PROP_VIDEO_ID.to_string(), Any::String(props.video_id.to_string()))?; + Ok(()) +} + +pub(super) fn apply_embed_iframe_block_props( + block: &mut Map, + props: &EmbedIframeBlockProps<'_>, +) -> Result<(), ParseError> { + block.insert(PROP_URL.to_string(), Any::String(props.url.to_string()))?; + Ok(()) +} + +pub(super) fn apply_table_block_props(block: &mut Map, rows: &[Vec]) -> Result<(), ParseError> { + clear_table_props(block); + + if rows.is_empty() { + return Ok(()); + } + + let column_count = rows.iter().map(|row| row.len()).max().unwrap_or(0); + let column_ids: Vec = (0..column_count).map(|_| nanoid::nanoid!()).collect(); + + for (col_idx, column_id) in column_ids.iter().enumerate() { + let order = format_table_order(col_idx); + block.insert(table_column_id_key(column_id), Any::String(column_id.to_string()))?; + block.insert(table_column_order_key(column_id), Any::String(order))?; + } + + for (row_idx, row) in rows.iter().enumerate() { + let row_id = nanoid::nanoid!(); + let order = format_table_order(row_idx); + block.insert(table_row_id_key(&row_id), Any::String(row_id.to_string()))?; + block.insert(table_row_order_key(&row_id), Any::String(order))?; + + for (col_idx, column_id) in column_ids.iter().enumerate() { + let cell_text = row.get(col_idx).cloned().unwrap_or_default(); + block.insert(table_cell_text_key(&row_id, column_id), Any::String(cell_text))?; + } + } + + Ok(()) +} + +pub(super) struct ApplyBlockOptions { + pub preserve_text: bool, + pub clear_missing: bool, +} + +pub(super) fn apply_block_spec( + doc: &Doc, + block: &mut Map, + spec: &BlockSpec, + options: ApplyBlockOptions, +) -> Result<(), ParseError> { + match spec.flavour { + BlockFlavour::Image => { + if options.preserve_text { + return Ok(()); + } + let image = spec + .image + .as_ref() + .ok_or_else(|| ParseError::ParserError("image spec missing".into()))?; + let props = ImageBlockProps { + source_id: &image.source_id, + caption: image.caption.as_deref(), + width: image.width, + height: image.height, + }; + apply_image_block_props(block, &props, options.clear_missing)?; + } + BlockFlavour::Bookmark => { + if options.preserve_text { + return Ok(()); + } + let bookmark = spec + .bookmark + .as_ref() + .ok_or_else(|| ParseError::ParserError("bookmark spec missing".into()))?; + let props = BookmarkBlockProps { + url: &bookmark.url, + caption: bookmark.caption.as_deref(), + }; + apply_bookmark_block_props(block, &props, options.clear_missing)?; + } + BlockFlavour::EmbedYoutube => { + if options.preserve_text { + return Ok(()); + } + let embed = spec + .embed_youtube + .as_ref() + .ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?; + let props = EmbedYoutubeBlockProps { + video_id: &embed.video_id, + }; + apply_embed_youtube_block_props(block, &props)?; + } + BlockFlavour::EmbedIframe => { + if options.preserve_text { + return Ok(()); + } + let embed = spec + .embed_iframe + .as_ref() + .ok_or_else(|| ParseError::ParserError("embed spec missing".into()))?; + let props = EmbedIframeBlockProps { url: &embed.url }; + apply_embed_iframe_block_props(block, &props)?; + } + BlockFlavour::Callout => { + return Ok(()); + } + BlockFlavour::Table => { + if options.preserve_text { + return Ok(()); + } + let table = spec + .table + .as_ref() + .ok_or_else(|| ParseError::ParserError("table spec missing".into()))?; + apply_table_block_props(block, &table.rows)?; + } + _ => { + let props = TextBlockProps { + block_type: spec.block_type_str(), + checked: spec.checked, + language: spec.language.as_deref(), + order: spec.order, + text: &spec.text, + }; + apply_text_block_props(doc, block, &props, options.preserve_text, options.clear_missing)?; + } + } + + Ok(()) +} + +pub(super) fn insert_block_tree(doc: &Doc, blocks_map: &mut Map, node: &BlockNode) -> Result { + let block_id = nanoid::nanoid!(); + let mut block_map = insert_block_map(doc, blocks_map, &block_id)?; + + insert_sys_fields(&mut block_map, &block_id, node.spec.flavour.as_str())?; + apply_block_spec( + doc, + &mut block_map, + &node.spec, + ApplyBlockOptions { + preserve_text: false, + clear_missing: false, + }, + )?; + + let child_ids = node + .children + .iter() + .map(|child| insert_block_tree(doc, blocks_map, child)) + .collect::, _>>()?; + insert_children(doc, &mut block_map, &child_ids)?; + + Ok(block_id) +} + +fn clear_table_props(block: &mut Map) { + let keys = block + .keys() + .filter(|key| { + key.starts_with(PROP_ROWS_PREFIX) || key.starts_with(PROP_COLUMNS_PREFIX) || key.starts_with(PROP_CELLS_PREFIX) + }) + .map(|s| s.to_string()) + .collect::>(); + for key in keys { + block.remove(&key); + } +} + +fn format_table_order(index: usize) -> String { + format!("{index:0width$}", width = TABLE_ORDER_WIDTH) +} + +pub(super) fn boxed_empty_map(doc: &Doc) -> Result { + doc.create_map().map_err(ParseError::from) +} + +pub(super) fn note_background_map(doc: &Doc) -> Result { + doc.create_map().map_err(ParseError::from) +} diff --git a/packages/common/native/src/doc_parser/write/create.rs b/packages/common/native/src/doc_parser/write/create.rs new file mode 100644 index 000000000..c7d7022f6 --- /dev/null +++ b/packages/common/native/src/doc_parser/write/create.rs @@ -0,0 +1,281 @@ +//! Markdown to YDoc conversion module +//! +//! Converts markdown content into AFFiNE-compatible y-octo document binary +//! format. + +use y_octo::DocOptions; + +use super::{ + super::{ + markdown::parse_markdown_blocks, + schema::{PROP_BACKGROUND, PROP_DISPLAY_MODE, PROP_ELEMENTS, PROP_HIDDEN, PROP_INDEX, PROP_XYWH, SURFACE_FLAVOUR}, + }, + builder::{ + BOXED_NATIVE_TYPE, NOTE_BG_DARK, NOTE_BG_LIGHT, boxed_empty_map, insert_block_map, insert_block_tree, + insert_children, insert_sys_fields, insert_text, note_background_map, text_ops_from_plain, + }, + *, +}; + +/// Converts markdown into an AFFiNE-compatible y-octo document binary. +/// +/// # Arguments +/// * `title` - The document title +/// * `markdown` - The markdown content to convert +/// * `doc_id` - The document ID to use +/// +/// # Returns +/// A binary vector containing the y-octo encoded document update +pub fn build_full_doc(title: &str, markdown: &str, doc_id: &str) -> Result, ParseError> { + let nodes = parse_markdown_blocks(markdown)?; + build_doc_update(doc_id, title, &nodes) +} + +fn build_doc_update(doc_id: &str, title: &str, blocks: &[BlockNode]) -> Result, ParseError> { + let doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + let mut blocks_map = doc.get_or_create_map("blocks")?; + + let page_id = nanoid::nanoid!(); + let surface_id = nanoid::nanoid!(); + let note_id = nanoid::nanoid!(); + + // Insert root blocks first to establish stable IDs. + let mut page_map = insert_block_map(&doc, &mut blocks_map, &page_id)?; + let mut surface_map = insert_block_map(&doc, &mut blocks_map, &surface_id)?; + let mut note_map = insert_block_map(&doc, &mut blocks_map, ¬e_id)?; + + // Create content blocks under note. + let content_ids = insert_block_trees(&doc, &mut blocks_map, blocks)?; + + // Page block + insert_sys_fields(&mut page_map, &page_id, PAGE_FLAVOUR)?; + insert_children(&doc, &mut page_map, &[surface_id.clone(), note_id.clone()])?; + insert_text(&doc, &mut page_map, PROP_TITLE, &text_ops_from_plain(title))?; + + // Surface block + insert_sys_fields(&mut surface_map, &surface_id, SURFACE_FLAVOUR)?; + insert_children(&doc, &mut surface_map, &[])?; + let mut boxed = boxed_empty_map(&doc)?; + surface_map.insert(PROP_ELEMENTS.to_string(), Value::Map(boxed.clone()))?; + boxed.insert("type".to_string(), Any::String(BOXED_NATIVE_TYPE.to_string()))?; + let value = doc.create_map()?; + boxed.insert("value".to_string(), Value::Map(value))?; + + // Note block + insert_sys_fields(&mut note_map, ¬e_id, NOTE_FLAVOUR)?; + insert_children(&doc, &mut note_map, &content_ids)?; + let mut background = note_background_map(&doc)?; + note_map.insert(PROP_BACKGROUND.to_string(), Value::Map(background.clone()))?; + background.insert("light".to_string(), Any::String(NOTE_BG_LIGHT.to_string()))?; + background.insert("dark".to_string(), Any::String(NOTE_BG_DARK.to_string()))?; + note_map.insert(PROP_XYWH.to_string(), Any::String("[0,0,800,95]".to_string()))?; + note_map.insert(PROP_INDEX.to_string(), Any::String("a0".to_string()))?; + note_map.insert(PROP_HIDDEN.to_string(), Any::False)?; + note_map.insert(PROP_DISPLAY_MODE.to_string(), Any::String("both".to_string()))?; + + Ok(doc.encode_update_v1()?) +} + +fn insert_block_trees(doc: &Doc, blocks_map: &mut Map, blocks: &[BlockNode]) -> Result, ParseError> { + let mut ids = Vec::with_capacity(blocks.len()); + for block in blocks { + let id = insert_block_tree(doc, blocks_map, block)?; + ids.push(id); + } + Ok(ids) +} + +#[cfg(test)] +mod tests { + use y_octo::{Any, DocOptions}; + + use super::{ + super::super::{ + blocksuite::get_string, + markdown::{MAX_BLOCKS, MAX_MARKDOWN_CHARS}, + schema::PAGE_FLAVOUR, + }, + *, + }; + + #[test] + fn test_simple_markdown() { + let markdown = "# Hello World\n\nThis is a test paragraph."; + let result = build_full_doc("Hello World", markdown, "test-doc-id"); + assert!(result.is_ok()); + let bin = result.unwrap(); + assert!(!bin.is_empty()); + } + + #[test] + fn test_title_from_param() { + let markdown = "# Markdown Title\n\nContent."; + let doc_id = "title-param-test"; + let bin = build_full_doc("External Title", markdown, doc_id).expect("create doc"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc.apply_update_from_binary_v1(&bin).expect("apply update"); + + let blocks_map = doc.get_map("blocks").expect("blocks map"); + let mut title = None; + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() + && get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR) + { + title = get_string(&block_map, "prop:title"); + break; + } + } + + assert_eq!(title.as_deref(), Some("External Title")); + } + + #[test] + fn test_markdown_with_list() { + let markdown = "# Test List\n\n- Item 1\n- Item 2\n- Item 3"; + let result = build_full_doc("Test List", markdown, "test-doc-id"); + assert!(result.is_ok()); + } + + #[test] + fn test_markdown_with_code() { + let markdown = "# Code Example\n\n```rust\nfn main() {\n println!(\"Hello\");\n}\n```"; + let result = build_full_doc("Code Example", markdown, "test-doc-id"); + assert!(result.is_ok()); + } + + #[test] + fn test_markdown_with_headings() { + let markdown = "# H1\n\n## H2\n\n### H3\n\nParagraph text."; + let result = build_full_doc("H1", markdown, "test-doc-id"); + assert!(result.is_ok()); + } + + #[test] + fn test_empty_markdown() { + let result = build_full_doc("Untitled", "", "test-doc-id"); + assert!(result.is_ok()); + let bin = result.unwrap(); + assert!(!bin.is_empty()); + } + + #[test] + fn test_whitespace_only_markdown() { + let result = build_full_doc("Untitled", " \n\n\t\n ", "test-doc-id"); + assert!(result.is_ok()); + let bin = result.unwrap(); + assert!(!bin.is_empty()); + } + + #[test] + fn test_markdown_without_h1() { + let markdown = "## Secondary Heading\n\nSome content without H1."; + let result = build_full_doc("Title", markdown, "test-doc-id"); + assert!(result.is_ok()); + } + + #[test] + fn test_nested_lists() { + let markdown = "# Nested Lists\n\n- Item 1\n - Nested 1.1\n - Nested 1.2\n- Item 2\n - Nested 2.1"; + let result = build_full_doc("Nested Lists", markdown, "test-doc-id"); + assert!(result.is_ok()); + } + + #[test] + fn test_blockquote() { + let markdown = "# Title\n\n> A blockquote"; + let result = build_full_doc("Title", markdown, "test-doc-id"); + assert!(result.is_ok()); + } + + #[test] + fn test_divider() { + let markdown = "# Title\n\nBefore divider\n\n---\n\nAfter divider"; + let result = build_full_doc("Title", markdown, "test-doc-id"); + assert!(result.is_ok()); + } + + #[test] + fn test_numbered_list() { + let markdown = "# Title\n\n1. First item\n2. Second item"; + let result = build_full_doc("Title", markdown, "test-doc-id"); + assert!(result.is_ok()); + } + + #[test] + fn test_markdown_too_large() { + let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1); + let result = build_full_doc("Title", &markdown, "test-doc-id"); + assert!(result.is_err()); + } + + #[test] + fn test_markdown_block_limit() { + let mut markdown = String::from("# Title\n\n"); + for i in 0..=MAX_BLOCKS { + markdown.push_str(&format!("Paragraph {i}\n\n")); + } + let result = build_full_doc("Title", &markdown, "test-doc-id"); + assert!(result.is_err()); + } + + #[test] + fn test_markdown_with_image() { + let markdown = "![Alt](blob://image-id)"; + let doc_id = "image-doc"; + let bin = build_full_doc("Title", markdown, doc_id).expect("create doc"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc.apply_update_from_binary_v1(&bin).expect("apply update"); + + let blocks_map = doc.get_map("blocks").expect("blocks map"); + let mut found = false; + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() + && get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image") + { + let source_id = get_string(&block_map, "prop:sourceId"); + assert_eq!(source_id.as_deref(), Some("image-id")); + found = true; + break; + } + } + + assert!(found); + } + + #[test] + fn test_markdown_with_table() { + let markdown = "| A | B |\n| --- | --- |\n| 1 | 2 |"; + let doc_id = "table-doc"; + let bin = build_full_doc("Title", markdown, doc_id).expect("create doc"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc.apply_update_from_binary_v1(&bin).expect("apply update"); + + let blocks_map = doc.get_map("blocks").expect("blocks map"); + let mut found_cell = false; + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() + && get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table") + { + for key in block_map.keys() { + if key.starts_with("prop:cells.") && key.ends_with(".text") { + let value = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a { + Any::String(value) => Some(value), + _ => None, + }); + if let Some(value) = value + && (value == "A" || value == "1") + { + found_cell = true; + break; + } + } + } + } + } + + assert!(found_cell); + } +} diff --git a/packages/common/native/src/doc_parser/write/doc_meta.rs b/packages/common/native/src/doc_parser/write/doc_meta.rs new file mode 100644 index 000000000..f988f01be --- /dev/null +++ b/packages/common/native/src/doc_parser/write/doc_meta.rs @@ -0,0 +1,132 @@ +use super::{ + builder::{insert_text, text_ops_from_plain}, + root_doc::ensure_pages_array, + *, +}; + +pub fn update_doc_title(existing_binary: &[u8], doc_id: &str, title: &str) -> Result, ParseError> { + let doc = load_doc(existing_binary, Some(doc_id))?; + + let state_before = doc.get_state_vector(); + let blocks_map = doc.get_map("blocks")?; + if blocks_map.is_empty() { + return Err(ParseError::ParserError("blocks map is empty".into())); + } + + let mut page_block = find_page_block(&blocks_map)?; + let current = get_string(&page_block, PROP_TITLE).unwrap_or_default(); + if current != title { + insert_text(&doc, &mut page_block, PROP_TITLE, &text_ops_from_plain(title))?; + } + + Ok(doc.encode_state_as_update_v1(&state_before)?) +} + +pub fn update_root_doc_meta_title(root_doc_bin: &[u8], doc_id: &str, title: &str) -> Result, ParseError> { + let doc = load_doc_or_new(root_doc_bin)?; + + let state_before = doc.get_state_vector(); + let mut meta = doc.get_or_create_map("meta")?; + let mut pages = ensure_pages_array(&doc, &mut meta)?; + + let mut found = false; + for idx in 0..pages.len() { + let Some(mut page) = pages.get(idx).and_then(|v| v.to_map()) else { + continue; + }; + if get_string(&page, "id").as_deref() == Some(doc_id) { + page.insert("title".to_string(), Any::String(title.to_string()))?; + found = true; + break; + } + } + + if !found { + let page_map = doc.create_map()?; + + let idx = pages.len(); + pages.insert(idx, Value::Map(page_map))?; + + if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) { + inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?; + inserted_page.insert("title".to_string(), Any::String(title.to_string()))?; + + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0); + inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?; + + let tags = doc.create_array()?; + inserted_page.insert("tags".to_string(), Value::Array(tags))?; + } + } + + Ok(doc.encode_state_as_update_v1(&state_before)?) +} + +fn find_page_block(blocks_map: &Map) -> Result { + let index = build_block_index(blocks_map); + let page_id = find_block_id_by_flavour(&index.block_pool, PAGE_FLAVOUR) + .ok_or_else(|| ParseError::ParserError("page block not found".into()))?; + blocks_map + .get(&page_id) + .and_then(|value| value.to_map()) + .ok_or_else(|| ParseError::ParserError("page block not found".into())) +} + +#[cfg(test)] +mod tests { + use y_octo::DocOptions; + + use super::*; + use crate::doc_parser::{add_doc_to_root_doc, build_full_doc}; + + #[test] + fn test_update_doc_title() { + let doc_id = "doc-meta-title-test"; + let initial = build_full_doc("Old Title", "Content.", doc_id).expect("create doc"); + let delta = update_doc_title(&initial, doc_id, "New Title").expect("update title"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc.apply_update_from_binary_v1(&initial).expect("apply initial"); + doc.apply_update_from_binary_v1(&delta).expect("apply delta"); + + let blocks_map = doc.get_map("blocks").expect("blocks map"); + let mut title = None; + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() + && get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR) + { + title = get_string(&block_map, "prop:title"); + break; + } + } + + assert_eq!(title.as_deref(), Some("New Title")); + } + + #[test] + fn test_update_root_doc_meta_title() { + let doc_id = "root-meta-title-test"; + let root_bin = add_doc_to_root_doc(Vec::new(), doc_id, Some("Old Title")).expect("create root meta"); + let delta = update_root_doc_meta_title(&root_bin, doc_id, "New Title").expect("update meta"); + + let mut doc = DocOptions::new().build(); + doc.apply_update_from_binary_v1(&root_bin).expect("apply root"); + doc.apply_update_from_binary_v1(&delta).expect("apply delta"); + + let meta = doc.get_map("meta").expect("meta map"); + let pages = meta.get("pages").and_then(|v| v.to_array()).expect("pages array"); + let mut title = None; + for page in pages.iter() { + if let Some(page_map) = page.to_map() + && get_string(&page_map, "id").as_deref() == Some(doc_id) + { + title = get_string(&page_map, "title"); + break; + } + } + assert_eq!(title.as_deref(), Some("New Title")); + } +} diff --git a/packages/common/native/src/doc_parser/write/doc_properties.rs b/packages/common/native/src/doc_parser/write/doc_properties.rs new file mode 100644 index 000000000..53f67dd04 --- /dev/null +++ b/packages/common/native/src/doc_parser/write/doc_properties.rs @@ -0,0 +1,95 @@ +use y_octo::{Any, DocOptions, Map}; + +use super::{ + super::{doc_loader::is_empty_doc, value::value_to_string}, + ParseError, +}; + +pub fn update_doc_properties( + existing_binary: &[u8], + properties_doc_id: &str, + target_doc_id: &str, + created_by: Option<&str>, + updated_by: Option<&str>, +) -> Result, ParseError> { + let doc = if is_empty_doc(existing_binary) { + DocOptions::new().with_guid(properties_doc_id.to_string()).build() + } else { + super::load_doc(existing_binary, Some(properties_doc_id))? + }; + + let state_before = doc.get_state_vector(); + let mut record = doc.get_or_create_map(target_doc_id)?; + let mut changed = false; + + if record.get("id").is_none() { + record.insert("id".to_string(), Any::String(target_doc_id.to_string()))?; + changed = true; + } + + if let Some(created_by) = created_by + && get_record_string(&record, "createdBy").as_deref() != Some(created_by) + { + record.insert("createdBy".to_string(), Any::String(created_by.to_string()))?; + changed = true; + } + + if let Some(updated_by) = updated_by + && get_record_string(&record, "updatedBy").as_deref() != Some(updated_by) + { + record.insert("updatedBy".to_string(), Any::String(updated_by.to_string()))?; + changed = true; + } + + if !changed { + return Ok(Vec::new()); + } + + Ok(doc.encode_state_as_update_v1(&state_before)?) +} + +fn get_record_string(record: &Map, key: &str) -> Option { + record.get(key).and_then(|value| value_to_string(&value)) +} + +#[cfg(test)] +mod tests { + use y_octo::DocOptions; + + use super::*; + + #[test] + fn test_update_doc_properties_creates_record() { + let properties_doc_id = "doc-properties"; + let target_doc_id = "doc-1"; + let update = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-1")) + .expect("update properties"); + + let mut doc = DocOptions::new().with_guid(properties_doc_id.to_string()).build(); + doc.apply_update_from_binary_v1(&update).expect("apply"); + + let record = doc.get_map(target_doc_id).expect("record map"); + assert_eq!(get_record_string(&record, "id").as_deref(), Some(target_doc_id)); + assert_eq!(get_record_string(&record, "createdBy").as_deref(), Some("user-1")); + assert_eq!(get_record_string(&record, "updatedBy").as_deref(), Some("user-1")); + } + + #[test] + fn test_update_doc_properties_no_change() { + let properties_doc_id = "doc-properties-no-change"; + let target_doc_id = "doc-2"; + let initial = update_doc_properties(&[], properties_doc_id, target_doc_id, Some("user-1"), Some("user-2")) + .expect("initial update"); + + let delta = update_doc_properties( + &initial, + properties_doc_id, + target_doc_id, + Some("user-1"), + Some("user-2"), + ) + .expect("no change update"); + + assert!(delta.is_empty()); + } +} diff --git a/packages/common/native/src/doc_parser/write/mod.rs b/packages/common/native/src/doc_parser/write/mod.rs new file mode 100644 index 000000000..c631d53a0 --- /dev/null +++ b/packages/common/native/src/doc_parser/write/mod.rs @@ -0,0 +1,21 @@ +pub mod builder; +mod create; +mod doc_meta; +mod doc_properties; +mod root_doc; +mod update; + +pub use create::build_full_doc; +pub use doc_meta::{update_doc_title, update_root_doc_meta_title}; +pub use doc_properties::update_doc_properties; +pub use root_doc::add_doc_to_root_doc; +pub use update::update_doc; +use y_octo::{Any, Doc, Map, Value}; + +use super::{ + ParseError, + block_spec::{BlockFlavour, BlockNode, BlockSpec}, + blocksuite::{build_block_index, find_block_id_by_flavour, get_string}, + doc_loader::{load_doc, load_doc_or_new}, + schema::{NOTE_FLAVOUR, PAGE_FLAVOUR, PROP_TITLE}, +}; diff --git a/packages/common/native/src/doc_parser/write/root_doc.rs b/packages/common/native/src/doc_parser/write/root_doc.rs new file mode 100644 index 000000000..3afa33863 --- /dev/null +++ b/packages/common/native/src/doc_parser/write/root_doc.rs @@ -0,0 +1,106 @@ +use y_octo::Array; + +use super::*; + +const DEFAULT_DOC_TITLE: &str = "Untitled"; + +fn any_to_value(doc: &Doc, any: Any) -> Result { + match any { + Any::Array(values) => { + let mut array = doc.create_array()?; + for value in values { + let item = any_to_value(doc, value)?; + array.push(item)?; + } + Ok(Value::Array(array)) + } + Any::Object(values) => { + let mut map = doc.create_map()?; + for (key, value) in values { + let item = any_to_value(doc, value)?; + map.insert(key, item)?; + } + Ok(Value::Map(map)) + } + _ => Ok(Value::Any(any)), + } +} + +pub(super) fn ensure_pages_array(doc: &Doc, meta: &mut Map) -> Result { + let pages_value = meta.get("pages"); + if let Some(pages) = pages_value.as_ref().and_then(|value| value.to_array()) { + return Ok(pages); + } + + if let Some(Any::Array(entries)) = pages_value.and_then(|value| value.to_any()) { + let mut pages = doc.create_array()?; + for entry in entries { + let value = any_to_value(doc, entry)?; + pages.push(value)?; + } + meta.insert("pages".to_string(), Value::Array(pages.clone()))?; + return Ok(pages); + } + + let pages = doc.create_array()?; + meta.insert("pages".to_string(), Value::Array(pages.clone()))?; + Ok(pages) +} + +/// Adds a document ID to the root doc's meta.pages array. +/// Returns a binary update that can be applied to the root doc. +/// +/// # Arguments +/// * `root_doc_bin` - The current root doc binary +/// * `doc_id` - The document ID to add +/// * `title` - Optional title for the document +/// +/// # Returns +/// A Vec containing the y-octo update binary to add the doc +pub fn add_doc_to_root_doc(root_doc_bin: Vec, doc_id: &str, title: Option<&str>) -> Result, ParseError> { + // Handle empty or minimal root doc - create a new one + let doc = load_doc_or_new(&root_doc_bin)?; + + // Capture state before modifications to encode only the delta + let state_before = doc.get_state_vector(); + + // Get or create the meta map + let mut meta = doc.get_or_create_map("meta")?; + + let mut pages = ensure_pages_array(&doc, &mut meta)?; + + // Check if doc already exists + let doc_exists = pages.iter().any(|page_val| { + page_val + .to_map() + .and_then(|page| get_string(&page, "id")) + .map(|id| id == doc_id) + .unwrap_or(false) + }); + + if !doc_exists { + let page_map = doc.create_map()?; + + let idx = pages.len(); + pages.insert(idx, Value::Map(page_map))?; + + if let Some(mut inserted_page) = pages.get(idx).and_then(|v| v.to_map()) { + inserted_page.insert("id".to_string(), Any::String(doc_id.to_string()))?; + + let page_title = title.unwrap_or(DEFAULT_DOC_TITLE); + inserted_page.insert("title".to_string(), Any::String(page_title.to_string()))?; + + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0); + inserted_page.insert("createDate".to_string(), Any::Float64((timestamp as f64).into()))?; + + let tags = doc.create_array()?; + inserted_page.insert("tags".to_string(), Value::Array(tags))?; + } + } + + // Encode only the changes (delta) since state_before + Ok(doc.encode_state_as_update_v1(&state_before)?) +} diff --git a/packages/common/native/src/doc_parser/write/update.rs b/packages/common/native/src/doc_parser/write/update.rs new file mode 100644 index 000000000..ddf47e21e --- /dev/null +++ b/packages/common/native/src/doc_parser/write/update.rs @@ -0,0 +1,671 @@ +//! Update YDoc module +//! +//! Provides functionality to update existing AFFiNE documents by applying +//! surgical y-octo operations based on content differences. + +use std::collections::HashMap; + +use super::{ + super::{ + block_spec::{TreeNode, count_tree_nodes, text_delta_eq}, + blocksuite::{collect_child_ids, find_child_id_by_flavour}, + markdown::{MAX_BLOCKS, parse_markdown_blocks}, + }, + builder::{ApplyBlockOptions, apply_block_spec, insert_block_tree, insert_children}, + *, +}; + +const MAX_LCS_CELLS: usize = 2_000_000; + +#[derive(Debug, Clone)] +struct StoredNode { + id: String, + spec: BlockSpec, + children: Vec, +} + +impl TreeNode for StoredNode { + fn children(&self) -> &[StoredNode] { + &self.children + } +} + +struct DocState { + doc: Doc, + note_id: String, + blocks: Vec, +} + +#[derive(Debug)] +enum PatchOp { + Keep(usize, usize), + Delete(usize), + Insert(usize), + Update(usize, usize), +} + +/// Updates an existing document with new markdown content. +/// +/// This function performs structural diffing between the existing document +/// and the new markdown content, then applies block-level replacements +/// for changed blocks. This enables proper CRDT merging with concurrent +/// edits from other clients. +/// +/// # Arguments +/// * `existing_binary` - The current document binary +/// * `new_markdown` - The new markdown content (document title is not updated) +/// * `doc_id` - The document ID +/// +/// # Returns +/// A binary vector representing only the delta (changes) to apply +pub fn update_doc(existing_binary: &[u8], new_markdown: &str, doc_id: &str) -> Result, ParseError> { + let mut new_nodes = parse_markdown_blocks(new_markdown)?; + let state = load_doc_state(existing_binary, doc_id)?; + + check_limits(&state.blocks, &new_nodes)?; + + let state_before = state.doc.get_state_vector(); + + let mut blocks_map = state.doc.get_map("blocks")?; + + let new_children = sync_nodes(&state.doc, &mut blocks_map, &state.blocks, &mut new_nodes)?; + sync_children(&state.doc, &mut blocks_map, &state.note_id, &new_children)?; + + Ok(state.doc.encode_state_as_update_v1(&state_before)?) +} + +fn load_doc_state(binary: &[u8], doc_id: &str) -> Result { + let doc = load_doc(binary, Some(doc_id))?; + + let blocks_map = doc.get_map("blocks")?; + if blocks_map.is_empty() { + return Err(ParseError::ParserError("blocks map is empty".into())); + } + + let block_index = build_block_index(&blocks_map); + let page_id = find_block_id_by_flavour(&block_index.block_pool, PAGE_FLAVOUR) + .ok_or_else(|| ParseError::ParserError("page block not found".into()))?; + let page_block = block_index + .block_pool + .get(&page_id) + .ok_or_else(|| ParseError::ParserError("page block not found".into()))?; + let note_id = find_child_id_by_flavour(page_block, &block_index.block_pool, NOTE_FLAVOUR) + .ok_or_else(|| ParseError::ParserError("note block not found".into()))?; + let note_block = block_index + .block_pool + .get(¬e_id) + .ok_or_else(|| ParseError::ParserError("note block not found".into()))?; + let content_ids = collect_child_ids(note_block); + + let mut blocks = Vec::new(); + for block_id in content_ids { + let block = block_index + .block_pool + .get(&block_id) + .ok_or_else(|| ParseError::ParserError("content block not found".into()))?; + blocks.push(build_stored_tree(&block_id, block, &block_index.block_pool)?); + } + + Ok(DocState { doc, note_id, blocks }) +} + +fn build_stored_tree(block_id: &str, block: &Map, pool: &HashMap) -> Result { + let spec = BlockSpec::from_block_map(block)?; + + let child_ids = collect_child_ids(block); + if !child_ids.is_empty() && !matches!(spec.flavour, BlockFlavour::List | BlockFlavour::Callout) { + return Err(ParseError::ParserError(format!( + "unsupported children on block: {block_id}" + ))); + } + let mut children = Vec::new(); + for child_id in child_ids { + let child_block = pool + .get(&child_id) + .ok_or_else(|| ParseError::ParserError("child block not found".into()))?; + children.push(build_stored_tree(&child_id, child_block, pool)?); + } + + Ok(StoredNode { + id: block_id.to_string(), + spec, + children, + }) +} + +fn sync_nodes( + doc: &Doc, + blocks_map: &mut Map, + current: &[StoredNode], + target: &mut [BlockNode], +) -> Result, ParseError> { + let ops = diff_blocks(current, target); + let mut new_children = Vec::new(); + let mut to_remove = Vec::new(); + + for op in ops { + match op { + PatchOp::Keep(old_idx, new_idx) => { + let old_node = ¤t[old_idx]; + let new_node = &target[new_idx]; + update_block_props(doc, blocks_map, old_node, &new_node.spec, true)?; + let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?; + sync_children(doc, blocks_map, &old_node.id, &child_ids)?; + new_children.push(old_node.id.clone()); + } + PatchOp::Update(old_idx, new_idx) => { + let old_node = ¤t[old_idx]; + let new_node = &target[new_idx]; + update_block_props(doc, blocks_map, old_node, &new_node.spec, false)?; + let child_ids = sync_nodes(doc, blocks_map, &old_node.children, &mut new_node.children.clone())?; + sync_children(doc, blocks_map, &old_node.id, &child_ids)?; + new_children.push(old_node.id.clone()); + } + PatchOp::Insert(new_idx) => { + let new_id = insert_block_tree(doc, blocks_map, &target[new_idx])?; + new_children.push(new_id); + } + PatchOp::Delete(old_idx) => { + let node = ¤t[old_idx]; + if node.spec.flavour == BlockFlavour::Callout { + new_children.push(node.id.clone()); + } else { + collect_tree_ids(node, &mut to_remove); + } + } + } + } + + for id in to_remove { + blocks_map.remove(&id); + } + + Ok(new_children) +} + +fn diff_blocks(current: &[StoredNode], target: &[BlockNode]) -> Vec { + let old_len = current.len(); + let new_len = target.len(); + + if old_len == 0 { + return (0..new_len).map(PatchOp::Insert).collect(); + } + if new_len == 0 { + return (0..old_len).map(PatchOp::Delete).collect(); + } + + let mut lcs = vec![vec![0usize; new_len + 1]; old_len + 1]; + + for i in 1..=old_len { + for j in 1..=new_len { + let old_spec = ¤t[i - 1].spec; + let new_spec = &target[j - 1].spec; + + if old_spec.is_exact(new_spec) { + lcs[i][j] = lcs[i - 1][j - 1] + 1; + } else { + lcs[i][j] = std::cmp::max(lcs[i - 1][j], lcs[i][j - 1]); + } + } + } + + let mut ops = Vec::new(); + let mut i = old_len; + let mut j = new_len; + + while i > 0 || j > 0 { + if i > 0 && j > 0 { + let old_spec = ¤t[i - 1].spec; + let new_spec = &target[j - 1].spec; + + if old_spec.is_exact(new_spec) { + ops.push(PatchOp::Keep(i - 1, j - 1)); + i -= 1; + j -= 1; + } else if old_spec.is_similar(new_spec) + && lcs[i - 1][j - 1] >= lcs[i - 1][j] + && lcs[i - 1][j - 1] >= lcs[i][j - 1] + { + ops.push(PatchOp::Update(i - 1, j - 1)); + i -= 1; + j -= 1; + } else if lcs[i][j - 1] >= lcs[i - 1][j] { + ops.push(PatchOp::Insert(j - 1)); + j -= 1; + } else { + ops.push(PatchOp::Delete(i - 1)); + i -= 1; + } + } else if j > 0 { + ops.push(PatchOp::Insert(j - 1)); + j -= 1; + } else { + ops.push(PatchOp::Delete(i - 1)); + i -= 1; + } + } + + ops.reverse(); + ops +} + +fn update_block_props( + doc: &Doc, + blocks_map: &mut Map, + node: &StoredNode, + target: &BlockSpec, + preserve_text: bool, +) -> Result<(), ParseError> { + let Some(mut block) = blocks_map.get(&node.id).and_then(|v| v.to_map()) else { + return Err(ParseError::ParserError(format!("Block {} not found", node.id))); + }; + + let preserve = match target.flavour { + BlockFlavour::Image + | BlockFlavour::Table + | BlockFlavour::Bookmark + | BlockFlavour::EmbedYoutube + | BlockFlavour::EmbedIframe => preserve_text, + _ => preserve_text || text_delta_eq(&node.spec.text, &target.text), + }; + + apply_block_spec( + doc, + &mut block, + target, + ApplyBlockOptions { + preserve_text: preserve, + clear_missing: true, + }, + )?; + + Ok(()) +} + +fn sync_children(doc: &Doc, blocks_map: &mut Map, block_id: &str, children: &[String]) -> Result<(), ParseError> { + let Some(mut block) = blocks_map.get(block_id).and_then(|v| v.to_map()) else { + return Err(ParseError::ParserError("Block not found".into())); + }; + + let current_children = collect_child_ids(&block); + if current_children != children { + insert_children(doc, &mut block, children)?; + } + + Ok(()) +} + +fn collect_tree_ids(node: &StoredNode, output: &mut Vec) { + output.push(node.id.clone()); + for child in &node.children { + collect_tree_ids(child, output); + } +} + +fn check_limits(current: &[StoredNode], target: &[BlockNode]) -> Result<(), ParseError> { + let current_count = count_tree_nodes(current); + let target_count = count_tree_nodes(target); + + if current_count > MAX_BLOCKS || target_count > MAX_BLOCKS { + return Err(ParseError::ParserError("block_count_too_large".into())); + } + + if current_count.saturating_mul(target_count) > MAX_LCS_CELLS { + return Err(ParseError::ParserError("diff_matrix_too_large".into())); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use y_octo::{Any, DocOptions, TextDeltaOp, TextInsert}; + + use super::{super::builder::text_ops_from_plain, *}; + use crate::doc_parser::{ + block_spec::BlockType, blocksuite::get_string, build_full_doc, markdown::MAX_MARKDOWN_CHARS, parse_doc_to_markdown, + }; + + #[test] + fn test_compute_text_diff_simple() { + let ops = text_ops_from_plain("hello world"); + assert_eq!(ops.len(), 1); + match &ops[0] { + TextDeltaOp::Insert { + insert: TextInsert::Text(text), + format: None, + } => { + assert_eq!(text, "hello world"); + } + _ => panic!("unexpected delta op"), + } + } + + #[test] + fn test_content_block_similarity() { + let b1 = BlockSpec { + flavour: BlockFlavour::Paragraph, + block_type: Some(BlockType::H1), + text: text_ops_from_plain("Hello"), + checked: None, + language: None, + order: None, + image: None, + table: None, + bookmark: None, + embed_youtube: None, + embed_iframe: None, + }; + let b2 = BlockSpec { + flavour: BlockFlavour::Paragraph, + block_type: Some(BlockType::H1), + text: text_ops_from_plain("World"), + checked: None, + language: None, + order: None, + image: None, + table: None, + bookmark: None, + embed_youtube: None, + embed_iframe: None, + }; + let b3 = BlockSpec { + flavour: BlockFlavour::Paragraph, + block_type: Some(BlockType::H2), + text: text_ops_from_plain("Hello"), + checked: None, + language: None, + order: None, + image: None, + table: None, + bookmark: None, + embed_youtube: None, + embed_iframe: None, + }; + + assert!(b1.is_similar(&b2)); + assert!(!b1.is_similar(&b3)); + } + + #[test] + fn test_update_ydoc_roundtrip() { + let initial_md = "# Test Document\n\nFirst paragraph.\n\nSecond paragraph."; + let doc_id = "update-test"; + + let initial_bin = build_full_doc("Test Document", initial_md, doc_id).expect("Should create initial doc"); + + let updated_md = "# Test Document\n\nFirst paragraph.\n\nModified second paragraph.\n\nNew third paragraph."; + + let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); + assert!(!delta.is_empty(), "Delta should contain changes"); + } + + #[test] + fn test_update_ydoc_does_not_update_page_title() { + let initial_md = "# Original Title\n\nContent here."; + let doc_id = "title-test"; + + let initial_bin = build_full_doc("Original Title", initial_md, doc_id).expect("Should create initial doc"); + + let updated_md = "# New Title\n\nContent here."; + let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc + .apply_update_from_binary_v1(&initial_bin) + .expect("Should apply initial"); + doc.apply_update_from_binary_v1(&delta).expect("Should apply delta"); + + let blocks_map = doc.get_map("blocks").expect("blocks map exists"); + let mut title = None; + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() + && get_string(&block_map, "sys:flavour").as_deref() == Some(PAGE_FLAVOUR) + { + title = get_string(&block_map, "prop:title"); + break; + } + } + + assert_eq!(title.as_deref(), Some("Original Title")); + } + + #[test] + fn test_update_ydoc_no_changes() { + let markdown = "# Same Title\n\nSame content."; + let doc_id = "no-change-test"; + + let initial_bin = build_full_doc("Same Title", markdown, doc_id).expect("Should create initial doc"); + let delta = update_doc(&initial_bin, markdown, doc_id).expect("Should compute delta"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc + .apply_update_from_binary_v1(&initial_bin) + .expect("Should apply initial"); + doc + .apply_update_from_binary_v1(&delta) + .expect("Should apply delta even with no changes"); + } + + #[test] + fn test_update_ydoc_ignores_ai_editable_comments() { + let markdown = "Plain paragraph."; + let doc_id = "ai-comment-test"; + + let initial_bin = build_full_doc("Title", markdown, doc_id).expect("Should create initial doc"); + + let ai_markdown = parse_doc_to_markdown(initial_bin.clone(), doc_id.to_string(), true, None) + .expect("parse doc") + .markdown; + assert!(ai_markdown.contains("block_id=")); + + let delta = update_doc(&initial_bin, &ai_markdown, doc_id).expect("Should compute delta"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc + .apply_update_from_binary_v1(&initial_bin) + .expect("Should apply initial"); + doc.apply_update_from_binary_v1(&delta).expect("Should apply delta"); + + let before = parse_doc_to_markdown(initial_bin, doc_id.to_string(), false, None) + .expect("parse before") + .markdown; + let after = parse_doc_to_markdown(doc.encode_update_v1().unwrap(), doc_id.to_string(), false, None) + .expect("parse after") + .markdown; + + assert_eq!(after, before); + } + + #[test] + fn test_update_ydoc_add_block() { + let initial_md = "# Add Block Test\n\nOriginal paragraph."; + let doc_id = "add-block-test"; + + let initial_bin = build_full_doc("Add Block Test", initial_md, doc_id).expect("Should create initial doc"); + + let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + initial_doc + .apply_update_from_binary_v1(&initial_bin) + .expect("Should apply initial"); + let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len(); + + let updated_md = "# Add Block Test\n\nOriginal paragraph.\n\nNew paragraph added."; + let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); + assert!(!delta.is_empty(), "Delta should contain changes"); + + let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + updated_doc + .apply_update_from_binary_v1(&initial_bin) + .expect("Should apply initial"); + updated_doc + .apply_update_from_binary_v1(&delta) + .expect("Should apply delta with new block"); + + let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len(); + assert!( + updated_count > initial_count, + "Expected more blocks after insert, got {updated_count} vs {initial_count}" + ); + } + + #[test] + fn test_update_ydoc_delete_block() { + let initial_md = "# Delete Block Test\n\nFirst paragraph.\n\nSecond paragraph to delete."; + let doc_id = "delete-block-test"; + + let initial_bin = build_full_doc("Delete Block Test", initial_md, doc_id).expect("Should create initial doc"); + + let mut initial_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + initial_doc + .apply_update_from_binary_v1(&initial_bin) + .expect("Should apply initial"); + let initial_count = initial_doc.get_map("blocks").expect("blocks map exists").len(); + + let updated_md = "# Delete Block Test\n\nFirst paragraph."; + let delta = update_doc(&initial_bin, updated_md, doc_id).expect("Should compute delta"); + assert!(!delta.is_empty(), "Delta should contain changes"); + + let mut updated_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + updated_doc + .apply_update_from_binary_v1(&initial_bin) + .expect("Should apply initial"); + updated_doc + .apply_update_from_binary_v1(&delta) + .expect("Should apply delta with block deletion"); + + let updated_count = updated_doc.get_map("blocks").expect("blocks map exists").len(); + assert!( + updated_count < initial_count, + "Expected fewer blocks after deletion, got {updated_count} vs {initial_count}" + ); + } + + #[test] + fn test_update_ydoc_update_image_caption() { + let initial_md = "![Alt](blob://image-id)"; + let doc_id = "image-update-test"; + let initial_bin = build_full_doc("Image", initial_md, doc_id).expect("create doc"); + + let updated_md = "![New Caption](blob://image-id)"; + let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial"); + doc.apply_update_from_binary_v1(&delta).expect("apply delta"); + + let blocks_map = doc.get_map("blocks").expect("blocks map"); + let mut caption = None; + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() + && get_string(&block_map, "sys:flavour").as_deref() == Some("affine:image") + { + caption = get_string(&block_map, "prop:caption"); + break; + } + } + + assert_eq!(caption.as_deref(), Some("New Caption")); + } + + #[test] + fn test_update_ydoc_update_table_cell() { + let initial_md = "| A | B |\n| --- | --- |\n| 1 | 2 |"; + let doc_id = "table-update-test"; + let initial_bin = build_full_doc("Table", initial_md, doc_id).expect("create doc"); + + let updated_md = "| A | B |\n| --- | --- |\n| 1 | 9 |"; + let delta = update_doc(&initial_bin, updated_md, doc_id).expect("delta"); + + let mut doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + doc.apply_update_from_binary_v1(&initial_bin).expect("apply initial"); + doc.apply_update_from_binary_v1(&delta).expect("apply delta"); + + let blocks_map = doc.get_map("blocks").expect("blocks map"); + let mut found = false; + for (_, value) in blocks_map.iter() { + if let Some(block_map) = value.to_map() + && get_string(&block_map, "sys:flavour").as_deref() == Some("affine:table") + { + for key in block_map.keys() { + if key.starts_with("prop:cells.") + && key.ends_with(".text") + && let Some(value) = block_map.get(key).and_then(|v| v.to_any()).and_then(|a| match a { + Any::String(value) => Some(value), + _ => None, + }) + && value == "9" + { + found = true; + break; + } + } + } + } + + assert!(found); + } + + #[test] + fn test_update_ydoc_concurrent_merge_simulation() { + let base_md = "# Concurrent Test\n\nBase paragraph."; + let doc_id = "concurrent-test"; + + let base_bin = build_full_doc("Concurrent Test", base_md, doc_id).expect("Should create base doc"); + + let mut base_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + base_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base"); + let base_count = base_doc.get_map("blocks").expect("blocks map exists").len(); + + let client_a_md = "# Concurrent Test\n\nModified by client A."; + let delta_a = update_doc(&base_bin, client_a_md, doc_id).expect("Delta A"); + + let client_b_md = "# Concurrent Test\n\nBase paragraph.\n\nAdded by client B."; + let delta_b = update_doc(&base_bin, client_b_md, doc_id).expect("Delta B"); + + let mut final_doc = DocOptions::new().with_guid(doc_id.to_string()).build(); + final_doc.apply_update_from_binary_v1(&base_bin).expect("Apply base"); + final_doc.apply_update_from_binary_v1(&delta_a).expect("Apply delta A"); + final_doc.apply_update_from_binary_v1(&delta_b).expect("Apply delta B"); + + let final_count = final_doc.get_map("blocks").expect("blocks map exists").len(); + assert!( + final_count > base_count, + "Expected merged blocks after concurrent updates, got {final_count} vs {base_count}" + ); + } + + #[test] + fn test_update_ydoc_empty_binary_errors() { + let markdown = "# New Document\n\nCreated from empty binary."; + let doc_id = "empty-fallback-test"; + + let result = update_doc(&[], markdown, doc_id); + assert!(result.is_err()); + + let result = update_doc(&[0, 0], markdown, doc_id); + assert!(result.is_err()); + } + + #[test] + fn test_update_ydoc_markdown_too_large() { + let initial_md = "# Title\n\nContent."; + let doc_id = "size-limit-test"; + let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc"); + + let markdown = "a".repeat(MAX_MARKDOWN_CHARS + 1); + let result = update_doc(&initial_bin, &markdown, doc_id); + assert!(result.is_err()); + } + + #[test] + fn test_update_ydoc_rejects_unsupported_markdown() { + let initial_md = "# Title\n\nContent."; + let doc_id = "unsupported-test"; + let initial_bin = build_full_doc("Title", initial_md, doc_id).expect("Should create initial doc"); + + let markdown = "# Title\n\n
HTML
"; + let result = update_doc(&initial_bin, markdown, doc_id); + assert!(result.is_err()); + } +} diff --git a/packages/frontend/core/src/blocksuite/ai/components/ai-message-content/stream-objects.ts b/packages/frontend/core/src/blocksuite/ai/components/ai-message-content/stream-objects.ts index 3db5beb9e..704452fd2 100644 --- a/packages/frontend/core/src/blocksuite/ai/components/ai-message-content/stream-objects.ts +++ b/packages/frontend/core/src/blocksuite/ai/components/ai-message-content/stream-objects.ts @@ -131,6 +131,16 @@ export class ChatContentStreamObjects extends WithDisposable( .data=${streamObject} .width=${this.width} >`; + case 'doc_create': + case 'doc_update': + case 'doc_update_meta': + return html``; case 'section_edit': return html` `; + case 'doc_create': + case 'doc_update': + case 'doc_update_meta': + return html``; case 'section_edit': return html` + !!result && + typeof result === 'object' && + 'type' in result && + (result as ToolError).type === 'error'; + +export class DocWriteTool extends WithDisposable(ShadowlessElement) { + @property({ attribute: false }) + accessor data!: DocWriteToolCall | DocWriteToolResult; + + @property({ attribute: false }) + accessor width: Signal | undefined; + + @property({ attribute: false }) + accessor peekViewService!: PeekViewService; + + @property({ attribute: false }) + accessor docDisplayService!: DocDisplayConfig; + + @property({ attribute: false }) + accessor onOpenDoc!: (docId: string, sessionId?: string) => void; + + private getDocId() { + const { data } = this; + if ( + data.type === 'tool-result' && + data.result && + !isToolError(data.result) + ) { + const docId = + typeof data.result.docId === 'string' ? data.result.docId : undefined; + if (docId) return docId; + } + const docId = data.args.doc_id; + return typeof docId === 'string' && docId.trim() ? docId : undefined; + } + + private getDocTitle(docId?: string) { + const { data } = this; + if (data.toolName === 'doc_create' || data.toolName === 'doc_update_meta') { + const title = data.args.title; + if (title) return title; + } + if (docId && this.docDisplayService) { + const title = this.docDisplayService.getTitle(docId); + if (title) return title; + } + return undefined; + } + + private getToolIcon() { + return this.data.toolName === 'doc_create' ? PageIcon() : PenIcon(); + } + + private getCallLabel(title?: string) { + switch (this.data.toolName) { + case 'doc_create': + return title ? `Creating "${title}"` : 'Creating document'; + case 'doc_update': + return title ? `Updating "${title}"` : 'Updating document'; + case 'doc_update_meta': + return title ? `Renaming to "${title}"` : 'Updating document title'; + default: + return 'Updating document'; + } + } + + private getResultLabel(title?: string) { + switch (this.data.toolName) { + case 'doc_create': + return title ? `Created "${title}"` : 'Document created'; + case 'doc_update': + return title ? `Updated "${title}"` : 'Document updated'; + case 'doc_update_meta': + return title ? `Renamed "${title}"` : 'Document title updated'; + default: + return 'Document updated'; + } + } + + private openDoc(docId?: string) { + if (!docId) return; + if (this.peekViewService) { + this.peekViewService.peekView + .open({ type: 'doc', docRef: { docId } }) + .catch(console.error); + return; + } + this.onOpenDoc?.(docId); + } + + renderToolCall() { + const docId = this.getDocId(); + const title = this.getDocTitle(docId); + return html``; + } + + renderToolResult() { + if (this.data.type !== 'tool-result') { + return nothing; + } + + const result = this.data.result; + if (!result || isToolError(result)) { + const name = isToolError(result) ? result.name : 'Document action failed'; + return html``; + } + + const docId = this.getDocId(); + const title = this.getDocTitle(docId) ?? 'Document'; + const parts: string[] = []; + if (result.message) parts.push(result.message); + if (docId) parts.push(`Doc ID: ${docId}`); + const content = parts.length ? parts.join('\n') : undefined; + + return html` this.openDoc(docId), + }, + ]} + >`; + } + + protected override render() { + if (this.data.type === 'tool-call') { + return this.renderToolCall(); + } + if (this.data.type === 'tool-result') { + return this.renderToolResult(); + } + return nothing; + } +} + +declare global { + interface HTMLElementTagNameMap { + 'doc-write-tool': DocWriteTool; + } +} diff --git a/packages/frontend/core/src/blocksuite/ai/effects.ts b/packages/frontend/core/src/blocksuite/ai/effects.ts index c0a40cdc4..c330480e2 100644 --- a/packages/frontend/core/src/blocksuite/ai/effects.ts +++ b/packages/frontend/core/src/blocksuite/ai/effects.ts @@ -64,6 +64,7 @@ import { DocEditTool } from './components/ai-tools/doc-edit'; import { DocKeywordSearchResult } from './components/ai-tools/doc-keyword-search-result'; import { DocReadResult } from './components/ai-tools/doc-read-result'; import { DocSemanticSearchResult } from './components/ai-tools/doc-semantic-search-result'; +import { DocWriteTool } from './components/ai-tools/doc-write'; import { SectionEditTool } from './components/ai-tools/section-edit'; import { ToolCallCard } from './components/ai-tools/tool-call-card'; import { ToolFailedCard } from './components/ai-tools/tool-failed-card'; @@ -222,6 +223,7 @@ export function registerAIEffects() { customElements.define('doc-semantic-search-result', DocSemanticSearchResult); customElements.define('doc-keyword-search-result', DocKeywordSearchResult); customElements.define('doc-read-result', DocReadResult); + customElements.define('doc-write-tool', DocWriteTool); customElements.define('web-crawl-tool', WebCrawlTool); customElements.define('web-search-tool', WebSearchTool); customElements.define('section-edit-tool', SectionEditTool); diff --git a/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx b/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx index c9cc43c7f..5ae661f7a 100644 --- a/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx +++ b/packages/frontend/core/src/blocksuite/ai/provider/setup-provider.tsx @@ -102,6 +102,7 @@ export function setupAIProvider( selectedSnapshot: contexts?.selectedSnapshot, selectedMarkdown: contexts?.selectedMarkdown, html: contexts?.html, + ...(options.docId ? { currentDocId: options.docId } : {}), }, endpoint: Endpoint.StreamObject, });