diff --git a/blocksuite/framework/store/src/model/workspace.ts b/blocksuite/framework/store/src/model/workspace.ts index 66c171c89..bf0a5b962 100644 --- a/blocksuite/framework/store/src/model/workspace.ts +++ b/blocksuite/framework/store/src/model/workspace.ts @@ -1,5 +1,6 @@ import type { Slot } from '@blocksuite/global/utils'; import type { BlobEngine } from '@blocksuite/sync'; +import type { Awareness } from 'y-protocols/awareness.js'; import type * as Y from 'yjs'; import type { Schema } from '../schema/schema.js'; @@ -15,6 +16,8 @@ export interface Workspace { readonly idGenerator: IdGenerator; readonly blobSync: BlobEngine; readonly awarenessStore: AwarenessStore; + readonly onLoadDoc?: (doc: Y.Doc) => void; + readonly onLoadAwareness?: (awareness: Awareness) => void; get schema(): Schema; get doc(): Y.Doc; diff --git a/blocksuite/framework/sync/src/doc/impl/broadcast.ts b/blocksuite/framework/sync/src/doc/impl/broadcast.ts index 96b3246cb..42f896d69 100644 --- a/blocksuite/framework/sync/src/doc/impl/broadcast.ts +++ b/blocksuite/framework/sync/src/doc/impl/broadcast.ts @@ -1,6 +1,7 @@ import { assertExists } from '@blocksuite/global/utils'; import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs'; +import { MANUALLY_STOP } from '../../utils/throw-if-aborted.js'; import type { DocSource } from '../source.js'; type ChannelMessage = @@ -85,7 +86,7 @@ export class BroadcastChannelDocSource implements DocSource { { signal: abortController.signal } ); return () => { - abortController.abort(); + abortController.abort(MANUALLY_STOP); }; } } diff --git a/packages/common/infra/src/op/__tests__/message.spec.ts b/packages/common/infra/src/op/__tests__/message.spec.ts index 677ab0fc2..710198125 100644 --- a/packages/common/infra/src/op/__tests__/message.spec.ts +++ b/packages/common/infra/src/op/__tests__/message.spec.ts @@ -50,7 +50,6 @@ describe('message', () => { removeEventListener: vi.fn(), }; ctx.handler = new CustomMessageHandler(ctx.receivePort); - ctx.handler.listen(); }); it('should ignore unknown message type', ctx => { diff --git a/packages/common/infra/src/op/client.ts b/packages/common/infra/src/op/client.ts index 3a368913c..806b681e1 100644 --- a/packages/common/infra/src/op/client.ts +++ b/packages/common/infra/src/op/client.ts @@ -162,16 +162,19 @@ export class OpClient extends AutoMessageHandler { op: Op, ...args: OpInput ): Observable { - const payload = args[0]; - - const msg = { - type: 'subscribe', - id: this.nextCallId(op), - name: op as string, - payload, - } satisfies SubscribeMessage; - const sub$ = new Observable(ob => { + const payload = args[0]; + + const msg = { + type: 'subscribe', + id: this.nextCallId(op), + name: op as string, + payload, + } satisfies SubscribeMessage; + + const transferables = fetchTransferables(payload); + this.port.postMessage(msg, { transfer: transferables }); + this.obs.set(msg.id, ob); return () => { @@ -184,9 +187,6 @@ export class OpClient extends AutoMessageHandler { }; }); - const transferables = fetchTransferables(payload); - this.port.postMessage(msg, { transfer: transferables }); - return sub$; } diff --git a/packages/common/infra/src/op/consumer.ts b/packages/common/infra/src/op/consumer.ts index 4a0d2cc28..9d420e311 100644 --- a/packages/common/infra/src/op/consumer.ts +++ b/packages/common/infra/src/op/consumer.ts @@ -1,6 +1,7 @@ import EventEmitter2 from 'eventemitter2'; import { defer, from, fromEvent, Observable, of, take, takeUntil } from 'rxjs'; +import { MANUALLY_STOP } from '../utils'; import { AutoMessageHandler, type CallMessage, @@ -45,7 +46,7 @@ export class OpConsumer extends AutoMessageHandler { }; } - private readonly handleCallMessage: MessageHandlers['call'] = async msg => { + private readonly handleCallMessage: MessageHandlers['call'] = msg => { const abortController = new AbortController(); this.processing.set(msg.id, abortController); @@ -119,7 +120,7 @@ export class OpConsumer extends AutoMessageHandler { return; } - abortController.abort(); + abortController.abort(MANUALLY_STOP); }; register>(op: Op, handler: OpHandler) { @@ -181,7 +182,7 @@ export class OpConsumer extends AutoMessageHandler { super.close(); this.registeredOpHandlers.clear(); this.processing.forEach(controller => { - controller.abort(); + controller.abort(MANUALLY_STOP); }); this.processing.clear(); this.eventBus.removeAllListeners(); diff --git a/packages/common/infra/src/op/message.ts b/packages/common/infra/src/op/message.ts index aa58fd3b0..a177e2d74 100644 --- a/packages/common/infra/src/op/message.ts +++ b/packages/common/infra/src/op/message.ts @@ -134,7 +134,9 @@ export abstract class AutoMessageHandler { private listening = false; protected abstract handlers: Partial; - constructor(protected readonly port: MessageCommunicapable) {} + constructor(protected readonly port: MessageCommunicapable) { + this.listen(); + } protected handleMessage = ignoreUnknownEvent((msg: Messages) => { const handler = this.handlers[msg.type]; @@ -145,7 +147,7 @@ export abstract class AutoMessageHandler { handler(msg as any); }); - listen() { + protected listen() { if (this.listening) { return; } diff --git a/packages/common/infra/src/orm/core/__tests__/sync.spec.ts b/packages/common/infra/src/orm/core/__tests__/sync.spec.ts deleted file mode 100644 index 831ea4b28..000000000 --- a/packages/common/infra/src/orm/core/__tests__/sync.spec.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { nanoid } from 'nanoid'; -import { - afterEach, - beforeEach, - describe, - expect, - test as t, - type TestAPI, - vitest, -} from 'vitest'; -import { Doc } from 'yjs'; - -import { DocEngine } from '../../../sync'; -import { MiniSyncServer } from '../../../sync/doc/__tests__/utils'; -import { MemoryStorage } from '../../../sync/doc/storage'; -import { createORMClient, type DBSchemaBuilder, f, YjsDBAdapter } from '../'; - -const TEST_SCHEMA = { - tags: { - id: f.string().primaryKey().default(nanoid), - name: f.string(), - color: f.string().optional(), - colors: f.json().optional(), - }, -} satisfies DBSchemaBuilder; - -const ORMClient = createORMClient(TEST_SCHEMA); - -type Context = { - server: MiniSyncServer; - user1: { - client: InstanceType; - engine: DocEngine; - }; - user2: { - client: InstanceType; - engine: DocEngine; - }; -}; - -function createEngine(server: MiniSyncServer) { - return new DocEngine(new MemoryStorage(), server.client()); -} - -async function createClient(server: MiniSyncServer, clientId: number) { - const engine = createEngine(server); - const Client = createORMClient(TEST_SCHEMA); - - // define the hooks - Client.defineHook('tags', 'migrate field `color` to field `colors`', { - deserialize(data) { - if (!data.colors && data.color) { - data.colors = [data.color]; - } - - return data; - }, - }); - - const client = new Client( - new YjsDBAdapter(TEST_SCHEMA, { - getDoc(guid: string) { - const doc = new Doc({ guid }); - doc.clientID = clientId; - engine.addDoc(doc); - return doc; - }, - }) - ); - - return { - engine, - client, - }; -} - -beforeEach(async t => { - t.server = new MiniSyncServer(); - // we set user2's clientId greater than user1's clientId, - // so all conflicts will be resolved to user2's changes - t.user1 = await createClient(t.server, 1); - t.user2 = await createClient(t.server, 2); - - t.user1.engine.start(); - t.user2.engine.start(); -}); - -afterEach(async t => { - t.user1.engine.stop(); - t.user2.engine.stop(); -}); - -const test = t as TestAPI; - -describe('ORM compatibility in synchronization scenerio', () => { - test('2 clients create at the same time', async t => { - const { user1, user2 } = t; - const tag1 = user1.client.tags.create({ - name: 'tag1', - color: 'blue', - }); - - const tag2 = user2.client.tags.create({ - name: 'tag2', - color: 'red', - }); - - await vitest.waitFor(() => { - expect(user1.client.tags.keys()).toHaveLength(2); - expect(user2.client.tags.keys()).toHaveLength(2); - }); - - expect(user2.client.tags.get(tag1.id)).toStrictEqual(tag1); - expect(user1.client.tags.get(tag2.id)).toStrictEqual(tag2); - }); - - test('2 clients updating the same entity', async t => { - const { user1, user2 } = t; - const tag = user1.client.tags.create({ - name: 'tag1', - color: 'blue', - }); - - await vitest.waitFor(() => { - expect(user2.client.tags.keys()).toHaveLength(1); - }); - - user1.client.tags.update(tag.id, { color: 'red' }); - user2.client.tags.update(tag.id, { color: 'gray' }); - - await vitest.waitFor(() => { - expect(user1.client.tags.get(tag.id)).toHaveProperty('color', 'gray'); - expect(user2.client.tags.get(tag.id)).toHaveProperty('color', 'gray'); - }); - }); -}); diff --git a/packages/common/infra/src/sync/awareness.ts b/packages/common/infra/src/sync/awareness.ts deleted file mode 100644 index 395de22ea..000000000 --- a/packages/common/infra/src/sync/awareness.ts +++ /dev/null @@ -1,23 +0,0 @@ -import type { Awareness } from 'y-protocols/awareness.js'; - -export interface AwarenessConnection { - connect(awareness: Awareness): void; - disconnect(): void; - dispose?(): void; -} - -export class AwarenessEngine { - constructor(public readonly connections: AwarenessConnection[]) {} - - connect(awareness: Awareness) { - this.connections.forEach(connection => connection.connect(awareness)); - } - - disconnect() { - this.connections.forEach(connection => connection.disconnect()); - } - - dispose() { - this.connections.forEach(connection => connection.dispose?.()); - } -} diff --git a/packages/common/infra/src/sync/blob/blob.ts b/packages/common/infra/src/sync/blob/blob.ts deleted file mode 100644 index b289c968d..000000000 --- a/packages/common/infra/src/sync/blob/blob.ts +++ /dev/null @@ -1,264 +0,0 @@ -import { DebugLogger } from '@affine/debug'; -import EventEmitter2 from 'eventemitter2'; -import { difference } from 'lodash-es'; - -import { LiveData } from '../../livedata'; -import type { Memento } from '../../storage'; -import { MANUALLY_STOP } from '../../utils'; -import { BlobStorageOverCapacity } from './error'; - -const logger = new DebugLogger('affine:blob-engine'); - -export interface BlobStorage { - name: string; - readonly: boolean; - get: (key: string) => Promise; - set: (key: string, value: Blob) => Promise; - delete: (key: string) => Promise; - list: () => Promise; -} - -export interface BlobStatus { - isStorageOverCapacity: boolean; -} - -/** - * # BlobEngine - * - * sync blobs between storages in background. - * - * all operations priority use local, then use remote. - */ -export class BlobEngine { - readonly name = 'blob-engine'; - readonly readonly = this.local.readonly; - readonly event = new EventEmitter2(); - - private abort: AbortController | null = null; - - readonly isStorageOverCapacity$ = new LiveData(false); - - singleBlobSizeLimit: number = 100 * 1024 * 1024; - onAbortLargeBlob = (callback: (blob: Blob) => void) => { - this.event.on('abort-large-blob', callback); - return () => { - this.event.off('abort-large-blob', callback); - }; - }; - - constructor( - private readonly local: BlobStorage, - private readonly remotes: BlobStorage[] - ) {} - - start() { - if (this.abort || this.isStorageOverCapacity$.value) { - return; - } - this.abort = new AbortController(); - const abortSignal = this.abort.signal; - - const sync = () => { - if (abortSignal.aborted) { - return; - } - - this.sync() - .catch(error => { - logger.error('sync blob error', error); - }) - .finally(() => { - // sync every 1 minute - setTimeout(sync, 60000); - }); - }; - - sync(); - } - - stop() { - this.abort?.abort(MANUALLY_STOP); - this.abort = null; - } - - get storages() { - return [this.local, ...this.remotes]; - } - - async sync() { - if (this.local.readonly) { - return; - } - logger.debug('start syncing blob...'); - for (const remote of this.remotes) { - let localList: string[] = []; - let remoteList: string[] = []; - - if (!remote.readonly) { - try { - localList = await this.local.list(); - remoteList = await remote.list(); - } catch (err) { - logger.error(`error when sync`, err); - continue; - } - - const needUpload = difference(localList, remoteList); - for (const key of needUpload) { - try { - const data = await this.local.get(key); - if (data) { - await remote.set(key, data); - } - } catch (err) { - logger.error( - `error when sync ${key} from [${this.local.name}] to [${remote.name}]`, - err - ); - } - } - } - - const needDownload = difference(remoteList, localList); - - for (const key of needDownload) { - try { - const data = await remote.get(key); - if (data) { - await this.local.set(key, data); - } - } catch (err) { - if (err instanceof BlobStorageOverCapacity) { - this.isStorageOverCapacity$.value = true; - } - logger.error( - `error when sync ${key} from [${remote.name}] to [${this.local.name}]`, - err - ); - } - } - } - - logger.debug('finish syncing blob'); - } - - async get(key: string) { - logger.debug('get blob', key); - for (const storage of this.storages) { - const data = await storage.get(key); - if (data) { - return data; - } - } - return null; - } - - async set(key: string, value: Blob) { - if (this.local.readonly) { - throw new Error('local peer is readonly'); - } - - if (value.size > this.singleBlobSizeLimit) { - this.event.emit('abort-large-blob', value); - logger.error('blob over limit, abort set'); - return key; - } - - // await upload to the local peer - await this.local.set(key, value); - - // uploads to other peers in the background - Promise.allSettled( - this.remotes - .filter(r => !r.readonly) - .map(peer => - peer.set(key, value).catch(err => { - logger.error('Error when uploading to peer', err); - }) - ) - ) - .then(result => { - if (result.some(({ status }) => status === 'rejected')) { - logger.error( - `blob ${key} update finish, but some peers failed to update` - ); - } else { - logger.debug(`blob ${key} update finish`); - } - }) - .catch(() => { - // Promise.allSettled never reject - }); - - return key; - } - - async delete(_key: string) { - // not supported - } - - async list() { - const blobList = new Set(); - - for (const peer of this.storages) { - const list = await peer.list(); - if (list) { - for (const blob of list) { - blobList.add(blob); - } - } - } - - return Array.from(blobList); - } -} - -export const EmptyBlobStorage: BlobStorage = { - name: 'empty', - readonly: true, - async get(_key: string) { - return null; - }, - async set(_key: string, _value: Blob) { - throw new Error('not supported'); - }, - async delete(_key: string) { - throw new Error('not supported'); - }, - async list() { - return []; - }, -}; - -export class MemoryBlobStorage implements BlobStorage { - name = 'testing'; - readonly = false; - - constructor(private readonly state: Memento) {} - - get(key: string) { - return Promise.resolve(this.state.get(key) ?? null); - } - set(key: string, value: Blob) { - this.state.set(key, value); - - const list = this.state.get>('list') ?? new Set(); - list.add(key); - this.state.set('list', list); - - return Promise.resolve(key); - } - delete(key: string) { - this.state.set(key, null); - - const list = this.state.get>('list') ?? new Set(); - list.delete(key); - this.state.set('list', list); - - return Promise.resolve(); - } - list() { - const list = this.state.get>('list'); - return Promise.resolve(list ? Array.from(list) : []); - } -} diff --git a/packages/common/infra/src/sync/blob/error.ts b/packages/common/infra/src/sync/blob/error.ts deleted file mode 100644 index db74dc86f..000000000 --- a/packages/common/infra/src/sync/blob/error.ts +++ /dev/null @@ -1,5 +0,0 @@ -export class BlobStorageOverCapacity extends Error { - constructor(public originError?: any) { - super('Blob storage over capacity.'); - } -} diff --git a/packages/common/infra/src/sync/doc/README.md b/packages/common/infra/src/sync/doc/README.md deleted file mode 100644 index dcd49f394..000000000 --- a/packages/common/infra/src/sync/doc/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# DocEngine - -The synchronization algorithm for yjs docs. - -``` - ┌─────────┐ ┌───────────┐ ┌────────┐ - │ Storage ◄──┤ DocEngine ├──► Server │ - └─────────┘ └───────────┘ └────────┘ -``` - -# Core Components - -## DocStorage - -```ts -export interface DocStorage { - eventBus: DocEventBus; - doc: ByteKV; - syncMetadata: ByteKV; - serverClock: ByteKV; -} -``` - -Represents the local storage used, Specific implementations are replaceable, such as `IndexedDBDocStorage` on the `browser` and `SqliteDocStorage` on the `desktop`. - -### DocEventBus - -Each `DocStorage` contains a `DocEventBus`, which is used to communicate with other engines that share the same storage. - -With `DocEventBus` we can sync updates between engines without connecting to the server. - -For example, on the `browser`, we have multiple tabs, all tabs share the same `IndexedDBDocStorage`, so we use `BroadcastChannel` to implement `DocEventBus`, which allows us to broadcast events to all tabs. - -On the `desktop` app, if we have multiple Windows sharing the same `SqliteDocStorage`, we must build a mechanism to broadcast events between all Windows (currently not implemented). - -## DocServer - -```ts -export interface DocServer { - pullDoc( - docId: string, - stateVector: Uint8Array - ): Promise<{ - data: Uint8Array; - serverClock: number; - stateVector?: Uint8Array; - } | null>; - - pushDoc(docId: string, data: Uint8Array): Promise<{ serverClock: number }>; - - subscribeAllDocs(cb: (updates: { docId: string; data: Uint8Array; serverClock: number }) => void): Promise<() => void>; - - loadServerClock(after: number): Promise>; - - waitForConnectingServer(signal: AbortSignal): Promise; - disconnectServer(): void; - onInterrupted(cb: (reason: string) => void): void; -} -``` - -Represents the server we want to synchronize, there is a simulated implementation in `tests/sync.spec.ts`, and the real implementation is in `packages/backend/server`. - -### ServerClock - -`ServerClock` is a clock generated after each updates is stored in the Server. It is used to determine the order in which updates are stored in the Server. - -The `DocEngine` decides whether to pull updates from the server based on the `ServerClock`. - -The `ServerClock` written later must be **greater** than all previously. So on the client side, we can use `loadServerClock(the largest ServerClock previously received)` to obtain all changed `ServerClock`. - -## DocEngine - -The `DocEngine` is where all the synchronization logic actually happens. - -Due to the complexity of the implementation, we divide it into 2 parts. - -## DocEngine - LocalPart - -Synchronizing **the `YDoc` instance** and **storage**. - -The typical workflow is: - -1. load data from storage, apply to `YDoc` instance. -2. track `YDoc` changes -3. write the changes back to storage. - -### SeqNum - -There is a `SeqNum` on each Doc data in `Storage`. Every time `LocalPart` writes data, `SeqNum` will be +1. - -There is also a `PushedSeqNum`, which is used for RemotePart later. - -## DocEngine - RemotePart - -Synchronizing `Storage` and `Server`. - -The typical workflow is: - -1. Connect with the server, Load `ServerClocks` for all docs, Start subscribing to server-side updates. - -2. Check whether each doc requires `push` and `pull` - -3. Execute all push and pull - -4. Listen for updates from `LocalPart` and push the updates to the server - -5. Listen for server-side updates and write them to storage. - -### PushedSeqNum - -Each Doc will record a `PushedSeqNum`, used to determine whether the doc has unpush updates. - -After each `push` is completed, `PushedSeqNum` + 1 - -If `PushedSeqNum` and `SeqNum` are still different after we complete the push (usually means the previous `push` failed) - -Then do a full pull and push and set `pushedSeqNum` = `SeqNum` - -### PulledServerClock - -Each Doc also record `PulledServerClock`, Used to compare with ServerClock to determine whether to `pull` doc. - -When the `pull` is completed, set `PulledServerClock` = `ServerClock` returned by the server. - -### Retry - -The `RemotePart` may fail at any time, and `RemotePart`'s built-in retry mechanism will restart the process in 5 seconds after failure. diff --git a/packages/common/infra/src/sync/doc/__tests__/priority-queue.spec.ts b/packages/common/infra/src/sync/doc/__tests__/priority-queue.spec.ts deleted file mode 100644 index f840c7d65..000000000 --- a/packages/common/infra/src/sync/doc/__tests__/priority-queue.spec.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { describe, expect, test } from 'vitest'; - -import { PriorityQueue } from '../priority-queue'; - -describe('Priority Queue', () => { - test('priority', () => { - const queue = new PriorityQueue(); - - queue.push('foo', 1); - queue.push('bar', 2); - queue.push('baz', 0); - - expect(queue.pop()).toBe('bar'); - expect(queue.pop()).toBe('foo'); - expect(queue.pop()).toBe('baz'); - expect(queue.pop()).toBe(null); - - queue.push('B', 1); - queue.push('A', 1); - - // if priority same then follow id binary order - expect(queue.pop()).toBe('B'); - expect(queue.pop()).toBe('A'); - expect(queue.pop()).toBe(null); - - queue.push('A', 1); - queue.push('B', 2); - queue.push('A', 3); // same id but different priority, update the priority - - expect(queue.pop()).toBe('A'); - expect(queue.pop()).toBe('B'); - expect(queue.pop()).toBe(null); - - queue.push('A', 1); - queue.push('B', 2); - queue.remove('B'); - - expect(queue.pop()).toBe('A'); - expect(queue.pop()).toBe(null); - }); -}); diff --git a/packages/common/infra/src/sync/doc/__tests__/sync.spec.ts b/packages/common/infra/src/sync/doc/__tests__/sync.spec.ts deleted file mode 100644 index 437b21fd8..000000000 --- a/packages/common/infra/src/sync/doc/__tests__/sync.spec.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { describe, expect, test, vitest } from 'vitest'; -import { Doc as YDoc, encodeStateAsUpdate } from 'yjs'; - -import { DocEngine } from '..'; -import { MemoryStorage } from '../storage'; -import { MiniSyncServer } from './utils'; - -describe('sync', () => { - test('basic sync', async () => { - const storage = new MemoryStorage(); - const server = new MiniSyncServer(); - const engine = new DocEngine(storage, server.client()).start(); - const doc = new YDoc({ guid: 'a' }); - engine.addDoc(doc); - const map = doc.getMap('aaa'); - map.set('a', 1); - - await engine.waitForSynced(); - expect(server.db.size).toBe(1); - expect(storage.docDb.keys().length).toBe(1); - }); - - test('can pull from server', async () => { - const server = new MiniSyncServer(); - { - const engine = new DocEngine( - new MemoryStorage(), - server.client() - ).start(); - const doc = new YDoc({ guid: 'a' }); - engine.addDoc(doc); - const map = doc.getMap('aaa'); - map.set('a', 1); - await engine.waitForSynced(); - expect(server.db.size).toBe(1); - } - { - const engine = new DocEngine( - new MemoryStorage(), - server.client() - ).start(); - const doc = new YDoc({ guid: 'a' }); - engine.addDoc(doc); - await engine.waitForSynced(); - expect(doc.getMap('aaa').get('a')).toBe(1); - } - }); - - test('2 client', async () => { - const server = new MiniSyncServer(); - await Promise.all([ - (async () => { - const engine = new DocEngine( - new MemoryStorage(), - server.client() - ).start(); - const doc = new YDoc({ guid: 'a' }); - engine.addDoc(doc); - const map = doc.getMap('aaa'); - map.set('a', 1); - await vitest.waitUntil(() => { - return map.get('b') === 2; - }); - })(), - (async () => { - const engine = new DocEngine( - new MemoryStorage(), - server.client() - ).start(); - const doc = new YDoc({ guid: 'a' }); - engine.addDoc(doc); - const map = doc.getMap('aaa'); - map.set('b', 2); - await vitest.waitUntil(() => { - return map.get('a') === 1; - }); - })(), - ]); - }); - - test('2 client share storage and eventBus (simulate different tabs in same browser)', async () => { - const server = new MiniSyncServer(); - const storage = new MemoryStorage(); - - await Promise.all([ - (async () => { - const engine = new DocEngine(storage, server.client()).start(); - const doc = new YDoc({ guid: 'a' }); - engine.addDoc(doc); - - const map = doc.getMap('aaa'); - map.set('a', 1); - await vitest.waitUntil(() => map.get('b') === 2); - })(), - (async () => { - const engine = new DocEngine(storage, server.client()).start(); - const doc = new YDoc({ guid: 'a' }); - engine.addDoc(doc); - const map = doc.getMap('aaa'); - map.set('b', 2); - await vitest.waitUntil(() => map.get('a') === 1); - })(), - ]); - }); - - test('legacy data', async () => { - const server = new MiniSyncServer(); - const storage = new MemoryStorage(); - - { - // write legacy data to storage - const doc = new YDoc({ guid: 'a' }); - const map = doc.getMap('aaa'); - map.set('a', 1); - - await storage.doc.set('a', encodeStateAsUpdate(doc)); - } - - const engine = new DocEngine(storage, server.client()).start(); - const doc = new YDoc({ guid: 'a' }); - engine.addDoc(doc); - - // should load to ydoc and save to server - await vitest.waitUntil( - () => doc.getMap('aaa').get('a') === 1 && server.db.size === 1 - ); - }); -}); diff --git a/packages/common/infra/src/sync/doc/__tests__/utils.ts b/packages/common/infra/src/sync/doc/__tests__/utils.ts deleted file mode 100644 index fc1ffeee2..000000000 --- a/packages/common/infra/src/sync/doc/__tests__/utils.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { nanoid } from 'nanoid'; -import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs'; - -import { AsyncLock } from '../../../utils'; -import type { DocServer } from '../server'; -import { isEmptyUpdate } from '../utils'; - -export class MiniSyncServer { - lock = new AsyncLock(); - db = new Map(); - listeners = new Set<{ - cb: (updates: { - docId: string; - data: Uint8Array; - serverClock: number; - }) => void; - clientId: string; - }>(); - - client() { - return new MiniServerClient(nanoid(), this); - } -} - -export class MiniServerClient implements DocServer { - constructor( - private readonly id: string, - private readonly server: MiniSyncServer - ) {} - - async pullDoc(docId: string, stateVector: Uint8Array) { - using _lock = await this.server.lock.acquire(); - const doc = this.server.db.get(docId); - if (!doc) { - return null; - } - const data = doc.data; - return { - data: - !isEmptyUpdate(data) && stateVector.length > 0 - ? diffUpdate(data, stateVector) - : data, - serverClock: 0, - stateVector: !isEmptyUpdate(data) - ? encodeStateVectorFromUpdate(data) - : new Uint8Array(), - }; - } - - async pushDoc( - docId: string, - data: Uint8Array - ): Promise<{ serverClock: number }> { - using _lock = await this.server.lock.acquire(); - const doc = this.server.db.get(docId); - const oldData = doc?.data ?? new Uint8Array(); - const newClock = (doc?.clock ?? 0) + 1; - this.server.db.set(docId, { - data: !isEmptyUpdate(data) - ? !isEmptyUpdate(oldData) - ? mergeUpdates([oldData, data]) - : data - : oldData, - clock: newClock, - }); - for (const { clientId, cb } of this.server.listeners) { - if (clientId !== this.id) { - cb({ - docId, - data, - serverClock: newClock, - }); - } - } - return { serverClock: newClock }; - } - - async loadServerClock(after: number): Promise> { - using _lock = await this.server.lock.acquire(); - const map = new Map(); - - for (const [docId, { clock }] of this.server.db) { - if (clock > after) { - map.set(docId, clock); - } - } - - return map; - } - - async subscribeAllDocs( - cb: (updates: { - docId: string; - data: Uint8Array; - serverClock: number; - }) => void - ): Promise<() => void> { - const listener = { cb, clientId: this.id }; - this.server.listeners.add(listener); - return () => { - this.server.listeners.delete(listener); - }; - } - - async waitForConnectingServer(): Promise {} - disconnectServer(): void {} - onInterrupted(_cb: (reason: string) => void): void {} -} diff --git a/packages/common/infra/src/sync/doc/async-priority-queue.ts b/packages/common/infra/src/sync/doc/async-priority-queue.ts deleted file mode 100644 index 14ed54c99..000000000 --- a/packages/common/infra/src/sync/doc/async-priority-queue.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { PriorityQueue } from './priority-queue'; - -export class AsyncPriorityQueue extends PriorityQueue { - private _resolveUpdate: (() => void) | null = null; - private _waitForUpdate: Promise | null = null; - - async asyncPop(abort?: AbortSignal): Promise { - const update = this.pop(); - if (update) { - return update; - } else { - if (!this._waitForUpdate) { - this._waitForUpdate = new Promise(resolve => { - this._resolveUpdate = resolve; - }); - } - - await Promise.race([ - this._waitForUpdate, - new Promise((_, reject) => { - if (abort?.aborted) { - reject(abort?.reason); - } - abort?.addEventListener('abort', () => { - reject(abort.reason); - }); - }), - ]); - - return this.asyncPop(abort); - } - } - - override push(id: string, priority: number = 0) { - super.push(id, priority); - if (this._resolveUpdate) { - const resolve = this._resolveUpdate; - this._resolveUpdate = null; - this._waitForUpdate = null; - resolve(); - } - } -} diff --git a/packages/common/infra/src/sync/doc/clock.ts b/packages/common/infra/src/sync/doc/clock.ts deleted file mode 100644 index 42226c2de..000000000 --- a/packages/common/infra/src/sync/doc/clock.ts +++ /dev/null @@ -1,32 +0,0 @@ -export class ClockMap { - max: number = 0; - constructor(private readonly map: Map) { - for (const value of map.values()) { - if (value > this.max) { - this.max = value; - } - } - } - - get(id: string): number { - return this.map.get(id) ?? 0; - } - - set(id: string, value: number) { - this.map.set(id, value); - if (value > this.max) { - this.max = value; - } - } - - setIfBigger(id: string, value: number) { - if (value > this.get(id)) { - this.set(id, value); - } - } - - clear() { - this.map.clear(); - this.max = 0; - } -} diff --git a/packages/common/infra/src/sync/doc/event.ts b/packages/common/infra/src/sync/doc/event.ts deleted file mode 100644 index d8e77209d..000000000 --- a/packages/common/infra/src/sync/doc/event.ts +++ /dev/null @@ -1,50 +0,0 @@ -export type DocEvent = - | { - type: 'ClientUpdateCommitted'; - clientId: string; - docId: string; - update: Uint8Array; - seqNum: number; - } - | { - type: 'ServerUpdateCommitted'; - docId: string; - update: Uint8Array; - clientId: string; - }; - -export interface DocEventBus { - emit(event: DocEvent): void; - on(cb: (event: DocEvent) => void): () => void; -} - -export class MemoryDocEventBus implements DocEventBus { - listeners = new Set<(event: DocEvent) => void>(); - emit(event: DocEvent): void { - for (const listener of this.listeners) { - try { - listener(event); - } catch (e) { - console.error(e); - } - } - } - on(cb: (event: DocEvent) => void): () => void { - this.listeners.add(cb); - return () => { - this.listeners.delete(cb); - }; - } -} - -export class DocEventBusInner implements DocEventBus { - constructor(private readonly eventBusBehavior: DocEventBus) {} - - emit(event: DocEvent) { - this.eventBusBehavior.emit(event); - } - - on(cb: (event: DocEvent) => void) { - return this.eventBusBehavior.on(cb); - } -} diff --git a/packages/common/infra/src/sync/doc/index.ts b/packages/common/infra/src/sync/doc/index.ts deleted file mode 100644 index ec67e0ffc..000000000 --- a/packages/common/infra/src/sync/doc/index.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { DebugLogger } from '@affine/debug'; -import { nanoid } from 'nanoid'; -import { map } from 'rxjs'; -import type { Doc as YDoc } from 'yjs'; - -import { LiveData } from '../../livedata'; -import { MANUALLY_STOP } from '../../utils'; -import { DocEngineLocalPart } from './local'; -import { DocEngineRemotePart } from './remote'; -import type { DocServer } from './server'; -import type { DocStorage } from './storage'; -import { DocStorageInner } from './storage'; - -const logger = new DebugLogger('doc-engine'); - -export type { DocEvent, DocEventBus } from './event'; -export { MemoryDocEventBus } from './event'; -export type { DocServer } from './server'; -export type { DocStorage } from './storage'; -export { - MemoryStorage as MemoryDocStorage, - ReadonlyStorage as ReadonlyDocStorage, -} from './storage'; - -export interface DocEngineDocState { - /** - * is syncing with the server - */ - syncing: boolean; - /** - * is saving to local storage - */ - saving: boolean; - /** - * is loading from local storage - */ - loading: boolean; - retrying: boolean; - ready: boolean; - errorMessage: string | null; - serverClock: number | null; -} - -export class DocEngine { - readonly clientId: string; - localPart: DocEngineLocalPart; - remotePart: DocEngineRemotePart | null; - - storage: DocStorageInner; - - engineState$ = LiveData.computed(get => { - const localState = get(this.localPart.engineState$); - if (this.remotePart) { - const remoteState = get(this.remotePart?.engineState$); - return { - total: remoteState.total, - syncing: remoteState.syncing, - saving: localState.syncing, - retrying: remoteState.retrying, - errorMessage: remoteState.errorMessage, - }; - } - return { - total: localState.total, - syncing: localState.syncing, - saving: localState.syncing, - retrying: false, - errorMessage: null, - }; - }); - - docState$(docId: string) { - const localState$ = this.localPart.docState$(docId); - const remoteState$ = this.remotePart?.docState$(docId); - return LiveData.computed(get => { - const localState = get(localState$); - const remoteState = remoteState$ ? get(remoteState$) : null; - if (remoteState) { - return { - syncing: remoteState.syncing, - saving: localState.syncing, - loading: localState.syncing, - retrying: remoteState.retrying, - ready: localState.ready, - errorMessage: remoteState.errorMessage, - serverClock: remoteState.serverClock, - }; - } - return { - syncing: localState.syncing, - saving: localState.syncing, - loading: localState.syncing, - ready: localState.ready, - retrying: false, - errorMessage: null, - serverClock: null, - }; - }); - } - - markAsReady(docId: string) { - this.localPart.actions.markAsReady(docId); - } - - constructor( - storage: DocStorage, - private readonly server?: DocServer | null - ) { - this.clientId = nanoid(); - this.storage = new DocStorageInner(storage); - this.localPart = new DocEngineLocalPart(this.clientId, this.storage); - this.remotePart = this.server - ? new DocEngineRemotePart(this.clientId, this.storage, this.server) - : null; - } - - abort = new AbortController(); - - start() { - this.abort.abort(MANUALLY_STOP); - this.abort = new AbortController(); - Promise.all([ - this.localPart.mainLoop(this.abort.signal), - this.remotePart?.mainLoop(this.abort.signal), - ]).catch(err => { - if (err === MANUALLY_STOP) { - return; - } - logger.error('Doc engine error', err); - }); - return this; - } - - stop() { - this.abort.abort(MANUALLY_STOP); - } - - async resetSyncStatus() { - this.stop(); - await this.storage.clearSyncMetadata(); - await this.storage.clearServerClock(); - } - - addDoc(doc: YDoc, withSubDocs = true) { - this.remotePart?.actions.addDoc(doc.guid); - this.localPart.actions.addDoc(doc); - - if (withSubDocs) { - doc.on('subdocs', ({ added, loaded }) => { - // added: the subdocs that are existing on the ydoc - // loaded: the subdocs that have been called `ydoc.load()` - // - // we add all existing subdocs to remote part, let them sync between storage and server - // but only add loaded subdocs to local part, let them sync between storage and ydoc - // sync data to ydoc will consume more memory, so we only sync the ydoc that are necessary. - for (const subdoc of added) { - this.remotePart?.actions.addDoc(subdoc.guid); - } - for (const subdoc of loaded) { - this.localPart.actions.addDoc(subdoc); - } - }); - } - } - - setPriority(docId: string, priority: number) { - this.localPart.setPriority(docId, priority); - this.remotePart?.setPriority(docId, priority); - } - - /** - * ## Saved: - * YDoc changes have been saved to storage, and the browser can be safely closed without losing data. - */ - waitForSaved() { - return new Promise(resolve => { - this.engineState$ - .pipe(map(state => state.saving === 0)) - .subscribe(saved => { - if (saved) { - resolve(); - } - }); - }); - } - - /** - * ## Synced: - * is fully synchronized with the server - */ - waitForSynced() { - return new Promise(resolve => { - this.engineState$ - .pipe(map(state => state.syncing === 0 && state.saving === 0)) - .subscribe(synced => { - if (synced) { - resolve(); - } - }); - }); - } - - /** - * ## Ready: - * - * means that the doc has been loaded and the data can be modified. - * (is not force, you can still modify it if you know you are creating some new data) - * - * this is a temporary solution to deal with the yjs overwrite issue. - * - * if content is loaded from storage - * or if content is pulled from the server, it will be true, otherwise be false. - * - * For example, when opening a doc that is not in storage, ready = false until the content is pulled from the server. - */ - waitForReady(docId: string) { - return new Promise(resolve => { - this.docState$(docId) - .pipe(map(state => state.ready)) - .subscribe(ready => { - if (ready) { - resolve(); - } - }); - }); - } - - dispose() { - this.stop(); - this.server?.dispose?.(); - } -} diff --git a/packages/common/infra/src/sync/doc/local.ts b/packages/common/infra/src/sync/doc/local.ts deleted file mode 100644 index 0ec180b83..000000000 --- a/packages/common/infra/src/sync/doc/local.ts +++ /dev/null @@ -1,302 +0,0 @@ -import { DebugLogger } from '@affine/debug'; -import { Unreachable } from '@affine/env/constant'; -import { groupBy } from 'lodash-es'; -import { Observable, Subject } from 'rxjs'; -import type { Doc as YDoc } from 'yjs'; -import { applyUpdate, encodeStateAsUpdate, mergeUpdates } from 'yjs'; - -import { LiveData } from '../../livedata'; -import { throwIfAborted } from '../../utils'; -import { AsyncPriorityQueue } from './async-priority-queue'; -import type { DocEvent } from './event'; -import type { DocStorageInner } from './storage'; -import { isEmptyUpdate } from './utils'; - -type Job = - | { - type: 'load'; - docId: string; - } - | { - type: 'save'; - docId: string; - update: Uint8Array; - } - | { - type: 'apply'; - docId: string; - update: Uint8Array; - isInitialize: boolean; - }; - -const DOC_ENGINE_ORIGIN = 'doc-engine'; - -const logger = new DebugLogger('doc-engine:local'); - -export interface LocalEngineState { - total: number; - syncing: number; -} - -export interface LocalDocState { - ready: boolean; - loading: boolean; - syncing: boolean; -} - -/** - * never fail - */ -export class DocEngineLocalPart { - private readonly prioritySettings = new Map(); - private readonly statusUpdatedSubject$ = new Subject(); - - private readonly status = { - docs: new Map(), - connectedDocs: new Set(), - readyDocs: new Set(), - jobDocQueue: new AsyncPriorityQueue(), - jobMap: new Map(), - currentJob: null as { docId: string; jobs: Job[] } | null, - }; - - engineState$ = LiveData.from( - new Observable(subscribe => { - const next = () => { - subscribe.next({ - total: this.status.docs.size, - syncing: this.status.jobMap.size + (this.status.currentJob ? 1 : 0), - }); - }; - next(); - return this.statusUpdatedSubject$.subscribe(() => { - next(); - }); - }), - { syncing: 0, total: 0 } - ); - - docState$(docId: string) { - return LiveData.from( - new Observable(subscribe => { - const next = () => { - subscribe.next({ - ready: this.status.readyDocs.has(docId) ?? false, - loading: this.status.connectedDocs.has(docId), - syncing: - (this.status.jobMap.get(docId)?.length ?? 0) > 0 || - this.status.currentJob?.docId === docId, - }); - }; - next(); - return this.statusUpdatedSubject$.subscribe(updatedId => { - if (updatedId === docId) next(); - }); - }), - { ready: false, loading: false, syncing: false } - ); - } - - constructor( - private readonly clientId: string, - private readonly storage: DocStorageInner - ) {} - - async mainLoop(signal?: AbortSignal) { - const dispose = this.storage.eventBus.on(event => { - const handler = this.events[event.type]; - if (handler) { - handler(event as any); - } - }); - try { - // eslint-disable-next-line no-constant-condition - while (true) { - throwIfAborted(signal); - const docId = await this.status.jobDocQueue.asyncPop(signal); - const jobs = this.status.jobMap.get(docId); - this.status.jobMap.delete(docId); - - if (!jobs) { - continue; - } - - this.status.currentJob = { docId, jobs }; - this.statusUpdatedSubject$.next(docId); - - const { apply, load, save } = groupBy(jobs, job => job.type) as { - [key in Job['type']]?: Job[]; - }; - - if (load?.length) { - await this.jobs.load(load[0] as any, signal); - } - - for (const applyJob of apply ?? []) { - await this.jobs.apply(applyJob as any, signal); - } - - if (save?.length) { - await this.jobs.save(docId, save as any, signal); - } - - this.status.currentJob = null; - this.statusUpdatedSubject$.next(docId); - } - } finally { - dispose(); - - for (const docs of this.status.connectedDocs) { - const doc = this.status.docs.get(docs); - if (doc) { - doc.off('update', this.handleDocUpdate); - } - } - } - } - - readonly actions = { - addDoc: (doc: YDoc) => { - this.schedule({ - type: 'load', - docId: doc.guid, - }); - - this.status.docs.set(doc.guid, doc); - this.statusUpdatedSubject$.next(doc.guid); - }, - markAsReady: (docId: string) => { - this.status.readyDocs.add(docId); - this.statusUpdatedSubject$.next(docId); - }, - }; - - readonly jobs = { - load: async (job: Job & { type: 'load' }, signal?: AbortSignal) => { - const doc = this.status.docs.get(job.docId); - if (!doc) { - throw new Unreachable('doc not found'); - } - const existingData = encodeStateAsUpdate(doc); - - if (!isEmptyUpdate(existingData)) { - this.schedule({ - type: 'save', - docId: doc.guid, - update: existingData, - }); - } - - // mark doc as loaded - doc.emit('sync', [true, doc]); - doc.on('update', this.handleDocUpdate); - - this.status.connectedDocs.add(job.docId); - this.statusUpdatedSubject$.next(job.docId); - - const docData = await this.storage.loadDocFromLocal(job.docId, signal); - - if (!docData || isEmptyUpdate(docData)) { - return; - } - - this.applyUpdate(job.docId, docData); - this.status.readyDocs.add(job.docId); - this.statusUpdatedSubject$.next(job.docId); - }, - save: async ( - docId: string, - jobs: (Job & { type: 'save' })[], - signal?: AbortSignal - ) => { - if (this.status.connectedDocs.has(docId)) { - const merged = mergeUpdates( - jobs.map(j => j.update).filter(update => !isEmptyUpdate(update)) - ); - const newSeqNum = await this.storage.commitDocAsClientUpdate( - docId, - merged, - signal - ); - this.storage.eventBus.emit({ - type: 'ClientUpdateCommitted', - seqNum: newSeqNum, - docId: docId, - clientId: this.clientId, - update: merged, - }); - } - }, - apply: async (job: Job & { type: 'apply' }, signal?: AbortSignal) => { - throwIfAborted(signal); - if (this.status.connectedDocs.has(job.docId)) { - this.applyUpdate(job.docId, job.update); - } - if (job.isInitialize && !isEmptyUpdate(job.update)) { - this.status.readyDocs.add(job.docId); - this.statusUpdatedSubject$.next(job.docId); - } - }, - }; - - readonly events: { - [key in DocEvent['type']]?: (event: DocEvent & { type: key }) => void; - } = { - ServerUpdateCommitted: ({ docId, update, clientId }) => { - this.schedule({ - type: 'apply', - docId, - update, - isInitialize: clientId === this.clientId, - }); - }, - ClientUpdateCommitted: ({ docId, update, clientId }) => { - if (clientId !== this.clientId) { - this.schedule({ - type: 'apply', - docId, - update, - isInitialize: false, - }); - } - }, - }; - - handleDocUpdate = (update: Uint8Array, origin: any, doc: YDoc) => { - if (origin === DOC_ENGINE_ORIGIN) { - return; - } - - this.schedule({ - type: 'save', - docId: doc.guid, - update, - }); - }; - - applyUpdate(docId: string, update: Uint8Array) { - const doc = this.status.docs.get(docId); - if (doc && !isEmptyUpdate(update)) { - try { - applyUpdate(doc, update, DOC_ENGINE_ORIGIN); - } catch (err) { - logger.error('failed to apply update yjs doc', err); - } - } - } - - schedule(job: Job) { - const priority = this.prioritySettings.get(job.docId) ?? 0; - this.status.jobDocQueue.push(job.docId, priority); - - const existingJobs = this.status.jobMap.get(job.docId) ?? []; - existingJobs.push(job); - this.status.jobMap.set(job.docId, existingJobs); - this.statusUpdatedSubject$.next(job.docId); - } - - setPriority(docId: string, priority: number) { - this.prioritySettings.set(docId, priority); - this.status.jobDocQueue.updatePriority(docId, priority); - } -} diff --git a/packages/common/infra/src/sync/doc/old-id.md b/packages/common/infra/src/sync/doc/old-id.md deleted file mode 100644 index 87aed16c7..000000000 --- a/packages/common/infra/src/sync/doc/old-id.md +++ /dev/null @@ -1,24 +0,0 @@ -AFFiNE currently has a lot of data stored using the old ID format. Here, we record the usage of IDs to avoid forgetting. - -## Old ID Format - -The format is: - -- `{workspace-id}:space:{nanoid}` Common -- `{workspace-id}:space:page:{nanoid}` - -> Note: sometimes the `workspace-id` is not same with current workspace id. - -## Usage - -- Local Storage - - indexeddb: Both new and old IDs coexist - - sqlite: Both new and old IDs coexist - - server-clock: Only new IDs are stored - - sync-metadata: Both new and old IDs coexist -- Server Storage - - Only stores new IDs but accepts writes using old IDs -- Protocols - - When the client submits an update, both new and old IDs are used. - - When the server broadcasts updates sent by other clients, both new and old IDs are used. - - When the server responds to `client-pre-sync` (listing all updated docids), only new IDs are used. diff --git a/packages/common/infra/src/sync/doc/priority-queue.ts b/packages/common/infra/src/sync/doc/priority-queue.ts deleted file mode 100644 index 0c38fca44..000000000 --- a/packages/common/infra/src/sync/doc/priority-queue.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { BinarySearchTree } from '@datastructures-js/binary-search-tree'; - -export class PriorityQueue { - tree = new BinarySearchTree<{ id: string; priority: number }>((a, b) => { - return a.priority === b.priority - ? a.id === b.id - ? 0 - : a.id > b.id - ? 1 - : -1 - : a.priority - b.priority; - }); - priorityMap = new Map(); - - push(id: string, priority: number = 0) { - const oldPriority = this.priorityMap.get(id); - if (oldPriority === priority) { - return; - } - if (oldPriority !== undefined) { - this.remove(id); - } - this.tree.insert({ id, priority }); - this.priorityMap.set(id, priority); - } - - pop() { - const node = this.tree.max(); - - if (!node) { - return null; - } - - this.tree.removeNode(node); - - const { id } = node.getValue(); - this.priorityMap.delete(id); - - return id; - } - - remove(id: string, priority?: number) { - priority ??= this.priorityMap.get(id); - if (priority === undefined) { - return false; - } - const removed = this.tree.remove({ id, priority }); - if (removed) { - this.priorityMap.delete(id); - } - - return removed; - } - - clear() { - this.tree.clear(); - this.priorityMap.clear(); - } - - updatePriority(id: string, priority: number) { - if (this.remove(id)) { - this.push(id, priority); - } - } - - get length() { - return this.tree.count; - } -} diff --git a/packages/common/infra/src/sync/doc/remote.ts b/packages/common/infra/src/sync/doc/remote.ts deleted file mode 100644 index 92afc8489..000000000 --- a/packages/common/infra/src/sync/doc/remote.ts +++ /dev/null @@ -1,611 +0,0 @@ -import { DebugLogger } from '@affine/debug'; -import { remove } from 'lodash-es'; -import { Observable, Subject } from 'rxjs'; -import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs'; - -import { LiveData } from '../../livedata'; -import { throwIfAborted } from '../../utils'; -import { AsyncPriorityQueue } from './async-priority-queue'; -import { ClockMap } from './clock'; -import type { DocEvent } from './event'; -import type { DocServer } from './server'; -import type { DocStorageInner } from './storage'; -import { isEmptyUpdate } from './utils'; - -const logger = new DebugLogger('doc-engine:remote'); - -type Job = - | { - type: 'connect'; - docId: string; - } - | { - type: 'push'; - docId: string; - update: Uint8Array; - seqNum: number; - } - | { - type: 'pull'; - docId: string; - } - | { - type: 'pullAndPush'; - docId: string; - } - | { - type: 'save'; - docId: string; - update?: Uint8Array; - serverClock: number; - }; - -export interface Status { - docs: Set; - connectedDocs: Set; - jobDocQueue: AsyncPriorityQueue; - jobMap: Map; - serverClocks: ClockMap; - syncing: boolean; - retrying: boolean; - errorMessage: string | null; -} - -export interface RemoteEngineState { - total: number; - syncing: number; - retrying: boolean; - errorMessage: string | null; -} - -export interface RemoteDocState { - syncing: boolean; - retrying: boolean; - serverClock: number | null; - errorMessage: string | null; -} - -export class DocEngineRemotePart { - private readonly prioritySettings = new Map(); - - constructor( - private readonly clientId: string, - private readonly storage: DocStorageInner, - private readonly server: DocServer - ) {} - - private status: Status = { - docs: new Set(), - connectedDocs: new Set(), - jobDocQueue: new AsyncPriorityQueue(), - jobMap: new Map(), - serverClocks: new ClockMap(new Map()), - syncing: false, - retrying: false, - errorMessage: null, - }; - private readonly statusUpdatedSubject$ = new Subject(); - - engineState$ = LiveData.from( - new Observable(subscribe => { - const next = () => { - if (!this.status.syncing) { - // if syncing = false, jobMap is empty - subscribe.next({ - total: this.status.docs.size, - syncing: this.status.docs.size, - retrying: this.status.retrying, - errorMessage: this.status.errorMessage, - }); - } else { - const syncing = this.status.jobMap.size; - subscribe.next({ - total: this.status.docs.size, - syncing: syncing, - retrying: this.status.retrying, - errorMessage: this.status.errorMessage, - }); - } - }; - next(); - return this.statusUpdatedSubject$.subscribe(() => { - next(); - }); - }), - { - syncing: 0, - total: 0, - retrying: false, - errorMessage: null, - } - ); - - docState$(docId: string) { - return LiveData.from( - new Observable(subscribe => { - const next = () => { - subscribe.next({ - syncing: - !this.status.connectedDocs.has(docId) || - this.status.jobMap.has(docId), - serverClock: this.status.serverClocks.get(docId), - retrying: this.status.retrying, - errorMessage: this.status.errorMessage, - }); - }; - next(); - return this.statusUpdatedSubject$.subscribe(updatedId => { - if (updatedId === true || updatedId === docId) next(); - }); - }), - { syncing: false, retrying: false, errorMessage: null, serverClock: null } - ); - } - - readonly jobs = { - connect: async (docId: string, signal?: AbortSignal) => { - const pushedSeqNum = await this.storage.loadDocSeqNumPushed( - docId, - signal - ); - const seqNum = await this.storage.loadDocSeqNum(docId, signal); - - if (pushedSeqNum === null || pushedSeqNum !== seqNum) { - await this.jobs.pullAndPush(docId, signal); - } else { - const pulled = await this.storage.loadDocServerClockPulled(docId); - if ( - pulled === null || - pulled !== this.status.serverClocks.get(normalizeServerDocId(docId)) - ) { - await this.jobs.pull(docId, signal); - } - } - - this.status.connectedDocs.add(docId); - this.statusUpdatedSubject$.next(docId); - }, - push: async ( - docId: string, - jobs: (Job & { type: 'push' })[], - signal?: AbortSignal - ) => { - if (this.status.connectedDocs.has(docId)) { - const maxSeqNum = Math.max(...jobs.map(j => j.seqNum)); - const pushedSeqNum = - (await this.storage.loadDocSeqNumPushed(docId, signal)) ?? 0; - - if (maxSeqNum - pushedSeqNum === jobs.length) { - const merged = mergeUpdates( - jobs.map(j => j.update).filter(update => !isEmptyUpdate(update)) - ); - if (!isEmptyUpdate(merged)) { - const { serverClock } = await this.server.pushDoc(docId, merged); - this.schedule({ - type: 'save', - docId, - serverClock, - }); - } - await this.storage.saveDocPushedSeqNum( - docId, - { add: jobs.length }, - signal - ); - } else { - // maybe other tab is modifying the doc, do full pull and push for safety - await this.jobs.pullAndPush(docId, signal); - } - } - }, - pullAndPush: async (docId: string, signal?: AbortSignal) => { - const seqNum = await this.storage.loadDocSeqNum(docId, signal); - const data = await this.storage.loadDocFromLocal(docId, signal); - - const stateVector = - data && !isEmptyUpdate(data) - ? encodeStateVectorFromUpdate(data) - : new Uint8Array(); - const serverData = await this.server.pullDoc(docId, stateVector); - - if (serverData) { - const { - data: newData, - stateVector: serverStateVector, - serverClock, - } = serverData; - await this.storage.saveServerClock( - new Map([[normalizeServerDocId(docId), serverClock]]), - signal - ); - this.actions.updateServerClock( - normalizeServerDocId(docId), - serverClock - ); - await this.storage.commitDocAsServerUpdate( - docId, - newData, - serverClock, - signal - ); - this.storage.eventBus.emit({ - type: 'ServerUpdateCommitted', - docId, - clientId: this.clientId, - update: newData, - }); - const diff = - data && serverStateVector && serverStateVector.length > 0 - ? diffUpdate(data, serverStateVector) - : data; - if (diff && !isEmptyUpdate(diff)) { - const { serverClock } = await this.server.pushDoc(docId, diff); - this.schedule({ - type: 'save', - docId, - serverClock, - }); - } - await this.storage.saveDocPushedSeqNum(docId, seqNum, signal); - } else { - if (data && !isEmptyUpdate(data)) { - const { serverClock } = await this.server.pushDoc(docId, data); - await this.storage.saveDocServerClockPulled( - docId, - serverClock, - signal - ); - await this.storage.saveServerClock( - new Map([[normalizeServerDocId(docId), serverClock]]), - signal - ); - this.actions.updateServerClock( - normalizeServerDocId(docId), - serverClock - ); - } - await this.storage.saveDocPushedSeqNum(docId, seqNum, signal); - } - }, - pull: async (docId: string, signal?: AbortSignal) => { - const data = await this.storage.loadDocFromLocal(docId, signal); - - const stateVector = - data && !isEmptyUpdate(data) - ? encodeStateVectorFromUpdate(data) - : new Uint8Array(); - const serverDoc = await this.server.pullDoc(docId, stateVector); - if (!serverDoc) { - return; - } - const { data: newData, serverClock } = serverDoc; - await this.storage.commitDocAsServerUpdate( - docId, - newData, - serverClock, - signal - ); - this.storage.eventBus.emit({ - type: 'ServerUpdateCommitted', - docId, - clientId: this.clientId, - update: newData, - }); - await this.storage.saveServerClock( - new Map([[normalizeServerDocId(docId), serverClock]]), - signal - ); - this.actions.updateServerClock(normalizeServerDocId(docId), serverClock); - }, - save: async ( - docId: string, - jobs: (Job & { type: 'save' })[], - signal?: AbortSignal - ) => { - const serverClock = jobs.reduce((a, b) => Math.max(a, b.serverClock), 0); - await this.storage.saveServerClock( - new Map([[normalizeServerDocId(docId), serverClock]]), - signal - ); - this.actions.updateServerClock(normalizeServerDocId(docId), serverClock); - if (this.status.connectedDocs.has(docId)) { - const data = jobs - .map(j => j.update) - .filter((update): update is Uint8Array => - update ? !isEmptyUpdate(update) : false - ); - const update = data.length > 0 ? mergeUpdates(data) : new Uint8Array(); - await this.storage.commitDocAsServerUpdate( - docId, - update, - serverClock, - signal - ); - this.storage.eventBus.emit({ - type: 'ServerUpdateCommitted', - docId, - clientId: this.clientId, - update, - }); - } - }, - }; - - readonly actions = { - updateServerClock: (docId: string, serverClock: number) => { - this.status.serverClocks.setIfBigger(docId, serverClock); - this.statusUpdatedSubject$.next(docId); - }, - addDoc: (docId: string) => { - if (!this.status.docs.has(docId)) { - this.status.docs.add(docId); - this.statusUpdatedSubject$.next(docId); - this.schedule({ - type: 'connect', - docId, - }); - } - }, - }; - - readonly events: { - [key in DocEvent['type']]?: (event: DocEvent & { type: key }) => void; - } = { - ClientUpdateCommitted: ({ clientId, docId, seqNum, update }) => { - if (clientId !== this.clientId) { - return; - } - this.schedule({ - type: 'push', - docId, - update, - seqNum, - }); - }, - }; - - async mainLoop(signal?: AbortSignal) { - // eslint-disable-next-line no-constant-condition - while (true) { - try { - await this.retryLoop(signal); - } catch (err) { - if (signal?.aborted) { - return; - } - logger.error('Remote sync error, retry in 5s', err); - this.status.errorMessage = - err instanceof Error ? err.message : `${err}`; - this.statusUpdatedSubject$.next(true); - } finally { - this.status = { - docs: this.status.docs, - connectedDocs: new Set(), - jobDocQueue: new AsyncPriorityQueue(), - jobMap: new Map(), - serverClocks: new ClockMap(new Map()), - syncing: false, - retrying: true, - errorMessage: this.status.errorMessage, - }; - this.statusUpdatedSubject$.next(true); - } - await Promise.race([ - new Promise(resolve => { - setTimeout(resolve, 5 * 1000); - }), - new Promise((_, reject) => { - // exit if manually stopped - if (signal?.aborted) { - reject(signal.reason); - } - signal?.addEventListener('abort', () => { - reject(signal.reason); - }); - }), - ]); - } - } - - async retryLoop(signal?: AbortSignal) { - throwIfAborted(signal); - const abort = new AbortController(); - - signal?.addEventListener('abort', reason => { - abort.abort(reason); - }); - - signal = abort.signal; - - const disposes: (() => void)[] = []; - - try { - disposes.push( - this.storage.eventBus.on(event => { - const handler = this.events[event.type]; - handler?.(event as any); - }) - ); - throwIfAborted(signal); - - for (const doc of this.status.docs) { - this.schedule({ - type: 'connect', - docId: doc, - }); - } - - logger.info('Remote sync started'); - this.status.syncing = true; - this.statusUpdatedSubject$.next(true); - - this.server.onInterrupted(reason => { - abort.abort(reason); - }); - await Promise.race([ - this.server.waitForConnectingServer(signal), - new Promise((_, reject) => { - setTimeout(() => { - reject(new Error('Connect to server timeout')); - }, 1000 * 30); - }), - new Promise((_, reject) => { - signal?.addEventListener('abort', reason => { - reject(reason); - }); - }), - ]); - - // reset retrying flag after connected with server - this.status.retrying = false; - this.statusUpdatedSubject$.next(true); - - throwIfAborted(signal); - disposes.push( - await this.server.subscribeAllDocs(({ docId, data, serverClock }) => { - this.schedule({ - type: 'save', - docId: docId, - serverClock, - update: data, - }); - }) - ); - const cachedClocks = await this.storage.loadServerClock(signal); - for (const [id, v] of cachedClocks) { - this.actions.updateServerClock(id, v); - } - const maxClockValue = this.status.serverClocks.max; - const newClocks = await this.server.loadServerClock(maxClockValue); - for (const [id, v] of newClocks) { - this.actions.updateServerClock(id, v); - } - await this.storage.saveServerClock(newClocks, signal); - - // eslint-disable-next-line no-constant-condition - while (true) { - throwIfAborted(signal); - - const docId = await this.status.jobDocQueue.asyncPop(signal); - // eslint-disable-next-line no-constant-condition - while (true) { - const jobs = this.status.jobMap.get(docId); - if (!jobs || jobs.length === 0) { - this.status.jobMap.delete(docId); - this.statusUpdatedSubject$.next(docId); - break; - } - - const connect = remove(jobs, j => j.type === 'connect'); - if (connect && connect.length > 0) { - await this.jobs.connect(docId, signal); - continue; - } - - const pullAndPush = remove(jobs, j => j.type === 'pullAndPush'); - if (pullAndPush && pullAndPush.length > 0) { - await this.jobs.pullAndPush(docId, signal); - continue; - } - - const pull = remove(jobs, j => j.type === 'pull'); - if (pull && pull.length > 0) { - await this.jobs.pull(docId, signal); - continue; - } - - const push = remove(jobs, j => j.type === 'push'); - if (push && push.length > 0) { - await this.jobs.push( - docId, - push as (Job & { type: 'push' })[], - signal - ); - continue; - } - - const save = remove(jobs, j => j.type === 'save'); - if (save && save.length > 0) { - await this.jobs.save( - docId, - save as (Job & { type: 'save' })[], - signal - ); - continue; - } - } - } - } finally { - for (const dispose of disposes) { - dispose(); - } - try { - this.server.disconnectServer(); - } catch (err) { - logger.error('Error on disconnect server', err); - } - this.status.syncing = false; - logger.info('Remote sync ended'); - } - } - - schedule(job: Job) { - const priority = this.prioritySettings.get(job.docId) ?? 0; - this.status.jobDocQueue.push(job.docId, priority); - - const existingJobs = this.status.jobMap.get(job.docId) ?? []; - existingJobs.push(job); - this.status.jobMap.set(job.docId, existingJobs); - this.statusUpdatedSubject$.next(job.docId); - } - - setPriority(docId: string, priority: number) { - this.prioritySettings.set(docId, priority); - this.status.jobDocQueue.updatePriority(docId, priority); - } -} - -// use normalized id in server clock -function normalizeServerDocId(raw: string) { - enum DocVariant { - Workspace = 'workspace', - Page = 'page', - Space = 'space', - Settings = 'settings', - Unknown = 'unknown', - } - - try { - if (!raw.length) { - throw new Error('Invalid Empty Doc ID'); - } - - let parts = raw.split(':'); - - if (parts.length > 3) { - // special adapt case `wsId:space:page:pageId` - if (parts[1] === DocVariant.Space && parts[2] === DocVariant.Page) { - parts = [parts[0], DocVariant.Space, parts[3]]; - } else { - throw new Error(`Invalid format of Doc ID: ${raw}`); - } - } else if (parts.length === 2) { - // `${variant}:${guid}` - throw new Error('not supported'); - } else if (parts.length === 1) { - // ${ws} or ${pageId} - parts = ['', DocVariant.Unknown, parts[0]]; - } - - const docId = parts.at(2); - - if (!docId) { - throw new Error('ID is required'); - } - - return docId; - } catch (err) { - logger.error('Error on normalize docId ' + raw, err); - return raw; - } -} diff --git a/packages/common/infra/src/sync/doc/server.ts b/packages/common/infra/src/sync/doc/server.ts deleted file mode 100644 index 9fdade3e7..000000000 --- a/packages/common/infra/src/sync/doc/server.ts +++ /dev/null @@ -1,28 +0,0 @@ -export interface DocServer { - pullDoc( - docId: string, - stateVector: Uint8Array - ): Promise<{ - data: Uint8Array; - serverClock: number; - stateVector?: Uint8Array; - } | null>; - - pushDoc(docId: string, data: Uint8Array): Promise<{ serverClock: number }>; - - loadServerClock(after: number): Promise>; - - subscribeAllDocs( - cb: (updates: { - docId: string; - data: Uint8Array; - serverClock: number; - }) => void - ): Promise<() => void>; - - waitForConnectingServer(signal: AbortSignal): Promise; - disconnectServer(): void; - onInterrupted(cb: (reason: string) => void): void; - - dispose?(): void; -} diff --git a/packages/common/infra/src/sync/doc/storage.ts b/packages/common/infra/src/sync/doc/storage.ts deleted file mode 100644 index bc6880c3b..000000000 --- a/packages/common/infra/src/sync/doc/storage.ts +++ /dev/null @@ -1,361 +0,0 @@ -import { mergeUpdates } from 'yjs'; - -import type { ByteKV, Memento } from '../../storage'; -import { MemoryMemento, ReadonlyByteKV, wrapMemento } from '../../storage'; -import { AsyncLock, throwIfAborted } from '../../utils'; -import type { DocEventBus } from '.'; -import { DocEventBusInner, MemoryDocEventBus } from './event'; -import { isEmptyUpdate } from './utils'; - -export interface DocStorage { - eventBus: DocEventBus; - doc: ByteKV; - syncMetadata: ByteKV; - serverClock: ByteKV; -} - -const Keys = { - SeqNum: (docId: string) => `${docId}:seqNum`, - SeqNumPushed: (docId: string) => `${docId}:seqNumPushed`, - ServerClockPulled: (docId: string) => `${docId}:serverClockPulled`, - UpdatedTime: (docId: string) => `${docId}:updateTime`, -}; - -const Values = { - UInt64: { - parse: (buffer: Uint8Array) => { - const view = new DataView(buffer.buffer); - return Number(view.getBigUint64(0, false)); - }, - serialize: (value: number) => { - const buffer = new ArrayBuffer(8); - const view = new DataView(buffer); - view.setBigUint64(0, BigInt(value), false); - return new Uint8Array(buffer); - }, - }, -}; - -export class DocStorageInner { - public readonly eventBus = new DocEventBusInner(this.behavior.eventBus); - constructor(public readonly behavior: DocStorage) {} - - async loadServerClock(signal?: AbortSignal): Promise> { - throwIfAborted(signal); - const list = await this.behavior.serverClock.keys(); - - const map = new Map(); - for (const key of list) { - const docId = key; - const value = await this.behavior.serverClock.get(key); - if (value) { - map.set(docId, Values.UInt64.parse(value)); - } - } - - return map; - } - - async saveServerClock(map: Map, signal?: AbortSignal) { - throwIfAborted(signal); - await this.behavior.serverClock.transaction(async transaction => { - for (const [docId, value] of map) { - const key = docId; - const oldBuffer = await transaction.get(key); - const old = oldBuffer ? Values.UInt64.parse(oldBuffer) : 0; - if (old < value) { - await transaction.set(key, Values.UInt64.serialize(value)); - } - } - }); - } - - async loadDocSeqNum(docId: string, signal?: AbortSignal) { - throwIfAborted(signal); - const bytes = await this.behavior.syncMetadata.get(Keys.SeqNum(docId)); - if (bytes === null) { - return 0; - } - return Values.UInt64.parse(bytes); - } - - async saveDocSeqNum( - docId: string, - seqNum: number | true, - signal?: AbortSignal - ) { - throwIfAborted(signal); - return await this.behavior.syncMetadata.transaction(async transaction => { - const key = Keys.SeqNum(docId); - const oldBytes = await transaction.get(key); - const old = oldBytes ? Values.UInt64.parse(oldBytes) : 0; - if (seqNum === true) { - await transaction.set(key, Values.UInt64.serialize(old + 1)); - return old + 1; - } - if (old < seqNum) { - await transaction.set(key, Values.UInt64.serialize(seqNum)); - return seqNum; - } - return old; - }); - } - - async loadDocSeqNumPushed(docId: string, signal?: AbortSignal) { - throwIfAborted(signal); - const bytes = await this.behavior.syncMetadata.get( - Keys.SeqNumPushed(docId) - ); - if (bytes === null) { - return null; - } - return Values.UInt64.parse(bytes); - } - - async saveDocPushedSeqNum( - docId: string, - seqNum: number | { add: number }, - signal?: AbortSignal - ) { - throwIfAborted(signal); - await this.behavior.syncMetadata.transaction(async transaction => { - const key = Keys.SeqNumPushed(docId); - const oldBytes = await transaction.get(key); - const old = oldBytes ? Values.UInt64.parse(oldBytes) : null; - if (typeof seqNum === 'object') { - return transaction.set( - key, - Values.UInt64.serialize((old ?? 0) + seqNum.add) - ); - } - if (old === null || old < seqNum) { - return transaction.set(key, Values.UInt64.serialize(seqNum)); - } - }); - } - - async loadDocServerClockPulled(docId: string, signal?: AbortSignal) { - throwIfAborted(signal); - const bytes = await this.behavior.syncMetadata.get( - Keys.ServerClockPulled(docId) - ); - if (bytes === null) { - return null; - } - return bytes ? Values.UInt64.parse(bytes) : 0; - } - - async saveDocServerClockPulled( - docId: string, - serverClock: number, - signal?: AbortSignal - ) { - throwIfAborted(signal); - await this.behavior.syncMetadata.transaction(async transaction => { - const oldBytes = await transaction.get(Keys.ServerClockPulled(docId)); - const old = oldBytes ? Values.UInt64.parse(oldBytes) : null; - if (old === null || old < serverClock) { - await transaction.set( - Keys.ServerClockPulled(docId), - Values.UInt64.serialize(serverClock) - ); - } - }); - } - - async loadDocFromLocal(docId: string, signal?: AbortSignal) { - throwIfAborted(signal); - return await this.behavior.doc.get(docId); - } - - /** - * Confirm that server updates are applied in the order they occur!!! - */ - async commitDocAsServerUpdate( - docId: string, - update: Uint8Array, - serverClock: number, - signal?: AbortSignal - ) { - throwIfAborted(signal); - await this.behavior.doc.transaction(async tx => { - const data = await tx.get(docId); - await tx.set( - docId, - data && !isEmptyUpdate(data) - ? !isEmptyUpdate(update) - ? mergeUpdates([data, update]) - : data - : update - ); - }); - await this.saveDocServerClockPulled(docId, serverClock); - } - - async commitDocAsClientUpdate( - docId: string, - update: Uint8Array, - signal?: AbortSignal - ) { - throwIfAborted(signal); - - await this.behavior.doc.transaction(async tx => { - const data = await tx.get(docId); - await tx.set( - docId, - data && !isEmptyUpdate(data) - ? !isEmptyUpdate(update) - ? mergeUpdates([data, update]) - : data - : update - ); - }); - - return await this.saveDocSeqNum(docId, true); - } - - clearSyncMetadata() { - return this.behavior.syncMetadata.clear(); - } - - async clearServerClock() { - return this.behavior.serverClock.clear(); - } -} - -export class ReadonlyStorage implements DocStorage { - constructor( - private readonly map: { - [key: string]: Uint8Array; - } - ) {} - - eventBus = new MemoryDocEventBus(); - doc = new ReadonlyByteKV(new Map(Object.entries(this.map))); - serverClock = new ReadonlyByteKV(); - syncMetadata = new ReadonlyByteKV(); -} - -export class MemoryStorage implements DocStorage { - constructor(private readonly memo: Memento = new MemoryMemento()) {} - - eventBus = new MemoryDocEventBus(); - lock = new AsyncLock(); - readonly docDb = wrapMemento(this.memo, 'doc:'); - readonly syncMetadataDb = wrapMemento(this.memo, 'syncMetadata:'); - readonly serverClockDb = wrapMemento(this.memo, 'serverClock:'); - - readonly doc = { - transaction: async cb => { - using _lock = await this.lock.acquire(); - return await cb({ - get: async key => { - return this.docDb.get(key) ?? null; - }, - set: async (key, value) => { - this.docDb.set(key, value); - }, - keys: async () => { - return Array.from(this.docDb.keys()); - }, - clear: () => { - this.docDb.clear(); - }, - del: key => { - this.docDb.del(key); - }, - }); - }, - get(key) { - return this.transaction(async tx => tx.get(key)); - }, - set(key, value) { - return this.transaction(async tx => tx.set(key, value)); - }, - keys() { - return this.transaction(async tx => tx.keys()); - }, - clear() { - return this.transaction(async tx => tx.clear()); - }, - del(key) { - return this.transaction(async tx => tx.del(key)); - }, - } satisfies ByteKV; - - readonly syncMetadata = { - transaction: async cb => { - using _lock = await this.lock.acquire(); - return await cb({ - get: async key => { - return this.syncMetadataDb.get(key) ?? null; - }, - set: async (key, value) => { - this.syncMetadataDb.set(key, value); - }, - keys: async () => { - return Array.from(this.syncMetadataDb.keys()); - }, - clear: () => { - this.syncMetadataDb.clear(); - }, - del: key => { - this.syncMetadataDb.del(key); - }, - }); - }, - get(key) { - return this.transaction(async tx => tx.get(key)); - }, - set(key, value) { - return this.transaction(async tx => tx.set(key, value)); - }, - keys() { - return this.transaction(async tx => tx.keys()); - }, - clear() { - return this.transaction(async tx => tx.clear()); - }, - del(key) { - return this.transaction(async tx => tx.del(key)); - }, - } satisfies ByteKV; - - readonly serverClock = { - transaction: async cb => { - using _lock = await this.lock.acquire(); - return await cb({ - get: async key => { - return this.serverClockDb.get(key) ?? null; - }, - set: async (key, value) => { - this.serverClockDb.set(key, value); - }, - keys: async () => { - return Array.from(this.serverClockDb.keys()); - }, - clear: () => { - this.serverClockDb.clear(); - }, - del: key => { - this.serverClockDb.del(key); - }, - }); - }, - get(key) { - return this.transaction(async tx => tx.get(key)); - }, - set(key, value) { - return this.transaction(async tx => tx.set(key, value)); - }, - keys() { - return this.transaction(async tx => tx.keys()); - }, - clear() { - return this.transaction(async tx => tx.clear()); - }, - del(key) { - return this.transaction(async tx => tx.del(key)); - }, - } satisfies ByteKV; -} diff --git a/packages/common/infra/src/sync/doc/utils.ts b/packages/common/infra/src/sync/doc/utils.ts deleted file mode 100644 index 797a05eaa..000000000 --- a/packages/common/infra/src/sync/doc/utils.ts +++ /dev/null @@ -1,6 +0,0 @@ -export function isEmptyUpdate(binary: Uint8Array) { - return ( - binary.byteLength === 0 || - (binary.byteLength === 2 && binary[0] === 0 && binary[1] === 0) - ); -} diff --git a/packages/common/infra/src/sync/index.ts b/packages/common/infra/src/sync/index.ts index 9f8f3bb7e..bf2553b3d 100644 --- a/packages/common/infra/src/sync/index.ts +++ b/packages/common/infra/src/sync/index.ts @@ -1,9 +1,3 @@ -export type { AwarenessConnection } from './awareness'; -export { AwarenessEngine } from './awareness'; -export type { BlobStatus, BlobStorage } from './blob/blob'; -export { BlobEngine, EmptyBlobStorage } from './blob/blob'; -export { BlobStorageOverCapacity } from './blob/error'; -export * from './doc'; export * from './indexer'; export { IndexedDBIndex, diff --git a/packages/common/infra/src/sync/job/runner.ts b/packages/common/infra/src/sync/job/runner.ts index 3220e2170..d25dc1ba0 100644 --- a/packages/common/infra/src/sync/job/runner.ts +++ b/packages/common/infra/src/sync/job/runner.ts @@ -48,7 +48,10 @@ export class JobRunner { // TODO: retry logic await this.queue.return(jobs); } - logger.error('Error processing jobs', err); + logger.error( + 'Error processing jobs', + err instanceof Error ? (err.stack ?? err.message) : err + ); } } else { await new Promise(resolve => setTimeout(resolve, 1000)); diff --git a/packages/common/nbstore/package.json b/packages/common/nbstore/package.json index 264509205..c3c38bb2b 100644 --- a/packages/common/nbstore/package.json +++ b/packages/common/nbstore/package.json @@ -9,6 +9,7 @@ "./worker/client": "./src/worker/client.ts", "./worker/consumer": "./src/worker/consumer.ts", "./idb": "./src/impls/idb/index.ts", + "./broadcast-channel": "./src/impls/broadcast-channel/index.ts", "./idb/v1": "./src/impls/idb/v1/index.ts", "./cloud": "./src/impls/cloud/index.ts", "./sqlite": "./src/impls/sqlite/index.ts", diff --git a/packages/common/nbstore/src/__tests__/frontend.spec.ts b/packages/common/nbstore/src/__tests__/frontend.spec.ts index b0e5357e7..cb1446b62 100644 --- a/packages/common/nbstore/src/__tests__/frontend.spec.ts +++ b/packages/common/nbstore/src/__tests__/frontend.spec.ts @@ -30,7 +30,7 @@ test('doc', async () => { const frontend1 = new DocFrontend(docStorage, DocSyncImpl.dummy); frontend1.start(); - frontend1.addDoc(doc1); + frontend1.connectDoc(doc1); await vitest.waitFor(async () => { const doc = await docStorage.getDoc('test-doc'); expectYjsEqual(doc!.bin, { @@ -45,7 +45,7 @@ test('doc', async () => { }); const frontend2 = new DocFrontend(docStorage, DocSyncImpl.dummy); frontend2.start(); - frontend2.addDoc(doc2); + frontend2.connectDoc(doc2); await vitest.waitFor(async () => { expectYjsEqual(doc2, { @@ -94,8 +94,8 @@ test('awareness', async () => { }, }); const frontend = new AwarenessFrontend(sync); - frontend.connect(awarenessA); - frontend.connect(awarenessB); + frontend.connectAwareness(awarenessA); + frontend.connectAwareness(awarenessB); } { const sync = new AwarenessSyncImpl({ @@ -105,7 +105,7 @@ test('awareness', async () => { }, }); const frontend = new AwarenessFrontend(sync); - frontend.connect(awarenessC); + frontend.connectAwareness(awarenessC); } awarenessA.setLocalState({ diff --git a/packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts b/packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts new file mode 100644 index 000000000..f57d3886a --- /dev/null +++ b/packages/common/nbstore/src/connection/__tests__/auto-reconnection.spec.ts @@ -0,0 +1,200 @@ +import { expect, test, vitest } from 'vitest'; + +import { AutoReconnectConnection } from '../connection'; + +test('connect and disconnect', async () => { + class TestConnection extends AutoReconnectConnection<{ + disconnect: () => void; + }> { + connectCount = 0; + abortCount = 0; + disconnectCount = 0; + notListenAbort = false; + override async doConnect(signal?: AbortSignal) { + this.connectCount++; + return new Promise<{ disconnect: () => void }>((resolve, reject) => { + setTimeout(() => { + resolve({ + disconnect: () => { + this.disconnectCount++; + }, + }); + }, 300); + if (!this.notListenAbort) { + signal?.addEventListener('abort', reason => { + reject(reason); + }); + } + }).catch(err => { + this.abortCount++; + throw err; + }); + } + override doDisconnect(t: { disconnect: () => void }) { + return t.disconnect(); + } + } + + const connection = new TestConnection(); + connection.connect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(1); + expect(connection.disconnectCount).toBe(0); + expect(connection.abortCount).toBe(0); + expect(connection.status).toBe('connected'); + }); + + connection.disconnect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(1); + expect(connection.disconnectCount).toBe(1); + expect(connection.abortCount).toBe(0); + expect(connection.status).toBe('closed'); + }); + + // connect twice + connection.connect(); + connection.connect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(2); + expect(connection.disconnectCount).toBe(1); + expect(connection.abortCount).toBe(0); + expect(connection.status).toBe('connected'); + }); + + connection.disconnect(); + connection.disconnect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(2); + expect(connection.disconnectCount).toBe(2); + expect(connection.abortCount).toBe(0); + expect(connection.status).toBe('closed'); + }); + + // calling connect disconnect consecutively, the previous connect call will be aborted. + connection.connect(); + connection.disconnect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(3); + expect(connection.disconnectCount).toBe(2); + expect(connection.abortCount).toBe(1); + expect(connection.status).toBe('closed'); + }); + + connection.connect(); + connection.disconnect(); + connection.connect(); + connection.disconnect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(5); + expect(connection.disconnectCount).toBe(2); + expect(connection.abortCount).toBe(3); + expect(connection.status).toBe('closed'); + }); + + // if connection is not listening to abort event, disconnect will be called + connection.notListenAbort = true; + connection.connect(); + connection.disconnect(); + connection.connect(); + connection.disconnect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(7); + expect(connection.disconnectCount).toBe(4); + expect(connection.abortCount).toBe(3); + expect(connection.status).toBe('closed'); + }); +}); + +test('retry when connect failed', async () => { + class TestConnection extends AutoReconnectConnection { + override retryDelay = 300; + connectCount = 0; + override async doConnect() { + this.connectCount++; + if (this.connectCount === 3) { + return { hello: 'world' }; + } + throw new Error('not connected, count: ' + this.connectCount); + } + override doDisconnect() { + return Promise.resolve(); + } + } + + const connection = new TestConnection(); + connection.connect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(1); + expect(connection.status).toBe('error'); + expect(connection.error?.message).toContain('not connected, count: 1'); + }); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(2); + expect(connection.status).toBe('error'); + expect(connection.error?.message).toBe('not connected, count: 2'); + }); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(3); + expect(connection.status).toBe('connected'); + expect(connection.error).toBeUndefined(); + }); +}); + +test('retry when error', async () => { + class TestConnection extends AutoReconnectConnection { + override retryDelay = 300; + connectCount = 0; + disconnectCount = 0; + override async doConnect() { + this.connectCount++; + return { + hello: 'world', + }; + } + override doDisconnect(conn: any) { + this.disconnectCount++; + expect(conn).toEqual({ + hello: 'world', + }); + } + triggerError(error: Error) { + this.error = error; + } + } + + const connection = new TestConnection(); + connection.connect(); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(1); + expect(connection.status).toBe('connected'); + }); + + connection.triggerError(new Error('test error')); + + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(1); + expect(connection.disconnectCount).toBe(1); + expect(connection.status).toBe('error'); + expect(connection.error?.message).toBe('test error'); + }); + + // waitfor reconnect + await vitest.waitFor(() => { + expect(connection.connectCount).toBe(2); + expect(connection.disconnectCount).toBe(1); + expect(connection.status).toBe('connected'); + expect(connection.error).toBeUndefined(); + }); +}); diff --git a/packages/common/nbstore/src/connection/connection.ts b/packages/common/nbstore/src/connection/connection.ts index 8ceea47e1..28b7fe2f7 100644 --- a/packages/common/nbstore/src/connection/connection.ts +++ b/packages/common/nbstore/src/connection/connection.ts @@ -1,5 +1,6 @@ import EventEmitter2 from 'eventemitter2'; -import { throttle } from 'lodash-es'; + +import { MANUALLY_STOP } from '../utils/throw-if-aborted'; export type ConnectionStatus = | 'idle' @@ -10,6 +11,7 @@ export type ConnectionStatus = export interface Connection { readonly status: ConnectionStatus; + readonly error?: Error; readonly inner: T; connect(): void; disconnect(): void; @@ -23,16 +25,15 @@ export abstract class AutoReconnectConnection implements Connection { private readonly event = new EventEmitter2(); - private _inner: T | null = null; + private _inner: T | undefined = undefined; private _status: ConnectionStatus = 'idle'; - protected error?: Error; + private _error: Error | undefined = undefined; + retryDelay = 3000; private refCount = 0; - private _enableAutoReconnect = false; private connectingAbort?: AbortController; + private reconnectingAbort?: AbortController; - constructor() { - this.autoReconnect(); - } + constructor() {} get shareId(): string | undefined { return undefined; @@ -43,7 +44,7 @@ export abstract class AutoReconnectConnection } get inner(): T { - if (!this._inner) { + if (this._inner === undefined) { throw new Error( `Connection ${this.constructor.name} has not been established.` ); @@ -52,7 +53,7 @@ export abstract class AutoReconnectConnection return this._inner; } - protected set inner(inner: T | null) { + private set inner(inner: T | undefined) { this._inner = inner; } @@ -60,12 +61,23 @@ export abstract class AutoReconnectConnection return this._status; } - protected setStatus(status: ConnectionStatus, error?: Error) { - const shouldEmit = status !== this._status || error !== this.error; + get error() { + return this._error; + } + + protected set error(error: Error | undefined) { + this.handleError(error); + } + + private setStatus(status: ConnectionStatus, error?: Error) { + const shouldEmit = status !== this._status || error !== this._error; this._status = status; - this.error = error; + // we only clear-up error when status is connected + if (error || status === 'connected') { + this._error = error; + } if (shouldEmit) { - this.emitStatusChanged(status, error); + this.emitStatusChanged(status, this._error); } } @@ -73,15 +85,15 @@ export abstract class AutoReconnectConnection protected abstract doDisconnect(conn: T): void; private innerConnect() { - if (this.status === 'idle' || this.status === 'error') { - this._enableAutoReconnect = true; + if (this.status !== 'connecting') { this.setStatus('connecting'); this.connectingAbort = new AbortController(); - this.doConnect(this.connectingAbort.signal) + const signal = this.connectingAbort.signal; + this.doConnect(signal) .then(value => { - if (!this.connectingAbort?.signal.aborted) { - this.setStatus('connected'); + if (!signal.aborted) { this._inner = value; + this.setStatus('connected'); } else { try { this.doDisconnect(value); @@ -91,14 +103,45 @@ export abstract class AutoReconnectConnection } }) .catch(error => { - if (!this.connectingAbort?.signal.aborted) { + if (!signal.aborted) { console.error('failed to connect', error); - this.setStatus('error', error as any); + this.handleError(error as any); } }); } } + private innerDisconnect() { + this.connectingAbort?.abort(MANUALLY_STOP); + this.reconnectingAbort?.abort(MANUALLY_STOP); + try { + if (this._inner) { + this.doDisconnect(this._inner); + } + } catch (error) { + console.error('failed to disconnect', error); + } + this.reconnectingAbort = undefined; + this.connectingAbort = undefined; + this._inner = undefined; + } + + private handleError(reason?: Error) { + // on error + console.error('connection error, will reconnect', reason); + this.innerDisconnect(); + this.setStatus('error', reason); + // reconnect + + this.reconnectingAbort = new AbortController(); + const signal = this.reconnectingAbort.signal; + setTimeout(() => { + if (!signal.aborted) { + this.innerConnect(); + } + }, this.retryDelay); + } + connect() { this.refCount++; if (this.refCount === 1) { @@ -106,36 +149,16 @@ export abstract class AutoReconnectConnection } } - disconnect() { - this.refCount--; - if (this.refCount === 0) { - this._enableAutoReconnect = false; - this.connectingAbort?.abort(); - try { - if (this._inner) { - this.doDisconnect(this._inner); - } - } catch (error) { - console.error('failed to disconnect', error); - } - this.setStatus('closed'); - this._inner = null; + disconnect(force?: boolean) { + if (force) { + this.refCount = 0; + } else { + this.refCount = Math.max(this.refCount - 1, 0); + } + if (this.refCount === 0) { + this.innerDisconnect(); + this.setStatus('closed'); } - } - - private autoReconnect() { - // TODO: - // - maximum retry count - // - dynamic sleep time (attempt < 3 ? 1s : 1min)? - this.onStatusChanged( - throttle(() => { - () => { - if (this._enableAutoReconnect) { - this.innerConnect(); - } - }; - }, 1000) - ); } waitForConnected(signal?: AbortSignal) { diff --git a/packages/common/nbstore/src/frontend/awareness.ts b/packages/common/nbstore/src/frontend/awareness.ts index 76aaf7f5c..6b0f22bee 100644 --- a/packages/common/nbstore/src/frontend/awareness.ts +++ b/packages/common/nbstore/src/frontend/awareness.ts @@ -13,7 +13,7 @@ type AwarenessChanges = Record<'added' | 'updated' | 'removed', number[]>; export class AwarenessFrontend { constructor(private readonly sync: AwarenessSync) {} - connect(awareness: Awareness) { + connectAwareness(awareness: Awareness) { const uniqueId = nanoid(); const handleAwarenessUpdate = ( changes: AwarenessChanges, @@ -27,7 +27,6 @@ export class AwarenessFrontend { ); const update = encodeAwarenessUpdate(awareness, changedClients); - this.sync .update( { diff --git a/packages/common/nbstore/src/frontend/doc.ts b/packages/common/nbstore/src/frontend/doc.ts index 4265af902..b528dcb60 100644 --- a/packages/common/nbstore/src/frontend/doc.ts +++ b/packages/common/nbstore/src/frontend/doc.ts @@ -1,7 +1,14 @@ import { groupBy } from 'lodash-es'; import { nanoid } from 'nanoid'; import type { Subscription } from 'rxjs'; -import { combineLatest, map, Observable, Subject } from 'rxjs'; +import { + combineLatest, + map, + Observable, + ReplaySubject, + share, + Subject, +} from 'rxjs'; import { applyUpdate, type Doc as YDoc, @@ -173,7 +180,10 @@ export class DocFrontend { synced: sync.synced, syncRetrying: sync.retrying, syncErrorMessage: sync.errorMessage, - })) + })), + share({ + connector: () => new ReplaySubject(1), + }) ) satisfies Observable; start() { @@ -241,19 +251,11 @@ export class DocFrontend { } /** - * Add a doc to the frontend, the doc will sync with the doc storage. - * @param doc - The doc to add - * @param withSubDoc - Whether to add the subdocs of the doc + * Connect a doc to the frontend, the doc will sync with the doc storage. + * @param doc - The doc to connect */ - addDoc(doc: YDoc, withSubDoc: boolean = false) { - this._addDoc(doc); - if (withSubDoc) { - doc.on('subdocs', ({ loaded }) => { - for (const subdoc of loaded) { - this._addDoc(subdoc); - } - }); - } + connectDoc(doc: YDoc) { + this._connectDoc(doc); } readonly jobs = { @@ -275,18 +277,16 @@ export class DocFrontend { // mark doc as loaded doc.emit('sync', [true, doc]); - this.status.connectedDocs.add(job.docId); - this.statusUpdatedSubject$.next(job.docId); - const docRecord = await this.storage.getDoc(job.docId); throwIfAborted(signal); - if (!docRecord || isEmptyUpdate(docRecord.bin)) { - return; + if (docRecord && !isEmptyUpdate(docRecord.bin)) { + this.applyUpdate(job.docId, docRecord.bin); + + this.status.readyDocs.add(job.docId); } - this.applyUpdate(job.docId, docRecord.bin); - this.status.readyDocs.add(job.docId); + this.status.connectedDocs.add(job.docId); this.statusUpdatedSubject$.next(job.docId); }, save: async ( @@ -339,12 +339,12 @@ export class DocFrontend { }; /** - * Remove a doc from the frontend, the doc will stop syncing with the doc storage. + * Disconnect a doc from the frontend, the doc will stop syncing with the doc storage. * It's not recommended to use this method directly, better to use `doc.destroy()`. * - * @param doc - The doc to remove + * @param doc - The doc to disconnect */ - removeDoc(doc: YDoc) { + disconnectDoc(doc: YDoc) { this.status.docs.delete(doc.guid); this.status.connectedDocs.delete(doc.guid); this.status.readyDocs.delete(doc.guid); @@ -370,7 +370,10 @@ export class DocFrontend { }; } - private _addDoc(doc: YDoc) { + private _connectDoc(doc: YDoc) { + if (this.status.docs.has(doc.guid)) { + throw new Error('doc already connected'); + } this.schedule({ type: 'load', docId: doc.guid, @@ -382,7 +385,7 @@ export class DocFrontend { doc.on('update', this.handleDocUpdate); doc.on('destroy', () => { - this.removeDoc(doc); + this.disconnectDoc(doc); }); } diff --git a/packages/common/nbstore/src/impls/broadcast-channel/channel.ts b/packages/common/nbstore/src/impls/broadcast-channel/channel.ts index a29d1e830..9ff1340d5 100644 --- a/packages/common/nbstore/src/impls/broadcast-channel/channel.ts +++ b/packages/common/nbstore/src/impls/broadcast-channel/channel.ts @@ -15,12 +15,7 @@ export class BroadcastChannelConnection extends AutoReconnectConnection void, onCollect: () => Promise ): () => void { - // TODO: handle disconnect // leave awareness const leave = () => { + if (this.connection.status !== 'connected') return; + this.socket.off('space:collect-awareness', handleCollectAwareness); + this.socket.off( + 'space:broadcast-awareness-update', + handleBroadcastAwarenessUpdate + ); this.socket.emit('space:leave-awareness', { spaceType: this.options.type, spaceId: this.options.id, @@ -64,6 +61,11 @@ export class CloudAwarenessStorage extends AwarenessStorageBase { // join awareness, and collect awareness from others const joinAndCollect = async () => { + this.socket.on('space:collect-awareness', handleCollectAwareness); + this.socket.on( + 'space:broadcast-awareness-update', + handleBroadcastAwarenessUpdate + ); await this.socket.emitWithAck('space:join-awareness', { spaceType: this.options.type, spaceId: this.options.id, @@ -77,7 +79,11 @@ export class CloudAwarenessStorage extends AwarenessStorageBase { }); }; - joinAndCollect().catch(err => console.error('awareness join failed', err)); + if (this.connection.status === 'connected') { + joinAndCollect().catch(err => + console.error('awareness join failed', err) + ); + } const unsubscribeConnectionStatusChanged = this.connection.onStatusChanged( status => { @@ -141,18 +147,9 @@ export class CloudAwarenessStorage extends AwarenessStorageBase { } }; - this.socket.on('space:collect-awareness', handleCollectAwareness); - this.socket.on( - 'space:broadcast-awareness-update', - handleBroadcastAwarenessUpdate - ); return () => { leave(); - this.socket.off('space:collect-awareness', handleCollectAwareness); - this.socket.off( - 'space:broadcast-awareness-update', - handleBroadcastAwarenessUpdate - ); + unsubscribeConnectionStatusChanged(); }; } diff --git a/packages/common/nbstore/src/impls/cloud/doc-static.ts b/packages/common/nbstore/src/impls/cloud/doc-static.ts index 6d698067a..0381b5a3f 100644 --- a/packages/common/nbstore/src/impls/cloud/doc-static.ts +++ b/packages/common/nbstore/src/impls/cloud/doc-static.ts @@ -45,23 +45,28 @@ export class StaticCloudDocStorage extends DocStorageBase { - const arrayBuffer = await this.connection.fetchArrayBuffer( - `/api/workspaces/${this.spaceId}/docs/${docId}`, - { - priority: 'high', - headers: { - Accept: 'application/octet-stream', // this is necessary for ios native fetch to return arraybuffer - }, + try { + const arrayBuffer = await this.connection.fetchArrayBuffer( + `/api/workspaces/${this.spaceId}/docs/${docId}`, + { + priority: 'high', + headers: { + Accept: 'application/octet-stream', // this is necessary for ios native fetch to return arraybuffer + }, + } + ); + if (!arrayBuffer) { + return null; } - ); - if (!arrayBuffer) { + return { + docId: docId, + bin: new Uint8Array(arrayBuffer), + timestamp: new Date(), + }; + } catch (error) { + console.error(error); return null; } - return { - docId: docId, - bin: new Uint8Array(arrayBuffer), - timestamp: new Date(), - }; } protected override setDocSnapshot( _snapshot: DocRecord, diff --git a/packages/common/nbstore/src/impls/cloud/doc.ts b/packages/common/nbstore/src/impls/cloud/doc.ts index 34511b351..4f0ac0be9 100644 --- a/packages/common/nbstore/src/impls/cloud/doc.ts +++ b/packages/common/nbstore/src/impls/cloud/doc.ts @@ -1,10 +1,5 @@ -import type { Socket, SocketOptions } from 'socket.io-client'; +import type { Socket } from 'socket.io-client'; -import { - type Connection, - type ConnectionStatus, - share, -} from '../../connection'; import { type DocClock, type DocClocks, @@ -12,6 +7,7 @@ import { type DocStorageOptions, type DocUpdate, } from '../../storage'; +import { getIdConverter, type IdConverter } from '../../utils/id-converter'; import type { SpaceType } from '../../utils/universal-id'; import { base64ToUint8Array, @@ -21,7 +17,6 @@ import { } from './socket'; interface CloudDocStorageOptions extends DocStorageOptions { - socketOptions?: SocketOptions; serverBaseUrl: string; type: SpaceType; } @@ -32,7 +27,12 @@ export class CloudDocStorage extends DocStorageBase { get socket() { return this.connection.inner; } - + get idConverter() { + if (!this.connection.idConverter) { + throw new Error('Id converter not initialized'); + } + return this.connection.idConverter; + } readonly spaceType = this.options.type; onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => { @@ -41,7 +41,7 @@ export class CloudDocStorage extends DocStorageBase { this.spaceId === message.spaceId ) { this.emit('update', { - docId: message.docId, + docId: this.idConverter.oldIdToNewId(message.docId), bin: base64ToUint8Array(message.update), timestamp: new Date(message.timestamp), editor: message.editor, @@ -58,10 +58,13 @@ export class CloudDocStorage extends DocStorageBase { const response = await this.socket.emitWithAck('space:load-doc', { spaceType: this.spaceType, spaceId: this.spaceId, - docId, + docId: this.idConverter.newIdToOldId(docId), }); if ('error' in response) { + if (response.error.name === 'DOC_NOT_FOUND') { + return null; + } // TODO: use [UserFriendlyError] throw new Error(response.error.message); } @@ -77,11 +80,14 @@ export class CloudDocStorage extends DocStorageBase { const response = await this.socket.emitWithAck('space:load-doc', { spaceType: this.spaceType, spaceId: this.spaceId, - docId, + docId: this.idConverter.newIdToOldId(docId), stateVector: state ? await uint8ArrayToBase64(state) : void 0, }); if ('error' in response) { + if (response.error.name === 'DOC_NOT_FOUND') { + return null; + } // TODO: use [UserFriendlyError] throw new Error(response.error.message); } @@ -98,8 +104,8 @@ export class CloudDocStorage extends DocStorageBase { const response = await this.socket.emitWithAck('space:push-doc-update', { spaceType: this.spaceType, spaceId: this.spaceId, - docId: update.docId, - updates: await uint8ArrayToBase64(update.bin), + docId: this.idConverter.newIdToOldId(update.docId), + update: await uint8ArrayToBase64(update.bin), }); if ('error' in response) { @@ -120,7 +126,7 @@ export class CloudDocStorage extends DocStorageBase { const response = await this.socket.emitWithAck('space:load-doc', { spaceType: this.spaceType, spaceId: this.spaceId, - docId, + docId: this.idConverter.newIdToOldId(docId), }); if ('error' in response) { @@ -150,7 +156,7 @@ export class CloudDocStorage extends DocStorageBase { } return Object.entries(response.data).reduce((ret, [docId, timestamp]) => { - ret[docId] = new Date(timestamp); + ret[this.idConverter.oldIdToNewId(docId)] = new Date(timestamp); return ret; }, {} as DocClocks); } @@ -159,7 +165,7 @@ export class CloudDocStorage extends DocStorageBase { this.socket.emit('space:delete-doc', { spaceType: this.spaceType, spaceId: this.spaceId, - docId, + docId: this.idConverter.newIdToOldId(docId), }); } @@ -174,83 +180,74 @@ export class CloudDocStorage extends DocStorageBase { } } -class CloudDocStorageConnection implements Connection { - connection = share( - new SocketConnection( - `${this.options.serverBaseUrl}/`, - this.options.socketOptions - ) - ); - - private disposeConnectionStatusListener?: () => void; - - private get socket() { - return this.connection.inner; - } - +class CloudDocStorageConnection extends SocketConnection { constructor( private readonly options: CloudDocStorageOptions, private readonly onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] - ) {} - - get status() { - return this.connection.status; + ) { + super(`${options.serverBaseUrl}/`); } - get inner() { - return this.connection.inner; - } + idConverter: IdConverter | null = null; - connect(): void { - if (!this.disposeConnectionStatusListener) { - this.disposeConnectionStatusListener = this.connection.onStatusChanged( - status => { - if (status === 'connected') { - this.join().catch(err => { - console.error('doc storage join failed', err); - }); - this.socket.on('space:broadcast-doc-update', this.onServerUpdate); - } - } - ); - } - return this.connection.connect(); - } + override async doConnect(signal?: AbortSignal) { + const socket = await super.doConnect(signal); - async join() { try { - const res = await this.socket.emitWithAck('space:join', { + const res = await socket.emitWithAck('space:join', { spaceType: this.options.type, spaceId: this.options.id, clientVersion: BUILD_CONFIG.appVersion, }); if ('error' in res) { - this.connection.setStatus('closed', new Error(res.error.message)); + throw new Error(res.error.message); } + + if (!this.idConverter) { + this.idConverter = await this.getIdConverter(socket); + } + + socket.on('space:broadcast-doc-update', this.onServerUpdate); + + return socket; } catch (e) { - this.connection.setStatus('error', e as Error); + socket.close(); + throw e; } } - disconnect() { - if (this.disposeConnectionStatusListener) { - this.disposeConnectionStatusListener(); - } - this.socket.emit('space:leave', { + override doDisconnect(socket: Socket) { + socket.emit('space:leave', { spaceType: this.options.type, spaceId: this.options.id, }); - this.socket.off('space:broadcast-doc-update', this.onServerUpdate); - this.connection.disconnect(); + socket.off('space:broadcast-doc-update', this.onServerUpdate); + super.disconnect(); } - waitForConnected(signal?: AbortSignal): Promise { - return this.connection.waitForConnected(signal); - } - onStatusChanged( - cb: (status: ConnectionStatus, error?: Error) => void - ): () => void { - return this.connection.onStatusChanged(cb); + async getIdConverter(socket: Socket) { + return getIdConverter( + { + getDocBuffer: async id => { + const response = await socket.emitWithAck('space:load-doc', { + spaceType: this.options.type, + spaceId: this.options.id, + docId: id, + }); + + if ('error' in response) { + if (response.error.name === 'DOC_NOT_FOUND') { + return null; + } + // TODO: use [UserFriendlyError] + throw new Error(response.error.message); + } + + return base64ToUint8Array(response.data.missing); + }, + }, + this.options.id + ); } } diff --git a/packages/common/nbstore/src/impls/cloud/http.ts b/packages/common/nbstore/src/impls/cloud/http.ts index 560e76cb7..62e38a8db 100644 --- a/packages/common/nbstore/src/impls/cloud/http.ts +++ b/packages/common/nbstore/src/impls/cloud/http.ts @@ -23,6 +23,7 @@ export class HttpConnection extends DummyConnection { ...init, signal: abortController.signal, headers: { + ...this.requestHeaders, ...init?.headers, 'x-affine-version': BUILD_CONFIG.appVersion, }, @@ -35,7 +36,7 @@ export class HttpConnection extends DummyConnection { let reason: string | any = ''; if (res.headers.get('Content-Type')?.includes('application/json')) { try { - reason = await res.json(); + reason = JSON.stringify(await res.json()); } catch { // ignore } @@ -63,7 +64,10 @@ export class HttpConnection extends DummyConnection { this.fetch ); - constructor(private readonly serverBaseUrl: string) { + constructor( + private readonly serverBaseUrl: string, + private readonly requestHeaders?: Record + ) { super(); } } diff --git a/packages/common/nbstore/src/impls/cloud/socket.ts b/packages/common/nbstore/src/impls/cloud/socket.ts index 878333339..528d3be03 100644 --- a/packages/common/nbstore/src/impls/cloud/socket.ts +++ b/packages/common/nbstore/src/impls/cloud/socket.ts @@ -4,10 +4,8 @@ import { type SocketOptions, } from 'socket.io-client'; -import { - AutoReconnectConnection, - type ConnectionStatus, -} from '../../connection'; +import { AutoReconnectConnection } from '../../connection'; +import { throwIfAborted } from '../../utils/throw-if-aborted'; // TODO(@forehalo): use [UserFriendlyError] interface EventError { @@ -82,7 +80,7 @@ interface ClientEvents { }; 'space:push-doc-update': [ - { spaceType: string; spaceId: string; docId: string; updates: string }, + { spaceType: string; spaceId: string; docId: string; update: string }, { timestamp: number }, ]; 'space:load-doc-timestamps': [ @@ -153,12 +151,24 @@ export function base64ToUint8Array(base64: string) { return new Uint8Array(binaryArray); } +const SOCKET_IOMANAGER_CACHE = new Map(); +function getSocketIOManager(endpoint: string) { + let manager = SOCKET_IOMANAGER_CACHE.get(endpoint); + if (!manager) { + manager = new SocketIOManager(endpoint, { + autoConnect: false, + transports: ['websocket'], + secure: new URL(endpoint).protocol === 'https:', + // we will handle reconnection by ourselves + reconnection: false, + }); + SOCKET_IOMANAGER_CACHE.set(endpoint, manager); + } + return manager; +} + export class SocketConnection extends AutoReconnectConnection { - manager = new SocketIOManager(this.endpoint, { - autoConnect: false, - transports: ['websocket'], - secure: new URL(this.endpoint).protocol === 'https:', - }); + manager = getSocketIOManager(this.endpoint); constructor( private readonly endpoint: string, @@ -171,32 +181,42 @@ export class SocketConnection extends AutoReconnectConnection { return `socket:${this.endpoint}`; } - override async doConnect() { - const conn = this.manager.socket('/', this.socketOptions); + override async doConnect(signal?: AbortSignal) { + const socket = this.manager.socket('/', this.socketOptions); + try { + throwIfAborted(signal); + await Promise.race([ + new Promise((resolve, reject) => { + socket.once('connect', () => { + resolve(); + }); + socket.once('connect_error', err => { + reject(err); + }); + socket.open(); + }), + new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => { + reject(new Error('Aborted')); + }); + }), + ]); + } catch (err) { + socket.close(); + throw err; + } - await new Promise((resolve, reject) => { - conn.once('connect', () => { - resolve(); - }); - conn.once('connect_error', err => { - reject(err); - }); - conn.open(); - }); + socket.on('disconnect', this.handleDisconnect); - return conn; + return socket; } override doDisconnect(conn: Socket) { + conn.off('disconnect', this.handleDisconnect); conn.close(); } - /** - * Socket connection allow explicitly set status by user - * - * used when join space failed - */ - override setStatus(status: ConnectionStatus, error?: Error) { - super.setStatus(status, error); - } + handleDisconnect = (reason: SocketIO.DisconnectReason) => { + this.error = new Error(reason); + }; } diff --git a/packages/common/nbstore/src/impls/idb/db.ts b/packages/common/nbstore/src/impls/idb/db.ts index 130e619e1..0326ab91e 100644 --- a/packages/common/nbstore/src/impls/idb/db.ts +++ b/packages/common/nbstore/src/impls/idb/db.ts @@ -25,22 +25,14 @@ export class IDBConnection extends AutoReconnectConnection<{ } override async doConnect() { + // indexeddb will responsible for version control, so the db.version always match migrator.version + const db = await openDB(this.dbName, migrator.version, { + upgrade: migrator.migrate, + }); + db.addEventListener('versionchange', this.handleVersionChange); + return { - db: await openDB(this.dbName, migrator.version, { - upgrade: migrator.migrate, - blocking: () => { - // if, for example, an tab with newer version is opened, this function will be called. - // we should close current connection to allow the new version to upgrade the db. - this.setStatus( - 'closed', - new Error('Blocking a new version. Closing the connection.') - ); - }, - blocked: () => { - // fallback to retry auto retry - this.setStatus('error', new Error('Blocked by other tabs.')); - }, - }), + db, channel: new BroadcastChannel('idb:' + this.dbName), }; } @@ -49,7 +41,19 @@ export class IDBConnection extends AutoReconnectConnection<{ db: IDBPDatabase; channel: BroadcastChannel; }) { + db.db.removeEventListener('versionchange', this.handleVersionChange); db.channel.close(); db.db.close(); } + + handleVersionChange = (e: IDBVersionChangeEvent) => { + if (e.newVersion !== migrator.version) { + this.error = new Error( + 'Database version mismatch, expected ' + + migrator.version + + ' but got ' + + e.newVersion + ); + } + }; } diff --git a/packages/common/nbstore/src/impls/idb/doc.ts b/packages/common/nbstore/src/impls/idb/doc.ts index 3a239b018..e5abc63f2 100644 --- a/packages/common/nbstore/src/impls/idb/doc.ts +++ b/packages/common/nbstore/src/impls/idb/doc.ts @@ -29,26 +29,35 @@ export class IndexedDBDocStorage extends DocStorageBase { override locker = new IndexedDBLocker(this.connection); - private _lastTimestamp = new Date(0); - - private generateTimestamp() { - const timestamp = new Date(); - if (timestamp.getTime() <= this._lastTimestamp.getTime()) { - timestamp.setTime(this._lastTimestamp.getTime() + 1); - } - this._lastTimestamp = timestamp; - return timestamp; - } - override async pushDocUpdate(update: DocUpdate, origin?: string) { - const trx = this.db.transaction(['updates', 'clocks'], 'readwrite'); - const timestamp = this.generateTimestamp(); - await trx.objectStore('updates').add({ - ...update, - createdAt: timestamp, - }); + let timestamp = new Date(); - await trx.objectStore('clocks').put({ docId: update.docId, timestamp }); + let retry = 0; + + while (true) { + try { + const trx = this.db.transaction(['updates', 'clocks'], 'readwrite'); + + await trx.objectStore('updates').add({ + ...update, + createdAt: timestamp, + }); + + await trx.objectStore('clocks').put({ docId: update.docId, timestamp }); + + trx.commit(); + } catch (e) { + if (e instanceof Error && e.name === 'ConstraintError') { + retry++; + if (retry < 10) { + timestamp = new Date(timestamp.getTime() + 1); + continue; + } + } + throw e; + } + break; + } this.emit( 'update', @@ -191,9 +200,9 @@ export class IndexedDBDocStorage extends DocStorageBase { }; } - handleChannelMessage(event: MessageEvent) { + handleChannelMessage = (event: MessageEvent) => { if (event.data.type === 'update') { this.emit('update', event.data.update, event.data.origin); } - } + }; } diff --git a/packages/common/nbstore/src/impls/idb/index.ts b/packages/common/nbstore/src/impls/idb/index.ts index 8959391c4..f55137d04 100644 --- a/packages/common/nbstore/src/impls/idb/index.ts +++ b/packages/common/nbstore/src/impls/idb/index.ts @@ -2,7 +2,6 @@ import type { StorageConstructor } from '..'; import { IndexedDBBlobStorage } from './blob'; import { IndexedDBDocStorage } from './doc'; import { IndexedDBSyncStorage } from './sync'; -import { IndexedDBV1BlobStorage, IndexedDBV1DocStorage } from './v1'; export * from './blob'; export * from './doc'; @@ -13,8 +12,3 @@ export const idbStorages = [ IndexedDBBlobStorage, IndexedDBSyncStorage, ] satisfies StorageConstructor[]; - -export const idbv1Storages = [ - IndexedDBV1DocStorage, - IndexedDBV1BlobStorage, -] satisfies StorageConstructor[]; diff --git a/packages/common/nbstore/src/impls/idb/v1/blob.ts b/packages/common/nbstore/src/impls/idb/v1/blob.ts index 261911911..ba85d8c3e 100644 --- a/packages/common/nbstore/src/impls/idb/v1/blob.ts +++ b/packages/common/nbstore/src/impls/idb/v1/blob.ts @@ -19,6 +19,9 @@ export class IndexedDBV1BlobStorage extends BlobStorageBase { } override async get(key: string) { + if (!this.db) { + return null; + } const trx = this.db.transaction('blob', 'readonly'); const blob = await trx.store.get(key); if (!blob) { @@ -34,6 +37,9 @@ export class IndexedDBV1BlobStorage extends BlobStorageBase { } override async delete(key: string, permanently: boolean) { + if (!this.db) { + return; + } if (permanently) { const trx = this.db.transaction('blob', 'readwrite'); await trx.store.delete(key); @@ -41,6 +47,9 @@ export class IndexedDBV1BlobStorage extends BlobStorageBase { } override async list() { + if (!this.db) { + return []; + } const trx = this.db.transaction('blob', 'readonly'); const it = trx.store.iterate(); diff --git a/packages/common/nbstore/src/impls/idb/v1/db.ts b/packages/common/nbstore/src/impls/idb/v1/db.ts index 946eb76c4..5dd0e13d2 100644 --- a/packages/common/nbstore/src/impls/idb/v1/db.ts +++ b/packages/common/nbstore/src/impls/idb/v1/db.ts @@ -15,23 +15,26 @@ export interface DocDBSchema extends DBSchema { }; } -export class DocIDBConnection extends AutoReconnectConnection< - IDBPDatabase -> { +export class DocIDBConnection extends AutoReconnectConnection | null> { override get shareId() { return 'idb(old):affine-local'; } override async doConnect() { - return openDB('affine-local', 1, { - upgrade: db => { - db.createObjectStore('workspace', { keyPath: 'id' }); - }, - }); + const dbs = await indexedDB.databases(); + if (dbs.some(d => d.name === 'affine-local')) { + return openDB('affine-local', 1, { + upgrade: db => { + db.createObjectStore('workspace', { keyPath: 'id' }); + }, + }); + } else { + return null; + } } - override doDisconnect(conn: IDBPDatabase) { - conn.close(); + override doDisconnect(conn: IDBPDatabase | null) { + conn?.close(); } } @@ -46,9 +49,7 @@ export interface BlobIDBConnectionOptions { id: string; } -export class BlobIDBConnection extends AutoReconnectConnection< - IDBPDatabase -> { +export class BlobIDBConnection extends AutoReconnectConnection | null> { constructor(private readonly options: BlobIDBConnectionOptions) { super(); } @@ -58,14 +59,19 @@ export class BlobIDBConnection extends AutoReconnectConnection< } override async doConnect() { - return openDB(`${this.options.id}_blob`, 1, { - upgrade: db => { - db.createObjectStore('blob'); - }, - }); + const dbs = await indexedDB.databases(); + if (dbs.some(d => d.name === `${this.options.id}_blob`)) { + return openDB(`${this.options.id}_blob`, 1, { + upgrade: db => { + db.createObjectStore('blob'); + }, + }); + } else { + return null; + } } - override doDisconnect(conn: IDBPDatabase) { - conn.close(); + override doDisconnect(conn: IDBPDatabase | null) { + conn?.close(); } } diff --git a/packages/common/nbstore/src/impls/idb/v1/doc.ts b/packages/common/nbstore/src/impls/idb/v1/doc.ts index a19c0fa20..82b88384d 100644 --- a/packages/common/nbstore/src/impls/idb/v1/doc.ts +++ b/packages/common/nbstore/src/impls/idb/v1/doc.ts @@ -1,9 +1,20 @@ +import { once } from 'lodash-es'; +import { + applyUpdate, + type Array as YArray, + Doc as YDoc, + type Map as YMap, +} from 'yjs'; + import { share } from '../../../connection'; import { + type DocClocks, type DocRecord, DocStorageBase, + type DocStorageOptions, type DocUpdate, } from '../../../storage'; +import { getIdConverter } from '../../../utils/id-converter'; import { DocIDBConnection } from './db'; /** @@ -14,6 +25,13 @@ export class IndexedDBV1DocStorage extends DocStorageBase { readonly connection = share(new DocIDBConnection()); + constructor(opts: DocStorageOptions) { + super({ + ...opts, + readonlyMode: true, + }); + } + get db() { return this.connection.inner; } @@ -23,26 +41,11 @@ export class IndexedDBV1DocStorage extends DocStorageBase { } override async getDoc(docId: string) { - const trx = this.db.transaction('workspace', 'readonly'); - const record = await trx.store.get(docId); - - if (!record?.updates.length) { + if (!this.db) { return null; } - - if (record.updates.length === 1) { - return { - docId, - bin: record.updates[0].update, - timestamp: new Date(record.updates[0].timestamp), - }; - } - - return { - docId, - bin: await this.mergeUpdates(record.updates.map(update => update.update)), - timestamp: new Date(record.updates.at(-1)?.timestamp ?? Date.now()), - }; + const oldId = (await this.getIdConverter()).newIdToOldId(docId); + return this.rawGetDoc(oldId); } protected override async getDocSnapshot() { @@ -55,12 +58,60 @@ export class IndexedDBV1DocStorage extends DocStorageBase { } override async deleteDoc(docId: string) { + if (!this.db) { + return; + } + const oldId = (await this.getIdConverter()).newIdToOldId(docId); const trx = this.db.transaction('workspace', 'readwrite'); - await trx.store.delete(docId); + await trx.store.delete(oldId); } - override async getDocTimestamps() { - return {}; + override async getDocTimestamps(): Promise { + if (!this.db) { + return {}; + } + + const idConverter = await this.getIdConverter(); + + const oldIds: string[] = [this.spaceId]; + + const rootDocBuffer = await this.rawGetDoc(this.spaceId); + if (rootDocBuffer) { + const ydoc = new YDoc({ + guid: this.spaceId, + }); + applyUpdate(ydoc, rootDocBuffer.bin); + + // get all ids from rootDoc.meta.pages.[*].id, trust this id as normalized id + const normalizedDocIds = ( + (ydoc.getMap('meta') as YMap | undefined)?.get('pages') as + | YArray> + | undefined + ) + ?.map(i => i.get('id') as string) + .filter(i => !!i); + + const spaces = ydoc.getMap('spaces') as YMap | undefined; + for (const pageId of normalizedDocIds ?? []) { + const subdoc = spaces?.get(pageId); + if (subdoc && subdoc instanceof YDoc) { + oldIds.push(subdoc.guid); + } + } + } + + const trx = this.db.transaction('workspace', 'readonly'); + const allKeys = await trx.store.getAllKeys(); + oldIds.push(...allKeys.filter(k => k.startsWith(`db$${this.spaceId}$`))); + oldIds.push( + ...allKeys.filter(k => + k.match(new RegExp(`^userdata\\$[\\w-]+\\$${this.spaceId}$`)) + ) + ); + + return Object.fromEntries( + oldIds.map(id => [idConverter.oldIdToNewId(id), new Date(1)]) + ); } override async getDocTimestamp(_docId: string) { @@ -78,4 +129,59 @@ export class IndexedDBV1DocStorage extends DocStorageBase { protected override async markUpdatesMerged(): Promise { return 0; } + + private async rawGetDoc(id: string) { + if (!this.db) { + return null; + } + const trx = this.db.transaction('workspace', 'readonly'); + const record = await trx.store.get(id); + + if (!record?.updates.length) { + return null; + } + + if (record.updates.length === 1) { + return { + docId: id, + bin: record.updates[0].update, + timestamp: new Date(record.updates[0].timestamp), + }; + } + + return { + docId: id, + bin: await this.mergeUpdates(record.updates.map(update => update.update)), + timestamp: new Date(record.updates.at(-1)?.timestamp ?? Date.now()), + }; + } + + private readonly getIdConverter = once(async () => { + const idConverter = getIdConverter( + { + getDocBuffer: async id => { + if (!this.db) { + return null; + } + const trx = this.db.transaction('workspace', 'readonly'); + const record = await trx.store.get(id); + + if (!record?.updates.length) { + return null; + } + + if (record.updates.length === 1) { + return record.updates[0].update; + } + + return await this.mergeUpdates( + record.updates.map(update => update.update) + ); + }, + }, + this.spaceId + ); + + return await idConverter; + }); } diff --git a/packages/common/nbstore/src/impls/idb/v1/index.ts b/packages/common/nbstore/src/impls/idb/v1/index.ts index d476ae6eb..be5b61dbc 100644 --- a/packages/common/nbstore/src/impls/idb/v1/index.ts +++ b/packages/common/nbstore/src/impls/idb/v1/index.ts @@ -1,2 +1,11 @@ +import type { StorageConstructor } from '../..'; +import { IndexedDBV1BlobStorage } from './blob'; +import { IndexedDBV1DocStorage } from './doc'; + export * from './blob'; export * from './doc'; + +export const idbV1Storages = [ + IndexedDBV1DocStorage, + IndexedDBV1BlobStorage, +] satisfies StorageConstructor[]; diff --git a/packages/common/nbstore/src/impls/index.ts b/packages/common/nbstore/src/impls/index.ts index 4ff03bdae..5ac4d9b6a 100644 --- a/packages/common/nbstore/src/impls/index.ts +++ b/packages/common/nbstore/src/impls/index.ts @@ -1,8 +1,10 @@ import type { Storage } from '../storage'; import type { broadcastChannelStorages } from './broadcast-channel'; import type { cloudStorages } from './cloud'; -import type { idbStorages, idbv1Storages } from './idb'; +import type { idbStorages } from './idb'; +import type { idbV1Storages } from './idb/v1'; import type { sqliteStorages } from './sqlite'; +import type { sqliteV1Storages } from './sqlite/v1'; export type StorageConstructor = { new (...args: any[]): Storage; @@ -11,9 +13,10 @@ export type StorageConstructor = { type Storages = | typeof cloudStorages - | typeof idbv1Storages + | typeof idbV1Storages | typeof idbStorages | typeof sqliteStorages + | typeof sqliteV1Storages | typeof broadcastChannelStorages; // oxlint-disable-next-line no-redeclare diff --git a/packages/common/nbstore/src/impls/sqlite/db.ts b/packages/common/nbstore/src/impls/sqlite/db.ts index e9724949d..b89d52804 100644 --- a/packages/common/nbstore/src/impls/sqlite/db.ts +++ b/packages/common/nbstore/src/impls/sqlite/db.ts @@ -41,7 +41,7 @@ export type NativeDBApis = { id: string, peer: string, docId: string - ): Promise; + ): Promise; setPeerRemoteClock( id: string, peer: string, @@ -53,7 +53,7 @@ export type NativeDBApis = { id: string, peer: string, docId: string - ): Promise; + ): Promise; setPeerPulledRemoteClock( id: string, peer: string, @@ -65,7 +65,7 @@ export type NativeDBApis = { id: string, peer: string, docId: string - ): Promise; + ): Promise; setPeerPushedClock( id: string, peer: string, diff --git a/packages/common/nbstore/src/impls/sqlite/index.ts b/packages/common/nbstore/src/impls/sqlite/index.ts index e9c1ece1f..b409b12a9 100644 --- a/packages/common/nbstore/src/impls/sqlite/index.ts +++ b/packages/common/nbstore/src/impls/sqlite/index.ts @@ -7,7 +7,6 @@ export * from './blob'; export { bindNativeDBApis, type NativeDBApis } from './db'; export * from './doc'; export * from './sync'; -export * from './v1'; export const sqliteStorages = [ SqliteDocStorage, diff --git a/packages/common/nbstore/src/impls/sqlite/v1/blob.ts b/packages/common/nbstore/src/impls/sqlite/v1/blob.ts index 229a4c28a..22fadb410 100644 --- a/packages/common/nbstore/src/impls/sqlite/v1/blob.ts +++ b/packages/common/nbstore/src/impls/sqlite/v1/blob.ts @@ -7,6 +7,7 @@ import { apis } from './db'; * @deprecated readonly */ export class SqliteV1BlobStorage extends BlobStorageBase { + static identifier = 'SqliteV1BlobStorage'; override connection = new DummyConnection(); constructor(private readonly options: { type: SpaceType; id: string }) { diff --git a/packages/common/nbstore/src/impls/sqlite/v1/doc.ts b/packages/common/nbstore/src/impls/sqlite/v1/doc.ts index 203fcfa6a..9e852f35a 100644 --- a/packages/common/nbstore/src/impls/sqlite/v1/doc.ts +++ b/packages/common/nbstore/src/impls/sqlite/v1/doc.ts @@ -4,6 +4,8 @@ import { DocStorageBase, type DocUpdate, } from '../../../storage'; +import { getIdConverter, type IdConverter } from '../../../utils/id-converter'; +import { isEmptyUpdate } from '../../../utils/is-empty-update'; import type { SpaceType } from '../../../utils/universal-id'; import { apis } from './db'; @@ -14,8 +16,14 @@ export class SqliteV1DocStorage extends DocStorageBase<{ type: SpaceType; id: string; }> { + static identifier = 'SqliteV1DocStorage'; + cachedIdConverter: Promise | null = null; override connection = new DummyConnection(); + constructor(options: { type: SpaceType; id: string }) { + super({ ...options, readonlyMode: true }); + } + private get db() { if (!apis) { throw new Error('Not in electron context.'); @@ -26,17 +34,21 @@ export class SqliteV1DocStorage extends DocStorageBase<{ override async pushDocUpdate(update: DocUpdate) { // no more writes - return { docId: update.docId, timestamp: new Date() }; } override async getDoc(docId: string) { + const idConverter = await this.getIdConverter(); const bin = await this.db.getDocAsUpdates( this.options.type, this.options.id, - docId + idConverter.newIdToOldId(docId) ); + if (isEmptyUpdate(bin)) { + return null; + } + return { docId, bin, @@ -71,4 +83,37 @@ export class SqliteV1DocStorage extends DocStorageBase<{ protected override async markUpdatesMerged(): Promise { return 0; } + + private async getIdConverter() { + if (this.cachedIdConverter) { + return await this.cachedIdConverter; + } + this.cachedIdConverter = getIdConverter( + { + getDocBuffer: async id => { + if (!this.db) { + return null; + } + const updates = await this.db.getDocAsUpdates( + this.options.type, + this.options.id, + id + ); + + if (isEmptyUpdate(updates)) { + return null; + } + + if (!updates) { + return null; + } + + return updates; + }, + }, + this.spaceId + ); + + return await this.cachedIdConverter; + } } diff --git a/packages/common/nbstore/src/impls/sqlite/v1/index.ts b/packages/common/nbstore/src/impls/sqlite/v1/index.ts index 808cd280e..14a159676 100644 --- a/packages/common/nbstore/src/impls/sqlite/v1/index.ts +++ b/packages/common/nbstore/src/impls/sqlite/v1/index.ts @@ -1,3 +1,12 @@ +import type { StorageConstructor } from '../..'; +import { SqliteV1BlobStorage } from './blob'; +import { SqliteV1DocStorage } from './doc'; + export * from './blob'; export { bindNativeDBV1Apis } from './db'; export * from './doc'; + +export const sqliteV1Storages = [ + SqliteV1DocStorage, + SqliteV1BlobStorage, +] satisfies StorageConstructor[]; diff --git a/packages/common/nbstore/src/storage/doc.ts b/packages/common/nbstore/src/storage/doc.ts index f2541b31e..53cd8521e 100644 --- a/packages/common/nbstore/src/storage/doc.ts +++ b/packages/common/nbstore/src/storage/doc.ts @@ -151,7 +151,7 @@ export abstract class DocStorageBase implements DocStorage { return { docId, - missing: state ? diffUpdate(doc.bin, state) : doc.bin, + missing: state && state.length > 0 ? diffUpdate(doc.bin, state) : doc.bin, state: encodeStateVectorFromUpdate(doc.bin), timestamp: doc.timestamp, }; diff --git a/packages/common/nbstore/src/sync/awareness/index.ts b/packages/common/nbstore/src/sync/awareness/index.ts index 7a905647c..b0d867552 100644 --- a/packages/common/nbstore/src/sync/awareness/index.ts +++ b/packages/common/nbstore/src/sync/awareness/index.ts @@ -18,8 +18,11 @@ export class AwarenessSyncImpl implements AwarenessSync { async update(record: AwarenessRecord, origin?: string) { await Promise.all( - [this.storages.local, ...Object.values(this.storages.remotes)].map(peer => - peer.update(record, origin) + [this.storages.local, ...Object.values(this.storages.remotes)].map( + peer => + peer.connection.status === 'connected' + ? peer.update(record, origin) + : Promise.resolve() ) ); } diff --git a/packages/common/nbstore/src/sync/blob/index.ts b/packages/common/nbstore/src/sync/blob/index.ts index a561f94e5..f7964f3a1 100644 --- a/packages/common/nbstore/src/sync/blob/index.ts +++ b/packages/common/nbstore/src/sync/blob/index.ts @@ -73,10 +73,14 @@ export class BlobSyncImpl implements BlobSync { async fullSync(signal?: AbortSignal) { throwIfAborted(signal); + await this.storages.local.connection.waitForConnected(signal); + for (const [remotePeer, remote] of Object.entries(this.storages.remotes)) { let localList: string[] = []; let remoteList: string[] = []; + await remote.connection.waitForConnected(signal); + try { localList = (await this.storages.local.list(signal)).map(b => b.key); throwIfAborted(signal); @@ -150,7 +154,7 @@ export class BlobSyncImpl implements BlobSync { } stop() { - this.abort?.abort(); + this.abort?.abort(MANUALLY_STOP); this.abort = null; } diff --git a/packages/common/nbstore/src/sync/doc/index.ts b/packages/common/nbstore/src/sync/doc/index.ts index b179c2b99..db9e30e3d 100644 --- a/packages/common/nbstore/src/sync/doc/index.ts +++ b/packages/common/nbstore/src/sync/doc/index.ts @@ -1,5 +1,5 @@ import type { Observable } from 'rxjs'; -import { combineLatest, map, of } from 'rxjs'; +import { combineLatest, map, of, ReplaySubject, share } from 'rxjs'; import type { DocStorage, SyncStorage } from '../../storage'; import { DummyDocStorage } from '../../storage/dummy/doc'; @@ -38,18 +38,32 @@ export class DocSyncImpl implements DocSync { ); private abort: AbortController | null = null; - get state$() { - return combineLatest(this.peers.map(peer => peer.peerState$)).pipe( - map(allPeers => ({ - total: allPeers.reduce((acc, peer) => Math.max(acc, peer.total), 0), - syncing: allPeers.reduce((acc, peer) => Math.max(acc, peer.syncing), 0), - synced: allPeers.every(peer => peer.synced), - retrying: allPeers.some(peer => peer.retrying), - errorMessage: - allPeers.find(peer => peer.errorMessage)?.errorMessage ?? null, - })) - ) as Observable; - } + state$ = combineLatest(this.peers.map(peer => peer.peerState$)).pipe( + map(allPeers => + allPeers.length === 0 + ? { + total: 0, + syncing: 0, + synced: true, + retrying: false, + errorMessage: null, + } + : { + total: allPeers.reduce((acc, peer) => Math.max(acc, peer.total), 0), + syncing: allPeers.reduce( + (acc, peer) => Math.max(acc, peer.syncing), + 0 + ), + synced: allPeers.every(peer => peer.synced), + retrying: allPeers.some(peer => peer.retrying), + errorMessage: + allPeers.find(peer => peer.errorMessage)?.errorMessage ?? null, + } + ), + share({ + connector: () => new ReplaySubject(1), + }) + ) as Observable; constructor( readonly storages: PeerStorageOptions, @@ -105,7 +119,7 @@ export class DocSyncImpl implements DocSync { } stop() { - this.abort?.abort(); + this.abort?.abort(MANUALLY_STOP); this.abort = null; } diff --git a/packages/common/nbstore/src/sync/doc/peer.ts b/packages/common/nbstore/src/sync/doc/peer.ts index c12a4629f..b38e7ee48 100644 --- a/packages/common/nbstore/src/sync/doc/peer.ts +++ b/packages/common/nbstore/src/sync/doc/peer.ts @@ -1,6 +1,6 @@ import { remove } from 'lodash-es'; import { nanoid } from 'nanoid'; -import { Observable, Subject } from 'rxjs'; +import { Observable, ReplaySubject, share, Subject } from 'rxjs'; import { diffUpdate, encodeStateVectorFromUpdate, mergeUpdates } from 'yjs'; import type { DocStorage, SyncStorage } from '../../storage'; @@ -119,54 +119,65 @@ export class DocSyncPeer { }; private readonly statusUpdatedSubject$ = new Subject(); - get peerState$() { - return new Observable(subscribe => { - const next = () => { - if (this.status.skipped) { - subscribe.next({ - total: 0, - syncing: 0, - synced: true, - retrying: false, - errorMessage: null, - }); - } else if (!this.status.syncing) { - // if syncing = false, jobMap is empty - subscribe.next({ - total: this.status.docs.size, - syncing: this.status.docs.size, - synced: false, - retrying: this.status.retrying, - errorMessage: this.status.errorMessage, - }); - } else { - const syncing = this.status.jobMap.size; - subscribe.next({ - total: this.status.docs.size, - syncing: syncing, - retrying: this.status.retrying, - errorMessage: this.status.errorMessage, - synced: syncing === 0, - }); - } - }; + peerState$ = new Observable(subscribe => { + const next = () => { + if (this.status.skipped) { + subscribe.next({ + total: 0, + syncing: 0, + synced: true, + retrying: false, + errorMessage: null, + }); + } else if (!this.status.syncing) { + // if syncing = false, jobMap is empty + subscribe.next({ + total: this.status.docs.size, + syncing: this.status.docs.size, + synced: false, + retrying: this.status.retrying, + errorMessage: this.status.errorMessage, + }); + } else { + const syncing = this.status.jobMap.size; + subscribe.next({ + total: this.status.docs.size, + syncing: syncing, + retrying: this.status.retrying, + errorMessage: this.status.errorMessage, + synced: syncing === 0, + }); + } + }; + next(); + const dispose = this.statusUpdatedSubject$.subscribe(() => { next(); - return this.statusUpdatedSubject$.subscribe(() => { - next(); - }); }); - } + return () => { + dispose.unsubscribe(); + }; + }).pipe( + share({ + connector: () => new ReplaySubject(1), + }) + ); docState$(docId: string) { return new Observable(subscribe => { const next = () => { - const syncing = - !this.status.connectedDocs.has(docId) || - this.status.jobMap.has(docId); - + if (this.status.skipped) { + subscribe.next({ + syncing: false, + synced: true, + retrying: false, + errorMessage: null, + }); + } subscribe.next({ - syncing: syncing, - synced: !syncing, + syncing: + !this.status.connectedDocs.has(docId) || + this.status.jobMap.has(docId), + synced: !this.status.jobMap.has(docId), retrying: this.status.retrying, errorMessage: this.status.errorMessage, }); @@ -524,10 +535,6 @@ export class DocSyncPeer { const disposes: (() => void)[] = []; try { - console.info('Remote sync started'); - this.status.syncing = true; - this.statusUpdatedSubject$.next(true); - // wait for all storages to connect, timeout after 30s await Promise.race([ Promise.all([ @@ -547,6 +554,10 @@ export class DocSyncPeer { }), ]); + console.info('Remote sync started'); + this.status.syncing = true; + this.statusUpdatedSubject$.next(true); + // throw error if failed to connect for (const storage of [this.remote, this.local, this.syncMetadata]) { // abort if disconnected diff --git a/packages/common/nbstore/src/utils/id-converter.ts b/packages/common/nbstore/src/utils/id-converter.ts new file mode 100644 index 000000000..a2d96b41b --- /dev/null +++ b/packages/common/nbstore/src/utils/id-converter.ts @@ -0,0 +1,73 @@ +import { + applyUpdate, + type Array as YArray, + Doc as YDoc, + type Map as YMap, +} from 'yjs'; + +type PromiseResult = T extends Promise ? R : never; +export type IdConverter = PromiseResult>; + +export async function getIdConverter( + storage: { + getDocBuffer: (id: string) => Promise; + }, + spaceId: string +) { + const oldIdToNewId = { [spaceId]: spaceId }; + const newIdToOldId = { [spaceId]: spaceId }; + + const rootDocBuffer = await storage.getDocBuffer(spaceId); + if (rootDocBuffer) { + const ydoc = new YDoc({ + guid: spaceId, + }); + applyUpdate(ydoc, rootDocBuffer); + + // get all ids from rootDoc.meta.pages.[*].id, trust this id as normalized id + const normalizedDocIds = ( + (ydoc.getMap('meta') as YMap | undefined)?.get('pages') as + | YArray> + | undefined + ) + ?.map(i => i.get('id') as string) + .filter(i => !!i); + + const spaces = ydoc.getMap('spaces') as YMap | undefined; + for (const pageId of normalizedDocIds ?? []) { + const subdoc = spaces?.get(pageId); + if (subdoc && subdoc instanceof YDoc) { + oldIdToNewId[subdoc.guid] = pageId; + newIdToOldId[pageId] = subdoc.guid; + } + } + } + + return { + newIdToOldId(newId: string) { + if (newId.startsWith(`db$`)) { + // db$docId -> db$${spaceId}$docId + return newId.replace(`db$`, `db$${spaceId}$`); + } + if (newId.startsWith(`userdata$`)) { + // userdata$userId$docId -> userdata$userId$spaceId$docId + return newId.replace( + new RegExp(`^(userdata\\$[\\w-]+)\\$([^\\$]+)`), + (_, p1, p2) => `${p1}$${spaceId}$${p2}` + ); + } + return newIdToOldId[newId] ?? newId; + }, + oldIdToNewId(oldId: string) { + // db$${spaceId}$docId -> db$docId + if (oldId.startsWith(`db$${spaceId}$`)) { + return oldId.replace(`db$${spaceId}$`, `db$`); + } + // userdata$userId$spaceId$docId -> userdata$userId$docId + if (oldId.match(new RegExp(`^userdata\\$[\\w-]+\\$${spaceId}$`))) { + return oldId.replace(`$${spaceId}$`, '$'); + } + return oldIdToNewId[oldId] ?? oldId; + }, + }; +} diff --git a/packages/common/nbstore/src/worker/client.ts b/packages/common/nbstore/src/worker/client.ts index bc7be3c6f..8e92c62ca 100644 --- a/packages/common/nbstore/src/worker/client.ts +++ b/packages/common/nbstore/src/worker/client.ts @@ -23,7 +23,6 @@ export class WorkerClient { private readonly client: OpClient, options: WorkerInitOptions ) { - client.listen(); this.client.call('worker.init', options).catch(err => { console.error('error initializing worker', err); }); @@ -156,7 +155,9 @@ class WorkerBlobStorage implements BlobStorage { class WorkerDocSync implements DocSync { constructor(private readonly client: OpClient) {} - readonly state$ = this.client.ob$('docSync.state'); + get state$() { + return this.client.ob$('docSync.state'); + } docState$(docId: string) { return this.client.ob$('docSync.docState', docId); @@ -174,7 +175,9 @@ class WorkerDocSync implements DocSync { class WorkerBlobSync implements BlobSync { constructor(private readonly client: OpClient) {} - readonly state$ = this.client.ob$('blobSync.state'); + get state$() { + return this.client.ob$('blobSync.state'); + } setMaxBlobSize(size: number): void { this.client.call('blobSync.setMaxBlobSize', size).catch(err => { console.error('error setting max blob size', err); diff --git a/packages/common/nbstore/src/worker/consumer.ts b/packages/common/nbstore/src/worker/consumer.ts index fd8ba71cd..0cd2d0b77 100644 --- a/packages/common/nbstore/src/worker/consumer.ts +++ b/packages/common/nbstore/src/worker/consumer.ts @@ -1,3 +1,4 @@ +import { MANUALLY_STOP } from '@toeverything/infra'; import type { OpConsumer } from '@toeverything/infra/op'; import { Observable } from 'rxjs'; @@ -11,6 +12,7 @@ import type { WorkerInitOptions, WorkerOps } from './ops'; export type { WorkerOps }; export class WorkerConsumer { + private inited = false; private storages: PeerStorageOptions | null = null; private sync: Sync | null = null; @@ -57,14 +59,18 @@ export class WorkerConsumer { } constructor( - private readonly consumer: OpConsumer, private readonly availableStorageImplementations: StorageConstructor[] - ) { - this.registerHandlers(); - this.consumer.listen(); + ) {} + + bindConsumer(consumer: OpConsumer) { + this.registerHandlers(consumer); } init(init: WorkerInitOptions) { + if (this.inited) { + return; + } + this.inited = true; this.storages = { local: new SpaceStorage( Object.fromEntries( @@ -120,13 +126,13 @@ export class WorkerConsumer { } } - private registerHandlers() { + private registerHandlers(consumer: OpConsumer) { const collectJobs = new Map< string, (awareness: AwarenessRecord | null) => void >(); let collectId = 0; - this.consumer.registerAll({ + consumer.registerAll({ 'worker.init': this.init.bind(this), 'worker.destroy': this.destroy.bind(this), 'docStorage.getDoc': (docId: string) => this.docStorage.getDoc(docId), @@ -158,7 +164,7 @@ export class WorkerConsumer { .catch((error: any) => { subscriber.error(error); }); - return () => abortController.abort(); + return () => abortController.abort(MANUALLY_STOP); }), 'blobStorage.getBlob': key => this.blobStorage.get(key), 'blobStorage.setBlob': blob => this.blobStorage.set(blob), @@ -212,13 +218,7 @@ export class WorkerConsumer { }), 'awarenessStorage.collect': ({ collectId, awareness }) => collectJobs.get(collectId)?.(awareness), - 'docSync.state': () => - new Observable(subscriber => { - const subscription = this.docSync.state$.subscribe(state => { - subscriber.next(state); - }); - return () => subscription.unsubscribe(); - }), + 'docSync.state': () => this.docSync.state$, 'docSync.docState': docId => new Observable(subscriber => { const subscription = this.docSync @@ -247,7 +247,7 @@ export class WorkerConsumer { .catch(error => { subscriber.error(error); }); - return () => abortController.abort(); + return () => abortController.abort(MANUALLY_STOP); }), 'blobSync.state': () => this.blobSync.state$, 'blobSync.setMaxBlobSize': size => this.blobSync.setMaxBlobSize(size), @@ -262,7 +262,7 @@ export class WorkerConsumer { this.awarenessSync.update(awareness, origin), 'awarenessSync.subscribeUpdate': docId => new Observable(subscriber => { - return this.awarenessStorage.subscribeUpdate( + return this.awarenessSync.subscribeUpdate( docId, (update, origin) => { subscriber.next({ @@ -279,6 +279,10 @@ export class WorkerConsumer { collectJobs.delete(currentCollectId.toString()); }); }); + subscriber.next({ + type: 'awareness-collect', + collectId: currentCollectId.toString(), + }); return promise; } ); diff --git a/packages/frontend/apps/android/src/app.tsx b/packages/frontend/apps/android/src/app.tsx index 03f701202..9c662d47b 100644 --- a/packages/frontend/apps/android/src/app.tsx +++ b/packages/frontend/apps/android/src/app.tsx @@ -6,12 +6,8 @@ import { configureCommonModules } from '@affine/core/modules'; import { I18nProvider } from '@affine/core/modules/i18n'; import { LifecycleService } from '@affine/core/modules/lifecycle'; import { configureLocalStorageStateStorageImpls } from '@affine/core/modules/storage'; -import { configureIndexedDBUserspaceStorageProvider } from '@affine/core/modules/userspace'; import { configureBrowserWorkbenchModule } from '@affine/core/modules/workbench'; -import { - configureBrowserWorkspaceFlavours, - configureIndexedDBWorkspaceEngineStorageProvider, -} from '@affine/core/modules/workspace-engine'; +import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine'; import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra'; import { Suspense } from 'react'; import { RouterProvider } from 'react-router-dom'; @@ -25,8 +21,6 @@ configureCommonModules(framework); configureBrowserWorkbenchModule(framework); configureLocalStorageStateStorageImpls(framework); configureBrowserWorkspaceFlavours(framework); -configureIndexedDBWorkspaceEngineStorageProvider(framework); -configureIndexedDBUserspaceStorageProvider(framework); configureMobileModules(framework); const frameworkProvider = framework.provider(); diff --git a/packages/frontend/apps/electron-renderer/package.json b/packages/frontend/apps/electron-renderer/package.json index f868d6078..609225c93 100644 --- a/packages/frontend/apps/electron-renderer/package.json +++ b/packages/frontend/apps/electron-renderer/package.json @@ -12,11 +12,13 @@ "@affine/core": "workspace:*", "@affine/electron-api": "workspace:*", "@affine/i18n": "workspace:*", + "@affine/nbstore": "workspace:*", "@emotion/react": "^11.14.0", "@sentry/react": "^8.44.0", "@toeverything/infra": "workspace:*", "@toeverything/theme": "^1.1.3", "@vanilla-extract/css": "^1.16.1", + "async-call-rpc": "^6.4.2", "next-themes": "^0.4.4", "react": "^19.0.0", "react-dom": "^19.0.0", diff --git a/packages/frontend/apps/electron-renderer/src/app.tsx b/packages/frontend/apps/electron-renderer/src/app.tsx index 83a1a2add..7ec15d171 100644 --- a/packages/frontend/apps/electron-renderer/src/app.tsx +++ b/packages/frontend/apps/electron-renderer/src/app.tsx @@ -19,25 +19,26 @@ import { configureFindInPageModule } from '@affine/core/modules/find-in-page'; import { GlobalContextService } from '@affine/core/modules/global-context'; import { I18nProvider } from '@affine/core/modules/i18n'; import { LifecycleService } from '@affine/core/modules/lifecycle'; -import { configureElectronStateStorageImpls } from '@affine/core/modules/storage'; +import { + configureElectronStateStorageImpls, + NbstoreProvider, +} from '@affine/core/modules/storage'; import { ClientSchemeProvider, PopupWindowProvider, } from '@affine/core/modules/url'; -import { configureSqliteUserspaceStorageProvider } from '@affine/core/modules/userspace'; import { configureDesktopWorkbenchModule, WorkbenchService, } from '@affine/core/modules/workbench'; import { WorkspacesService } from '@affine/core/modules/workspace'; -import { - configureBrowserWorkspaceFlavours, - configureSqliteWorkspaceEngineStorageProvider, -} from '@affine/core/modules/workspace-engine'; +import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine'; import createEmotionCache from '@affine/core/utils/create-emotion-cache'; import { apis, events } from '@affine/electron-api'; +import { WorkerClient } from '@affine/nbstore/worker/client'; import { CacheProvider } from '@emotion/react'; import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra'; +import { OpClient } from '@toeverything/infra/op'; import { Suspense } from 'react'; import { RouterProvider } from 'react-router-dom'; @@ -71,14 +72,61 @@ const framework = new Framework(); configureCommonModules(framework); configureElectronStateStorageImpls(framework); configureBrowserWorkspaceFlavours(framework); -configureSqliteWorkspaceEngineStorageProvider(framework); -configureSqliteUserspaceStorageProvider(framework); configureDesktopWorkbenchModule(framework); configureAppTabsHeaderModule(framework); configureFindInPageModule(framework); configureDesktopApiModule(framework); configureSpellCheckSettingModule(framework); +framework.impl(NbstoreProvider, { + openStore(key, options) { + const { port1: portForOpClient, port2: portForWorker } = + new MessageChannel(); + let portFromWorker: MessagePort | null = null; + let portId = crypto.randomUUID(); + const handleMessage = (ev: MessageEvent) => { + if ( + ev.data.type === 'electron:worker-connect' && + ev.data.portId === portId + ) { + portFromWorker = ev.ports[0]; + // connect portForWorker and portFromWorker + portFromWorker.addEventListener('message', ev => { + portForWorker.postMessage(ev.data); + }); + portForWorker.addEventListener('message', ev => { + // oxlint-disable-next-line no-non-null-assertion + portFromWorker!.postMessage(ev.data); + }); + portForWorker.start(); + portFromWorker.start(); + } + }; + + window.addEventListener('message', handleMessage); + + // oxlint-disable-next-line no-non-null-assertion + apis!.worker.connectWorker(key, portId).catch(err => { + console.error('failed to connect worker', err); + }); + + const store = new WorkerClient(new OpClient(portForOpClient), options); + portForOpClient.start(); + return { + store, + dispose: () => { + window.removeEventListener('message', handleMessage); + portForOpClient.close(); + portForWorker.close(); + portFromWorker?.close(); + // oxlint-disable-next-line no-non-null-assertion + apis!.worker.disconnectWorker(key, portId).catch(err => { + console.error('failed to disconnect worker', err); + }); + }, + }; + }, +}); framework.impl(PopupWindowProvider, p => { const apis = p.get(DesktopApiService).api; return { diff --git a/packages/frontend/apps/electron-renderer/src/background-worker/index.ts b/packages/frontend/apps/electron-renderer/src/background-worker/index.ts new file mode 100644 index 000000000..f9b2aaff0 --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/background-worker/index.ts @@ -0,0 +1,36 @@ +import '@affine/core/bootstrap/electron'; + +import { apis } from '@affine/electron-api'; +import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel'; +import { cloudStorages } from '@affine/nbstore/cloud'; +import { bindNativeDBApis, sqliteStorages } from '@affine/nbstore/sqlite'; +import { + bindNativeDBV1Apis, + sqliteV1Storages, +} from '@affine/nbstore/sqlite/v1'; +import { + WorkerConsumer, + type WorkerOps, +} from '@affine/nbstore/worker/consumer'; +import { OpConsumer } from '@toeverything/infra/op'; + +// oxlint-disable-next-line no-non-null-assertion +bindNativeDBApis(apis!.nbstore); +// oxlint-disable-next-line no-non-null-assertion +bindNativeDBV1Apis(apis!.db); + +const worker = new WorkerConsumer([ + ...sqliteStorages, + ...sqliteV1Storages, + ...broadcastChannelStorages, + ...cloudStorages, +]); + +window.addEventListener('message', ev => { + if (ev.data.type === 'electron:worker-connect') { + const port = ev.ports[0]; + + const consumer = new OpConsumer(port); + worker.bindConsumer(consumer); + } +}); diff --git a/packages/frontend/apps/electron-renderer/src/nbstore.ts b/packages/frontend/apps/electron-renderer/src/nbstore.ts new file mode 100644 index 000000000..d4c33c551 --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/nbstore.ts @@ -0,0 +1,96 @@ +import '@affine/core/bootstrap/electron'; + +import type { ClientHandler } from '@affine/electron-api'; +import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel'; +import { cloudStorages } from '@affine/nbstore/cloud'; +import { bindNativeDBApis, sqliteStorages } from '@affine/nbstore/sqlite'; +import { + bindNativeDBV1Apis, + sqliteV1Storages, +} from '@affine/nbstore/sqlite/v1'; +import { + WorkerConsumer, + type WorkerOps, +} from '@affine/nbstore/worker/consumer'; +import { OpConsumer } from '@toeverything/infra/op'; +import { AsyncCall } from 'async-call-rpc'; + +const worker = new WorkerConsumer([ + ...sqliteStorages, + ...sqliteV1Storages, + ...broadcastChannelStorages, + ...cloudStorages, +]); + +let activeConnectionCount = 0; +let electronAPIsInitialized = false; + +function connectElectronAPIs(port: MessagePort) { + if (electronAPIsInitialized) { + return; + } + electronAPIsInitialized = true; + port.postMessage({ type: '__electron-apis-init__' }); + + const { promise, resolve } = Promise.withResolvers(); + port.addEventListener('message', event => { + if (event.data.type === '__electron-apis__') { + const [port] = event.ports; + resolve(port); + } + }); + + const rpc = AsyncCall>(null, { + channel: promise.then(p => ({ + on(listener) { + p.onmessage = e => { + listener(e.data); + }; + p.start(); + return () => { + p.onmessage = null; + try { + p.close(); + } catch (err) { + console.error('close port error', err); + } + }; + }, + send(data) { + p.postMessage(data); + }, + })), + log: false, + }); + + const electronAPIs = new Proxy(rpc as any, { + get(_, namespace: string) { + return new Proxy(rpc as any, { + get(_, method: string) { + return rpc[`${namespace}:${method}`]; + }, + }); + }, + }); + + bindNativeDBApis(electronAPIs.nbstore); + bindNativeDBV1Apis(electronAPIs.db); +} + +(globalThis as any).onconnect = (event: MessageEvent) => { + activeConnectionCount++; + const port = event.ports[0]; + port.addEventListener('message', (event: MessageEvent) => { + if (event.data.type === '__close__') { + activeConnectionCount--; + if (activeConnectionCount === 0) { + globalThis.close(); + } + } + }); + + connectElectronAPIs(port); + + const consumer = new OpConsumer(port); + worker.bindConsumer(consumer); +}; diff --git a/packages/frontend/apps/electron-renderer/src/setup.ts b/packages/frontend/apps/electron-renderer/src/setup.ts index 2b5935864..931f198f8 100644 --- a/packages/frontend/apps/electron-renderer/src/setup.ts +++ b/packages/frontend/apps/electron-renderer/src/setup.ts @@ -1,3 +1,12 @@ import '@affine/core/bootstrap/electron'; import '@affine/component/theme'; import './global.css'; + +import { apis } from '@affine/electron-api'; +import { bindNativeDBApis } from '@affine/nbstore/sqlite'; +import { bindNativeDBV1Apis } from '@affine/nbstore/sqlite/v1'; + +// oxlint-disable-next-line no-non-null-assertion +bindNativeDBApis(apis!.nbstore); +// oxlint-disable-next-line no-non-null-assertion +bindNativeDBV1Apis(apis!.db); diff --git a/packages/frontend/apps/electron-renderer/src/shell/app.tsx b/packages/frontend/apps/electron-renderer/src/shell/app.tsx index 0a4b964fc..f22a3d032 100644 --- a/packages/frontend/apps/electron-renderer/src/shell/app.tsx +++ b/packages/frontend/apps/electron-renderer/src/shell/app.tsx @@ -11,7 +11,7 @@ import { configureDesktopApiModule } from '@affine/core/modules/desktop-api'; import { configureI18nModule, I18nProvider } from '@affine/core/modules/i18n'; import { configureElectronStateStorageImpls, - configureGlobalStorageModule, + configureStorageModule, } from '@affine/core/modules/storage'; import { configureAppThemeModule } from '@affine/core/modules/theme'; import { Framework, FrameworkRoot } from '@toeverything/infra'; @@ -19,7 +19,7 @@ import { Framework, FrameworkRoot } from '@toeverything/infra'; import * as styles from './app.css'; const framework = new Framework(); -configureGlobalStorageModule(framework); +configureStorageModule(framework); configureElectronStateStorageImpls(framework); configureAppTabsHeaderModule(framework); configureAppSidebarModule(framework); diff --git a/packages/frontend/apps/electron-renderer/tsconfig.json b/packages/frontend/apps/electron-renderer/tsconfig.json index 58135b383..19af95cc4 100644 --- a/packages/frontend/apps/electron-renderer/tsconfig.json +++ b/packages/frontend/apps/electron-renderer/tsconfig.json @@ -12,6 +12,7 @@ { "path": "../../core" }, { "path": "../../electron-api" }, { "path": "../../i18n" }, + { "path": "../../../common/nbstore" }, { "path": "../../../common/infra" }, { "path": "../../../../tools/utils" } ] diff --git a/packages/frontend/apps/electron-renderer/webpack.config.ts b/packages/frontend/apps/electron-renderer/webpack.config.ts index ed6471690..07fee611f 100644 --- a/packages/frontend/apps/electron-renderer/webpack.config.ts +++ b/packages/frontend/apps/electron-renderer/webpack.config.ts @@ -2,5 +2,6 @@ export const config = { entry: { app: './src/index.tsx', shell: './src/shell/index.tsx', + backgroundWorker: './src/background-worker/index.ts', }, }; diff --git a/packages/frontend/apps/electron/src/main/application-menu/create.ts b/packages/frontend/apps/electron/src/main/application-menu/create.ts index eb3211f45..32b140f1c 100644 --- a/packages/frontend/apps/electron/src/main/application-menu/create.ts +++ b/packages/frontend/apps/electron/src/main/application-menu/create.ts @@ -15,6 +15,7 @@ import { switchToPreviousTab, undoCloseTab, } from '../windows-manager'; +import { WorkerManager } from '../worker/pool'; import { applicationMenuSubjects } from './subject'; // Unique id for menuitems @@ -113,6 +114,21 @@ export function createApplicationMenu() { showDevTools(); }, }, + { + label: 'Open worker devtools', + click: () => { + Menu.buildFromTemplate( + Array.from(WorkerManager.instance.workers.values()).map(item => ({ + label: `${item.key}`, + click: () => { + item.browserWindow.webContents.openDevTools({ + mode: 'undocked', + }); + }, + })) + ).popup(); + }, + }, { type: 'separator' }, { role: 'resetZoom' }, { role: 'zoomIn' }, @@ -199,7 +215,7 @@ export function createApplicationMenu() { { label: 'Learn More', click: async () => { - // oxlint-disable-next-line + // oxlint-disable-next-line no-var-requires const { shell } = require('electron'); await shell.openExternal('https://affine.pro/'); }, @@ -220,7 +236,7 @@ export function createApplicationMenu() { { label: 'Documentation', click: async () => { - // oxlint-disable-next-line + // oxlint-disable-next-line no-var-requires const { shell } = require('electron'); await shell.openExternal( 'https://docs.affine.pro/docs/hello-bonjour-aloha-你好' diff --git a/packages/frontend/apps/electron/src/main/constants.ts b/packages/frontend/apps/electron/src/main/constants.ts index 734db54f5..eecbb1c70 100644 --- a/packages/frontend/apps/electron/src/main/constants.ts +++ b/packages/frontend/apps/electron/src/main/constants.ts @@ -1,4 +1,5 @@ export const mainWindowOrigin = process.env.DEV_SERVER_URL || 'file://.'; export const onboardingViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith('/') ? '' : '/'}onboarding`; export const shellViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith('/') ? '' : '/'}shell.html`; +export const backgroundWorkerViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith('/') ? '' : '/'}background-worker.html`; export const customThemeViewUrl = `${mainWindowOrigin}${mainWindowOrigin.endsWith('/') ? '' : '/'}theme-editor`; diff --git a/packages/frontend/apps/electron/src/main/handlers.ts b/packages/frontend/apps/electron/src/main/handlers.ts index e67b39c64..59ef7a0d5 100644 --- a/packages/frontend/apps/electron/src/main/handlers.ts +++ b/packages/frontend/apps/electron/src/main/handlers.ts @@ -8,6 +8,7 @@ import { getLogFilePath, logger, revealLogFile } from './logger'; import { sharedStorageHandlers } from './shared-storage'; import { uiHandlers } from './ui/handlers'; import { updaterHandlers } from './updater'; +import { workerHandlers } from './worker/handlers'; export const debugHandlers = { revealLogFile: async () => { @@ -27,6 +28,7 @@ export const allHandlers = { configStorage: configStorageHandlers, findInPage: findInPageHandlers, sharedStorage: sharedStorageHandlers, + worker: workerHandlers, }; export const registerHandlers = () => { diff --git a/packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts b/packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts index b0c818938..f92648dd5 100644 --- a/packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts +++ b/packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts @@ -25,7 +25,6 @@ import { import { isMacOS } from '../../shared/utils'; import { beforeAppQuit } from '../cleanup'; -import { isDev } from '../config'; import { mainWindowOrigin, shellViewUrl } from '../constants'; import { ensureHelperProcess } from '../helper-process'; import { logger } from '../logger'; @@ -871,9 +870,6 @@ export class WebContentViewsManager { }); view.webContents.loadURL(shellViewUrl).catch(err => logger.error(err)); - if (isDev) { - view.webContents.openDevTools(); - } } view.webContents.on('destroyed', () => { diff --git a/packages/frontend/apps/electron/src/main/worker/handlers.ts b/packages/frontend/apps/electron/src/main/worker/handlers.ts new file mode 100644 index 000000000..ccb4fa9ec --- /dev/null +++ b/packages/frontend/apps/electron/src/main/worker/handlers.ts @@ -0,0 +1,19 @@ +import type { NamespaceHandlers } from '../type'; +import { WorkerManager } from './pool'; + +export const workerHandlers = { + connectWorker: async (e, key: string, portId: string) => { + const { portForRenderer } = await WorkerManager.instance.connectWorker( + key, + portId, + e.sender + ); + e.sender.postMessage('worker-connect', { portId }, [portForRenderer]); + return { + portId: portId, + }; + }, + disconnectWorker: async (_, key: string, portId: string) => { + WorkerManager.instance.disconnectWorker(key, portId); + }, +} satisfies NamespaceHandlers; diff --git a/packages/frontend/apps/electron/src/main/worker/pool.ts b/packages/frontend/apps/electron/src/main/worker/pool.ts new file mode 100644 index 000000000..2ee3c965c --- /dev/null +++ b/packages/frontend/apps/electron/src/main/worker/pool.ts @@ -0,0 +1,96 @@ +import { join } from 'node:path'; + +import { BrowserWindow, MessageChannelMain, type WebContents } from 'electron'; + +import { backgroundWorkerViewUrl } from '../constants'; +import { ensureHelperProcess } from '../helper-process'; +import { logger } from '../logger'; + +async function getAdditionalArguments() { + const { getExposedMeta } = await import('../exposed'); + const mainExposedMeta = getExposedMeta(); + const helperProcessManager = await ensureHelperProcess(); + const helperExposedMeta = await helperProcessManager.rpc?.getMeta(); + return [ + `--main-exposed-meta=` + JSON.stringify(mainExposedMeta), + `--helper-exposed-meta=` + JSON.stringify(helperExposedMeta), + `--window-name=worker`, + ]; +} + +export class WorkerManager { + static readonly instance = new WorkerManager(); + + workers = new Map< + string, + { browserWindow: BrowserWindow; ports: Set; key: string } + >(); + + private async getOrCreateWorker(key: string) { + const additionalArguments = await getAdditionalArguments(); + const helperProcessManager = await ensureHelperProcess(); + const exists = this.workers.get(key); + if (exists) { + return exists; + } else { + const worker = new BrowserWindow({ + width: 1200, + height: 600, + webPreferences: { + preload: join(__dirname, './preload.js'), + additionalArguments: additionalArguments, + }, + show: false, + }); + let disconnectHelperProcess: (() => void) | null = null; + worker.on('close', e => { + e.preventDefault(); + if (worker && !worker.isDestroyed()) { + worker.destroy(); + this.workers.delete(key); + disconnectHelperProcess?.(); + } + }); + worker.loadURL(backgroundWorkerViewUrl).catch(e => { + logger.error('failed to load url', e); + }); + worker.webContents.addListener('did-finish-load', () => { + disconnectHelperProcess = helperProcessManager.connectRenderer( + worker.webContents + ); + }); + const record = { browserWindow: worker, ports: new Set(), key }; + this.workers.set(key, record); + return record; + } + } + + async connectWorker( + key: string, + portId: string, + bindWebContent: WebContents + ) { + bindWebContent.addListener('destroyed', () => { + this.disconnectWorker(key, portId); + }); + const worker = await this.getOrCreateWorker(key); + const { port1: portForWorker, port2: portForRenderer } = + new MessageChannelMain(); + + worker.browserWindow.webContents.postMessage('worker-connect', { portId }, [ + portForWorker, + ]); + return { portForRenderer, portId }; + } + + disconnectWorker(key: string, portId: string) { + const worker = this.workers.get(key); + if (worker) { + worker.ports.delete(portId); + if (worker.ports.size === 0) { + worker.browserWindow.destroy(); + this.workers.delete(key); + } + } + } +} diff --git a/packages/frontend/apps/electron/src/preload/bootstrap.ts b/packages/frontend/apps/electron/src/preload/bootstrap.ts index 8bbce9752..2aee63cc2 100644 --- a/packages/frontend/apps/electron/src/preload/bootstrap.ts +++ b/packages/frontend/apps/electron/src/preload/bootstrap.ts @@ -2,11 +2,13 @@ import '@sentry/electron/preload'; import { contextBridge } from 'electron'; -import { apis, appInfo, events, requestWebWorkerPort } from './electron-api'; +import { apis, appInfo, events } from './electron-api'; import { sharedStorage } from './shared-storage'; +import { listenWorkerApis } from './worker'; contextBridge.exposeInMainWorld('__appInfo', appInfo); contextBridge.exposeInMainWorld('__apis', apis); contextBridge.exposeInMainWorld('__events', events); contextBridge.exposeInMainWorld('__sharedStorage', sharedStorage); -contextBridge.exposeInMainWorld('__requestWebWorkerPort', requestWebWorkerPort); + +listenWorkerApis(); diff --git a/packages/frontend/apps/electron/src/preload/electron-api.ts b/packages/frontend/apps/electron/src/preload/electron-api.ts index 26b864ecd..420d025e2 100644 --- a/packages/frontend/apps/electron/src/preload/electron-api.ts +++ b/packages/frontend/apps/electron/src/preload/electron-api.ts @@ -248,53 +248,3 @@ export const events = { ...mainAPIs.events, ...helperAPIs.events, }; - -/** - * Create MessagePort that can be used by web workers - * - * !!! - * SHOULD ONLY BE USED IN RENDERER PROCESS - * !!! - */ -export function requestWebWorkerPort() { - const ch = new MessageChannel(); - const localPort = ch.port1; - const remotePort = ch.port2; - - // todo: should be able to let the web worker use the electron APIs directly for better performance - const flattenedAPIs = Object.entries(apis).flatMap(([namespace, api]) => { - return Object.entries(api as any).map(([method, fn]) => [ - `${namespace}:${method}`, - fn, - ]); - }); - - AsyncCall(Object.fromEntries(flattenedAPIs), { - channel: createMessagePortChannel(localPort), - log: false, - }); - - const cleanup = () => { - remotePort.close(); - localPort.close(); - }; - - const portId = crypto.randomUUID(); - - setTimeout(() => { - // @ts-expect-error this function should only be evaluated in the renderer process - window.postMessage( - { - type: 'electron:request-api-port', - portId, - ports: [remotePort], - }, - '*', - [remotePort] - ); - }); - - localPort.start(); - - return { portId, cleanup }; -} diff --git a/packages/frontend/apps/electron/src/preload/worker.ts b/packages/frontend/apps/electron/src/preload/worker.ts new file mode 100644 index 000000000..de14de917 --- /dev/null +++ b/packages/frontend/apps/electron/src/preload/worker.ts @@ -0,0 +1,33 @@ +import { ipcRenderer } from 'electron'; + +export function listenWorkerApis() { + ipcRenderer.on('worker-connect', (ev, data) => { + const portForRenderer = ev.ports[0]; + + // @ts-expect-error this function should only be evaluated in the renderer process + if (document.readyState === 'complete') { + // @ts-expect-error this function should only be evaluated in the renderer process + window.postMessage( + { + type: 'electron:worker-connect', + portId: data.portId, + }, + '*', + [portForRenderer] + ); + } else { + // @ts-expect-error this function should only be evaluated in the renderer process + window.addEventListener('load', () => { + // @ts-expect-error this function should only be evaluated in the renderer process + window.postMessage( + { + type: 'electron:worker-connect', + portId: data.portId, + }, + '*', + [portForRenderer] + ); + }); + } + }); +} diff --git a/packages/frontend/apps/ios/App/App.xcodeproj/project.pbxproj b/packages/frontend/apps/ios/App/App.xcodeproj/project.pbxproj index 32d212cfd..8cb8cf30b 100644 --- a/packages/frontend/apps/ios/App/App.xcodeproj/project.pbxproj +++ b/packages/frontend/apps/ios/App/App.xcodeproj/project.pbxproj @@ -24,6 +24,9 @@ 9D90BE2B2CCB9876006677DB /* config.xml in Resources */ = {isa = PBXBuildFile; fileRef = 9D90BE1F2CCB9876006677DB /* config.xml */; }; 9D90BE2D2CCB9876006677DB /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 9D90BE222CCB9876006677DB /* Main.storyboard */; }; 9D90BE2E2CCB9876006677DB /* public in Resources */ = {isa = PBXBuildFile; fileRef = 9D90BE232CCB9876006677DB /* public */; }; + 9DEC593B2D3002E70027CEBD /* AffineHttpHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9DEC593A2D3002C70027CEBD /* AffineHttpHandler.swift */; }; + 9DEC593F2D30EFA40027CEBD /* AffineWsHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9DEC593E2D30EFA40027CEBD /* AffineWsHandler.swift */; }; + 9DEC59432D323EE40027CEBD /* Mutex.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9DEC59422D323EE00027CEBD /* Mutex.swift */; }; 9DFCD1462D27D1D70028C92B /* libaffine_mobile_native.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 9DFCD1452D27D1D70028C92B /* libaffine_mobile_native.a */; }; C4C413792CBE705D00337889 /* Pods_App.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = AF277DCFFFF123FFC6DF26C7 /* Pods_App.framework */; }; C4C97C7C2D030BE000BC2AD1 /* affine_mobile_native.swift in Sources */ = {isa = PBXBuildFile; fileRef = C4C97C6F2D0307B700BC2AD1 /* affine_mobile_native.swift */; }; @@ -52,6 +55,9 @@ 9D90BE202CCB9876006677DB /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 9D90BE212CCB9876006677DB /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; 9D90BE232CCB9876006677DB /* public */ = {isa = PBXFileReference; lastKnownFileType = folder; path = public; sourceTree = ""; }; + 9DEC593A2D3002C70027CEBD /* AffineHttpHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AffineHttpHandler.swift; sourceTree = ""; }; + 9DEC593E2D30EFA40027CEBD /* AffineWsHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AffineWsHandler.swift; sourceTree = ""; }; + 9DEC59422D323EE00027CEBD /* Mutex.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Mutex.swift; sourceTree = ""; }; 9DFCD1452D27D1D70028C92B /* libaffine_mobile_native.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = libaffine_mobile_native.a; sourceTree = ""; }; AF277DCFFFF123FFC6DF26C7 /* Pods_App.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_App.framework; sourceTree = BUILT_PRODUCTS_DIR; }; AF51FD2D460BCFE21FA515B2 /* Pods-App.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-App.release.xcconfig"; path = "Pods/Target Support Files/Pods-App/Pods-App.release.xcconfig"; sourceTree = ""; }; @@ -156,6 +162,9 @@ 9D90BE242CCB9876006677DB /* App */ = { isa = PBXGroup; children = ( + 9DEC59422D323EE00027CEBD /* Mutex.swift */, + 9DEC593A2D3002C70027CEBD /* AffineHttpHandler.swift */, + 9DEC593E2D30EFA40027CEBD /* AffineWsHandler.swift */, 9D52FC422D26CDB600105D0A /* JSValueContainerExt.swift */, 9D90BE1A2CCB9876006677DB /* Plugins */, 9D90BE1C2CCB9876006677DB /* AppDelegate.swift */, @@ -331,13 +340,16 @@ 9D52FC432D26CDBF00105D0A /* JSValueContainerExt.swift in Sources */, 5075136E2D1925BC00AD60C0 /* IntelligentsPlugin.swift in Sources */, 5075136A2D1924C600AD60C0 /* RootViewController.swift in Sources */, + 9DEC593B2D3002E70027CEBD /* AffineHttpHandler.swift in Sources */, C4C97C7C2D030BE000BC2AD1 /* affine_mobile_native.swift in Sources */, C4C97C7D2D030BE000BC2AD1 /* affine_mobile_nativeFFI.h in Sources */, C4C97C7E2D030BE000BC2AD1 /* affine_mobile_nativeFFI.modulemap in Sources */, E93B276C2CED92B1001409B8 /* NavigationGesturePlugin.swift in Sources */, + 9DEC59432D323EE40027CEBD /* Mutex.swift in Sources */, 9D90BE252CCB9876006677DB /* CookieManager.swift in Sources */, 9D90BE262CCB9876006677DB /* CookiePlugin.swift in Sources */, 9D6A85332CCF6DA700DAB35F /* HashcashPlugin.swift in Sources */, + 9DEC593F2D30EFA40027CEBD /* AffineWsHandler.swift in Sources */, 9D90BE272CCB9876006677DB /* AffineViewController.swift in Sources */, 9D90BE282CCB9876006677DB /* AppDelegate.swift in Sources */, ); diff --git a/packages/frontend/apps/ios/App/App/AffineHttpHandler.swift b/packages/frontend/apps/ios/App/App/AffineHttpHandler.swift new file mode 100644 index 000000000..6426858bf --- /dev/null +++ b/packages/frontend/apps/ios/App/App/AffineHttpHandler.swift @@ -0,0 +1,114 @@ +// +// RequestUrlSchemeHandler.swift +// App +// +// Created by EYHN on 2025/1/9. +// + +import WebKit + +enum AffineHttpError: Error { + case invalidOperation(reason: String), invalidState(reason: String) +} + +class AffineHttpHandler: NSObject, WKURLSchemeHandler { + func webView(_ webView: WKWebView, start urlSchemeTask: any WKURLSchemeTask) { + urlSchemeTask.stopped = Mutex.init(false) + guard let rawUrl = urlSchemeTask.request.url else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidOperation(reason: "bad request")) + return + } + guard let scheme = rawUrl.scheme else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidOperation(reason: "bad request")) + return + } + let httpProtocol = scheme == "affine-http" ? "http" : "https" + guard let urlComponents = URLComponents(url: rawUrl, resolvingAgainstBaseURL: true) else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidOperation(reason: "bad request")) + return + } + guard let host = urlComponents.host else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidOperation(reason: "bad url")) + return + } + let path = urlComponents.path + let query = urlComponents.query != nil ? "?\(urlComponents.query!)" : "" + guard let targetUrl = URL(string: "\(httpProtocol)://\(host)\(path)\(query)") else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidOperation(reason: "bad url")) + return + } + + var request = URLRequest(url: targetUrl); + request.httpMethod = urlSchemeTask.request.httpMethod; + request.httpShouldHandleCookies = true + request.httpBody = urlSchemeTask.request.httpBody + urlSchemeTask.request.allHTTPHeaderFields?.filter({ + key, value in + let normalizedKey = key.lowercased() + return normalizedKey == "content-type" || + normalizedKey == "content-length" || + normalizedKey == "accept" + }).forEach { + key, value in + request.setValue(value, forHTTPHeaderField: key) + } + + URLSession.shared.dataTask(with: request) { + rawData, rawResponse, error in + urlSchemeTask.stopped?.withLock({ + if $0 { + return + } + + if error != nil { + urlSchemeTask.didFailWithError(error!) + } else { + guard let httpResponse = rawResponse as? HTTPURLResponse else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidState(reason: "bad response")) + return + } + let inheritedHeaders = httpResponse.allHeaderFields.filter({ + key, value in + let normalizedKey = (key as? String)?.lowercased() + return normalizedKey == "content-type" || + normalizedKey == "content-length" + }) as? [String: String] ?? [:] + let newHeaders: [String: String] = [ + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "*" + ] + + guard let response = HTTPURLResponse.init(url: rawUrl, statusCode: httpResponse.statusCode, httpVersion: nil, headerFields: inheritedHeaders.merging(newHeaders, uniquingKeysWith: { (_, newHeaders) in newHeaders })) else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidState(reason: "failed to create response")) + return + } + + urlSchemeTask.didReceive(response) + if rawData != nil { + urlSchemeTask.didReceive(rawData!) + } + urlSchemeTask.didFinish() + } + }) + } + } + + func webView(_ webView: WKWebView, stop urlSchemeTask: WKURLSchemeTask) { + urlSchemeTask.stopped?.withLock({ + $0 = true + }) + } +} + +private extension WKURLSchemeTask { + var stopped: Mutex? { + get { + return objc_getAssociatedObject(self, &stoppedKey) as? Mutex ?? nil + } + set { + objc_setAssociatedObject(self, &stoppedKey, newValue, .OBJC_ASSOCIATION_ASSIGN) + } + } +} + +private var stoppedKey = malloc(1) diff --git a/packages/frontend/apps/ios/App/App/AffineViewController.swift b/packages/frontend/apps/ios/App/App/AffineViewController.swift index 131802418..357617a96 100644 --- a/packages/frontend/apps/ios/App/App/AffineViewController.swift +++ b/packages/frontend/apps/ios/App/App/AffineViewController.swift @@ -13,6 +13,19 @@ class AFFiNEViewController: CAPBridgeViewController { intelligentsButton.delegate = self dismissIntelligentsButton() } + + override func webViewConfiguration(for instanceConfiguration: InstanceConfiguration) -> WKWebViewConfiguration { + let configuration = super.webViewConfiguration(for: instanceConfiguration) + return configuration + } + + override func webView(with frame: CGRect, configuration: WKWebViewConfiguration) -> WKWebView { + configuration.setURLSchemeHandler(AffineHttpHandler(), forURLScheme: "affine-http") + configuration.setURLSchemeHandler(AffineHttpHandler(), forURLScheme: "affine-https") + configuration.setURLSchemeHandler(AffineWsHandler(), forURLScheme: "affine-ws") + configuration.setURLSchemeHandler(AffineWsHandler(), forURLScheme: "affine-wss") + return super.webView(with: frame, configuration: configuration) +} override func capacitorDidLoad() { let plugins: [CAPPlugin] = [ diff --git a/packages/frontend/apps/ios/App/App/AffineWsHandler.swift b/packages/frontend/apps/ios/App/App/AffineWsHandler.swift new file mode 100644 index 000000000..be72ad0a1 --- /dev/null +++ b/packages/frontend/apps/ios/App/App/AffineWsHandler.swift @@ -0,0 +1,197 @@ +// +// RequestUrlSchemeHandler.swift +// App +// +// Created by EYHN on 2025/1/9. +// + +import WebKit + +enum AffineWsError: Error { + case invalidOperation(reason: String), invalidState(reason: String) +} + +/** + this custom url scheme handler simulates websocket connection through an http request. + frontend open websocket connections and send messages by sending requests to affine-ws:// or affine-wss:// + the handler has two endpoints: + `affine-ws:///open?uuid={uuid}&url={wsUrl}`: open a websocket connection and return received data through the SSE protocol. If the front-end closes the http connection, the websocket connection will also be closed. + `affine-ws:///send?uuid={uuid}`: send the request body data to the websocket connection with the specified uuid. + */ +class AffineWsHandler: NSObject, WKURLSchemeHandler { + var wsTasks: [UUID: URLSessionWebSocketTask] = [:] + func webView(_ webView: WKWebView, start urlSchemeTask: any WKURLSchemeTask) { + urlSchemeTask.stopped = Mutex.init(false) + guard let rawUrl = urlSchemeTask.request.url else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "bad request")) + return + } + guard let urlComponents = URLComponents(url: rawUrl, resolvingAgainstBaseURL: true) else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "bad request")) + return + } + let path = urlComponents.path + if path == "/open" { + guard let targetUrlStr = urlComponents.queryItems?.first(where: { $0.name == "url" })?.value else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "url is request")) + return + } + + guard let targetUrl = URL(string: targetUrlStr) else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "failed to parse url")) + return + } + + guard let uuidStr = urlComponents.queryItems?.first(where: { $0.name == "uuid" })?.value else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "url is request")) + return + } + guard let uuid = UUID(uuidString: uuidStr) else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "invalid uuid")) + return + } + + guard let response = HTTPURLResponse.init(url: rawUrl, statusCode: 200, httpVersion: nil, headerFields: [ + "X-Accel-Buffering": "no", + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "*" + ]) else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidState(reason: "failed to create response")) + return + } + + urlSchemeTask.didReceive(response) + let jsonEncoder = JSONEncoder() + let json = String(data: try! jsonEncoder.encode(["type": "start"]), encoding: .utf8)! + urlSchemeTask.didReceive("data: \(json)\n\n".data(using: .utf8)!) + + var request = URLRequest(url: targetUrl); + request.httpShouldHandleCookies = true + + let webSocketTask = URLSession.shared.webSocketTask(with: targetUrl) + self.wsTasks[uuid] = webSocketTask + webSocketTask.resume() + + urlSchemeTask.wsTask = webSocketTask + + var completionHandler: ((Result) -> Void)! + completionHandler = { + let result = $0 + urlSchemeTask.stopped?.withLock({ + let stopped = $0 + if stopped { + return + } + let jsonEncoder = JSONEncoder() + switch result { + case .success(let message): + if case .string(let string) = message { + let json = String(data: try! jsonEncoder.encode(["type": "message", "data": string]), encoding: .utf8)! + urlSchemeTask.didReceive("data: \(json)\n\n".data(using: .utf8)!) + } + case .failure(let error): + let json = String(data: try! jsonEncoder.encode(["type": "error", "error": error.localizedDescription]), encoding: .utf8)! + urlSchemeTask.didReceive("data: \(json)\n\n".data(using: .utf8)!) + urlSchemeTask.didFinish() + } + }) + + // recursive calls + webSocketTask.receive(completionHandler: completionHandler) + } + + webSocketTask.receive(completionHandler: completionHandler) + } else if path == "/send" { + if urlSchemeTask.request.httpMethod != "POST" { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "Method should be POST")) + return + } + guard let uuidStr = urlComponents.queryItems?.first(where: { $0.name == "uuid" })?.value else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "url is request")) + return + } + guard let uuid = UUID(uuidString: uuidStr) else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "invalid uuid")) + return + } + guard let ContentType = urlSchemeTask.request.allHTTPHeaderFields?.first(where: {$0.key.lowercased() == "content-type"})?.value else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "content-type is request")) + return + } + if ContentType != "text/plain" { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "content-type not support")) + return + } + guard let body = urlSchemeTask.request.httpBody else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "no body")) + return + } + let stringBody = String(decoding: body, as: UTF8.self) + guard let webSocketTask = self.wsTasks[uuid] else { + urlSchemeTask.didFailWithError(AffineWsError.invalidOperation(reason: "connection not found")) + return + } + + guard let response = HTTPURLResponse.init(url: rawUrl, statusCode: 200, httpVersion: nil, headerFields: [ + "Content-Type": "application/json", + "Cache-Control": "no-cache", + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "*" + ]) else { + urlSchemeTask.didFailWithError(AffineHttpError.invalidState(reason: "failed to create response")) + return + } + + let jsonEncoder = JSONEncoder() + + webSocketTask.send(.string(stringBody), completionHandler: { + error in + urlSchemeTask.stopped?.withLock({ + if $0 { + return + } + if error != nil { + let json = try! jsonEncoder.encode(["error": error!.localizedDescription]) + urlSchemeTask.didReceive(response) + urlSchemeTask.didReceive(json) + } else { + urlSchemeTask.didReceive(response) + urlSchemeTask.didReceive(try! jsonEncoder.encode(["uuid": uuid.uuidString.data(using: .utf8)!])) + urlSchemeTask.didFinish() + } + }) + }) + } + } + + func webView(_ webView: WKWebView, stop urlSchemeTask: WKURLSchemeTask) { + urlSchemeTask.stopped?.withLock({ + $0 = false + }) + urlSchemeTask.wsTask?.cancel(with: .abnormalClosure, reason: "Closed".data(using: .utf8)) + } +} + +private extension WKURLSchemeTask { + var stopped: Mutex? { + get { + return objc_getAssociatedObject(self, &stoppedKey) as? Mutex ?? nil + } + set { + objc_setAssociatedObject(self, &stoppedKey, newValue, .OBJC_ASSOCIATION_ASSIGN) + } + } + var wsTask: URLSessionWebSocketTask? { + get { + return objc_getAssociatedObject(self, &wsTaskKey) as? URLSessionWebSocketTask + } + set { + objc_setAssociatedObject(self, &stoppedKey, newValue, .OBJC_ASSOCIATION_ASSIGN) + } + } +} + +private var stoppedKey = malloc(1) +private var wsTaskKey = malloc(1) diff --git a/packages/frontend/apps/ios/App/App/Mutex.swift b/packages/frontend/apps/ios/App/App/Mutex.swift new file mode 100644 index 000000000..f2e4a4c5b --- /dev/null +++ b/packages/frontend/apps/ios/App/App/Mutex.swift @@ -0,0 +1,23 @@ +// +// Mutex.swift +// App +// +// Created by EYHN on 2025/1/11. +// + +import Foundation + +final class Mutex: @unchecked Sendable { + private let lock = NSLock.init() + private var wrapped: Wrapped + + init(_ wrapped: Wrapped) { + self.wrapped = wrapped + } + + func withLock(_ body: @Sendable (inout Wrapped) throws -> R) rethrows -> R { + self.lock.lock() + defer { self.lock.unlock() } + return try body(&wrapped) + } +} diff --git a/packages/frontend/apps/ios/App/App/Plugins/NBStore/NBStorePlugin.swift b/packages/frontend/apps/ios/App/App/Plugins/NBStore/NBStorePlugin.swift index fd7120277..7177b2169 100644 --- a/packages/frontend/apps/ios/App/App/Plugins/NBStore/NBStorePlugin.swift +++ b/packages/frontend/apps/ios/App/App/Plugins/NBStore/NBStorePlugin.swift @@ -30,6 +30,7 @@ public class NbStorePlugin: CAPPlugin, CAPBridgedPlugin { CAPPluginMethod(name: "getPeerPulledRemoteClocks", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "getPeerPulledRemoteClock", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setPeerPulledRemoteClock", returnType: CAPPluginReturnPromise), + CAPPluginMethod(name: "getPeerPushedClock", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "getPeerPushedClocks", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "setPeerPushedClock", returnType: CAPPluginReturnPromise), CAPPluginMethod(name: "clearClocks", returnType: CAPPluginReturnPromise), @@ -334,11 +335,14 @@ public class NbStorePlugin: CAPPlugin, CAPBridgedPlugin { let peer = try call.getStringEnsure("peer") let docId = try call.getStringEnsure("docId") - let clock = try await docStoragePool.getPeerRemoteClock(universalId: id, peer: peer, docId: docId) - call.resolve([ - "docId": clock.docId, - "timestamp": clock.timestamp, - ]) + if let clock = try await docStoragePool.getPeerRemoteClock(universalId: id, peer: peer, docId: docId) { + call.resolve([ + "docId": clock.docId, + "timestamp": clock.timestamp, + ]) + } else { + call.resolve() + } } catch { call.reject("Failed to get peer remote clock, \(error)", nil, error) @@ -391,11 +395,14 @@ public class NbStorePlugin: CAPPlugin, CAPBridgedPlugin { let peer = try call.getStringEnsure("peer") let docId = try call.getStringEnsure("docId") - let clock = try await docStoragePool.getPeerPulledRemoteClock(universalId: id, peer: peer, docId: docId) - call.resolve([ - "docId": clock.docId, - "timestamp": clock.timestamp, - ]) + if let clock = try await docStoragePool.getPeerPulledRemoteClock(universalId: id, peer: peer, docId: docId) { + call.resolve([ + "docId": clock.docId, + "timestamp": clock.timestamp, + ]) + } else { + call.resolve() + } } catch { call.reject("Failed to get peer pulled remote clock, \(error)", nil, error) @@ -424,6 +431,26 @@ public class NbStorePlugin: CAPPlugin, CAPBridgedPlugin { } } + @objc func getPeerPushedClock(_ call: CAPPluginCall) { + Task { + do { + let id = try call.getStringEnsure("id") + let peer = try call.getStringEnsure("peer") + let docId = try call.getStringEnsure("docId") + if let clock = try await docStoragePool.getPeerPushedClock(universalId: id, peer: peer, docId: docId) { + call.resolve([ + "docId": clock.docId, + "timestamp": clock.timestamp, + ]) + } else { + call.resolve() + } + } catch { + call.reject("Failed to get peer pushed clock, \(error)", nil, error) + } + } + } + @objc func getPeerPushedClocks(_ call: CAPPluginCall) { Task { do { diff --git a/packages/frontend/apps/ios/App/App/SafeWKURLSchemeTask.swift b/packages/frontend/apps/ios/App/App/SafeWKURLSchemeTask.swift new file mode 100644 index 000000000..e8fccce2f --- /dev/null +++ b/packages/frontend/apps/ios/App/App/SafeWKURLSchemeTask.swift @@ -0,0 +1,36 @@ +// +// SafeWKURLSchemeTask.swift +// App +// +// Created by EYHN on 2025/1/11. +// + +import WebKit + +class SafeWKURLSchemeTask: WKURLSchemeTask, NSObject { + var origin: any WKURLSchemeTask + init(origin: any WKURLSchemeTask) { + self.origin = origin + self.request = origin.request + } + + var request: URLRequest + + func didReceive(_ response: URLResponse) { + <#code#> + } + + func didReceive(_ data: Data) { + self.origin.didReceive(<#T##response: URLResponse##URLResponse#>) + } + + func didFinish() { + self.origin.didFinish() + } + + func didFailWithError(_ error: any Error) { + self.origin.didFailWithError(error) + } + + +} diff --git a/packages/frontend/apps/ios/App/App/uniffi/affine_mobile_native.swift b/packages/frontend/apps/ios/App/App/uniffi/affine_mobile_native.swift index c96cec108..552f93b74 100644 --- a/packages/frontend/apps/ios/App/App/uniffi/affine_mobile_native.swift +++ b/packages/frontend/apps/ios/App/App/uniffi/affine_mobile_native.swift @@ -8,10 +8,10 @@ import Foundation // might be in a separate module, or it might be compiled inline into // this module. This is a bit of light hackery to work with both. #if canImport(affine_mobile_nativeFFI) - import affine_mobile_nativeFFI +import affine_mobile_nativeFFI #endif -private extension RustBuffer { +fileprivate extension RustBuffer { // Allocate a new buffer, copying the contents of a `UInt8` array. init(bytes: [UInt8]) { let rbuf = bytes.withUnsafeBufferPointer { ptr in @@ -21,7 +21,7 @@ private extension RustBuffer { } static func empty() -> RustBuffer { - RustBuffer(capacity: 0, len: 0, data: nil) + RustBuffer(capacity: 0, len:0, data: nil) } static func from(_ ptr: UnsafeBufferPointer) -> RustBuffer { @@ -35,7 +35,7 @@ private extension RustBuffer { } } -private extension ForeignBytes { +fileprivate extension ForeignBytes { init(bufferPointer: UnsafeBufferPointer) { self.init(len: Int32(bufferPointer.count), data: bufferPointer.baseAddress) } @@ -48,7 +48,7 @@ private extension ForeignBytes { // Helper classes/extensions that don't change. // Someday, this will be in a library of its own. -private extension Data { +fileprivate extension Data { init(rustBuffer: RustBuffer) { self.init( bytesNoCopy: rustBuffer.data!, @@ -72,15 +72,15 @@ private extension Data { // // Instead, the read() method and these helper functions input a tuple of data -private func createReader(data: Data) -> (data: Data, offset: Data.Index) { +fileprivate func createReader(data: Data) -> (data: Data, offset: Data.Index) { (data: data, offset: 0) } // Reads an integer at the current offset, in big-endian order, and advances // the offset on success. Throws if reading the integer would move the // offset past the end of the buffer. -private func readInt(_ reader: inout (data: Data, offset: Data.Index)) throws -> T { - let range = reader.offset ..< reader.offset + MemoryLayout.size +fileprivate func readInt(_ reader: inout (data: Data, offset: Data.Index)) throws -> T { + let range = reader.offset...size guard reader.data.count >= range.upperBound else { throw UniffiInternalError.bufferOverflow } @@ -90,38 +90,38 @@ private func readInt(_ reader: inout (data: Data, offset: return value as! T } var value: T = 0 - let _ = withUnsafeMutableBytes(of: &value) { reader.data.copyBytes(to: $0, from: range) } + let _ = withUnsafeMutableBytes(of: &value, { reader.data.copyBytes(to: $0, from: range)}) reader.offset = range.upperBound return value.bigEndian } // Reads an arbitrary number of bytes, to be used to read // raw bytes, this is useful when lifting strings -private func readBytes(_ reader: inout (data: Data, offset: Data.Index), count: Int) throws -> [UInt8] { - let range = reader.offset ..< (reader.offset + count) +fileprivate func readBytes(_ reader: inout (data: Data, offset: Data.Index), count: Int) throws -> Array { + let range = reader.offset..<(reader.offset+count) guard reader.data.count >= range.upperBound else { throw UniffiInternalError.bufferOverflow } var value = [UInt8](repeating: 0, count: count) - value.withUnsafeMutableBufferPointer { buffer in + value.withUnsafeMutableBufferPointer({ buffer in reader.data.copyBytes(to: buffer, from: range) - } + }) reader.offset = range.upperBound return value } // Reads a float at the current offset. -private func readFloat(_ reader: inout (data: Data, offset: Data.Index)) throws -> Float { - return try Float(bitPattern: readInt(&reader)) +fileprivate func readFloat(_ reader: inout (data: Data, offset: Data.Index)) throws -> Float { + return Float(bitPattern: try readInt(&reader)) } // Reads a float at the current offset. -private func readDouble(_ reader: inout (data: Data, offset: Data.Index)) throws -> Double { - return try Double(bitPattern: readInt(&reader)) +fileprivate func readDouble(_ reader: inout (data: Data, offset: Data.Index)) throws -> Double { + return Double(bitPattern: try readInt(&reader)) } // Indicates if the offset has reached the end of the buffer. -private func hasRemaining(_ reader: (data: Data, offset: Data.Index)) -> Bool { +fileprivate func hasRemaining(_ reader: (data: Data, offset: Data.Index)) -> Bool { return reader.offset < reader.data.count } @@ -129,11 +129,11 @@ private func hasRemaining(_ reader: (data: Data, offset: Data.Index)) -> Bool { // struct, but we use standalone functions instead in order to make external // types work. See the above discussion on Readers for details. -private func createWriter() -> [UInt8] { +fileprivate func createWriter() -> [UInt8] { return [] } -private func writeBytes(_ writer: inout [UInt8], _ byteArr: S) where S: Sequence, S.Element == UInt8 { +fileprivate func writeBytes(_ writer: inout [UInt8], _ byteArr: S) where S: Sequence, S.Element == UInt8 { writer.append(contentsOf: byteArr) } @@ -141,22 +141,22 @@ private func writeBytes(_ writer: inout [UInt8], _ byteArr: S) where S: Seque // // Warning: make sure what you are trying to write // is in the correct type! -private func writeInt(_ writer: inout [UInt8], _ value: T) { +fileprivate func writeInt(_ writer: inout [UInt8], _ value: T) { var value = value.bigEndian withUnsafeBytes(of: &value) { writer.append(contentsOf: $0) } } -private func writeFloat(_ writer: inout [UInt8], _ value: Float) { +fileprivate func writeFloat(_ writer: inout [UInt8], _ value: Float) { writeInt(&writer, value.bitPattern) } -private func writeDouble(_ writer: inout [UInt8], _ value: Double) { +fileprivate func writeDouble(_ writer: inout [UInt8], _ value: Double) { writeInt(&writer, value.bitPattern) } // Protocol for types that transfer other types across the FFI. This is // analogous to the Rust trait of the same name. -private protocol FfiConverter { +fileprivate protocol FfiConverter { associatedtype FfiType associatedtype SwiftType @@ -167,19 +167,19 @@ private protocol FfiConverter { } // Types conforming to `Primitive` pass themselves directly over the FFI. -private protocol FfiConverterPrimitive: FfiConverter where FfiType == SwiftType {} +fileprivate protocol FfiConverterPrimitive: FfiConverter where FfiType == SwiftType { } extension FfiConverterPrimitive { - #if swift(>=5.8) - @_documentation(visibility: private) - #endif +#if swift(>=5.8) + @_documentation(visibility: private) +#endif public static func lift(_ value: FfiType) throws -> SwiftType { return value } - #if swift(>=5.8) - @_documentation(visibility: private) - #endif +#if swift(>=5.8) + @_documentation(visibility: private) +#endif public static func lower(_ value: SwiftType) -> FfiType { return value } @@ -187,12 +187,12 @@ extension FfiConverterPrimitive { // Types conforming to `FfiConverterRustBuffer` lift and lower into a `RustBuffer`. // Used for complex types where it's hard to write a custom lift/lower. -private protocol FfiConverterRustBuffer: FfiConverter where FfiType == RustBuffer {} +fileprivate protocol FfiConverterRustBuffer: FfiConverter where FfiType == RustBuffer {} extension FfiConverterRustBuffer { - #if swift(>=5.8) - @_documentation(visibility: private) - #endif +#if swift(>=5.8) + @_documentation(visibility: private) +#endif public static func lift(_ buf: RustBuffer) throws -> SwiftType { var reader = createReader(data: Data(rustBuffer: buf)) let value = try read(from: &reader) @@ -203,19 +203,18 @@ extension FfiConverterRustBuffer { return value } - #if swift(>=5.8) - @_documentation(visibility: private) - #endif +#if swift(>=5.8) + @_documentation(visibility: private) +#endif public static func lower(_ value: SwiftType) -> RustBuffer { - var writer = createWriter() - write(value, into: &writer) - return RustBuffer(bytes: writer) + var writer = createWriter() + write(value, into: &writer) + return RustBuffer(bytes: writer) } } - // An error type for FFI errors. These errors occur at the UniFFI level, not // the library level. -private enum UniffiInternalError: LocalizedError { +fileprivate enum UniffiInternalError: LocalizedError { case bufferOverflow case incompleteData case unexpectedOptionalTag @@ -241,24 +240,24 @@ private enum UniffiInternalError: LocalizedError { } } -private extension NSLock { +fileprivate extension NSLock { func withLock(f: () throws -> T) rethrows -> T { - lock() + self.lock() defer { self.unlock() } return try f() } } -private let CALL_SUCCESS: Int8 = 0 -private let CALL_ERROR: Int8 = 1 -private let CALL_UNEXPECTED_ERROR: Int8 = 2 -private let CALL_CANCELLED: Int8 = 3 +fileprivate let CALL_SUCCESS: Int8 = 0 +fileprivate let CALL_ERROR: Int8 = 1 +fileprivate let CALL_UNEXPECTED_ERROR: Int8 = 2 +fileprivate let CALL_CANCELLED: Int8 = 3 -private extension RustCallStatus { +fileprivate extension RustCallStatus { init() { self.init( code: CALL_SUCCESS, - errorBuf: RustBuffer( + errorBuf: RustBuffer.init( capacity: 0, len: 0, data: nil @@ -274,8 +273,7 @@ private func rustCall(_ callback: (UnsafeMutablePointer) -> T private func rustCallWithError( _ errorHandler: @escaping (RustBuffer) throws -> E, - _ callback: (UnsafeMutablePointer) -> T -) throws -> T { + _ callback: (UnsafeMutablePointer) -> T) throws -> T { try makeRustCall(callback, errorHandler: errorHandler) } @@ -284,7 +282,7 @@ private func makeRustCall( errorHandler: ((RustBuffer) throws -> E)? ) throws -> T { uniffiEnsureInitialized() - var callStatus = RustCallStatus() + var callStatus = RustCallStatus.init() let returnedVal = callback(&callStatus) try uniffiCheckCallStatus(callStatus: callStatus, errorHandler: errorHandler) return returnedVal @@ -295,44 +293,44 @@ private func uniffiCheckCallStatus( errorHandler: ((RustBuffer) throws -> E)? ) throws { switch callStatus.code { - case CALL_SUCCESS: - return + case CALL_SUCCESS: + return - case CALL_ERROR: - if let errorHandler = errorHandler { - throw try errorHandler(callStatus.errorBuf) - } else { - callStatus.errorBuf.deallocate() - throw UniffiInternalError.unexpectedRustCallError - } + case CALL_ERROR: + if let errorHandler = errorHandler { + throw try errorHandler(callStatus.errorBuf) + } else { + callStatus.errorBuf.deallocate() + throw UniffiInternalError.unexpectedRustCallError + } - case CALL_UNEXPECTED_ERROR: - // When the rust code sees a panic, it tries to construct a RustBuffer - // with the message. But if that code panics, then it just sends back - // an empty buffer. - if callStatus.errorBuf.len > 0 { - throw try UniffiInternalError.rustPanic(FfiConverterString.lift(callStatus.errorBuf)) - } else { - callStatus.errorBuf.deallocate() - throw UniffiInternalError.rustPanic("Rust panic") - } + case CALL_UNEXPECTED_ERROR: + // When the rust code sees a panic, it tries to construct a RustBuffer + // with the message. But if that code panics, then it just sends back + // an empty buffer. + if callStatus.errorBuf.len > 0 { + throw UniffiInternalError.rustPanic(try FfiConverterString.lift(callStatus.errorBuf)) + } else { + callStatus.errorBuf.deallocate() + throw UniffiInternalError.rustPanic("Rust panic") + } - case CALL_CANCELLED: - fatalError("Cancellation not supported yet") + case CALL_CANCELLED: + fatalError("Cancellation not supported yet") - default: - throw UniffiInternalError.unexpectedRustCallStatusCode + default: + throw UniffiInternalError.unexpectedRustCallStatusCode } } private func uniffiTraitInterfaceCall( callStatus: UnsafeMutablePointer, makeCall: () throws -> T, - writeReturn: (T) -> Void + writeReturn: (T) -> () ) { do { try writeReturn(makeCall()) - } catch { + } catch let error { callStatus.pointee.code = CALL_UNEXPECTED_ERROR callStatus.pointee.errorBuf = FfiConverterString.lower(String(describing: error)) } @@ -341,7 +339,7 @@ private func uniffiTraitInterfaceCall( private func uniffiTraitInterfaceCallWithError( callStatus: UnsafeMutablePointer, makeCall: () throws -> T, - writeReturn: (T) -> Void, + writeReturn: (T) -> (), lowerError: (E) -> RustBuffer ) { do { @@ -354,8 +352,7 @@ private func uniffiTraitInterfaceCallWithError( callStatus.pointee.errorBuf = FfiConverterString.lower(String(describing: error)) } } - -private class UniffiHandleMap { +fileprivate class UniffiHandleMap { private var map: [UInt64: T] = [:] private let lock = NSLock() private var currentHandle: UInt64 = 1 @@ -369,7 +366,7 @@ private class UniffiHandleMap { } } - func get(handle: UInt64) throws -> T { + func get(handle: UInt64) throws -> T { try lock.withLock { guard let obj = map[handle] else { throw UniffiInternalError.unexpectedStaleHandle @@ -389,16 +386,20 @@ private class UniffiHandleMap { } var count: Int { - map.count + get { + map.count + } } } + // Public interface members begin here. + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterUInt32: FfiConverterPrimitive { +fileprivate struct FfiConverterUInt32: FfiConverterPrimitive { typealias FfiType = UInt32 typealias SwiftType = UInt32 @@ -412,9 +413,9 @@ private struct FfiConverterUInt32: FfiConverterPrimitive { } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterInt64: FfiConverterPrimitive { +fileprivate struct FfiConverterInt64: FfiConverterPrimitive { typealias FfiType = Int64 typealias SwiftType = Int64 @@ -428,9 +429,9 @@ private struct FfiConverterInt64: FfiConverterPrimitive { } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterBool: FfiConverter { +fileprivate struct FfiConverterBool : FfiConverter { typealias FfiType = Int8 typealias SwiftType = Bool @@ -452,9 +453,9 @@ private struct FfiConverterBool: FfiConverter { } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterString: FfiConverter { +fileprivate struct FfiConverterString: FfiConverter { typealias SwiftType = String typealias FfiType = RustBuffer @@ -482,7 +483,7 @@ private struct FfiConverterString: FfiConverter { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> String { let len: Int32 = try readInt(&buf) - return try String(bytes: readBytes(&buf, count: Int(len)), encoding: String.Encoding.utf8)! + return String(bytes: try readBytes(&buf, count: Int(len)), encoding: String.Encoding.utf8)! } public static func write(_ value: String, into buf: inout [UInt8]) { @@ -492,70 +493,76 @@ private struct FfiConverterString: FfiConverter { } } -public protocol DocStoragePoolProtocol: AnyObject { - func clearClocks(universalId: String) async throws + + +public protocol DocStoragePoolProtocol : AnyObject { + + func clearClocks(universalId: String) async throws + /** * Initialize the database and run migrations. */ - func connect(universalId: String, path: String) async throws - - func deleteBlob(universalId: String, key: String, permanently: Bool) async throws - - func deleteDoc(universalId: String, docId: String) async throws - - func disconnect(universalId: String) async throws - - func getBlob(universalId: String, key: String) async throws -> Blob? - - func getDocClock(universalId: String, docId: String) async throws -> DocClock? - - func getDocClocks(universalId: String, after: Int64?) async throws -> [DocClock] - - func getDocSnapshot(universalId: String, docId: String) async throws -> DocRecord? - - func getDocUpdates(universalId: String, docId: String) async throws -> [DocUpdate] - - func getPeerPulledRemoteClock(universalId: String, peer: String, docId: String) async throws -> DocClock - - func getPeerPulledRemoteClocks(universalId: String, peer: String) async throws -> [DocClock] - - func getPeerPushedClocks(universalId: String, peer: String) async throws -> [DocClock] - - func getPeerRemoteClock(universalId: String, peer: String, docId: String) async throws -> DocClock - - func getPeerRemoteClocks(universalId: String, peer: String) async throws -> [DocClock] - - func listBlobs(universalId: String) async throws -> [ListedBlob] - - func markUpdatesMerged(universalId: String, docId: String, updates: [Int64]) async throws -> UInt32 - - func pushUpdate(universalId: String, docId: String, update: String) async throws -> Int64 - - func releaseBlobs(universalId: String) async throws - - func setBlob(universalId: String, blob: SetBlob) async throws - - func setDocSnapshot(universalId: String, snapshot: DocRecord) async throws -> Bool - - func setPeerPulledRemoteClock(universalId: String, peer: String, docId: String, clock: Int64) async throws - - func setPeerPushedClock(universalId: String, peer: String, docId: String, clock: Int64) async throws - - func setPeerRemoteClock(universalId: String, peer: String, docId: String, clock: Int64) async throws - - func setSpaceId(universalId: String, spaceId: String) async throws + func connect(universalId: String, path: String) async throws + + func deleteBlob(universalId: String, key: String, permanently: Bool) async throws + + func deleteDoc(universalId: String, docId: String) async throws + + func disconnect(universalId: String) async throws + + func getBlob(universalId: String, key: String) async throws -> Blob? + + func getDocClock(universalId: String, docId: String) async throws -> DocClock? + + func getDocClocks(universalId: String, after: Int64?) async throws -> [DocClock] + + func getDocSnapshot(universalId: String, docId: String) async throws -> DocRecord? + + func getDocUpdates(universalId: String, docId: String) async throws -> [DocUpdate] + + func getPeerPulledRemoteClock(universalId: String, peer: String, docId: String) async throws -> DocClock? + + func getPeerPulledRemoteClocks(universalId: String, peer: String) async throws -> [DocClock] + + func getPeerPushedClock(universalId: String, peer: String, docId: String) async throws -> DocClock? + + func getPeerPushedClocks(universalId: String, peer: String) async throws -> [DocClock] + + func getPeerRemoteClock(universalId: String, peer: String, docId: String) async throws -> DocClock? + + func getPeerRemoteClocks(universalId: String, peer: String) async throws -> [DocClock] + + func listBlobs(universalId: String) async throws -> [ListedBlob] + + func markUpdatesMerged(universalId: String, docId: String, updates: [Int64]) async throws -> UInt32 + + func pushUpdate(universalId: String, docId: String, update: String) async throws -> Int64 + + func releaseBlobs(universalId: String) async throws + + func setBlob(universalId: String, blob: SetBlob) async throws + + func setDocSnapshot(universalId: String, snapshot: DocRecord) async throws -> Bool + + func setPeerPulledRemoteClock(universalId: String, peer: String, docId: String, clock: Int64) async throws + + func setPeerPushedClock(universalId: String, peer: String, docId: String, clock: Int64) async throws + + func setPeerRemoteClock(universalId: String, peer: String, docId: String, clock: Int64) async throws + + func setSpaceId(universalId: String, spaceId: String) async throws + } open class DocStoragePool: - DocStoragePoolProtocol -{ + DocStoragePoolProtocol { fileprivate let pointer: UnsafeMutableRawPointer! /// Used to instantiate a [FFIObject] without an actual pointer, for fakes in tests, mostly. - #if swift(>=5.8) - @_documentation(visibility: private) - #endif +#if swift(>=5.8) + @_documentation(visibility: private) +#endif public struct NoPointer { public init() {} } @@ -563,7 +570,7 @@ open class DocStoragePool: // TODO: We'd like this to be `private` but for Swifty reasons, // we can't implement `FfiConverter` without making this `required` and we can't // make it `required` without making it `public`. - public required init(unsafeFromRawPointer pointer: UnsafeMutableRawPointer) { + required public init(unsafeFromRawPointer pointer: UnsafeMutableRawPointer) { self.pointer = pointer } @@ -572,20 +579,19 @@ open class DocStoragePool: // // - Warning: // Any object instantiated with this constructor cannot be passed to an actual Rust-backed object. Since there isn't a backing [Pointer] the FFI lower functions will crash. - #if swift(>=5.8) - @_documentation(visibility: private) - #endif - public init(noPointer _: NoPointer) { - pointer = nil +#if swift(>=5.8) + @_documentation(visibility: private) +#endif + public init(noPointer: NoPointer) { + self.pointer = nil } - #if swift(>=5.8) - @_documentation(visibility: private) - #endif +#if swift(>=5.8) + @_documentation(visibility: private) +#endif public func uniffiClonePointer() -> UnsafeMutableRawPointer { return try! rustCall { uniffi_affine_mobile_native_fn_clone_docstoragepool(self.pointer, $0) } } - // No primary constructor declared for this class. deinit { @@ -596,439 +602,462 @@ open class DocStoragePool: try! rustCall { uniffi_affine_mobile_native_fn_free_docstoragepool(pointer, $0) } } - open func clearClocks(universalId: String) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_clear_clocks( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } + + +open func clearClocks(universalId: String)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_clear_clocks( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + /** * Initialize the database and run migrations. */ - open func connect(universalId: String, path: String) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_connect( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(path) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } +open func connect(universalId: String, path: String)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_connect( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(path) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func deleteBlob(universalId: String, key: String, permanently: Bool)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_delete_blob( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(key),FfiConverterBool.lower(permanently) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func deleteDoc(universalId: String, docId: String)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_delete_doc( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(docId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func disconnect(universalId: String)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_disconnect( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getBlob(universalId: String, key: String)async throws -> Blob? { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_blob( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(key) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterOptionTypeBlob.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getDocClock(universalId: String, docId: String)async throws -> DocClock? { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_clock( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(docId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterOptionTypeDocClock.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getDocClocks(universalId: String, after: Int64?)async throws -> [DocClock] { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_clocks( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterOptionInt64.lower(after) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterSequenceTypeDocClock.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getDocSnapshot(universalId: String, docId: String)async throws -> DocRecord? { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_snapshot( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(docId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterOptionTypeDocRecord.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getDocUpdates(universalId: String, docId: String)async throws -> [DocUpdate] { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_updates( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(docId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterSequenceTypeDocUpdate.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getPeerPulledRemoteClock(universalId: String, peer: String, docId: String)async throws -> DocClock? { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pulled_remote_clock( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer),FfiConverterString.lower(docId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterOptionTypeDocClock.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getPeerPulledRemoteClocks(universalId: String, peer: String)async throws -> [DocClock] { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pulled_remote_clocks( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterSequenceTypeDocClock.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getPeerPushedClock(universalId: String, peer: String, docId: String)async throws -> DocClock? { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pushed_clock( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer),FfiConverterString.lower(docId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterOptionTypeDocClock.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getPeerPushedClocks(universalId: String, peer: String)async throws -> [DocClock] { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pushed_clocks( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterSequenceTypeDocClock.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getPeerRemoteClock(universalId: String, peer: String, docId: String)async throws -> DocClock? { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_remote_clock( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer),FfiConverterString.lower(docId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterOptionTypeDocClock.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func getPeerRemoteClocks(universalId: String, peer: String)async throws -> [DocClock] { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_remote_clocks( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterSequenceTypeDocClock.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func listBlobs(universalId: String)async throws -> [ListedBlob] { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_list_blobs( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, + completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, + freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, + liftFunc: FfiConverterSequenceTypeListedBlob.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func markUpdatesMerged(universalId: String, docId: String, updates: [Int64])async throws -> UInt32 { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_mark_updates_merged( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(docId),FfiConverterSequenceInt64.lower(updates) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_u32, + completeFunc: ffi_affine_mobile_native_rust_future_complete_u32, + freeFunc: ffi_affine_mobile_native_rust_future_free_u32, + liftFunc: FfiConverterUInt32.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func pushUpdate(universalId: String, docId: String, update: String)async throws -> Int64 { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_push_update( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(docId),FfiConverterString.lower(update) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_i64, + completeFunc: ffi_affine_mobile_native_rust_future_complete_i64, + freeFunc: ffi_affine_mobile_native_rust_future_free_i64, + liftFunc: FfiConverterInt64.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func releaseBlobs(universalId: String)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_release_blobs( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func setBlob(universalId: String, blob: SetBlob)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_set_blob( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterTypeSetBlob.lower(blob) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func setDocSnapshot(universalId: String, snapshot: DocRecord)async throws -> Bool { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_set_doc_snapshot( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterTypeDocRecord.lower(snapshot) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_i8, + completeFunc: ffi_affine_mobile_native_rust_future_complete_i8, + freeFunc: ffi_affine_mobile_native_rust_future_free_i8, + liftFunc: FfiConverterBool.lift, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func setPeerPulledRemoteClock(universalId: String, peer: String, docId: String, clock: Int64)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_set_peer_pulled_remote_clock( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer),FfiConverterString.lower(docId),FfiConverterInt64.lower(clock) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func setPeerPushedClock(universalId: String, peer: String, docId: String, clock: Int64)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_set_peer_pushed_clock( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer),FfiConverterString.lower(docId),FfiConverterInt64.lower(clock) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func setPeerRemoteClock(universalId: String, peer: String, docId: String, clock: Int64)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_set_peer_remote_clock( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(peer),FfiConverterString.lower(docId),FfiConverterInt64.lower(clock) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + +open func setSpaceId(universalId: String, spaceId: String)async throws { + return + try await uniffiRustCallAsync( + rustFutureFunc: { + uniffi_affine_mobile_native_fn_method_docstoragepool_set_space_id( + self.uniffiClonePointer(), + FfiConverterString.lower(universalId),FfiConverterString.lower(spaceId) + ) + }, + pollFunc: ffi_affine_mobile_native_rust_future_poll_void, + completeFunc: ffi_affine_mobile_native_rust_future_complete_void, + freeFunc: ffi_affine_mobile_native_rust_future_free_void, + liftFunc: { $0 }, + errorHandler: FfiConverterTypeUniffiError.lift + ) +} + - open func deleteBlob(universalId: String, key: String, permanently: Bool) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_delete_blob( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(key), FfiConverterBool.lower(permanently) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func deleteDoc(universalId: String, docId: String) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_delete_doc( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(docId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func disconnect(universalId: String) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_disconnect( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getBlob(universalId: String, key: String) async throws -> Blob? { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_blob( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(key) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterOptionTypeBlob.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getDocClock(universalId: String, docId: String) async throws -> DocClock? { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_clock( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(docId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterOptionTypeDocClock.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getDocClocks(universalId: String, after: Int64?) async throws -> [DocClock] { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_clocks( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterOptionInt64.lower(after) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterSequenceTypeDocClock.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getDocSnapshot(universalId: String, docId: String) async throws -> DocRecord? { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_snapshot( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(docId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterOptionTypeDocRecord.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getDocUpdates(universalId: String, docId: String) async throws -> [DocUpdate] { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_doc_updates( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(docId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterSequenceTypeDocUpdate.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getPeerPulledRemoteClock(universalId: String, peer: String, docId: String) async throws -> DocClock { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pulled_remote_clock( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(peer), FfiConverterString.lower(docId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterTypeDocClock.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getPeerPulledRemoteClocks(universalId: String, peer: String) async throws -> [DocClock] { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pulled_remote_clocks( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(peer) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterSequenceTypeDocClock.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getPeerPushedClocks(universalId: String, peer: String) async throws -> [DocClock] { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pushed_clocks( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(peer) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterSequenceTypeDocClock.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getPeerRemoteClock(universalId: String, peer: String, docId: String) async throws -> DocClock { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_remote_clock( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(peer), FfiConverterString.lower(docId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterTypeDocClock.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func getPeerRemoteClocks(universalId: String, peer: String) async throws -> [DocClock] { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_remote_clocks( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(peer) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterSequenceTypeDocClock.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func listBlobs(universalId: String) async throws -> [ListedBlob] { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_list_blobs( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_rust_buffer, - completeFunc: ffi_affine_mobile_native_rust_future_complete_rust_buffer, - freeFunc: ffi_affine_mobile_native_rust_future_free_rust_buffer, - liftFunc: FfiConverterSequenceTypeListedBlob.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func markUpdatesMerged(universalId: String, docId: String, updates: [Int64]) async throws -> UInt32 { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_mark_updates_merged( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(docId), FfiConverterSequenceInt64.lower(updates) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_u32, - completeFunc: ffi_affine_mobile_native_rust_future_complete_u32, - freeFunc: ffi_affine_mobile_native_rust_future_free_u32, - liftFunc: FfiConverterUInt32.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func pushUpdate(universalId: String, docId: String, update: String) async throws -> Int64 { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_push_update( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(docId), FfiConverterString.lower(update) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_i64, - completeFunc: ffi_affine_mobile_native_rust_future_complete_i64, - freeFunc: ffi_affine_mobile_native_rust_future_free_i64, - liftFunc: FfiConverterInt64.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func releaseBlobs(universalId: String) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_release_blobs( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func setBlob(universalId: String, blob: SetBlob) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_set_blob( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterTypeSetBlob.lower(blob) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func setDocSnapshot(universalId: String, snapshot: DocRecord) async throws -> Bool { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_set_doc_snapshot( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterTypeDocRecord.lower(snapshot) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_i8, - completeFunc: ffi_affine_mobile_native_rust_future_complete_i8, - freeFunc: ffi_affine_mobile_native_rust_future_free_i8, - liftFunc: FfiConverterBool.lift, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func setPeerPulledRemoteClock(universalId: String, peer: String, docId: String, clock: Int64) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_set_peer_pulled_remote_clock( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(peer), FfiConverterString.lower(docId), FfiConverterInt64.lower(clock) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func setPeerPushedClock(universalId: String, peer: String, docId: String, clock: Int64) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_set_peer_pushed_clock( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(peer), FfiConverterString.lower(docId), FfiConverterInt64.lower(clock) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func setPeerRemoteClock(universalId: String, peer: String, docId: String, clock: Int64) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_set_peer_remote_clock( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(peer), FfiConverterString.lower(docId), FfiConverterInt64.lower(clock) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } - - open func setSpaceId(universalId: String, spaceId: String) async throws { - return - try await uniffiRustCallAsync( - rustFutureFunc: { - uniffi_affine_mobile_native_fn_method_docstoragepool_set_space_id( - self.uniffiClonePointer(), - FfiConverterString.lower(universalId), FfiConverterString.lower(spaceId) - ) - }, - pollFunc: ffi_affine_mobile_native_rust_future_poll_void, - completeFunc: ffi_affine_mobile_native_rust_future_complete_void, - freeFunc: ffi_affine_mobile_native_rust_future_free_void, - liftFunc: { $0 }, - errorHandler: FfiConverterTypeUniffiError.lift - ) - } } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public struct FfiConverterTypeDocStoragePool: FfiConverter { + typealias FfiType = UnsafeMutableRawPointer typealias SwiftType = DocStoragePool @@ -1045,7 +1074,7 @@ public struct FfiConverterTypeDocStoragePool: FfiConverter { // The Rust code won't compile if a pointer won't fit in a UInt64. // We have to go via `UInt` because that's the thing that's the size of a pointer. let ptr = UnsafeMutableRawPointer(bitPattern: UInt(truncatingIfNeeded: v)) - if ptr == nil { + if (ptr == nil) { throw UniffiInternalError.unexpectedNullPointer } return try lift(ptr!) @@ -1058,20 +1087,24 @@ public struct FfiConverterTypeDocStoragePool: FfiConverter { } } + + + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeDocStoragePool_lift(_ pointer: UnsafeMutableRawPointer) throws -> DocStoragePool { return try FfiConverterTypeDocStoragePool.lift(pointer) } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeDocStoragePool_lower(_ value: DocStoragePool) -> UnsafeMutableRawPointer { return FfiConverterTypeDocStoragePool.lower(value) } + public struct Blob { public var key: String public var data: String @@ -1090,8 +1123,10 @@ public struct Blob { } } + + extension Blob: Equatable, Hashable { - public static func == (lhs: Blob, rhs: Blob) -> Bool { + public static func ==(lhs: Blob, rhs: Blob) -> Bool { if lhs.key != rhs.key { return false } @@ -1119,19 +1154,20 @@ extension Blob: Equatable, Hashable { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public struct FfiConverterTypeBlob: FfiConverterRustBuffer { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> Blob { return try Blob( - key: FfiConverterString.read(from: &buf), - data: FfiConverterString.read(from: &buf), - mime: FfiConverterString.read(from: &buf), - size: FfiConverterInt64.read(from: &buf), + key: FfiConverterString.read(from: &buf), + data: FfiConverterString.read(from: &buf), + mime: FfiConverterString.read(from: &buf), + size: FfiConverterInt64.read(from: &buf), createdAt: FfiConverterInt64.read(from: &buf) - ) + ) } public static func write(_ value: Blob, into buf: inout [UInt8]) { @@ -1143,20 +1179,22 @@ public struct FfiConverterTypeBlob: FfiConverterRustBuffer { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeBlob_lift(_ buf: RustBuffer) throws -> Blob { return try FfiConverterTypeBlob.lift(buf) } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeBlob_lower(_ value: Blob) -> RustBuffer { return FfiConverterTypeBlob.lower(value) } + public struct DocClock { public var docId: String public var timestamp: Int64 @@ -1169,8 +1207,10 @@ public struct DocClock { } } + + extension DocClock: Equatable, Hashable { - public static func == (lhs: DocClock, rhs: DocClock) -> Bool { + public static func ==(lhs: DocClock, rhs: DocClock) -> Bool { if lhs.docId != rhs.docId { return false } @@ -1186,16 +1226,17 @@ extension DocClock: Equatable, Hashable { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public struct FfiConverterTypeDocClock: FfiConverterRustBuffer { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DocClock { return try DocClock( - docId: FfiConverterString.read(from: &buf), + docId: FfiConverterString.read(from: &buf), timestamp: FfiConverterInt64.read(from: &buf) - ) + ) } public static func write(_ value: DocClock, into buf: inout [UInt8]) { @@ -1204,20 +1245,22 @@ public struct FfiConverterTypeDocClock: FfiConverterRustBuffer { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeDocClock_lift(_ buf: RustBuffer) throws -> DocClock { return try FfiConverterTypeDocClock.lift(buf) } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeDocClock_lower(_ value: DocClock) -> RustBuffer { return FfiConverterTypeDocClock.lower(value) } + public struct DocRecord { public var docId: String public var bin: String @@ -1232,8 +1275,10 @@ public struct DocRecord { } } + + extension DocRecord: Equatable, Hashable { - public static func == (lhs: DocRecord, rhs: DocRecord) -> Bool { + public static func ==(lhs: DocRecord, rhs: DocRecord) -> Bool { if lhs.docId != rhs.docId { return false } @@ -1253,17 +1298,18 @@ extension DocRecord: Equatable, Hashable { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public struct FfiConverterTypeDocRecord: FfiConverterRustBuffer { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DocRecord { return try DocRecord( - docId: FfiConverterString.read(from: &buf), - bin: FfiConverterString.read(from: &buf), + docId: FfiConverterString.read(from: &buf), + bin: FfiConverterString.read(from: &buf), timestamp: FfiConverterInt64.read(from: &buf) - ) + ) } public static func write(_ value: DocRecord, into buf: inout [UInt8]) { @@ -1273,20 +1319,22 @@ public struct FfiConverterTypeDocRecord: FfiConverterRustBuffer { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeDocRecord_lift(_ buf: RustBuffer) throws -> DocRecord { return try FfiConverterTypeDocRecord.lift(buf) } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeDocRecord_lower(_ value: DocRecord) -> RustBuffer { return FfiConverterTypeDocRecord.lower(value) } + public struct DocUpdate { public var docId: String public var timestamp: Int64 @@ -1301,8 +1349,10 @@ public struct DocUpdate { } } + + extension DocUpdate: Equatable, Hashable { - public static func == (lhs: DocUpdate, rhs: DocUpdate) -> Bool { + public static func ==(lhs: DocUpdate, rhs: DocUpdate) -> Bool { if lhs.docId != rhs.docId { return false } @@ -1322,17 +1372,18 @@ extension DocUpdate: Equatable, Hashable { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public struct FfiConverterTypeDocUpdate: FfiConverterRustBuffer { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> DocUpdate { return try DocUpdate( - docId: FfiConverterString.read(from: &buf), - timestamp: FfiConverterInt64.read(from: &buf), + docId: FfiConverterString.read(from: &buf), + timestamp: FfiConverterInt64.read(from: &buf), bin: FfiConverterString.read(from: &buf) - ) + ) } public static func write(_ value: DocUpdate, into buf: inout [UInt8]) { @@ -1342,20 +1393,22 @@ public struct FfiConverterTypeDocUpdate: FfiConverterRustBuffer { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeDocUpdate_lift(_ buf: RustBuffer) throws -> DocUpdate { return try FfiConverterTypeDocUpdate.lift(buf) } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeDocUpdate_lower(_ value: DocUpdate) -> RustBuffer { return FfiConverterTypeDocUpdate.lower(value) } + public struct ListedBlob { public var key: String public var size: Int64 @@ -1372,8 +1425,10 @@ public struct ListedBlob { } } + + extension ListedBlob: Equatable, Hashable { - public static func == (lhs: ListedBlob, rhs: ListedBlob) -> Bool { + public static func ==(lhs: ListedBlob, rhs: ListedBlob) -> Bool { if lhs.key != rhs.key { return false } @@ -1397,18 +1452,19 @@ extension ListedBlob: Equatable, Hashable { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public struct FfiConverterTypeListedBlob: FfiConverterRustBuffer { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> ListedBlob { return try ListedBlob( - key: FfiConverterString.read(from: &buf), - size: FfiConverterInt64.read(from: &buf), - mime: FfiConverterString.read(from: &buf), + key: FfiConverterString.read(from: &buf), + size: FfiConverterInt64.read(from: &buf), + mime: FfiConverterString.read(from: &buf), createdAt: FfiConverterInt64.read(from: &buf) - ) + ) } public static func write(_ value: ListedBlob, into buf: inout [UInt8]) { @@ -1419,20 +1475,22 @@ public struct FfiConverterTypeListedBlob: FfiConverterRustBuffer { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeListedBlob_lift(_ buf: RustBuffer) throws -> ListedBlob { return try FfiConverterTypeListedBlob.lift(buf) } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeListedBlob_lower(_ value: ListedBlob) -> RustBuffer { return FfiConverterTypeListedBlob.lower(value) } + public struct SetBlob { public var key: String public var data: String @@ -1447,8 +1505,10 @@ public struct SetBlob { } } + + extension SetBlob: Equatable, Hashable { - public static func == (lhs: SetBlob, rhs: SetBlob) -> Bool { + public static func ==(lhs: SetBlob, rhs: SetBlob) -> Bool { if lhs.key != rhs.key { return false } @@ -1468,17 +1528,18 @@ extension SetBlob: Equatable, Hashable { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public struct FfiConverterTypeSetBlob: FfiConverterRustBuffer { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> SetBlob { return try SetBlob( - key: FfiConverterString.read(from: &buf), - data: FfiConverterString.read(from: &buf), + key: FfiConverterString.read(from: &buf), + data: FfiConverterString.read(from: &buf), mime: FfiConverterString.read(from: &buf) - ) + ) } public static func write(_ value: SetBlob, into buf: inout [UInt8]) { @@ -1488,21 +1549,26 @@ public struct FfiConverterTypeSetBlob: FfiConverterRustBuffer { } } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeSetBlob_lift(_ buf: RustBuffer) throws -> SetBlob { return try FfiConverterTypeSetBlob.lift(buf) } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public func FfiConverterTypeSetBlob_lower(_ value: SetBlob) -> RustBuffer { return FfiConverterTypeSetBlob.lower(value) } + public enum UniffiError { + + + case Err(String ) case Base64DecodingError(String @@ -1510,8 +1576,9 @@ public enum UniffiError { case TimestampDecodingError } + #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif public struct FfiConverterTypeUniffiError: FfiConverterRustBuffer { typealias SwiftType = UniffiError @@ -1519,33 +1586,47 @@ public struct FfiConverterTypeUniffiError: FfiConverterRustBuffer { public static func read(from buf: inout (data: Data, offset: Data.Index)) throws -> UniffiError { let variant: Int32 = try readInt(&buf) switch variant { - case 1: return try .Err( - FfiConverterString.read(from: &buf) + + + + + case 1: return .Err( + try FfiConverterString.read(from: &buf) ) - case 2: return try .Base64DecodingError( - FfiConverterString.read(from: &buf) + case 2: return .Base64DecodingError( + try FfiConverterString.read(from: &buf) ) case 3: return .TimestampDecodingError - default: throw UniffiInternalError.unexpectedEnumCase + + default: throw UniffiInternalError.unexpectedEnumCase } } public static func write(_ value: UniffiError, into buf: inout [UInt8]) { switch value { + + + + + case let .Err(v1): writeInt(&buf, Int32(1)) FfiConverterString.write(v1, into: &buf) - + + case let .Base64DecodingError(v1): writeInt(&buf, Int32(2)) FfiConverterString.write(v1, into: &buf) - + + case .TimestampDecodingError: writeInt(&buf, Int32(3)) + } } } + extension UniffiError: Equatable, Hashable {} extension UniffiError: Foundation.LocalizedError { @@ -1555,9 +1636,9 @@ extension UniffiError: Foundation.LocalizedError { } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterOptionInt64: FfiConverterRustBuffer { +fileprivate struct FfiConverterOptionInt64: FfiConverterRustBuffer { typealias SwiftType = Int64? public static func write(_ value: SwiftType, into buf: inout [UInt8]) { @@ -1579,9 +1660,9 @@ private struct FfiConverterOptionInt64: FfiConverterRustBuffer { } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterOptionTypeBlob: FfiConverterRustBuffer { +fileprivate struct FfiConverterOptionTypeBlob: FfiConverterRustBuffer { typealias SwiftType = Blob? public static func write(_ value: SwiftType, into buf: inout [UInt8]) { @@ -1603,9 +1684,9 @@ private struct FfiConverterOptionTypeBlob: FfiConverterRustBuffer { } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterOptionTypeDocClock: FfiConverterRustBuffer { +fileprivate struct FfiConverterOptionTypeDocClock: FfiConverterRustBuffer { typealias SwiftType = DocClock? public static func write(_ value: SwiftType, into buf: inout [UInt8]) { @@ -1627,9 +1708,9 @@ private struct FfiConverterOptionTypeDocClock: FfiConverterRustBuffer { } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterOptionTypeDocRecord: FfiConverterRustBuffer { +fileprivate struct FfiConverterOptionTypeDocRecord: FfiConverterRustBuffer { typealias SwiftType = DocRecord? public static func write(_ value: SwiftType, into buf: inout [UInt8]) { @@ -1651,9 +1732,9 @@ private struct FfiConverterOptionTypeDocRecord: FfiConverterRustBuffer { } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterSequenceInt64: FfiConverterRustBuffer { +fileprivate struct FfiConverterSequenceInt64: FfiConverterRustBuffer { typealias SwiftType = [Int64] public static func write(_ value: [Int64], into buf: inout [UInt8]) { @@ -1669,16 +1750,16 @@ private struct FfiConverterSequenceInt64: FfiConverterRustBuffer { var seq = [Int64]() seq.reserveCapacity(Int(len)) for _ in 0 ..< len { - try seq.append(FfiConverterInt64.read(from: &buf)) + seq.append(try FfiConverterInt64.read(from: &buf)) } return seq } } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterSequenceTypeDocClock: FfiConverterRustBuffer { +fileprivate struct FfiConverterSequenceTypeDocClock: FfiConverterRustBuffer { typealias SwiftType = [DocClock] public static func write(_ value: [DocClock], into buf: inout [UInt8]) { @@ -1694,16 +1775,16 @@ private struct FfiConverterSequenceTypeDocClock: FfiConverterRustBuffer { var seq = [DocClock]() seq.reserveCapacity(Int(len)) for _ in 0 ..< len { - try seq.append(FfiConverterTypeDocClock.read(from: &buf)) + seq.append(try FfiConverterTypeDocClock.read(from: &buf)) } return seq } } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterSequenceTypeDocUpdate: FfiConverterRustBuffer { +fileprivate struct FfiConverterSequenceTypeDocUpdate: FfiConverterRustBuffer { typealias SwiftType = [DocUpdate] public static func write(_ value: [DocUpdate], into buf: inout [UInt8]) { @@ -1719,16 +1800,16 @@ private struct FfiConverterSequenceTypeDocUpdate: FfiConverterRustBuffer { var seq = [DocUpdate]() seq.reserveCapacity(Int(len)) for _ in 0 ..< len { - try seq.append(FfiConverterTypeDocUpdate.read(from: &buf)) + seq.append(try FfiConverterTypeDocUpdate.read(from: &buf)) } return seq } } #if swift(>=5.8) - @_documentation(visibility: private) +@_documentation(visibility: private) #endif -private struct FfiConverterSequenceTypeListedBlob: FfiConverterRustBuffer { +fileprivate struct FfiConverterSequenceTypeListedBlob: FfiConverterRustBuffer { typealias SwiftType = [ListedBlob] public static func write(_ value: [ListedBlob], into buf: inout [UInt8]) { @@ -1744,22 +1825,21 @@ private struct FfiConverterSequenceTypeListedBlob: FfiConverterRustBuffer { var seq = [ListedBlob]() seq.reserveCapacity(Int(len)) for _ in 0 ..< len { - try seq.append(FfiConverterTypeListedBlob.read(from: &buf)) + seq.append(try FfiConverterTypeListedBlob.read(from: &buf)) } return seq } } - private let UNIFFI_RUST_FUTURE_POLL_READY: Int8 = 0 private let UNIFFI_RUST_FUTURE_POLL_MAYBE_READY: Int8 = 1 -private let uniffiContinuationHandleMap = UniffiHandleMap>() +fileprivate let uniffiContinuationHandleMap = UniffiHandleMap>() -private func uniffiRustCallAsync( +fileprivate func uniffiRustCallAsync( rustFutureFunc: () -> UInt64, - pollFunc: (UInt64, @escaping UniffiRustFutureContinuationCallback, UInt64) -> Void, + pollFunc: (UInt64, @escaping UniffiRustFutureContinuationCallback, UInt64) -> (), completeFunc: (UInt64, UnsafeMutablePointer) -> F, - freeFunc: (UInt64) -> Void, + freeFunc: (UInt64) -> (), liftFunc: (F) throws -> T, errorHandler: ((RustBuffer) throws -> Swift.Error)? ) async throws -> T { @@ -1770,7 +1850,7 @@ private func uniffiRustCallAsync( defer { freeFunc(rustFuture) } - var pollResult: Int8 + var pollResult: Int8; repeat { pollResult = await withUnsafeContinuation { pollFunc( @@ -1789,28 +1869,26 @@ private func uniffiRustCallAsync( // Callback handlers for an async calls. These are invoked by Rust when the future is ready. They // lift the return value or error and resume the suspended function. -private func uniffiFutureContinuationCallback(handle: UInt64, pollResult: Int8) { +fileprivate func uniffiFutureContinuationCallback(handle: UInt64, pollResult: Int8) { if let continuation = try? uniffiContinuationHandleMap.remove(handle: handle) { continuation.resume(returning: pollResult) } else { print("uniffiFutureContinuationCallback invalid handle") } } - public func hashcashMint(resource: String, bits: UInt32) -> String { - return try! FfiConverterString.lift(try! rustCall { - uniffi_affine_mobile_native_fn_func_hashcash_mint( - FfiConverterString.lower(resource), - FfiConverterUInt32.lower(bits), $0 - ) - }) + return try! FfiConverterString.lift(try! rustCall() { + uniffi_affine_mobile_native_fn_func_hashcash_mint( + FfiConverterString.lower(resource), + FfiConverterUInt32.lower(bits),$0 + ) +}) } - public func newDocStoragePool() -> DocStoragePool { - return try! FfiConverterTypeDocStoragePool.lift(try! rustCall { - uniffi_affine_mobile_native_fn_func_new_doc_storage_pool($0 - ) - }) + return try! FfiConverterTypeDocStoragePool.lift(try! rustCall() { + uniffi_affine_mobile_native_fn_func_new_doc_storage_pool($0 + ) +}) } private enum InitializationResult { @@ -1818,7 +1896,6 @@ private enum InitializationResult { case contractVersionMismatch case apiChecksumMismatch } - // Use a global variable to perform the versioning checks. Swift ensures that // the code inside is only computed once. private var initializationResult: InitializationResult = { @@ -1829,85 +1906,88 @@ private var initializationResult: InitializationResult = { if bindings_contract_version != scaffolding_contract_version { return InitializationResult.contractVersionMismatch } - if uniffi_affine_mobile_native_checksum_func_hashcash_mint() != 23633 { + if (uniffi_affine_mobile_native_checksum_func_hashcash_mint() != 23633) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_func_new_doc_storage_pool() != 32882 { + if (uniffi_affine_mobile_native_checksum_func_new_doc_storage_pool() != 32882) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_clear_clocks() != 51151 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_clear_clocks() != 51151) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_connect() != 19047 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_connect() != 19047) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_delete_blob() != 53695 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_delete_blob() != 53695) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_delete_doc() != 4005 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_delete_doc() != 4005) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_disconnect() != 20410 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_disconnect() != 20410) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_blob() != 56927 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_blob() != 56927) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_clock() != 48394 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_clock() != 48394) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_clocks() != 46082 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_clocks() != 46082) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_snapshot() != 31220 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_snapshot() != 31220) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_updates() != 65430 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_doc_updates() != 65430) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pulled_remote_clock() != 40122 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pulled_remote_clock() != 56577) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pulled_remote_clocks() != 13441 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pulled_remote_clocks() != 13441) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pushed_clocks() != 47148 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pushed_clock() != 34705) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_remote_clock() != 17458 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pushed_clocks() != 47148) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_remote_clocks() != 14523 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_remote_clock() != 47662) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_list_blobs() != 6777 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_remote_clocks() != 14523) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_mark_updates_merged() != 42713 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_list_blobs() != 6777) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_push_update() != 20688 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_mark_updates_merged() != 42713) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_release_blobs() != 2203 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_push_update() != 20688) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_set_blob() != 31398 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_release_blobs() != 2203) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_set_doc_snapshot() != 5287 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_set_blob() != 31398) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_set_peer_pulled_remote_clock() != 33923 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_set_doc_snapshot() != 5287) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_set_peer_pushed_clock() != 16565 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_set_peer_pulled_remote_clock() != 33923) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_set_peer_remote_clock() != 46506 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_set_peer_pushed_clock() != 16565) { return InitializationResult.apiChecksumMismatch } - if uniffi_affine_mobile_native_checksum_method_docstoragepool_set_space_id() != 21955 { + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_set_peer_remote_clock() != 46506) { + return InitializationResult.apiChecksumMismatch + } + if (uniffi_affine_mobile_native_checksum_method_docstoragepool_set_space_id() != 21955) { return InitializationResult.apiChecksumMismatch } @@ -1925,4 +2005,4 @@ private func uniffiEnsureInitialized() { } } -// swiftlint:enable all +// swiftlint:enable all \ No newline at end of file diff --git a/packages/frontend/apps/ios/App/App/uniffi/affine_mobile_nativeFFI.h b/packages/frontend/apps/ios/App/App/uniffi/affine_mobile_nativeFFI.h index 53e771035..4b7c59eee 100644 --- a/packages/frontend/apps/ios/App/App/uniffi/affine_mobile_nativeFFI.h +++ b/packages/frontend/apps/ios/App/App/uniffi/affine_mobile_nativeFFI.h @@ -321,6 +321,11 @@ uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pulled_re uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pulled_remote_clocks(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer peer ); #endif +#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_PEER_PUSHED_CLOCK +#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_PEER_PUSHED_CLOCK +uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pushed_clock(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer peer, RustBuffer doc_id +); +#endif #ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_PEER_PUSHED_CLOCKS #define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_FN_METHOD_DOCSTORAGEPOOL_GET_PEER_PUSHED_CLOCKS uint64_t uniffi_affine_mobile_native_fn_method_docstoragepool_get_peer_pushed_clocks(void*_Nonnull ptr, RustBuffer universal_id, RustBuffer peer @@ -759,6 +764,12 @@ uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pul #define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_PEER_PULLED_REMOTE_CLOCKS uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pulled_remote_clocks(void +); +#endif +#ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_PEER_PUSHED_CLOCK +#define UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_PEER_PUSHED_CLOCK +uint16_t uniffi_affine_mobile_native_checksum_method_docstoragepool_get_peer_pushed_clock(void + ); #endif #ifndef UNIFFI_FFIDEF_UNIFFI_AFFINE_MOBILE_NATIVE_CHECKSUM_METHOD_DOCSTORAGEPOOL_GET_PEER_PUSHED_CLOCKS diff --git a/packages/frontend/apps/ios/capacitor.config.ts b/packages/frontend/apps/ios/capacitor.config.ts index 951847bd1..a2ffe91b3 100644 --- a/packages/frontend/apps/ios/capacitor.config.ts +++ b/packages/frontend/apps/ios/capacitor.config.ts @@ -14,10 +14,10 @@ const config: CapacitorConfig = { }, plugins: { CapacitorCookies: { - enabled: true, + enabled: false, }, CapacitorHttp: { - enabled: true, + enabled: false, }, Keyboard: { resize: KeyboardResize.Native, diff --git a/packages/frontend/apps/ios/package.json b/packages/frontend/apps/ios/package.json index 65bfddd59..dedce498e 100644 --- a/packages/frontend/apps/ios/package.json +++ b/packages/frontend/apps/ios/package.json @@ -26,6 +26,7 @@ "@capacitor/keyboard": "^6.0.3", "@sentry/react": "^8.44.0", "@toeverything/infra": "workspace:^", + "async-call-rpc": "^6.4.2", "next-themes": "^0.4.4", "react": "^19.0.0", "react-dom": "^19.0.0", diff --git a/packages/frontend/apps/ios/src/app.tsx b/packages/frontend/apps/ios/src/app.tsx index 91fb67f76..c304210e2 100644 --- a/packages/frontend/apps/ios/src/app.tsx +++ b/packages/frontend/apps/ios/src/app.tsx @@ -12,23 +12,22 @@ import { DefaultServerService, ServersService, ValidatorProvider, - WebSocketAuthProvider, } from '@affine/core/modules/cloud'; import { DocsService } from '@affine/core/modules/doc'; import { GlobalContextService } from '@affine/core/modules/global-context'; import { I18nProvider } from '@affine/core/modules/i18n'; import { LifecycleService } from '@affine/core/modules/lifecycle'; -import { configureLocalStorageStateStorageImpls } from '@affine/core/modules/storage'; +import { + configureLocalStorageStateStorageImpls, + NbstoreProvider, +} from '@affine/core/modules/storage'; import { PopupWindowProvider } from '@affine/core/modules/url'; import { ClientSchemeProvider } from '@affine/core/modules/url/providers/client-schema'; -import { configureIndexedDBUserspaceStorageProvider } from '@affine/core/modules/userspace'; import { configureBrowserWorkbenchModule } from '@affine/core/modules/workbench'; import { WorkspacesService } from '@affine/core/modules/workspace'; -import { - configureBrowserWorkspaceFlavours, - configureIndexedDBWorkspaceEngineStorageProvider, -} from '@affine/core/modules/workspace-engine'; +import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine'; import { I18n } from '@affine/i18n'; +import { WorkerClient } from '@affine/nbstore/worker/client'; import { defaultBlockMarkdownAdapterMatchers, docLinkBaseURLMiddleware, @@ -44,16 +43,17 @@ import { Browser } from '@capacitor/browser'; import { Haptics } from '@capacitor/haptics'; import { Keyboard, KeyboardStyle } from '@capacitor/keyboard'; import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra'; +import { OpClient } from '@toeverything/infra/op'; +import { AsyncCall } from 'async-call-rpc'; import { useTheme } from 'next-themes'; import { Suspense, useEffect } from 'react'; import { RouterProvider } from 'react-router-dom'; import { BlocksuiteMenuConfigProvider } from './bs-menu-config'; -import { configureFetchProvider } from './fetch'; import { ModalConfigProvider } from './modal-config'; -import { Cookie } from './plugins/cookie'; import { Hashcash } from './plugins/hashcash'; import { Intelligents } from './plugins/intelligents'; +import { NbStoreNativeDBApis } from './plugins/nbstore'; import { enableNavigationGesture$ } from './web-navigation-control'; const future = { @@ -65,9 +65,52 @@ configureCommonModules(framework); configureBrowserWorkbenchModule(framework); configureLocalStorageStateStorageImpls(framework); configureBrowserWorkspaceFlavours(framework); -configureIndexedDBWorkspaceEngineStorageProvider(framework); -configureIndexedDBUserspaceStorageProvider(framework); configureMobileModules(framework); +framework.impl(NbstoreProvider, { + openStore(_key, options) { + const worker = new Worker( + new URL( + /* webpackChunkName: "nbstore-worker" */ './worker.ts', + import.meta.url + ) + ); + const { port1: nativeDBApiChannelServer, port2: nativeDBApiChannelClient } = + new MessageChannel(); + AsyncCall(NbStoreNativeDBApis, { + channel: { + on(listener) { + const f = (e: MessageEvent) => { + listener(e.data); + }; + nativeDBApiChannelServer.addEventListener('message', f); + return () => { + nativeDBApiChannelServer.removeEventListener('message', f); + }; + }, + send(data) { + nativeDBApiChannelServer.postMessage(data); + }, + }, + log: false, + }); + nativeDBApiChannelServer.start(); + worker.postMessage( + { + type: 'native-db-api-channel', + port: nativeDBApiChannelClient, + }, + [nativeDBApiChannelClient] + ); + const client = new WorkerClient(new OpClient(worker), options); + return { + store: client, + dispose: () => { + worker.terminate(); + nativeDBApiChannelServer.close(); + }, + }; + }, +}); framework.impl(PopupWindowProvider, { open: (url: string) => { Browser.open({ @@ -81,18 +124,6 @@ framework.impl(ClientSchemeProvider, { return 'affine'; }, }); -configureFetchProvider(framework); -framework.impl(WebSocketAuthProvider, { - getAuthToken: async url => { - const cookies = await Cookie.getCookies({ - url, - }); - return { - userId: cookies['affine_user_id'], - token: cookies['affine_session'], - }; - }, -}); framework.impl(ValidatorProvider, { async validate(_challenge, resource) { const res = await Hashcash.hash({ challenge: resource }); diff --git a/packages/frontend/apps/ios/src/fetch.ts b/packages/frontend/apps/ios/src/fetch.ts deleted file mode 100644 index cc7061e4c..000000000 --- a/packages/frontend/apps/ios/src/fetch.ts +++ /dev/null @@ -1,191 +0,0 @@ -/** - * this file is modified from part of https://github.com/ionic-team/capacitor/blob/74c3e9447e1e32e73f818d252eb12f453d849e8d/ios/Capacitor/Capacitor/assets/native-bridge.js#L466 - * - * for support arraybuffer response type - */ -import { RawFetchProvider } from '@affine/core/modules/cloud/provider/fetch'; -import { CapacitorHttp } from '@capacitor/core'; -import type { Framework } from '@toeverything/infra'; - -const readFileAsBase64 = (file: File) => - new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onloadend = () => { - const data = reader.result; - if (data === null) { - reject(new Error('Failed to read file')); - } else { - resolve(btoa(data as string)); - } - }; - reader.onerror = reject; - reader.readAsBinaryString(file); - }); -const convertFormData = async (formData: FormData) => { - const newFormData = []; - for (const pair of formData.entries()) { - const [key, value] = pair; - if (value instanceof File) { - const base64File = await readFileAsBase64(value); - newFormData.push({ - key, - value: base64File, - type: 'base64File', - contentType: value.type, - fileName: value.name, - }); - } else { - newFormData.push({ key, value, type: 'string' }); - } - } - return newFormData; -}; -const convertBody = async (body: unknown, contentType: string) => { - if (body instanceof ReadableStream || body instanceof Uint8Array) { - let encodedData; - if (body instanceof ReadableStream) { - const reader = body.getReader(); - const chunks = []; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - } - const concatenated = new Uint8Array( - chunks.reduce((acc, chunk) => acc + chunk.length, 0) - ); - let position = 0; - for (const chunk of chunks) { - concatenated.set(chunk, position); - position += chunk.length; - } - encodedData = concatenated; - } else { - encodedData = body; - } - let data = new TextDecoder().decode(encodedData); - let type; - if (contentType === 'application/json') { - try { - data = JSON.parse(data); - } catch { - // ignore - } - type = 'json'; - } else if (contentType === 'multipart/form-data') { - type = 'formData'; - } else if ( - contentType === null || contentType === void 0 - ? void 0 - : contentType.startsWith('image') - ) { - type = 'image'; - } else if (contentType === 'application/octet-stream') { - type = 'binary'; - } else { - type = 'text'; - } - return { - data, - type, - headers: { 'Content-Type': contentType || 'application/octet-stream' }, - }; - } else if (body instanceof URLSearchParams) { - return { - data: body.toString(), - type: 'text', - }; - } else if (body instanceof FormData) { - const formData = await convertFormData(body); - return { - data: formData, - type: 'formData', - }; - } else if (body instanceof File) { - const fileData = await readFileAsBase64(body); - return { - data: fileData, - type: 'file', - headers: { 'Content-Type': body.type }, - }; - } - return { data: body, type: 'json' }; -}; -function base64ToUint8Array(base64: string) { - const binaryString = atob(base64); - const binaryArray = [...binaryString].map(function (char) { - return char.charCodeAt(0); - }); - return new Uint8Array(binaryArray); -} -export function configureFetchProvider(framework: Framework) { - framework.override(RawFetchProvider, { - fetch: async (input, init) => { - const request = new Request(input, init); - const { method } = request; - const tag = `CapacitorHttp fetch ${Date.now()} ${input}`; - console.time(tag); - try { - const { body } = request; - const optionHeaders = Object.fromEntries(request.headers.entries()); - const { - data: requestData, - type, - headers, - } = await convertBody( - (init === null || init === void 0 ? void 0 : init.body) || - body || - undefined, - optionHeaders['Content-Type'] || optionHeaders['content-type'] - ); - const accept = optionHeaders['Accept'] || optionHeaders['accept']; - const nativeResponse = await CapacitorHttp.request({ - url: request.url, - method: method, - data: requestData, - dataType: type as any, - responseType: - accept === 'application/octet-stream' ? 'arraybuffer' : undefined, - headers: Object.assign(Object.assign({}, headers), optionHeaders), - }); - const contentType = - nativeResponse.headers['Content-Type'] || - nativeResponse.headers['content-type']; - let data = - accept === 'application/octet-stream' - ? base64ToUint8Array(nativeResponse.data) - : contentType === null || contentType === void 0 - ? void 0 - : contentType.startsWith('application/json') - ? JSON.stringify(nativeResponse.data) - : contentType === 'application/octet-stream' - ? base64ToUint8Array(nativeResponse.data) - : nativeResponse.data; - - // use null data for 204 No Content HTTP response - if (nativeResponse.status === 204) { - data = null; - } - // intercept & parse response before returning - const response = new Response(new Blob([data], { type: contentType }), { - headers: nativeResponse.headers, - status: nativeResponse.status, - }); - /* - * copy url to response, `cordova-plugin-ionic` uses this url from the response - * we need `Object.defineProperty` because url is an inherited getter on the Response - * see: https://stackoverflow.com/a/57382543 - * */ - Object.defineProperty(response, 'url', { - value: nativeResponse.url, - }); - console.timeEnd(tag); - return response; - } catch (error) { - console.timeEnd(tag); - throw error; - } - }, - }); -} diff --git a/packages/frontend/apps/ios/src/index.tsx b/packages/frontend/apps/ios/src/index.tsx index f6f8a7473..5b8205ad6 100644 --- a/packages/frontend/apps/ios/src/index.tsx +++ b/packages/frontend/apps/ios/src/index.tsx @@ -1,5 +1,8 @@ import './setup'; +import '@affine/component/theme'; +import '@affine/core/mobile/styles/mobile.css'; +import { bindNativeDBApis } from '@affine/nbstore/sqlite'; import { init, reactRouterV6BrowserTracingIntegration, @@ -15,18 +18,15 @@ import { } from 'react-router-dom'; import { App } from './app'; +import { NbStoreNativeDBApis } from './plugins/nbstore'; + +bindNativeDBApis(NbStoreNativeDBApis); + +// TODO(@L-Sun) Uncomment this when the `show` method implement by `@capacitor/keyboard` in ios +// import './virtual-keyboard'; function main() { if (BUILD_CONFIG.debug || window.SENTRY_RELEASE) { - // workaround for Capacitor HttpPlugin - // capacitor-http-plugin will replace window.XMLHttpRequest with its own implementation - // but XMLHttpRequest.prototype is not defined which is used by sentry - // see: https://github.com/ionic-team/capacitor/blob/74c3e9447e1e32e73f818d252eb12f453d849e8d/core/native-bridge.ts#L581 - if ('CapacitorWebXMLHttpRequest' in window) { - window.XMLHttpRequest.prototype = ( - window.CapacitorWebXMLHttpRequest as any - ).prototype; - } // https://docs.sentry.io/platforms/javascript/guides/react/#configure init({ dsn: process.env.SENTRY_DSN, diff --git a/packages/frontend/apps/ios/src/plugins/cookie/definitions.ts b/packages/frontend/apps/ios/src/plugins/cookie/definitions.ts deleted file mode 100644 index 125c661c1..000000000 --- a/packages/frontend/apps/ios/src/plugins/cookie/definitions.ts +++ /dev/null @@ -1,6 +0,0 @@ -export interface CookiePlugin { - /** - * Returns the screen's current orientation. - */ - getCookies(options: { url: string }): Promise>; -} diff --git a/packages/frontend/apps/ios/src/plugins/cookie/index.ts b/packages/frontend/apps/ios/src/plugins/cookie/index.ts deleted file mode 100644 index e156c0de4..000000000 --- a/packages/frontend/apps/ios/src/plugins/cookie/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { registerPlugin } from '@capacitor/core'; - -import type { CookiePlugin } from './definitions'; - -const Cookie = registerPlugin('Cookie'); - -export * from './definitions'; -export { Cookie }; diff --git a/packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts b/packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts index 916606c6c..faf31b3d7 100644 --- a/packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts +++ b/packages/frontend/apps/ios/src/plugins/nbstore/definitions.ts @@ -70,12 +70,12 @@ export interface NbStorePlugin { timestamps: number[]; }) => Promise<{ count: number }>; deleteDoc: (options: { id: string; docId: string }) => Promise; - getDocClocks: (options: { id: string; after?: number | null }) => Promise< - { + getDocClocks: (options: { id: string; after?: number | null }) => Promise<{ + clocks: { docId: string; timestamp: number; - }[] - >; + }[]; + }>; getDocClock: (options: { id: string; docId: string }) => Promise< | { docId: string; @@ -95,47 +95,47 @@ export interface NbStorePlugin { getPeerRemoteClocks: (options: { id: string; peer: string; - }) => Promise>; + }) => Promise<{ clocks: Array }>; getPeerRemoteClock: (options: { id: string; peer: string; docId: string; - }) => Promise; + }) => Promise; setPeerRemoteClock: (options: { id: string; peer: string; docId: string; - clock: number; + timestamp: number; }) => Promise; getPeerPushedClocks: (options: { id: string; peer: string; - }) => Promise>; + }) => Promise<{ clocks: Array }>; getPeerPushedClock: (options: { id: string; peer: string; docId: string; - }) => Promise; + }) => Promise; setPeerPushedClock: (options: { id: string; peer: string; docId: string; - clock: number; + timestamp: number; }) => Promise; getPeerPulledRemoteClocks: (options: { id: string; peer: string; - }) => Promise>; + }) => Promise<{ clocks: Array }>; getPeerPulledRemoteClock: (options: { id: string; peer: string; docId: string; - }) => Promise; + }) => Promise; setPeerPulledRemoteClock: (options: { id: string; peer: string; docId: string; - clock: number; + timestamp: number; }) => Promise; clearClocks: (options: { id: string }) => Promise; } diff --git a/packages/frontend/apps/ios/src/plugins/nbstore/index.ts b/packages/frontend/apps/ios/src/plugins/nbstore/index.ts index 2895ab4d2..230a9f50c 100644 --- a/packages/frontend/apps/ios/src/plugins/nbstore/index.ts +++ b/packages/frontend/apps/ios/src/plugins/nbstore/index.ts @@ -96,10 +96,12 @@ export const NbStoreNativeDBApis: NativeDBApis = { id: string, after?: Date | undefined | null ): Promise { - const clocks = await NbStore.getDocClocks({ - id, - after: after?.getTime(), - }); + const clocks = ( + await NbStore.getDocClocks({ + id, + after: after?.getTime(), + }) + ).clocks; return clocks.map(c => ({ docId: c.docId, timestamp: new Date(c.timestamp), @@ -176,30 +178,30 @@ export const NbStoreNativeDBApis: NativeDBApis = { id: string, peer: string ): Promise { - const clocks = await NbStore.getPeerRemoteClocks({ - id, - peer, - }); + const clocks = ( + await NbStore.getPeerRemoteClocks({ + id, + peer, + }) + ).clocks; return clocks.map(c => ({ docId: c.docId, timestamp: new Date(c.timestamp), })); }, - getPeerRemoteClock: async function ( - id: string, - peer: string, - docId: string - ): Promise { + getPeerRemoteClock: async function (id: string, peer: string, docId: string) { const clock = await NbStore.getPeerRemoteClock({ id, peer, docId, }); - return { - docId: clock.docId, - timestamp: new Date(clock.timestamp), - }; + return clock + ? { + docId: clock.docId, + timestamp: new Date(clock.timestamp), + } + : null; }, setPeerRemoteClock: async function ( id: string, @@ -211,17 +213,19 @@ export const NbStoreNativeDBApis: NativeDBApis = { id, peer, docId, - clock: clock.getTime(), + timestamp: clock.getTime(), }); }, getPeerPulledRemoteClocks: async function ( id: string, peer: string ): Promise { - const clocks = await NbStore.getPeerPulledRemoteClocks({ - id, - peer, - }); + const clocks = ( + await NbStore.getPeerPulledRemoteClocks({ + id, + peer, + }) + ).clocks; return clocks.map(c => ({ docId: c.docId, timestamp: new Date(c.timestamp), @@ -231,16 +235,18 @@ export const NbStoreNativeDBApis: NativeDBApis = { id: string, peer: string, docId: string - ): Promise { + ) { const clock = await NbStore.getPeerPulledRemoteClock({ id, peer, docId, }); - return { - docId: clock.docId, - timestamp: new Date(clock.timestamp), - }; + return clock + ? { + docId: clock.docId, + timestamp: new Date(clock.timestamp), + } + : null; }, setPeerPulledRemoteClock: async function ( id: string, @@ -252,17 +258,19 @@ export const NbStoreNativeDBApis: NativeDBApis = { id, peer, docId, - clock: clock.getTime(), + timestamp: clock.getTime(), }); }, getPeerPushedClocks: async function ( id: string, peer: string ): Promise { - const clocks = await NbStore.getPeerPushedClocks({ - id, - peer, - }); + const clocks = ( + await NbStore.getPeerPushedClocks({ + id, + peer, + }) + ).clocks; return clocks.map(c => ({ docId: c.docId, timestamp: new Date(c.timestamp), @@ -272,16 +280,18 @@ export const NbStoreNativeDBApis: NativeDBApis = { id: string, peer: string, docId: string - ): Promise { + ): Promise { const clock = await NbStore.getPeerPushedClock({ id, peer, docId, }); - return { - docId: clock.docId, - timestamp: new Date(clock.timestamp), - }; + return clock + ? { + docId: clock.docId, + timestamp: new Date(clock.timestamp), + } + : null; }, setPeerPushedClock: async function ( id: string, @@ -293,7 +303,7 @@ export const NbStoreNativeDBApis: NativeDBApis = { id, peer, docId, - clock: clock.getTime(), + timestamp: clock.getTime(), }); }, clearClocks: async function (id: string): Promise { diff --git a/packages/frontend/apps/ios/src/setup.ts b/packages/frontend/apps/ios/src/setup.ts index 6cf4000d5..68904eaf0 100644 --- a/packages/frontend/apps/ios/src/setup.ts +++ b/packages/frontend/apps/ios/src/setup.ts @@ -1,6 +1,191 @@ import '@affine/core/bootstrap/browser'; -import '@affine/component/theme'; -import '@affine/core/mobile/styles/mobile.css'; -// TODO(@L-Sun) Uncomment this when the `show` method implement by `@capacitor/keyboard` in ios -// import './virtual-keyboard'; +/** + * the below code includes the custom fetch and websocket implementation for ios webview. + * should be included in the entry file of the app or webworker. + */ + +/* + * we override the browser's fetch function with our custom fetch function to + * overcome the restrictions of cross-domain and third-party cookies in ios webview. + * + * the custom fetch function will convert the request to `affine-http://` or `affine-https://` + * and send the request to the server. + */ +const rawFetch = globalThis.fetch; +globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL( + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : input.url, + globalThis.location.origin + ); + + if (url.protocol === 'capacitor:') { + return rawFetch(input, init); + } + + if (url.protocol === 'http:') { + url.protocol = 'affine-http:'; + } + + if (url.protocol === 'https:') { + url.protocol = 'affine-https:'; + } + + return rawFetch(url, input instanceof Request ? input : init); +}; + +/** + * we create a custom websocket class to simulate the browser's websocket connection + * through the custom url scheme handler. + * + * to overcome the restrictions of cross-domain and third-party cookies in ios webview, + * the front-end opens a websocket connection and sends a message by sending a request + * to `affine-ws://` or `affine-wss://`. + * + * the scheme has two endpoints: + * + * `affine-ws:///open?uuid={uuid}&url={wsUrl}`: opens a websocket connection and returns + * the received data via the SSE protocol. + * If the front-end closes the http connection, the websocket connection will also be closed. + * + * `affine-ws:///send?uuid={uuid}`: sends the request body data to the websocket connection + * with the specified uuid. + */ +class WrappedWebSocket { + static CLOSED = WebSocket.CLOSED; + static CLOSING = WebSocket.CLOSING; + static CONNECTING = WebSocket.CONNECTING; + static OPEN = WebSocket.OPEN; + readonly isWss: boolean; + readonly uuid = crypto.randomUUID(); + readyState: number = WebSocket.CONNECTING; + events: Record void)[]> = {}; + onopen: ((event: any) => void) | undefined = undefined; + onclose: ((event: any) => void) | undefined = undefined; + onerror: ((event: any) => void) | undefined = undefined; + onmessage: ((event: any) => void) | undefined = undefined; + eventSource: EventSource; + constructor( + readonly url: string, + _protocols?: string | string[] // not supported yet + ) { + const parsedUrl = new URL(url); + this.isWss = parsedUrl.protocol === 'wss:'; + this.eventSource = new EventSource( + `${this.isWss ? 'affine-wss' : 'affine-ws'}:///open?uuid=${this.uuid}&url=${encodeURIComponent(this.url)}` + ); + this.eventSource.addEventListener('open', () => { + this.emitOpen(new Event('open')); + }); + this.eventSource.addEventListener('error', () => { + this.eventSource.close(); + this.emitError(new Event('error')); + this.emitClose(new CloseEvent('close')); + }); + this.eventSource.addEventListener('message', data => { + const decodedData = JSON.parse(data.data); + if (decodedData.type === 'message') { + this.emitMessage( + new MessageEvent('message', { data: decodedData.data }) + ); + } + }); + } + + send(data: string) { + rawFetch( + `${this.isWss ? 'affine-wss' : 'affine-ws'}:///send?uuid=${this.uuid}`, + { + method: 'POST', + headers: { + 'Content-Type': 'text/plain', + }, + body: data, + } + ).catch(e => { + console.error('Failed to send message', e); + }); + } + + close() { + this.eventSource.close(); + this.emitClose(new CloseEvent('close')); + } + + addEventListener(type: string, listener: (event: any) => void) { + this.events[type] = this.events[type] || []; + this.events[type].push(listener); + } + + removeEventListener(type: string, listener: (event: any) => void) { + this.events[type] = this.events[type] || []; + this.events[type] = this.events[type].filter(l => l !== listener); + } + + private emitOpen(event: Event) { + this.readyState = WebSocket.OPEN; + this.events['open']?.forEach(listener => { + try { + listener(event); + } catch (e) { + console.error(e); + } + }); + try { + this.onopen?.(event); + } catch (e) { + console.error(e); + } + } + + private emitClose(event: CloseEvent) { + this.readyState = WebSocket.CLOSED; + this.events['close']?.forEach(listener => { + try { + listener(event); + } catch (e) { + console.error(e); + } + }); + try { + this.onclose?.(event); + } catch (e) { + console.error(e); + } + } + + private emitMessage(event: MessageEvent) { + this.events['message']?.forEach(listener => { + try { + listener(event); + } catch (e) { + console.error(e); + } + }); + try { + this.onmessage?.(event); + } catch (e) { + console.error(e); + } + } + + private emitError(event: Event) { + this.events['error']?.forEach(listener => { + try { + listener(event); + } catch (e) { + console.error(e); + } + }); + try { + this.onerror?.(event); + } catch (e) { + console.error(e); + } + } +} +globalThis.WebSocket = WrappedWebSocket as any; diff --git a/packages/frontend/apps/ios/src/worker.ts b/packages/frontend/apps/ios/src/worker.ts new file mode 100644 index 000000000..65f7ffe38 --- /dev/null +++ b/packages/frontend/apps/ios/src/worker.ts @@ -0,0 +1,52 @@ +import './setup'; + +import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel'; +import { cloudStorages } from '@affine/nbstore/cloud'; +import { + bindNativeDBApis, + type NativeDBApis, + sqliteStorages, +} from '@affine/nbstore/sqlite'; +import { + WorkerConsumer, + type WorkerOps, +} from '@affine/nbstore/worker/consumer'; +import { type MessageCommunicapable, OpConsumer } from '@toeverything/infra/op'; +import { AsyncCall } from 'async-call-rpc'; + +globalThis.addEventListener('message', e => { + if (e.data.type === 'native-db-api-channel') { + const port = e.ports[0] as MessagePort; + const rpc = AsyncCall( + {}, + { + channel: { + on(listener) { + const f = (e: MessageEvent) => { + listener(e.data); + }; + port.addEventListener('message', f); + return () => { + port.removeEventListener('message', f); + }; + }, + send(data) { + port.postMessage(data); + }, + }, + } + ); + bindNativeDBApis(rpc); + port.start(); + } +}); + +const consumer = new OpConsumer(globalThis as MessageCommunicapable); + +const worker = new WorkerConsumer([ + ...sqliteStorages, + ...broadcastChannelStorages, + ...cloudStorages, +]); + +worker.bindConsumer(consumer); diff --git a/packages/frontend/apps/mobile/package.json b/packages/frontend/apps/mobile/package.json index ec9c01be7..5be77cfee 100644 --- a/packages/frontend/apps/mobile/package.json +++ b/packages/frontend/apps/mobile/package.json @@ -12,6 +12,7 @@ "@affine/component": "workspace:*", "@affine/core": "workspace:*", "@affine/i18n": "workspace:*", + "@affine/nbstore": "workspace:*", "@blocksuite/affine": "workspace:*", "@blocksuite/icons": "2.2.2", "@sentry/react": "^8.44.0", diff --git a/packages/frontend/apps/mobile/src/app.tsx b/packages/frontend/apps/mobile/src/app.tsx index e099a50f5..e0d4997e3 100644 --- a/packages/frontend/apps/mobile/src/app.tsx +++ b/packages/frontend/apps/mobile/src/app.tsx @@ -6,15 +6,16 @@ import { router } from '@affine/core/mobile/router'; import { configureCommonModules } from '@affine/core/modules'; import { I18nProvider } from '@affine/core/modules/i18n'; import { LifecycleService } from '@affine/core/modules/lifecycle'; -import { configureLocalStorageStateStorageImpls } from '@affine/core/modules/storage'; -import { PopupWindowProvider } from '@affine/core/modules/url'; -import { configureIndexedDBUserspaceStorageProvider } from '@affine/core/modules/userspace'; -import { configureBrowserWorkbenchModule } from '@affine/core/modules/workbench'; import { - configureBrowserWorkspaceFlavours, - configureIndexedDBWorkspaceEngineStorageProvider, -} from '@affine/core/modules/workspace-engine'; + configureLocalStorageStateStorageImpls, + NbstoreProvider, +} from '@affine/core/modules/storage'; +import { PopupWindowProvider } from '@affine/core/modules/url'; +import { configureBrowserWorkbenchModule } from '@affine/core/modules/workbench'; +import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine'; +import { WorkerClient } from '@affine/nbstore/worker/client'; import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra'; +import { OpClient } from '@toeverything/infra/op'; import { Suspense } from 'react'; import { RouterProvider } from 'react-router-dom'; @@ -27,9 +28,43 @@ configureCommonModules(framework); configureBrowserWorkbenchModule(framework); configureLocalStorageStateStorageImpls(framework); configureBrowserWorkspaceFlavours(framework); -configureIndexedDBWorkspaceEngineStorageProvider(framework); -configureIndexedDBUserspaceStorageProvider(framework); configureMobileModules(framework); +framework.impl(NbstoreProvider, { + openStore(key, options) { + if (window.SharedWorker) { + const worker = new SharedWorker( + new URL( + /* webpackChunkName: "nbstore" */ './nbstore.ts', + import.meta.url + ), + { name: key } + ); + const client = new WorkerClient(new OpClient(worker.port), options); + worker.port.start(); + return { + store: client, + dispose: () => { + worker.port.postMessage({ type: '__close__' }); + worker.port.close(); + }, + }; + } else { + const worker = new Worker( + new URL( + /* webpackChunkName: "nbstore" */ './nbstore.ts', + import.meta.url + ) + ); + const client = new WorkerClient(new OpClient(worker), options); + return { + store: client, + dispose: () => { + worker.terminate(); + }, + }; + } + }, +}); framework.impl(PopupWindowProvider, { open: (target: string) => { const targetUrl = new URL(target); diff --git a/packages/frontend/apps/mobile/src/index.tsx b/packages/frontend/apps/mobile/src/index.tsx index 4d279a07a..df106ac8f 100644 --- a/packages/frontend/apps/mobile/src/index.tsx +++ b/packages/frontend/apps/mobile/src/index.tsx @@ -41,7 +41,7 @@ function main() { } function mountApp() { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + // oxlint-disable-next-line no-non-null-assertion const root = document.getElementById('app')!; createRoot(root).render( diff --git a/packages/frontend/apps/mobile/src/nbstore.ts b/packages/frontend/apps/mobile/src/nbstore.ts new file mode 100644 index 000000000..df075f940 --- /dev/null +++ b/packages/frontend/apps/mobile/src/nbstore.ts @@ -0,0 +1,46 @@ +import '@affine/core/bootstrap/browser'; + +import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel'; +import { cloudStorages } from '@affine/nbstore/cloud'; +import { idbStorages } from '@affine/nbstore/idb'; +import { idbV1Storages } from '@affine/nbstore/idb/v1'; +import { + WorkerConsumer, + type WorkerOps, +} from '@affine/nbstore/worker/consumer'; +import { type MessageCommunicapable, OpConsumer } from '@toeverything/infra/op'; + +const consumer = new WorkerConsumer([ + ...idbStorages, + ...idbV1Storages, + ...broadcastChannelStorages, + ...cloudStorages, +]); + +if ('onconnect' in globalThis) { + // if in shared worker + let activeConnectionCount = 0; + + (globalThis as any).onconnect = (event: MessageEvent) => { + activeConnectionCount++; + const port = event.ports[0]; + port.addEventListener('message', (event: MessageEvent) => { + if (event.data.type === '__close__') { + activeConnectionCount--; + if (activeConnectionCount === 0) { + globalThis.close(); + } + } + }); + + const opConsumer = new OpConsumer(port); + consumer.bindConsumer(opConsumer); + }; +} else { + // if in worker + const opConsumer = new OpConsumer( + globalThis as MessageCommunicapable + ); + + consumer.bindConsumer(opConsumer); +} diff --git a/packages/frontend/apps/mobile/tsconfig.json b/packages/frontend/apps/mobile/tsconfig.json index 263e2a387..7bdf03aa0 100644 --- a/packages/frontend/apps/mobile/tsconfig.json +++ b/packages/frontend/apps/mobile/tsconfig.json @@ -10,6 +10,7 @@ { "path": "../../component" }, { "path": "../../core" }, { "path": "../../i18n" }, + { "path": "../../../common/nbstore" }, { "path": "../../../../blocksuite/affine/all" }, { "path": "../../../common/infra" } ] diff --git a/packages/frontend/apps/web/package.json b/packages/frontend/apps/web/package.json index b5fd1705b..c16093aae 100644 --- a/packages/frontend/apps/web/package.json +++ b/packages/frontend/apps/web/package.json @@ -12,6 +12,7 @@ "@affine/component": "workspace:*", "@affine/core": "workspace:*", "@affine/i18n": "workspace:*", + "@affine/nbstore": "workspace:*", "@emotion/react": "^11.14.0", "@sentry/react": "^8.44.0", "@toeverything/infra": "workspace:*", diff --git a/packages/frontend/apps/web/src/app.tsx b/packages/frontend/apps/web/src/app.tsx index 95c16d191..70869ccc5 100644 --- a/packages/frontend/apps/web/src/app.tsx +++ b/packages/frontend/apps/web/src/app.tsx @@ -5,17 +5,18 @@ import { configureCommonModules } from '@affine/core/modules'; import { I18nProvider } from '@affine/core/modules/i18n'; import { LifecycleService } from '@affine/core/modules/lifecycle'; import { OpenInAppGuard } from '@affine/core/modules/open-in-app'; -import { configureLocalStorageStateStorageImpls } from '@affine/core/modules/storage'; -import { PopupWindowProvider } from '@affine/core/modules/url'; -import { configureIndexedDBUserspaceStorageProvider } from '@affine/core/modules/userspace'; -import { configureBrowserWorkbenchModule } from '@affine/core/modules/workbench'; import { - configureBrowserWorkspaceFlavours, - configureIndexedDBWorkspaceEngineStorageProvider, -} from '@affine/core/modules/workspace-engine'; + configureLocalStorageStateStorageImpls, + NbstoreProvider, +} from '@affine/core/modules/storage'; +import { PopupWindowProvider } from '@affine/core/modules/url'; +import { configureBrowserWorkbenchModule } from '@affine/core/modules/workbench'; +import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine'; import createEmotionCache from '@affine/core/utils/create-emotion-cache'; +import { WorkerClient } from '@affine/nbstore/worker/client'; import { CacheProvider } from '@emotion/react'; import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra'; +import { OpClient } from '@toeverything/infra/op'; import { Suspense } from 'react'; import { RouterProvider } from 'react-router-dom'; @@ -30,8 +31,41 @@ configureCommonModules(framework); configureBrowserWorkbenchModule(framework); configureLocalStorageStateStorageImpls(framework); configureBrowserWorkspaceFlavours(framework); -configureIndexedDBWorkspaceEngineStorageProvider(framework); -configureIndexedDBUserspaceStorageProvider(framework); +framework.impl(NbstoreProvider, { + openStore(key, options) { + if (window.SharedWorker) { + const worker = new SharedWorker( + new URL( + /* webpackChunkName: "nbstore" */ './nbstore.ts', + import.meta.url + ), + { name: key } + ); + const client = new WorkerClient(new OpClient(worker.port), options); + return { + store: client, + dispose: () => { + worker.port.postMessage({ type: '__close__' }); + worker.port.close(); + }, + }; + } else { + const worker = new Worker( + new URL( + /* webpackChunkName: "nbstore" */ './nbstore.ts', + import.meta.url + ) + ); + const client = new WorkerClient(new OpClient(worker), options); + return { + store: client, + dispose: () => { + worker.terminate(); + }, + }; + } + }, +}); framework.impl(PopupWindowProvider, { open: (target: string) => { const targetUrl = new URL(target); diff --git a/packages/frontend/apps/web/src/nbstore.ts b/packages/frontend/apps/web/src/nbstore.ts new file mode 100644 index 000000000..df075f940 --- /dev/null +++ b/packages/frontend/apps/web/src/nbstore.ts @@ -0,0 +1,46 @@ +import '@affine/core/bootstrap/browser'; + +import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel'; +import { cloudStorages } from '@affine/nbstore/cloud'; +import { idbStorages } from '@affine/nbstore/idb'; +import { idbV1Storages } from '@affine/nbstore/idb/v1'; +import { + WorkerConsumer, + type WorkerOps, +} from '@affine/nbstore/worker/consumer'; +import { type MessageCommunicapable, OpConsumer } from '@toeverything/infra/op'; + +const consumer = new WorkerConsumer([ + ...idbStorages, + ...idbV1Storages, + ...broadcastChannelStorages, + ...cloudStorages, +]); + +if ('onconnect' in globalThis) { + // if in shared worker + let activeConnectionCount = 0; + + (globalThis as any).onconnect = (event: MessageEvent) => { + activeConnectionCount++; + const port = event.ports[0]; + port.addEventListener('message', (event: MessageEvent) => { + if (event.data.type === '__close__') { + activeConnectionCount--; + if (activeConnectionCount === 0) { + globalThis.close(); + } + } + }); + + const opConsumer = new OpConsumer(port); + consumer.bindConsumer(opConsumer); + }; +} else { + // if in worker + const opConsumer = new OpConsumer( + globalThis as MessageCommunicapable + ); + + consumer.bindConsumer(opConsumer); +} diff --git a/packages/frontend/apps/web/tsconfig.json b/packages/frontend/apps/web/tsconfig.json index 9ff1e9ff3..f79b34df5 100644 --- a/packages/frontend/apps/web/tsconfig.json +++ b/packages/frontend/apps/web/tsconfig.json @@ -10,6 +10,7 @@ { "path": "../../component" }, { "path": "../../core" }, { "path": "../../i18n" }, + { "path": "../../../common/nbstore" }, { "path": "../../../common/infra" } ] } diff --git a/packages/frontend/core/package.json b/packages/frontend/core/package.json index b4c5c682e..35b01428c 100644 --- a/packages/frontend/core/package.json +++ b/packages/frontend/core/package.json @@ -14,6 +14,7 @@ "@affine/env": "workspace:*", "@affine/graphql": "workspace:*", "@affine/i18n": "workspace:*", + "@affine/nbstore": "workspace:*", "@affine/templates": "workspace:*", "@affine/track": "workspace:*", "@blocksuite/affine": "workspace:*", diff --git a/packages/frontend/core/src/bootstrap/browser.ts b/packages/frontend/core/src/bootstrap/browser.ts index 35073fbc8..5e824eaa7 100644 --- a/packages/frontend/core/src/bootstrap/browser.ts +++ b/packages/frontend/core/src/bootstrap/browser.ts @@ -1,4 +1,5 @@ // ORDER MATTERS import './env'; import './public-path'; +import './shared-worker'; import './polyfill/browser'; diff --git a/packages/frontend/core/src/bootstrap/electron.ts b/packages/frontend/core/src/bootstrap/electron.ts index f0391ff46..5a8e151d8 100644 --- a/packages/frontend/core/src/bootstrap/electron.ts +++ b/packages/frontend/core/src/bootstrap/electron.ts @@ -1,4 +1,5 @@ // ORDER MATTERS import './env'; import './public-path'; +import './shared-worker'; import './polyfill/electron'; diff --git a/packages/frontend/core/src/bootstrap/polyfill/browser.ts b/packages/frontend/core/src/bootstrap/polyfill/browser.ts index f92ff94f2..bdf4adf1c 100644 --- a/packages/frontend/core/src/bootstrap/polyfill/browser.ts +++ b/packages/frontend/core/src/bootstrap/polyfill/browser.ts @@ -1,11 +1,9 @@ -import { polyfillDispose } from './dispose'; -import { polyfillIteratorHelpers } from './iterator-helpers'; -import { polyfillPromise } from './promise-with-resolvers'; +import './dispose'; +import './iterator-helpers'; +import './promise-with-resolvers'; + import { polyfillEventLoop } from './request-idle-callback'; import { polyfillResizeObserver } from './resize-observer'; polyfillResizeObserver(); polyfillEventLoop(); -await polyfillPromise(); -await polyfillDispose(); -await polyfillIteratorHelpers(); diff --git a/packages/frontend/core/src/bootstrap/polyfill/dispose.ts b/packages/frontend/core/src/bootstrap/polyfill/dispose.ts index 8a6f63659..615ed233c 100644 --- a/packages/frontend/core/src/bootstrap/polyfill/dispose.ts +++ b/packages/frontend/core/src/bootstrap/polyfill/dispose.ts @@ -1,8 +1,2 @@ -export async function polyfillDispose() { - if (typeof Symbol.dispose !== 'symbol') { - // @ts-expect-error ignore - await import('core-js/modules/esnext.symbol.async-dispose'); - // @ts-expect-error ignore - await import('core-js/modules/esnext.symbol.dispose'); - } -} +import 'core-js/modules/esnext.symbol.async-dispose'; +import 'core-js/modules/esnext.symbol.dispose'; diff --git a/packages/frontend/core/src/bootstrap/polyfill/iterator-helpers.ts b/packages/frontend/core/src/bootstrap/polyfill/iterator-helpers.ts index 59ccde6c4..92e051090 100644 --- a/packages/frontend/core/src/bootstrap/polyfill/iterator-helpers.ts +++ b/packages/frontend/core/src/bootstrap/polyfill/iterator-helpers.ts @@ -1,7 +1 @@ -export async function polyfillIteratorHelpers() { - if (typeof globalThis['Iterator'] !== 'function') { - // @ts-expect-error ignore - // https://github.com/zloirock/core-js/blob/master/packages/core-js/proposals/iterator-helpers-stage-3.js - await import('core-js/proposals/iterator-helpers-stage-3'); - } -} +import 'core-js/proposals/iterator-helpers-stage-3'; diff --git a/packages/frontend/core/src/bootstrap/polyfill/promise-with-resolvers.ts b/packages/frontend/core/src/bootstrap/polyfill/promise-with-resolvers.ts index d9c2cef52..6d806c3a0 100644 --- a/packages/frontend/core/src/bootstrap/polyfill/promise-with-resolvers.ts +++ b/packages/frontend/core/src/bootstrap/polyfill/promise-with-resolvers.ts @@ -1,6 +1 @@ -export async function polyfillPromise() { - if (typeof Promise.withResolvers !== 'function') { - // @ts-expect-error ignore - await import('core-js/features/promise/with-resolvers'); - } -} +import 'core-js/features/promise/with-resolvers'; diff --git a/packages/frontend/core/src/bootstrap/polyfill/request-idle-callback.ts b/packages/frontend/core/src/bootstrap/polyfill/request-idle-callback.ts index 98b99e828..0664b5e93 100644 --- a/packages/frontend/core/src/bootstrap/polyfill/request-idle-callback.ts +++ b/packages/frontend/core/src/bootstrap/polyfill/request-idle-callback.ts @@ -1,6 +1,6 @@ export function polyfillEventLoop() { - window.requestIdleCallback = - window.requestIdleCallback || + globalThis.requestIdleCallback = + globalThis.requestIdleCallback || function (cb) { const start = Date.now(); return setTimeout(function () { @@ -13,8 +13,8 @@ export function polyfillEventLoop() { }, 1); }; - window.cancelIdleCallback = - window.cancelIdleCallback || + globalThis.cancelIdleCallback = + globalThis.cancelIdleCallback || function (id) { clearTimeout(id); }; diff --git a/packages/frontend/core/src/bootstrap/polyfill/resize-observer.ts b/packages/frontend/core/src/bootstrap/polyfill/resize-observer.ts index 6f878419f..9659c83cd 100644 --- a/packages/frontend/core/src/bootstrap/polyfill/resize-observer.ts +++ b/packages/frontend/core/src/bootstrap/polyfill/resize-observer.ts @@ -1,5 +1,7 @@ import { ResizeObserver } from '@juggle/resize-observer'; export function polyfillResizeObserver() { - window.ResizeObserver = ResizeObserver; + if (typeof window !== 'undefined') { + window.ResizeObserver = ResizeObserver; + } } diff --git a/packages/frontend/core/src/bootstrap/shared-worker.ts b/packages/frontend/core/src/bootstrap/shared-worker.ts new file mode 100644 index 000000000..8ae7500a1 --- /dev/null +++ b/packages/frontend/core/src/bootstrap/shared-worker.ts @@ -0,0 +1,25 @@ +/** + * This is a wrapper for SharedWorker, + * added the `name` parameter to the `SharedWorker` URL so that + * multiple `SharedWorkers` can share one script file. + */ +const rawSharedWorker = globalThis.SharedWorker; + +// TODO(@eyhn): remove this when we can use single shared worker for all workspaces +function PatchedSharedWorker( + urlParam: URL | string, + options?: string | { name: string } +) { + const url = typeof urlParam === 'string' ? new URL(urlParam) : urlParam; + if (options) { + url.searchParams.append( + typeof options === 'string' ? options : options.name, + '' + ); + } + return new rawSharedWorker(url, options); +} +// if SharedWorker is not supported, do nothing +if (rawSharedWorker) { + globalThis.SharedWorker = PatchedSharedWorker as any; +} diff --git a/packages/frontend/core/src/components/affine/page-history-modal/data.ts b/packages/frontend/core/src/components/affine/page-history-modal/data.ts index d9781ec22..ed95bde3f 100644 --- a/packages/frontend/core/src/components/affine/page-history-modal/data.ts +++ b/packages/frontend/core/src/components/affine/page-history-modal/data.ts @@ -1,7 +1,12 @@ import { useDocMetaHelper } from '@affine/core/components/hooks/use-block-suite-page-meta'; import { useDocCollectionPage } from '@affine/core/components/hooks/use-block-suite-workspace-page'; import { FetchService, GraphQLService } from '@affine/core/modules/cloud'; -import { getAFFiNEWorkspaceSchema } from '@affine/core/modules/workspace'; +import { + getAFFiNEWorkspaceSchema, + type WorkspaceFlavourProvider, + WorkspaceService, + WorkspacesService, +} from '@affine/core/modules/workspace'; import { WorkspaceImpl } from '@affine/core/modules/workspace/impls/workspace'; import { DebugLogger } from '@affine/debug'; import type { ListHistoryQuery } from '@affine/graphql'; @@ -25,7 +30,6 @@ import { useMutation, } from '../../../components/hooks/use-mutation'; import { useQueryInfinite } from '../../../components/hooks/use-query'; -import { CloudBlobStorage } from '../../../modules/workspace-engine/impls/engine/blob-cloud'; const logger = new DebugLogger('page-history'); @@ -105,19 +109,28 @@ const docCollectionMap = new Map(); // assume the workspace is a cloud workspace since the history feature is only enabled for cloud workspace const getOrCreateShellWorkspace = ( workspaceId: string, - fetchService: FetchService, - graphQLService: GraphQLService + flavourProvider?: WorkspaceFlavourProvider ) => { let docCollection = docCollectionMap.get(workspaceId); if (!docCollection) { - const blobStorage = new CloudBlobStorage( - workspaceId, - fetchService, - graphQLService - ); docCollection = new WorkspaceImpl({ id: workspaceId, - blobSource: blobStorage, + blobSource: { + name: 'cloud', + readonly: true, + async get(key) { + return flavourProvider?.getWorkspaceBlob(workspaceId, key) ?? null; + }, + set() { + return Promise.resolve(''); + }, + delete() { + return Promise.resolve(); + }, + list() { + return Promise.resolve([]); + }, + }, schema: getAFFiNEWorkspaceSchema(), }); docCollectionMap.set(workspaceId, docCollection); @@ -150,6 +163,8 @@ export const useSnapshotPage = ( pageDocId: string, ts?: string ) => { + const affineWorkspace = useService(WorkspaceService).workspace; + const workspacesService = useService(WorkspacesService); const fetchService = useService(FetchService); const graphQLService = useService(GraphQLService); const snapshot = usePageHistory(docCollection.id, pageDocId, ts); @@ -160,8 +175,7 @@ export const useSnapshotPage = ( const pageId = pageDocId + '-' + ts; const historyShellWorkspace = getOrCreateShellWorkspace( docCollection.id, - fetchService, - graphQLService + workspacesService.getWorkspaceFlavourProvider(affineWorkspace.meta) ); let page = historyShellWorkspace.getDoc(pageId); if (!page && snapshot) { @@ -175,19 +189,31 @@ export const useSnapshotPage = ( }); // must load before applyUpdate } return page ?? undefined; - }, [ts, pageDocId, docCollection.id, fetchService, graphQLService, snapshot]); + }, [ + ts, + pageDocId, + docCollection.id, + workspacesService, + affineWorkspace.meta, + snapshot, + ]); useEffect(() => { const historyShellWorkspace = getOrCreateShellWorkspace( docCollection.id, - fetchService, - graphQLService + workspacesService.getWorkspaceFlavourProvider(affineWorkspace.meta) ); // apply the rootdoc's update to the current workspace // this makes sure the page reference links are not deleted ones in the preview const update = encodeStateAsUpdate(docCollection.doc); applyUpdate(historyShellWorkspace.doc, update); - }, [docCollection, fetchService, graphQLService]); + }, [ + affineWorkspace.meta, + docCollection, + fetchService, + graphQLService, + workspacesService, + ]); return page; }; diff --git a/packages/frontend/core/src/components/affine/quota-reached-modal/cloud-quota-modal.tsx b/packages/frontend/core/src/components/affine/quota-reached-modal/cloud-quota-modal.tsx index 52775be33..fe3005e60 100644 --- a/packages/frontend/core/src/components/affine/quota-reached-modal/cloud-quota-modal.tsx +++ b/packages/frontend/core/src/components/affine/quota-reached-modal/cloud-quota-modal.tsx @@ -68,11 +68,11 @@ export const CloudQuotaModal = () => { }, [userQuota, isOwner, workspaceQuota, t]); const onAbortLargeBlob = useAsyncCallback( - async (blob: Blob) => { + async (byteSize: number) => { // wait for quota revalidation await workspaceQuotaService.quota.waitForRevalidation(); if ( - blob.size > (workspaceQuotaService.quota.quota$.value?.blobLimit ?? 0) + byteSize > (workspaceQuotaService.quota.quota$.value?.blobLimit ?? 0) ) { setOpen(true); } @@ -85,10 +85,10 @@ export const CloudQuotaModal = () => { return; } - currentWorkspace.engine.blob.singleBlobSizeLimit = workspaceQuota.blobLimit; + currentWorkspace.engine.blob.setMaxBlobSize(workspaceQuota.blobLimit); const disposable = - currentWorkspace.engine.blob.onAbortLargeBlob(onAbortLargeBlob); + currentWorkspace.engine.blob.onReachedMaxBlobSize(onAbortLargeBlob); return () => { disposable(); }; diff --git a/packages/frontend/core/src/components/affine/quota-reached-modal/local-quota-modal.tsx b/packages/frontend/core/src/components/affine/quota-reached-modal/local-quota-modal.tsx index 06f7d8333..db1292cef 100644 --- a/packages/frontend/core/src/components/affine/quota-reached-modal/local-quota-modal.tsx +++ b/packages/frontend/core/src/components/affine/quota-reached-modal/local-quota-modal.tsx @@ -16,7 +16,7 @@ export const LocalQuotaModal = () => { }, [setOpen]); useEffect(() => { - const disposable = currentWorkspace.engine.blob.onAbortLargeBlob(() => { + const disposable = currentWorkspace.engine.blob.onReachedMaxBlobSize(() => { setOpen(true); }); return () => { diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/lit-adaper.tsx b/packages/frontend/core/src/components/blocksuite/block-suite-editor/lit-adaper.tsx index 1b561a02e..1df1799df 100644 --- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/lit-adaper.tsx +++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/lit-adaper.tsx @@ -57,7 +57,6 @@ import { patchForClipboardInElectron, patchForEdgelessNoteConfig, patchForMobile, - patchForSharedPage, patchGenerateDocUrlExtension, patchNotificationService, patchOpenDocExtension, @@ -93,7 +92,7 @@ interface BlocksuiteEditorProps { defaultOpenProperty?: DefaultOpenProperty; } -const usePatchSpecs = (shared: boolean, mode: DocMode) => { +const usePatchSpecs = (mode: DocMode) => { const [reactToLit, portals] = useLitPortalFactory(); const { peekViewService, @@ -168,9 +167,6 @@ const usePatchSpecs = (shared: boolean, mode: DocMode) => { patched = patched.concat(patchParseDocUrlExtension(framework)); patched = patched.concat(patchGenerateDocUrlExtension(framework)); patched = patched.concat(patchQuickSearchService(framework)); - if (shared) { - patched = patched.concat(patchForSharedPage()); - } if (BUILD_CONFIG.isMobileEdition) { patched = patched.concat(patchForMobile()); } @@ -190,7 +186,6 @@ const usePatchSpecs = (shared: boolean, mode: DocMode) => { peekViewService, reactToLit, referenceRenderer, - shared, specs, featureFlagService, ]); @@ -261,7 +256,7 @@ export const BlocksuiteDocEditor = forwardRef< [externalTitleRef] ); - const [specs, portals] = usePatchSpecs(!!shared, 'page'); + const [specs, portals] = usePatchSpecs('page'); const displayBiDirectionalLink = useLiveData( editorSettingService.editorSetting.settings$.selector( @@ -349,8 +344,8 @@ export const BlocksuiteDocEditor = forwardRef< export const BlocksuiteEdgelessEditor = forwardRef< EdgelessEditor, BlocksuiteEditorProps ->(function BlocksuiteEdgelessEditor({ page, shared }, ref) { - const [specs, portals] = usePatchSpecs(!!shared, 'edgeless'); +>(function BlocksuiteEdgelessEditor({ page }, ref) { + const [specs, portals] = usePatchSpecs('edgeless'); const editorRef = useRef(null); const onDocRef = useCallback( diff --git a/packages/frontend/core/src/components/over-capacity/index.tsx b/packages/frontend/core/src/components/over-capacity/index.tsx index faa8f237a..72feef5bd 100644 --- a/packages/frontend/core/src/components/over-capacity/index.tsx +++ b/packages/frontend/core/src/components/over-capacity/index.tsx @@ -3,6 +3,7 @@ import { WorkspaceDialogService } from '@affine/core/modules/dialogs'; import { WorkspacePermissionService } from '@affine/core/modules/permissions'; import { WorkspaceService } from '@affine/core/modules/workspace'; import { useI18n } from '@affine/i18n'; +import type { BlobSyncState } from '@affine/nbstore'; import { useLiveData, useService } from '@toeverything/infra'; import { debounce } from 'lodash-es'; import { useCallback, useEffect } from 'react'; @@ -31,8 +32,8 @@ export const OverCapacityNotification = () => { // debounce sync engine status useEffect(() => { const disposableOverCapacity = - currentWorkspace.engine.blob.isStorageOverCapacity$.subscribe( - debounce((isStorageOverCapacity: boolean) => { + currentWorkspace.engine.blob.state$.subscribe( + debounce(({ isStorageOverCapacity }: BlobSyncState) => { const isOver = isStorageOverCapacity; if (!isOver) { return; diff --git a/packages/frontend/core/src/components/workspace-selector/workspace-card/index.tsx b/packages/frontend/core/src/components/workspace-selector/workspace-card/index.tsx index 6f22c88e8..3613f8112 100644 --- a/packages/frontend/core/src/components/workspace-selector/workspace-card/index.tsx +++ b/packages/frontend/core/src/components/workspace-selector/workspace-card/index.tsx @@ -20,11 +20,11 @@ import { TeamWorkspaceIcon, UnsyncIcon, } from '@blocksuite/icons/rc'; -import { useLiveData } from '@toeverything/infra'; +import { LiveData, useLiveData } from '@toeverything/infra'; import { cssVar } from '@toeverything/theme'; import clsx from 'clsx'; import type { HTMLAttributes } from 'react'; -import { forwardRef, useCallback, useEffect, useState } from 'react'; +import { forwardRef, useCallback, useEffect, useMemo, useState } from 'react'; import { useCatchEventCallback } from '../../hooks/use-catch-event-hook'; import { WorkspaceAvatar } from '../../workspace-avatar'; @@ -85,7 +85,11 @@ const useSyncEngineSyncProgress = (meta: WorkspaceMetadata) => { const workspace = useWorkspace(meta); const engineState = useLiveData( - workspace?.engine.docEngineState$.throttleTime(100) + useMemo(() => { + return workspace + ? LiveData.from(workspace.engine.doc.state$, null).throttleTime(100) + : null; + }, [workspace]) ); if (!engineState || !workspace) { @@ -94,7 +98,7 @@ const useSyncEngineSyncProgress = (meta: WorkspaceMetadata) => { const progress = (engineState.total - engineState.syncing) / engineState.total; - const syncing = engineState.syncing > 0 || engineState.retrying; + const syncing = engineState.syncing > 0 || engineState.syncRetrying; let content; // TODO(@eyhn): add i18n @@ -106,9 +110,9 @@ const useSyncEngineSyncProgress = (meta: WorkspaceMetadata) => { } } else if (!isOnline) { content = 'Disconnected, please check your network connection'; - } else if (engineState.retrying && engineState.errorMessage) { - content = `${engineState.errorMessage}, reconnecting.`; - } else if (engineState.retrying) { + } else if (engineState.syncRetrying && engineState.syncErrorMessage) { + content = `${engineState.syncErrorMessage}, reconnecting.`; + } else if (engineState.syncRetrying) { content = 'Sync disconnected due to unexpected issues, reconnecting.'; } else if (syncing) { content = @@ -123,7 +127,7 @@ const useSyncEngineSyncProgress = (meta: WorkspaceMetadata) => { return SyncingWorkspaceStatus({ progress: progress ? Math.max(progress, 0.2) : undefined, }); - } else if (engineState.retrying) { + } else if (engineState.syncRetrying) { return UnSyncWorkspaceStatus(); } else { return CloudWorkspaceStatus(); @@ -145,7 +149,7 @@ const useSyncEngineSyncProgress = (meta: WorkspaceMetadata) => { progress, active: workspace.flavour !== 'local' && - ((syncing && progress !== undefined) || engineState.retrying), // active if syncing or retrying, + ((syncing && progress !== undefined) || engineState.syncRetrying), // active if syncing or retrying, }; }; diff --git a/packages/frontend/core/src/desktop/dialogs/doc-info/time-row.tsx b/packages/frontend/core/src/desktop/dialogs/doc-info/time-row.tsx index bfcced521..612bc8b47 100644 --- a/packages/frontend/core/src/desktop/dialogs/doc-info/time-row.tsx +++ b/packages/frontend/core/src/desktop/dialogs/doc-info/time-row.tsx @@ -1,8 +1,7 @@ import { PropertyName, PropertyRoot, PropertyValue } from '@affine/component'; import { DocsService } from '@affine/core/modules/doc'; -import { WorkspaceService } from '@affine/core/modules/workspace'; import { i18nTime, useI18n } from '@affine/i18n'; -import { DateTimeIcon, HistoryIcon } from '@blocksuite/icons/rc'; +import { DateTimeIcon } from '@blocksuite/icons/rc'; import { useLiveData, useService } from '@toeverything/infra'; import clsx from 'clsx'; import type { ConfigType } from 'dayjs'; @@ -19,11 +18,7 @@ export const TimeRow = ({ className?: string; }) => { const t = useI18n(); - const workspaceService = useService(WorkspaceService); const docsService = useService(DocsService); - const { syncing, retrying, serverClock } = useLiveData( - workspaceService.workspace.engine.doc.docState$(docId) - ); const docRecord = useLiveData(docsService.list.doc$(docId)); const docMeta = useLiveData(docRecord?.meta$); @@ -43,38 +38,14 @@ export const TimeRow = ({ : null; return ( - <> - - } /> - - {docMeta ? formatI18nTime(docMeta.createDate) : localizedCreateTime} - - - {serverClock ? ( - - } - /> - - {!syncing && !retrying - ? formatI18nTime(serverClock) - : docMeta?.updatedDate - ? formatI18nTime(docMeta.updatedDate) - : null} - - - ) : docMeta?.updatedDate ? ( - - } /> - {formatI18nTime(docMeta.updatedDate)} - - ) : null} - + + } /> + + {docMeta ? formatI18nTime(docMeta.createDate) : localizedCreateTime} + + ); - }, [docMeta, retrying, serverClock, syncing, t]); + }, [docMeta, t]); const dTimestampElement = useDebouncedValue(timestampElement, 500); diff --git a/packages/frontend/core/src/desktop/dialogs/setting/index.tsx b/packages/frontend/core/src/desktop/dialogs/setting/index.tsx index 131cd2700..67e18ce80 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/index.tsx +++ b/packages/frontend/core/src/desktop/dialogs/setting/index.tsx @@ -63,7 +63,6 @@ const SettingModalInner = ({ const currentServerId = useLiveData( globalContextService.globalContext.serverId.$ ); - console.log(currentServerId); const serversService = useService(ServersService); const defaultServerService = useService(DefaultServerService); const currentServer = diff --git a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/export.tsx b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/export.tsx index 8006e5b8a..8d23a5178 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/export.tsx +++ b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/export.tsx @@ -39,8 +39,8 @@ export const DesktopExportPanel = ({ workspace }: ExportPanelProps) => { type: 'workspace', }); if (isOnline) { - await workspace.engine.waitForDocSynced(); - await workspace.engine.blob.sync(); + await workspace.engine.doc.waitForSynced(); + await workspace.engine.blob.fullSync(); } const result = await desktopApi.handler?.dialog.saveDBFileAs(workspaceId); diff --git a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/index.tsx b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/index.tsx index 6d99d56e4..e1459dbfe 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/index.tsx +++ b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/index.tsx @@ -8,9 +8,7 @@ import { WorkspaceServerService } from '@affine/core/modules/cloud'; import { WorkspaceService } from '@affine/core/modules/workspace'; import { UNTITLED_WORKSPACE_NAME } from '@affine/env/constant'; import { useI18n } from '@affine/i18n'; -import { ArrowRightSmallIcon } from '@blocksuite/icons/rc'; import { FrameworkScope, useService } from '@toeverything/infra'; -import { useCallback } from 'react'; import { DeleteLeaveWorkspace } from './delete-leave-workspace'; import { EnableCloudPanel } from './enable-cloud'; @@ -34,17 +32,6 @@ export const WorkspaceSettingDetail = ({ const workspaceInfo = useWorkspaceInfo(workspace); - const handleResetSyncStatus = useCallback(() => { - workspace?.engine.doc - .resetSyncStatus() - .then(() => { - window.location.reload(); - }) - .catch(err => { - console.error(err); - }); - }, [workspace]); - if (!workspace) { return null; } @@ -71,8 +58,10 @@ export const WorkspaceSettingDetail = ({ - - + {workspace.flavour !== 'local' && } + {workspace.flavour !== 'local' && ( + + )} {BUILD_CONFIG.isElectron && ( @@ -82,19 +71,6 @@ export const WorkspaceSettingDetail = ({ )} - - {t['com.affine.resetSyncStatus.button']()} - - } - desc={t['com.affine.resetSyncStatus.description']()} - style={{ cursor: 'pointer' }} - onClick={handleResetSyncStatus} - data-testid="reset-sync-status" - > - - diff --git a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/profile.tsx b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/profile.tsx index 70268c961..f84e69de4 100644 --- a/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/profile.tsx +++ b/packages/frontend/core/src/desktop/dialogs/setting/workspace-setting/new-workspace-setting-detail/profile.tsx @@ -9,9 +9,10 @@ import { validateAndReduceImage } from '@affine/core/utils/reduce-image'; import { UNTITLED_WORKSPACE_NAME } from '@affine/env/constant'; import { useI18n } from '@affine/i18n'; import { CameraIcon } from '@blocksuite/icons/rc'; -import { useLiveData, useService } from '@toeverything/infra'; +import { LiveData, useLiveData, useService } from '@toeverything/infra'; import type { KeyboardEvent } from 'react'; -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { map } from 'rxjs'; import * as style from './style.css'; @@ -24,8 +25,18 @@ export const ProfilePanel = () => { useEffect(() => { permissionService.permission.revalidate(); }, [permissionService]); - const workspaceIsReady = useLiveData(workspace?.engine.rootDocState$)?.ready; - + const workspaceIsReady = useLiveData( + useMemo(() => { + return workspace + ? LiveData.from( + workspace.engine.doc + .docState$(workspace.id) + .pipe(map(v => v.ready)), + false + ) + : null; + }, [workspace]) + ); const [name, setName] = useState(''); useEffect(() => { diff --git a/packages/frontend/core/src/desktop/pages/workspace/attachment/index.tsx b/packages/frontend/core/src/desktop/pages/workspace/attachment/index.tsx index da04ac40a..7e6e00b8e 100644 --- a/packages/frontend/core/src/desktop/pages/workspace/attachment/index.tsx +++ b/packages/frontend/core/src/desktop/pages/workspace/attachment/index.tsx @@ -33,7 +33,7 @@ const useLoadAttachment = (pageId: string, attachmentId: string) => { if (!doc.blockSuiteDoc.ready) { doc.blockSuiteDoc.load(); } - doc.setPriorityLoad(10); + const dispose = doc.addPriorityLoad(10); doc .waitForSyncReady() @@ -47,6 +47,7 @@ const useLoadAttachment = (pageId: string, attachmentId: string) => { return () => { release(); + dispose(); }; }, [docRecord, docsService, pageId, attachmentId]); diff --git a/packages/frontend/core/src/desktop/pages/workspace/detail-page/detail-page-wrapper.tsx b/packages/frontend/core/src/desktop/pages/workspace/detail-page/detail-page-wrapper.tsx index dc63d1619..2ceb81f39 100644 --- a/packages/frontend/core/src/desktop/pages/workspace/detail-page/detail-page-wrapper.tsx +++ b/packages/frontend/core/src/desktop/pages/workspace/detail-page/detail-page-wrapper.tsx @@ -41,10 +41,7 @@ const useLoadDoc = (pageId: string) => { // set sync engine priority target useEffect(() => { - currentWorkspace.engine.doc.setPriority(pageId, 10); - return () => { - currentWorkspace.engine.doc.setPriority(pageId, 5); - }; + return currentWorkspace.engine.doc.addPriority(pageId, 10); }, [currentWorkspace, pageId]); const isInTrash = useLiveData(doc?.meta$.map(meta => meta.trash)); diff --git a/packages/frontend/core/src/desktop/pages/workspace/index.tsx b/packages/frontend/core/src/desktop/pages/workspace/index.tsx index 91cb69dbe..e74b87fc0 100644 --- a/packages/frontend/core/src/desktop/pages/workspace/index.tsx +++ b/packages/frontend/core/src/desktop/pages/workspace/index.tsx @@ -16,6 +16,7 @@ import { import { ZipTransformer } from '@blocksuite/affine/blocks'; import { FrameworkScope, + LiveData, useLiveData, useService, useServices, @@ -28,6 +29,7 @@ import { useParams, useSearchParams, } from 'react-router-dom'; +import { map } from 'rxjs'; import * as _Y from 'yjs'; import { AffineErrorBoundary } from '../../../components/affine/affine-error-boundary'; @@ -247,7 +249,20 @@ const WorkspacePage = ({ meta }: { meta: WorkspaceMetadata }) => { }, [meta, workspacesService]); const isRootDocReady = - useLiveData(workspace?.engine.rootDocState$.map(v => v.ready)) ?? false; + useLiveData( + useMemo( + () => + workspace + ? LiveData.from( + workspace.engine.doc + .docState$(workspace.id) + .pipe(map(v => v.ready)), + false + ) + : null, + [workspace] + ) + ) ?? false; useEffect(() => { if (workspace) { diff --git a/packages/frontend/core/src/desktop/pages/workspace/layouts/workspace-layout.tsx b/packages/frontend/core/src/desktop/pages/workspace/layouts/workspace-layout.tsx index f287d6671..ee6471e65 100644 --- a/packages/frontend/core/src/desktop/pages/workspace/layouts/workspace-layout.tsx +++ b/packages/frontend/core/src/desktop/pages/workspace/layouts/workspace-layout.tsx @@ -25,13 +25,13 @@ export const WorkspaceLayout = function WorkspaceLayout({ {/* ---- some side-effect components ---- */} - {currentWorkspace?.flavour === 'local' ? ( - - ) : ( + {currentWorkspace?.flavour !== 'local' ? ( <> + ) : ( + )} diff --git a/packages/frontend/core/src/desktop/pages/workspace/share/share-page.tsx b/packages/frontend/core/src/desktop/pages/workspace/share/share-page.tsx index f84926ef2..26ade694b 100644 --- a/packages/frontend/core/src/desktop/pages/workspace/share/share-page.tsx +++ b/packages/frontend/core/src/desktop/pages/workspace/share/share-page.tsx @@ -5,12 +5,7 @@ import { usePageDocumentTitle } from '@affine/core/components/hooks/use-global-s import { useNavigateHelper } from '@affine/core/components/hooks/use-navigate-helper'; import { PageDetailEditor } from '@affine/core/components/page-detail-editor'; import { AppContainer } from '@affine/core/desktop/components/app-container'; -import { - AuthService, - FetchService, - GraphQLService, - ServerService, -} from '@affine/core/modules/cloud'; +import { AuthService, ServerService } from '@affine/core/modules/cloud'; import { type Doc, DocsService } from '@affine/core/modules/doc'; import { type Editor, @@ -19,13 +14,11 @@ import { EditorsService, } from '@affine/core/modules/editor'; import { PeekViewManagerModal } from '@affine/core/modules/peek-view'; -import { ShareReaderService } from '@affine/core/modules/share-doc'; import { ViewIcon, ViewTitle } from '@affine/core/modules/workbench'; import { type Workspace, WorkspacesService, } from '@affine/core/modules/workspace'; -import { CloudBlobStorage } from '@affine/core/modules/workspace-engine'; import { useI18n } from '@affine/i18n'; import { type DocMode, @@ -35,22 +28,9 @@ import { import type { AffineEditorContainer } from '@blocksuite/affine/presets'; import { DisposableGroup } from '@blocksuite/global/utils'; import { Logo1Icon } from '@blocksuite/icons/rc'; -import { - EmptyBlobStorage, - FrameworkScope, - ReadonlyDocStorage, - useLiveData, - useService, - useServices, -} from '@toeverything/infra'; +import { FrameworkScope, useLiveData, useService } from '@toeverything/infra'; import clsx from 'clsx'; -import { - type ReactNode, - useCallback, - useEffect, - useMemo, - useState, -} from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { useLocation } from 'react-router-dom'; import { PageNotFound } from '../../404'; @@ -65,15 +45,6 @@ export const SharePage = ({ workspaceId: string; docId: string; }) => { - const { shareReaderService, serverService } = useServices({ - ShareReaderService, - ServerService, - }); - - const isLoading = useLiveData(shareReaderService.reader.isLoading$); - const error = useLiveData(shareReaderService.reader.error$); - const data = useLiveData(shareReaderService.reader.data$); - const location = useLocation(); const { mode, selector, isTemplate, templateName, templateSnapshotUrl } = @@ -105,47 +76,26 @@ export const SharePage = ({ }; }, [location.search]); - useEffect(() => { - shareReaderService.reader.loadShare({ - serverId: serverService.server.id, - workspaceId, - docId, - }); - }, [shareReaderService, docId, workspaceId, serverService.server.id]); - - let element: ReactNode = null; - if (isLoading) { - element = null; - } else if (data) { - element = ( + return ( + - ); - } else if (error) { - // TODO(@JimmFly): handle error - element = ; - } else { - element = ; - } - - return {element}; + + ); }; const SharePageInner = ({ workspaceId, docId, - workspaceBinary, - docBinary, - publishMode = 'page' as DocMode, + publishMode = 'page', selector, isTemplate, templateName, @@ -153,20 +103,18 @@ const SharePageInner = ({ }: { workspaceId: string; docId: string; - workspaceBinary: Uint8Array; - docBinary: Uint8Array; publishMode?: DocMode; selector?: EditorSelector; isTemplate?: boolean; templateName?: string; templateSnapshotUrl?: string; }) => { + const serverService = useService(ServerService); const workspacesService = useService(WorkspacesService); - const fetchService = useService(FetchService); - const graphQLService = useService(GraphQLService); const [workspace, setWorkspace] = useState(null); const [page, setPage] = useState(null); const [editor, setEditor] = useState(null); + const [noPermission, setNoPermission] = useState(false); const [editorContainer, setActiveBlocksuiteEditor] = useActiveBlocksuiteEditor(); @@ -181,38 +129,41 @@ const SharePageInner = ({ isSharedMode: true, }, { - getDocStorage() { - return new ReadonlyDocStorage({ - [workspaceId]: workspaceBinary, - [docId]: docBinary, - }); - }, - getAwarenessConnections() { - return []; - }, - getDocServer() { - return null; - }, - getLocalBlobStorage() { - return EmptyBlobStorage; - }, - getRemoteBlobStorages() { - return [ - new CloudBlobStorage(workspaceId, fetchService, graphQLService), - ]; + local: { + doc: { + name: 'StaticCloudDocStorage', + opts: { + id: workspaceId, + serverBaseUrl: serverService.server.baseUrl, + }, + }, + blob: { + name: 'CloudBlobStorage', + opts: { + id: workspaceId, + serverBaseUrl: serverService.server.baseUrl, + }, + }, }, + remotes: {}, } ); setWorkspace(workspace); - workspace.engine - .waitForRootDocReady() - .then(() => { + workspace.engine.doc + .waitForDocLoaded(workspace.id) + .then(async () => { const { doc } = workspace.scope.get(DocsService).open(docId); - + doc.blockSuiteDoc.load(); doc.blockSuiteDoc.readonly = true; + await workspace.engine.doc.waitForDocLoaded(docId); + + if (!doc.blockSuiteDoc.root) { + throw new Error('Doc is empty'); + } + setPage(doc); const editor = doc.scope.get(EditorsService).createEditor(); @@ -226,6 +177,7 @@ const SharePageInner = ({ }) .catch(err => { console.error(err); + setNoPermission(true); }); }, [ docId, @@ -233,10 +185,7 @@ const SharePageInner = ({ workspacesService, publishMode, selector, - workspaceBinary, - docBinary, - fetchService, - graphQLService, + serverService.server.baseUrl, ]); const t = useI18n(); @@ -281,6 +230,10 @@ const SharePageInner = ({ [editor, setActiveBlocksuiteEditor, jumpToPageBlock, openPage, workspaceId] ); + if (noPermission) { + return ; + } + if (!workspace || !page || !editor) { return; } diff --git a/packages/frontend/core/src/mobile/pages/workspace/layout.tsx b/packages/frontend/core/src/mobile/pages/workspace/layout.tsx index e1f699cdf..411741d15 100644 --- a/packages/frontend/core/src/mobile/pages/workspace/layout.tsx +++ b/packages/frontend/core/src/mobile/pages/workspace/layout.tsx @@ -17,13 +17,20 @@ import type { WorkspaceMetadata, } from '@affine/core/modules/workspace'; import { WorkspacesService } from '@affine/core/modules/workspace'; -import { FrameworkScope, useLiveData, useServices } from '@toeverything/infra'; +import { + FrameworkScope, + LiveData, + useLiveData, + useServices, +} from '@toeverything/infra'; import { type PropsWithChildren, useEffect, useLayoutEffect, + useMemo, useState, } from 'react'; +import { map } from 'rxjs'; import { AppFallback } from '../../components/app-fallback'; import { WorkspaceDialogs } from '../../dialogs'; @@ -33,11 +40,11 @@ declare global { /** * @internal debug only */ - // eslint-disable-next-line no-var + // oxlint-disable-next-line no-var var currentWorkspace: Workspace | undefined; - // eslint-disable-next-line no-var + // oxlint-disable-next-line no-var var exportWorkspaceSnapshot: (docs?: string[]) => Promise; - // eslint-disable-next-line no-var + // oxlint-disable-next-line no-var var importWorkspaceSnapshot: () => Promise; interface WindowEventMap { 'affine:workspace:change': CustomEvent<{ id: string }>; @@ -106,7 +113,20 @@ export const WorkspaceLayout = ({ ]); const isRootDocReady = - useLiveData(workspace?.engine.rootDocState$.map(v => v.ready)) ?? false; + useLiveData( + useMemo( + () => + workspace + ? LiveData.from( + workspace.engine.doc + .docState$(workspace.id) + .pipe(map(v => v.ready)), + false + ) + : null, + [workspace] + ) + ) ?? false; if (!workspace) { return null; // skip this, workspace will be set in layout effect @@ -125,10 +145,10 @@ export const WorkspaceLayout = ({ {/* ---- some side-effect components ---- */} - {workspace?.flavour === 'local' ? ( - - ) : ( + {workspace?.flavour !== 'local' ? ( + ) : ( + )} diff --git a/packages/frontend/core/src/modules/cloud/index.ts b/packages/frontend/core/src/modules/cloud/index.ts index 9d6604ec4..91abe89d0 100644 --- a/packages/frontend/core/src/modules/cloud/index.ts +++ b/packages/frontend/core/src/modules/cloud/index.ts @@ -11,9 +11,7 @@ export { AccountChanged } from './events/account-changed'; export { AccountLoggedIn } from './events/account-logged-in'; export { AccountLoggedOut } from './events/account-logged-out'; export { ServerInitialized } from './events/server-initialized'; -export { RawFetchProvider } from './provider/fetch'; export { ValidatorProvider } from './provider/validator'; -export { WebSocketAuthProvider } from './provider/websocket-auth'; export { AuthService } from './services/auth'; export { CaptchaService } from './services/captcha'; export { DefaultServerService } from './services/default-server'; @@ -27,7 +25,6 @@ export { SubscriptionService } from './services/subscription'; export { UserCopilotQuotaService } from './services/user-copilot-quota'; export { UserFeatureService } from './services/user-feature'; export { UserQuotaService } from './services/user-quota'; -export { WebSocketService } from './services/websocket'; export { WorkspaceInvoicesService } from './services/workspace-invoices'; export { WorkspaceServerService } from './services/workspace-server'; export { WorkspaceSubscriptionService } from './services/workspace-subscription'; @@ -51,9 +48,7 @@ import { UserFeature } from './entities/user-feature'; import { UserQuota } from './entities/user-quota'; import { WorkspaceInvoices } from './entities/workspace-invoices'; import { WorkspaceSubscription } from './entities/workspace-subscription'; -import { DefaultRawFetchProvider, RawFetchProvider } from './provider/fetch'; import { ValidatorProvider } from './provider/validator'; -import { WebSocketAuthProvider } from './provider/websocket-auth'; import { ServerScope } from './scopes/server'; import { AuthService } from './services/auth'; import { CaptchaService } from './services/captcha'; @@ -69,7 +64,6 @@ import { SubscriptionService } from './services/subscription'; import { UserCopilotQuotaService } from './services/user-copilot-quota'; import { UserFeatureService } from './services/user-feature'; import { UserQuotaService } from './services/user-quota'; -import { WebSocketService } from './services/websocket'; import { WorkspaceInvoicesService } from './services/workspace-invoices'; import { WorkspaceServerService } from './services/workspace-server'; import { WorkspaceSubscriptionService } from './services/workspace-subscription'; @@ -85,26 +79,16 @@ import { UserQuotaStore } from './stores/user-quota'; export function configureCloudModule(framework: Framework) { framework - .impl(RawFetchProvider, DefaultRawFetchProvider) .service(ServersService, [ServerListStore, ServerConfigStore]) .service(DefaultServerService, [ServersService]) .store(ServerListStore, [GlobalStateService]) - .store(ServerConfigStore, [RawFetchProvider]) + .store(ServerConfigStore) .entity(Server, [ServerListStore]) .scope(ServerScope) .service(ServerService, [ServerScope]) - .service(FetchService, [RawFetchProvider, ServerService]) + .service(FetchService, [ServerService]) .service(EventSourceService, [ServerService]) .service(GraphQLService, [FetchService]) - .service( - WebSocketService, - f => - new WebSocketService( - f.get(ServerService), - f.get(AuthService), - f.getOptional(WebSocketAuthProvider) - ) - ) .service(CaptchaService, f => { return new CaptchaService( f.get(ServerService), diff --git a/packages/frontend/core/src/modules/cloud/provider/fetch.ts b/packages/frontend/core/src/modules/cloud/provider/fetch.ts deleted file mode 100644 index 6fc49ea75..000000000 --- a/packages/frontend/core/src/modules/cloud/provider/fetch.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { createIdentifier } from '@toeverything/infra'; - -import type { FetchInit } from '../services/fetch'; - -export interface RawFetchProvider { - /** - * standard fetch, in ios&android, we can use native fetch to implement this - */ - fetch: (input: string | URL, init?: FetchInit) => Promise; -} - -export const RawFetchProvider = - createIdentifier('FetchProvider'); - -export const DefaultRawFetchProvider = { - fetch: globalThis.fetch.bind(globalThis), -}; diff --git a/packages/frontend/core/src/modules/cloud/provider/websocket-auth.ts b/packages/frontend/core/src/modules/cloud/provider/websocket-auth.ts deleted file mode 100644 index f458bb7e5..000000000 --- a/packages/frontend/core/src/modules/cloud/provider/websocket-auth.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { createIdentifier } from '@toeverything/infra'; - -export interface WebSocketAuthProvider { - /** - * Returns the token and userId for WebSocket authentication - * - * Useful when cookies are not available for WebSocket connections - * - * @param url - The URL of the WebSocket endpoint - */ - getAuthToken: (url: string) => Promise< - | { - token?: string; - userId?: string; - } - | undefined - >; -} - -export const WebSocketAuthProvider = createIdentifier( - 'WebSocketAuthProvider' -); diff --git a/packages/frontend/core/src/modules/cloud/services/fetch.ts b/packages/frontend/core/src/modules/cloud/services/fetch.ts index 668b25c6e..09ca356b5 100644 --- a/packages/frontend/core/src/modules/cloud/services/fetch.ts +++ b/packages/frontend/core/src/modules/cloud/services/fetch.ts @@ -3,7 +3,6 @@ import { UserFriendlyError } from '@affine/graphql'; import { fromPromise, Service } from '@toeverything/infra'; import { BackendError, NetworkError } from '../error'; -import type { RawFetchProvider } from '../provider/fetch'; import type { ServerService } from './server'; const logger = new DebugLogger('affine:fetch'); @@ -11,10 +10,7 @@ const logger = new DebugLogger('affine:fetch'); export type FetchInit = RequestInit & { timeout?: number }; export class FetchService extends Service { - constructor( - private readonly fetchProvider: RawFetchProvider, - private readonly serverService: ServerService - ) { + constructor(private readonly serverService: ServerService) { super(); } rxFetch = ( @@ -50,7 +46,7 @@ export class FetchService extends Service { abortController.abort('timeout'); }, timeout); - const res = await this.fetchProvider + const res = await globalThis .fetch(new URL(input, this.serverService.server.serverMetadata.baseUrl), { ...init, signal: abortController.signal, diff --git a/packages/frontend/core/src/modules/cloud/services/websocket.ts b/packages/frontend/core/src/modules/cloud/services/websocket.ts deleted file mode 100644 index 22db7ea20..000000000 --- a/packages/frontend/core/src/modules/cloud/services/websocket.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { OnEvent, Service } from '@toeverything/infra'; -import { Manager } from 'socket.io-client'; - -import { ApplicationStarted } from '../../lifecycle'; -import { AccountChanged } from '../events/account-changed'; -import type { WebSocketAuthProvider } from '../provider/websocket-auth'; -import type { AuthService } from './auth'; -import type { ServerService } from './server'; - -@OnEvent(AccountChanged, e => e.update) -@OnEvent(ApplicationStarted, e => e.update) -export class WebSocketService extends Service { - ioManager: Manager = new Manager(`${this.serverService.server.baseUrl}/`, { - autoConnect: false, - transports: ['websocket'], - secure: location.protocol === 'https:', - }); - socket = this.ioManager.socket('/', { - auth: this.webSocketAuthProvider - ? cb => { - this.webSocketAuthProvider - ?.getAuthToken(`${this.serverService.server.baseUrl}/`) - .then(v => { - cb(v ?? {}); - }) - .catch(e => { - console.error('Failed to get auth token for websocket', e); - }); - } - : undefined, - }); - refCount = 0; - - constructor( - private readonly serverService: ServerService, - private readonly authService: AuthService, - private readonly webSocketAuthProvider?: WebSocketAuthProvider - ) { - super(); - } - - /** - * Connect socket, with automatic connect and reconnect logic. - * External code should not call `socket.connect()` or `socket.disconnect()` manually. - * When socket is no longer needed, call `dispose()` to clean up resources. - */ - connect() { - this.refCount++; - this.update(); - return { - socket: this.socket, - dispose: () => { - this.refCount--; - this.update(); - }, - }; - } - - update(): void { - if (this.authService.session.account$.value && this.refCount > 0) { - this.socket.connect(); - } else { - this.socket.disconnect(); - } - } -} diff --git a/packages/frontend/core/src/modules/cloud/stores/server-config.ts b/packages/frontend/core/src/modules/cloud/stores/server-config.ts index 21469d509..109c16913 100644 --- a/packages/frontend/core/src/modules/cloud/stores/server-config.ts +++ b/packages/frontend/core/src/modules/cloud/stores/server-config.ts @@ -8,13 +8,11 @@ import { } from '@affine/graphql'; import { Store } from '@toeverything/infra'; -import type { RawFetchProvider } from '../provider/fetch'; - export type ServerConfigType = ServerConfigQuery['serverConfig'] & OauthProvidersQuery['serverConfig']; export class ServerConfigStore extends Store { - constructor(private readonly fetcher: RawFetchProvider) { + constructor() { super(); } @@ -22,10 +20,7 @@ export class ServerConfigStore extends Store { serverBaseUrl: string, abortSignal?: AbortSignal ): Promise { - const gql = gqlFetcherFactory( - `${serverBaseUrl}/graphql`, - this.fetcher.fetch - ); + const gql = gqlFetcherFactory(`${serverBaseUrl}/graphql`, globalThis.fetch); const serverConfigData = await gql({ query: serverConfigQuery, context: { diff --git a/packages/frontend/core/src/modules/db/entities/table.ts b/packages/frontend/core/src/modules/db/entities/table.ts index 27ba94d11..3957582a3 100644 --- a/packages/frontend/core/src/modules/db/entities/table.ts +++ b/packages/frontend/core/src/modules/db/entities/table.ts @@ -2,7 +2,8 @@ import type { Table as OrmTable, TableSchemaBuilder, } from '@toeverything/infra'; -import { Entity } from '@toeverything/infra'; +import { Entity, LiveData } from '@toeverything/infra'; +import { map } from 'rxjs'; import type { WorkspaceService } from '../../workspace'; @@ -18,13 +19,19 @@ export class WorkspaceDBTable< super(); } - isSyncing$ = this.workspaceService.workspace.engine.doc - .docState$(this.props.storageDocId) - .map(docState => docState.syncing); + isSyncing$ = LiveData.from( + this.workspaceService.workspace.engine.doc + .docState$(this.props.storageDocId) + .pipe(map(docState => docState.syncing)), + false + ); - isLoading$ = this.workspaceService.workspace.engine.doc - .docState$(this.props.storageDocId) - .map(docState => docState.loading); + isLoading$ = LiveData.from( + this.workspaceService.workspace.engine.doc + .docState$(this.props.storageDocId) + .pipe(map(docState => !docState.loaded)), + false + ); create = this.table.create.bind(this.table) as typeof this.table.create; update = this.table.update.bind(this.table) as typeof this.table.update; diff --git a/packages/frontend/core/src/modules/db/services/db.ts b/packages/frontend/core/src/modules/db/services/db.ts index ccad594d4..b01137808 100644 --- a/packages/frontend/core/src/modules/db/services/db.ts +++ b/packages/frontend/core/src/modules/db/services/db.ts @@ -46,11 +46,11 @@ export class WorkspaceDBService extends Service { new YjsDBAdapter(AFFiNE_WORKSPACE_DB_SCHEMA, { getDoc: guid => { const ydoc = new YDoc({ - // guid format: db${workspaceId}${guid} - guid: `db$${this.workspaceService.workspace.id}$${guid}`, + // guid format: db${guid} + guid: `db$${guid}`, }); - this.workspaceService.workspace.engine.doc.addDoc(ydoc, false); - this.workspaceService.workspace.engine.doc.setPriority( + this.workspaceService.workspace.engine.doc.connectDoc(ydoc); + this.workspaceService.workspace.engine.doc.addPriority( ydoc.guid, 50 ); @@ -59,8 +59,7 @@ export class WorkspaceDBService extends Service { }) ), schema: AFFiNE_WORKSPACE_DB_SCHEMA, - storageDocId: tableName => - `db$${this.workspaceService.workspace.id}$${tableName}`, + storageDocId: tableName => `db$${tableName}`, } ) as WorkspaceDBWithTables; } @@ -79,11 +78,11 @@ export class WorkspaceDBService extends Service { new YjsDBAdapter(AFFiNE_WORKSPACE_USERDATA_DB_SCHEMA, { getDoc: guid => { const ydoc = new YDoc({ - // guid format: userdata${userId}${workspaceId}${guid} - guid: `userdata$${userId}$${this.workspaceService.workspace.id}$${guid}`, + // guid format: userdata${userId}${guid} + guid: `userdata$${userId}$${guid}`, }); - this.workspaceService.workspace.engine.doc.addDoc(ydoc, false); - this.workspaceService.workspace.engine.doc.setPriority( + this.workspaceService.workspace.engine.doc.connectDoc(ydoc); + this.workspaceService.workspace.engine.doc.addPriority( ydoc.guid, 50 ); @@ -92,8 +91,7 @@ export class WorkspaceDBService extends Service { }) ), schema: AFFiNE_WORKSPACE_USERDATA_DB_SCHEMA, - storageDocId: tableName => - `userdata$${userId}$${this.workspaceService.workspace.id}$${tableName}`, + storageDocId: tableName => `userdata$${userId}$${tableName}`, } ); diff --git a/packages/frontend/core/src/modules/db/utils.ts b/packages/frontend/core/src/modules/db/utils.ts index a55891a42..9e986ab28 100644 --- a/packages/frontend/core/src/modules/db/utils.ts +++ b/packages/frontend/core/src/modules/db/utils.ts @@ -1,4 +1,4 @@ -import type { DocStorage } from '@toeverything/infra'; +import type { DocStorage } from '@affine/nbstore'; import { AFFiNE_WORKSPACE_DB_SCHEMA, @@ -6,27 +6,33 @@ import { } from './schema'; export async function transformWorkspaceDBLocalToCloud( - localWorkspaceId: string, - cloudWorkspaceId: string, + _localWorkspaceId: string, + _cloudWorkspaceId: string, localDocStorage: DocStorage, cloudDocStorage: DocStorage, accountId: string ) { for (const tableName of Object.keys(AFFiNE_WORKSPACE_DB_SCHEMA)) { - const localDocName = `db$${localWorkspaceId}$${tableName}`; - const localDoc = await localDocStorage.doc.get(localDocName); + const localDocName = `db$${tableName}`; + const localDoc = await localDocStorage.getDoc(localDocName); if (localDoc) { - const cloudDocName = `db$${cloudWorkspaceId}$${tableName}`; - await cloudDocStorage.doc.set(cloudDocName, localDoc); + const cloudDocName = `db$${tableName}`; + await cloudDocStorage.pushDocUpdate({ + docId: cloudDocName, + bin: localDoc.bin, + }); } } for (const tableName of Object.keys(AFFiNE_WORKSPACE_USERDATA_DB_SCHEMA)) { - const localDocName = `userdata$__local__$${localWorkspaceId}$${tableName}`; - const localDoc = await localDocStorage.doc.get(localDocName); + const localDocName = `userdata$__local__$${tableName}`; + const localDoc = await localDocStorage.getDoc(localDocName); if (localDoc) { - const cloudDocName = `userdata$${accountId}$${cloudWorkspaceId}$${tableName}`; - await cloudDocStorage.doc.set(cloudDocName, localDoc); + const cloudDocName = `userdata$${accountId}$${tableName}`; + await cloudDocStorage.pushDocUpdate({ + docId: cloudDocName, + bin: localDoc.bin, + }); } } } diff --git a/packages/frontend/core/src/modules/doc-info/services/doc-database-backlinks.ts b/packages/frontend/core/src/modules/doc-info/services/doc-database-backlinks.ts index 083d0523a..a14cb7e55 100644 --- a/packages/frontend/core/src/modules/doc-info/services/doc-database-backlinks.ts +++ b/packages/frontend/core/src/modules/doc-info/services/doc-database-backlinks.ts @@ -28,8 +28,9 @@ export class DocDatabaseBacklinksService extends Service { if (!docRef.doc.blockSuiteDoc.ready) { docRef.doc.blockSuiteDoc.load(); } - docRef.doc.setPriorityLoad(10); + const disposePriorityLoad = docRef.doc.addPriorityLoad(10); await docRef.doc.waitForSyncReady(); + disposePriorityLoad(); return docRef; } diff --git a/packages/frontend/core/src/modules/doc/entities/doc.ts b/packages/frontend/core/src/modules/doc/entities/doc.ts index 3df73d740..1a3a518b6 100644 --- a/packages/frontend/core/src/modules/doc/entities/doc.ts +++ b/packages/frontend/core/src/modules/doc/entities/doc.ts @@ -81,8 +81,8 @@ export class Doc extends Entity { return this.store.waitForDocLoadReady(this.id); } - setPriorityLoad(priority: number) { - return this.store.setPriorityLoad(this.id, priority); + addPriorityLoad(priority: number) { + return this.store.addPriorityLoad(this.id, priority); } changeDocTitle(newTitle: string) { diff --git a/packages/frontend/core/src/modules/doc/services/docs.ts b/packages/frontend/core/src/modules/doc/services/docs.ts index d4d45d82c..035375f83 100644 --- a/packages/frontend/core/src/modules/doc/services/docs.ts +++ b/packages/frontend/core/src/modules/doc/services/docs.ts @@ -107,7 +107,6 @@ export class DocsService extends Service { ) { const doc = this.store.createBlockSuiteDoc(); initDocFromProps(doc, options.docProps); - this.store.markDocSyncStateAsReady(doc.id); const docRecord = this.list.doc$(doc.id).value; if (!docRecord) { throw new Unreachable(); @@ -124,8 +123,9 @@ export class DocsService extends Service { async addLinkedDoc(targetDocId: string, linkedDocId: string) { const { doc, release } = this.open(targetDocId); - doc.setPriorityLoad(10); + const disposePriorityLoad = doc.addPriorityLoad(10); await doc.waitForSyncReady(); + disposePriorityLoad(); const text = new Text([ { insert: ' ', @@ -149,8 +149,9 @@ export class DocsService extends Service { async changeDocTitle(docId: string, newTitle: string) { const { doc, release } = this.open(docId); - doc.setPriorityLoad(10); + const disposePriorityLoad = doc.addPriorityLoad(10); await doc.waitForSyncReady(); + disposePriorityLoad(); doc.changeDocTitle(newTitle); release(); } diff --git a/packages/frontend/core/src/modules/doc/stores/docs.ts b/packages/frontend/core/src/modules/doc/stores/docs.ts index e37df8e19..352e0a138 100644 --- a/packages/frontend/core/src/modules/doc/stores/docs.ts +++ b/packages/frontend/core/src/modules/doc/stores/docs.ts @@ -126,9 +126,9 @@ export class DocsStore extends Store { } watchDocListReady() { - return this.workspaceService.workspace.engine.rootDocState$ - .map(state => !state.syncing) - .asObservable(); + return this.workspaceService.workspace.engine.doc + .docState$(this.workspaceService.workspace.id) + .pipe(map(state => state.synced)); } setDocMeta(id: string, meta: Partial) { @@ -153,14 +153,10 @@ export class DocsStore extends Store { } waitForDocLoadReady(id: string) { - return this.workspaceService.workspace.engine.doc.waitForReady(id); + return this.workspaceService.workspace.engine.doc.waitForDocLoaded(id); } - setPriorityLoad(id: string, priority: number) { - return this.workspaceService.workspace.engine.doc.setPriority(id, priority); - } - - markDocSyncStateAsReady(id: string) { - this.workspaceService.workspace.engine.doc.markAsReady(id); + addPriorityLoad(id: string, priority: number) { + return this.workspaceService.workspace.engine.doc.addPriority(id, priority); } } diff --git a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts index 94d421d42..d5ea5e727 100644 --- a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts +++ b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts @@ -25,7 +25,7 @@ const logger = new DebugLogger('crawler'); const WORKSPACE_DOCS_INDEXER_VERSION_KEY = 'docs-indexer-version'; interface IndexerJobPayload { - storageDocId: string; + docId: string; } export class DocsIndexer extends Entity { @@ -81,26 +81,31 @@ export class DocsIndexer extends Entity { } setupListener() { - this.disposables.push( - this.workspaceEngine.doc.storage.eventBus.on(event => { - if (WorkspaceDBService.isDBDocId(event.docId)) { - // skip db doc - return; - } - if (event.clientId === this.workspaceEngine.doc.clientId) { - this.jobQueue - .enqueue([ - { - batchKey: event.docId, - payload: { storageDocId: event.docId }, - }, - ]) - .catch(err => { - console.error('Error enqueueing job', err); - }); - } + this.workspaceEngine.doc.storage.connection + .waitForConnected() + .then(() => { + this.disposables.push( + this.workspaceEngine.doc.storage.subscribeDocUpdate(updated => { + if (WorkspaceDBService.isDBDocId(updated.docId)) { + // skip db doc + return; + } + this.jobQueue + .enqueue([ + { + batchKey: updated.docId, + payload: { docId: updated.docId }, + }, + ]) + .catch(err => { + console.error('Error enqueueing job', err); + }); + }) + ); }) - ); + .catch(err => { + console.error('Error waiting for doc storage connection', err); + }); } async execJob(jobs: Job[], signal: AbortSignal) { @@ -119,20 +124,19 @@ export class DocsIndexer extends Entity { const isUpgrade = dbVersion < DocsIndexer.INDEXER_VERSION; // jobs should have the same storage docId, so we just pick the first one - const storageDocId = jobs[0].payload.storageDocId; + const docId = jobs[0].payload.docId; const worker = await this.ensureWorker(signal); const startTime = performance.now(); - logger.debug('Start crawling job for storageDocId:', storageDocId); + logger.debug('Start crawling job for docId:', docId); let workerOutput; - if (storageDocId === this.workspaceId) { - const rootDocBuffer = - await this.workspaceEngine.doc.storage.loadDocFromLocal( - this.workspaceId - ); + if (docId === this.workspaceId) { + const rootDocBuffer = ( + await this.workspaceEngine.doc.storage.getDoc(this.workspaceId) + )?.bin; if (!rootDocBuffer) { return; } @@ -147,15 +151,13 @@ export class DocsIndexer extends Entity { rootDocId: this.workspaceId, }); } else { - const rootDocBuffer = - await this.workspaceEngine.doc.storage.loadDocFromLocal( - this.workspaceId - ); + const rootDocBuffer = ( + await this.workspaceEngine.doc.storage.getDoc(this.workspaceId) + )?.bin; const docBuffer = - (await this.workspaceEngine.doc.storage.loadDocFromLocal( - storageDocId - )) ?? new Uint8Array(0); + (await this.workspaceEngine.doc.storage.getDoc(docId))?.bin ?? + new Uint8Array(0); if (!rootDocBuffer) { return; @@ -164,7 +166,7 @@ export class DocsIndexer extends Entity { workerOutput = await worker.run({ type: 'doc', docBuffer, - storageDocId, + docId, rootDocBuffer, rootDocId: this.workspaceId, }); @@ -231,9 +233,9 @@ export class DocsIndexer extends Entity { if (workerOutput.reindexDoc) { await this.jobQueue.enqueue( - workerOutput.reindexDoc.map(({ storageDocId }) => ({ - batchKey: storageDocId, - payload: { storageDocId }, + workerOutput.reindexDoc.map(({ docId }) => ({ + batchKey: docId, + payload: { docId }, })) ); } @@ -244,11 +246,7 @@ export class DocsIndexer extends Entity { const duration = performance.now() - startTime; logger.debug( - 'Finish crawling job for storageDocId:' + - storageDocId + - ' in ' + - duration + - 'ms ' + 'Finish crawling job for docId:' + docId + ' in ' + duration + 'ms ' ); } @@ -259,7 +257,7 @@ export class DocsIndexer extends Entity { .enqueue([ { batchKey: this.workspaceId, - payload: { storageDocId: this.workspaceId }, + payload: { docId: this.workspaceId }, }, ]) .catch(err => { diff --git a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts index b962de193..d8504df43 100644 --- a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts +++ b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts @@ -1,4 +1,3 @@ -import { getElectronAPIs } from '@affine/electron-api/web-worker'; import type { AttachmentBlockModel, BookmarkBlockModel, @@ -50,11 +49,6 @@ const LRU_CACHE_SIZE = 5; // lru cache for ydoc instances, last used at the end of the array const lruCache = [] as { doc: YDoc; hash: string }[]; -const electronAPIs = BUILD_CONFIG.isElectron ? getElectronAPIs() : null; - -// @ts-expect-error test -globalThis.__electronAPIs = electronAPIs; - async function digest(data: Uint8Array) { if ( globalThis.crypto && @@ -478,7 +472,7 @@ function unindentMarkdown(markdown: string) { async function crawlingDocData({ docBuffer, - storageDocId, + docId, rootDocBuffer, rootDocId, }: WorkerInput & { type: 'doc' }): Promise { @@ -489,18 +483,6 @@ async function crawlingDocData({ const yRootDoc = await getOrCreateCachedYDoc(rootDocBuffer); - let docId = null; - for (const [id, subdoc] of yRootDoc.getMap('spaces')) { - if (subdoc instanceof YDoc && storageDocId === subdoc.guid) { - docId = id; - break; - } - } - - if (docId === null) { - return {}; - } - let docExists: boolean | null = null; ( diff --git a/packages/frontend/core/src/modules/docs-search/worker/out-worker.ts b/packages/frontend/core/src/modules/docs-search/worker/out-worker.ts index 97e45dc2b..8d1acee8a 100644 --- a/packages/frontend/core/src/modules/docs-search/worker/out-worker.ts +++ b/packages/frontend/core/src/modules/docs-search/worker/out-worker.ts @@ -1,5 +1,4 @@ import { DebugLogger } from '@affine/debug'; -import { connectWebWorker } from '@affine/electron-api/web-worker'; import { MANUALLY_STOP, throwIfAborted } from '@toeverything/infra'; import type { @@ -13,7 +12,6 @@ const logger = new DebugLogger('affine:indexer-worker'); export async function createWorker(abort: AbortSignal) { let worker: Worker | null = null; - let electronApiCleanup: (() => void) | null = null; while (throwIfAborted(abort)) { try { worker = await new Promise((resolve, reject) => { @@ -32,10 +30,6 @@ export async function createWorker(abort: AbortSignal) { }); worker.postMessage({ type: 'init', msgId: 0 } as WorkerIngoingMessage); - if (BUILD_CONFIG.isElectron) { - electronApiCleanup = connectWebWorker(worker); - } - setTimeout(() => { reject('timeout'); }, 1000 * 30 /* 30 sec */); @@ -104,7 +98,6 @@ export async function createWorker(abort: AbortSignal) { dispose: () => { terminateAbort.abort(MANUALLY_STOP); worker.terminate(); - electronApiCleanup?.(); }, }; } diff --git a/packages/frontend/core/src/modules/docs-search/worker/types.ts b/packages/frontend/core/src/modules/docs-search/worker/types.ts index 82d88b324..566b252c3 100644 --- a/packages/frontend/core/src/modules/docs-search/worker/types.ts +++ b/packages/frontend/core/src/modules/docs-search/worker/types.ts @@ -36,14 +36,14 @@ export type WorkerInput = } | { type: 'doc'; - storageDocId: string; + docId: string; rootDocId: string; rootDocBuffer: Uint8Array; docBuffer: Uint8Array; }; export interface WorkerOutput { - reindexDoc?: { docId: string; storageDocId: string }[]; + reindexDoc?: { docId: string }[]; addedDoc?: { id: string; blocks: Document[]; diff --git a/packages/frontend/core/src/modules/import-template/index.ts b/packages/frontend/core/src/modules/import-template/index.ts index 04836fe4b..c9d5ff326 100644 --- a/packages/frontend/core/src/modules/import-template/index.ts +++ b/packages/frontend/core/src/modules/import-template/index.ts @@ -1,6 +1,5 @@ import { type Framework } from '@toeverything/infra'; -import { RawFetchProvider } from '../cloud'; import { WorkspacesService } from '../workspace'; import { ImportTemplateDialog } from './entities/dialog'; import { TemplateDownloader } from './entities/downloader'; @@ -16,6 +15,6 @@ export function configureImportTemplateModule(framework: Framework) { .entity(ImportTemplateDialog) .service(TemplateDownloaderService) .entity(TemplateDownloader, [TemplateDownloaderStore]) - .store(TemplateDownloaderStore, [RawFetchProvider]) + .store(TemplateDownloaderStore) .service(ImportTemplateService, [WorkspacesService]); } diff --git a/packages/frontend/core/src/modules/import-template/services/import.ts b/packages/frontend/core/src/modules/import-template/services/import.ts index 1cb24b3b7..f1cf516ec 100644 --- a/packages/frontend/core/src/modules/import-template/services/import.ts +++ b/packages/frontend/core/src/modules/import-template/services/import.ts @@ -18,7 +18,7 @@ export class ImportTemplateService extends Service { this.workspacesService.open({ metadata: workspaceMetadata, }); - await workspace.engine.waitForRootDocReady(); + await workspace.engine.doc.waitForDocReady(workspace.id); // wait for root doc ready const [importedDoc] = await ZipTransformer.importDocs( workspace.docCollection, new Blob([docBinary], { @@ -42,7 +42,7 @@ export class ImportTemplateService extends Service { docBinary: Uint8Array // todo: support doc mode on init ) { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + // oxlint-disable-next-line @typescript-eslint/no-non-null-assertion let docId: string = null!; const { id: workspaceId } = await this.workspacesService.create( flavour, @@ -51,7 +51,10 @@ export class ImportTemplateService extends Service { docCollection.meta.setName(workspaceName); const doc = docCollection.createDoc(); docId = doc.id; - await docStorage.doc.set(doc.spaceDoc.guid, docBinary); + await docStorage.pushDocUpdate({ + docId: doc.spaceDoc.guid, + bin: docBinary, + }); } ); return { workspaceId, docId }; diff --git a/packages/frontend/core/src/modules/import-template/store/downloader.ts b/packages/frontend/core/src/modules/import-template/store/downloader.ts index 0c655cf02..44a57fadb 100644 --- a/packages/frontend/core/src/modules/import-template/store/downloader.ts +++ b/packages/frontend/core/src/modules/import-template/store/downloader.ts @@ -1,14 +1,12 @@ import { Store } from '@toeverything/infra'; -import type { RawFetchProvider } from '../../cloud'; - export class TemplateDownloaderStore extends Store { - constructor(private readonly fetchProvider: RawFetchProvider) { + constructor() { super(); } async download(snapshotUrl: string) { - const response = await this.fetchProvider.fetch(snapshotUrl, { + const response = await globalThis.fetch(snapshotUrl, { priority: 'high', } as any); const arrayBuffer = await response.arrayBuffer(); diff --git a/packages/frontend/core/src/modules/index.ts b/packages/frontend/core/src/modules/index.ts index 88b1569d5..649cd840c 100644 --- a/packages/frontend/core/src/modules/index.ts +++ b/packages/frontend/core/src/modules/index.ts @@ -38,7 +38,7 @@ import { configureShareDocsModule } from './share-doc'; import { configureShareSettingModule } from './share-setting'; import { configureCommonGlobalStorageImpls, - configureGlobalStorageModule, + configureStorageModule, } from './storage'; import { configureSystemFontFamilyModule } from './system-font-family'; import { configureTagModule } from './tag'; @@ -55,7 +55,7 @@ export function configureCommonModules(framework: Framework) { configureWorkspaceModule(framework); configureDocModule(framework); configureWorkspaceDBModule(framework); - configureGlobalStorageModule(framework); + configureStorageModule(framework); configureGlobalContextModule(framework); configureLifecycleModule(framework); configureFeatureFlagModule(framework); diff --git a/packages/frontend/core/src/modules/pdf/entities/pdf.ts b/packages/frontend/core/src/modules/pdf/entities/pdf.ts index bcf12d33b..9cd009d50 100644 --- a/packages/frontend/core/src/modules/pdf/entities/pdf.ts +++ b/packages/frontend/core/src/modules/pdf/entities/pdf.ts @@ -49,7 +49,6 @@ export class PDF extends Entity { constructor() { super(); - this.renderer.listen(); this.disposables.push(() => this.pages.clear()); } diff --git a/packages/frontend/core/src/modules/pdf/renderer/worker.ts b/packages/frontend/core/src/modules/pdf/renderer/worker.ts index 14dd1987a..ec01ae04c 100644 --- a/packages/frontend/core/src/modules/pdf/renderer/worker.ts +++ b/packages/frontend/core/src/modules/pdf/renderer/worker.ts @@ -1,4 +1,8 @@ -import { OpConsumer, transfer } from '@toeverything/infra/op'; +import { + type MessageCommunicapable, + OpConsumer, + transfer, +} from '@toeverything/infra/op'; import type { Document } from '@toeverything/pdf-viewer'; import { createPDFium, @@ -23,6 +27,11 @@ import type { ClientOps } from './ops'; import type { PDFMeta, RenderPageOpts } from './types'; class PDFRendererBackend extends OpConsumer { + constructor(port: MessageCommunicapable) { + super(port); + this.register('open', this.open.bind(this)); + this.register('render', this.render.bind(this)); + } private readonly viewer$: Observable = from( createPDFium().then(pdfium => { return new Viewer(new Runtime(pdfium)); @@ -147,13 +156,6 @@ class PDFRendererBackend extends OpConsumer { return imageBitmap; } - - override listen(): void { - this.register('open', this.open.bind(this)); - this.register('render', this.render.bind(this)); - super.listen(); - } } -// @ts-expect-error how could we get correct postMessage signature for worker, exclude `window.postMessage` -new PDFRendererBackend(self).listen(); +new PDFRendererBackend(self as MessageCommunicapable); diff --git a/packages/frontend/core/src/modules/peek-view/view/utils.ts b/packages/frontend/core/src/modules/peek-view/view/utils.ts index 183cbaa4b..ed452ef5f 100644 --- a/packages/frontend/core/src/modules/peek-view/view/utils.ts +++ b/packages/frontend/core/src/modules/peek-view/view/utils.ts @@ -52,10 +52,7 @@ export const useEditor = ( // set sync engine priority target useEffect(() => { - currentWorkspace.engine.doc.setPriority(pageId, 10); - return () => { - currentWorkspace.engine.doc.setPriority(pageId, 5); - }; + return currentWorkspace.engine.doc.addPriority(pageId, 10); }, [currentWorkspace, pageId]); return { doc, editor, workspace: currentWorkspace, loading: !docListReady }; diff --git a/packages/frontend/core/src/modules/quota/entities/quota.ts b/packages/frontend/core/src/modules/quota/entities/quota.ts index 19e2ed4ad..7b56d9777 100644 --- a/packages/frontend/core/src/modules/quota/entities/quota.ts +++ b/packages/frontend/core/src/modules/quota/entities/quota.ts @@ -5,7 +5,7 @@ import { catchErrorInto, effect, Entity, - exhaustMapSwitchUntilChanged, + exhaustMapWithTrailing, fromPromise, LiveData, onComplete, @@ -13,7 +13,7 @@ import { } from '@toeverything/infra'; import { cssVarV2 } from '@toeverything/theme/v2'; import bytes from 'bytes'; -import { EMPTY, map, mergeMap } from 'rxjs'; +import { EMPTY, mergeMap } from 'rxjs'; import { isBackendError, isNetworkError } from '../../cloud'; import type { WorkspaceService } from '../../workspace'; @@ -68,49 +68,40 @@ export class WorkspaceQuota extends Entity { } revalidate = effect( - map(() => ({ - workspaceId: this.workspaceService.workspace.id, - })), - exhaustMapSwitchUntilChanged( - (a, b) => a.workspaceId === b.workspaceId, - ({ workspaceId }) => { - return fromPromise(async signal => { - if (!workspaceId) { - return; // no quota if no workspace - } - const data = await this.store.fetchWorkspaceQuota( - this.workspaceService.workspace.id, - signal - ); - return { quota: data, used: data.usedSize }; - }).pipe( - backoffRetry({ - when: isNetworkError, - count: Infinity, - }), - backoffRetry({ - when: isBackendError, - count: 3, - }), - mergeMap(data => { - if (data) { - const { quota, used } = data; - this.quota$.next(quota); - this.used$.next(used); - } else { - this.quota$.next(null); - this.used$.next(null); - } - return EMPTY; - }), - catchErrorInto(this.error$, error => { - logger.error('Failed to fetch workspace quota', error); - }), - onStart(() => this.isRevalidating$.setValue(true)), - onComplete(() => this.isRevalidating$.setValue(false)) + exhaustMapWithTrailing(() => { + return fromPromise(async signal => { + const data = await this.store.fetchWorkspaceQuota( + this.workspaceService.workspace.id, + signal ); - } - ) + return { quota: data, used: data.usedSize }; + }).pipe( + backoffRetry({ + when: isNetworkError, + count: Infinity, + }), + backoffRetry({ + when: isBackendError, + count: 3, + }), + mergeMap(data => { + if (data) { + const { quota, used } = data; + this.quota$.next(quota); + this.used$.next(used); + } else { + this.quota$.next(null); + this.used$.next(null); + } + return EMPTY; + }), + catchErrorInto(this.error$, error => { + logger.error('Failed to fetch workspace quota', error); + }), + onStart(() => this.isRevalidating$.setValue(true)), + onComplete(() => this.isRevalidating$.setValue(false)) + ); + }) ); waitForRevalidation(signal?: AbortSignal) { diff --git a/packages/frontend/core/src/modules/share-doc/entities/share-reader.ts b/packages/frontend/core/src/modules/share-doc/entities/share-reader.ts deleted file mode 100644 index b9740e4cb..000000000 --- a/packages/frontend/core/src/modules/share-doc/entities/share-reader.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { UserFriendlyError } from '@affine/graphql'; -import type { DocMode } from '@blocksuite/affine/blocks'; -import { - effect, - Entity, - fromPromise, - LiveData, - onComplete, - onStart, -} from '@toeverything/infra'; -import { catchError, EMPTY, mergeMap, switchMap } from 'rxjs'; - -import type { ShareReaderStore } from '../stores/share-reader'; - -export class ShareReader extends Entity { - isLoading$ = new LiveData(false); - error$ = new LiveData(null); - data$ = new LiveData<{ - workspaceId: string; - docId: string; - workspaceBinary: Uint8Array; - docBinary: Uint8Array; - - // Used for old share server-side mode control - publishMode?: DocMode; - } | null>(null); - - constructor(private readonly store: ShareReaderStore) { - super(); - } - - loadShare = effect( - switchMap( - ({ - serverId, - workspaceId, - docId, - }: { - serverId: string; - workspaceId: string; - docId: string; - }) => { - return fromPromise( - this.store.loadShare(serverId, workspaceId, docId) - ).pipe( - mergeMap(data => { - if (!data) { - this.data$.next(null); - } else { - this.data$.next({ - workspaceId, - docId, - workspaceBinary: data.workspace, - docBinary: data.doc, - publishMode: data.publishMode ?? undefined, - }); - } - return EMPTY; - }), - catchError((error: any) => { - this.error$.next(UserFriendlyError.fromAnyError(error)); - return EMPTY; - }), - onStart(() => { - this.isLoading$.next(true); - this.data$.next(null); - this.error$.next(null); - }), - onComplete(() => { - this.isLoading$.next(false); - }) - ); - } - ) - ); -} diff --git a/packages/frontend/core/src/modules/share-doc/index.ts b/packages/frontend/core/src/modules/share-doc/index.ts index 816c6a8c4..772b11b6c 100644 --- a/packages/frontend/core/src/modules/share-doc/index.ts +++ b/packages/frontend/core/src/modules/share-doc/index.ts @@ -1,11 +1,9 @@ -export type { ShareReader } from './entities/share-reader'; export { ShareDocsListService } from './services/share-docs-list'; export { ShareInfoService } from './services/share-info'; -export { ShareReaderService } from './services/share-reader'; import { type Framework } from '@toeverything/infra'; -import { ServersService, WorkspaceServerService } from '../cloud'; +import { WorkspaceServerService } from '../cloud'; import { DocScope, DocService } from '../doc'; import { WorkspaceLocalCache, @@ -14,19 +12,13 @@ import { } from '../workspace'; import { ShareDocsList } from './entities/share-docs-list'; import { ShareInfo } from './entities/share-info'; -import { ShareReader } from './entities/share-reader'; import { ShareDocsListService } from './services/share-docs-list'; import { ShareInfoService } from './services/share-info'; -import { ShareReaderService } from './services/share-reader'; import { ShareStore } from './stores/share'; import { ShareDocsStore } from './stores/share-docs'; -import { ShareReaderStore } from './stores/share-reader'; export function configureShareDocsModule(framework: Framework) { framework - .service(ShareReaderService) - .entity(ShareReader, [ShareReaderStore]) - .store(ShareReaderStore, [ServersService]) .scope(WorkspaceScope) .service(ShareDocsListService, [WorkspaceService]) .store(ShareDocsStore, [WorkspaceServerService]) diff --git a/packages/frontend/core/src/modules/share-doc/services/share-reader.ts b/packages/frontend/core/src/modules/share-doc/services/share-reader.ts deleted file mode 100644 index 774dbbb04..000000000 --- a/packages/frontend/core/src/modules/share-doc/services/share-reader.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Service } from '@toeverything/infra'; - -import { ShareReader } from '../entities/share-reader'; - -export class ShareReaderService extends Service { - reader = this.framework.createEntity(ShareReader); -} diff --git a/packages/frontend/core/src/modules/share-doc/stores/share-reader.ts b/packages/frontend/core/src/modules/share-doc/stores/share-reader.ts deleted file mode 100644 index 3159d4cef..000000000 --- a/packages/frontend/core/src/modules/share-doc/stores/share-reader.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { ErrorNames, UserFriendlyError } from '@affine/graphql'; -import type { DocMode } from '@blocksuite/affine/blocks'; -import { Store } from '@toeverything/infra'; - -import type { ServersService } from '../../cloud'; -import { isBackendError } from '../../cloud'; - -export class ShareReaderStore extends Store { - constructor(private readonly serversService: ServersService) { - super(); - } - - async loadShare(serverId: string, workspaceId: string, docId: string) { - const server = this.serversService.server$(serverId).value; - if (!server) { - throw new Error(`Server ${serverId} not found`); - } - try { - const docResponse = await server.fetch( - `/api/workspaces/${workspaceId}/docs/${docId}` - ); - const publishMode = docResponse.headers.get( - 'publish-mode' - ) as DocMode | null; - const docBinary = await docResponse.arrayBuffer(); - - const workspaceResponse = await server.fetch( - `/api/workspaces/${workspaceId}/docs/${workspaceId}` - ); - const workspaceBinary = await workspaceResponse.arrayBuffer(); - - return { - doc: new Uint8Array(docBinary), - workspace: new Uint8Array(workspaceBinary), - publishMode, - }; - } catch (error) { - if ( - error instanceof Error && - isBackendError(error) && - UserFriendlyError.fromAnyError(error).name === ErrorNames.ACCESS_DENIED - ) { - return null; - } - throw error; - } - } -} diff --git a/packages/frontend/core/src/modules/storage/index.ts b/packages/frontend/core/src/modules/storage/index.ts index 7387b70c8..5996a8f3e 100644 --- a/packages/frontend/core/src/modules/storage/index.ts +++ b/packages/frontend/core/src/modules/storage/index.ts @@ -3,11 +3,13 @@ export { GlobalSessionState, GlobalState, } from './providers/global'; +export { NbstoreProvider } from './providers/nbstore'; export { GlobalCacheService, GlobalSessionStateService, GlobalStateService, } from './services/global'; +export { NbstoreService } from './services/nbstore'; import { type Framework } from '@toeverything/infra'; @@ -23,16 +25,19 @@ import { GlobalSessionState, GlobalState, } from './providers/global'; +import { NbstoreProvider } from './providers/nbstore'; import { GlobalCacheService, GlobalSessionStateService, GlobalStateService, } from './services/global'; +import { NbstoreService } from './services/nbstore'; -export const configureGlobalStorageModule = (framework: Framework) => { +export const configureStorageModule = (framework: Framework) => { framework.service(GlobalStateService, [GlobalState]); framework.service(GlobalCacheService, [GlobalCache]); framework.service(GlobalSessionStateService, [GlobalSessionState]); + framework.service(NbstoreService, [NbstoreProvider]); }; export function configureLocalStorageStateStorageImpls(framework: Framework) { diff --git a/packages/frontend/core/src/modules/storage/providers/nbstore.ts b/packages/frontend/core/src/modules/storage/providers/nbstore.ts new file mode 100644 index 000000000..9d2a3cac9 --- /dev/null +++ b/packages/frontend/core/src/modules/storage/providers/nbstore.ts @@ -0,0 +1,26 @@ +import type { + WorkerClient, + WorkerInitOptions, +} from '@affine/nbstore/worker/client'; +import { createIdentifier } from '@toeverything/infra'; + +export interface NbstoreProvider { + /** + * Open a nbstore with the given options, if the store with the given key already exists, it will be returned. + * + * in environment with SharedWorker support, the store also can be shared with other tabs/windows. + * + * @param key - the key of the store, can used to share the store with other tabs/windows. + * @param options - the options to open the store. + */ + openStore( + key: string, + options: WorkerInitOptions + ): { + store: WorkerClient; + dispose: () => void; + }; +} + +export const NbstoreProvider = + createIdentifier('NbstoreProvider'); diff --git a/packages/frontend/core/src/modules/storage/services/nbstore.ts b/packages/frontend/core/src/modules/storage/services/nbstore.ts new file mode 100644 index 000000000..a5a95db32 --- /dev/null +++ b/packages/frontend/core/src/modules/storage/services/nbstore.ts @@ -0,0 +1,14 @@ +import type { WorkerInitOptions } from '@affine/nbstore/worker/client'; +import { Service } from '@toeverything/infra'; + +import type { NbstoreProvider } from '../providers/nbstore'; + +export class NbstoreService extends Service { + constructor(private readonly nbstoreProvider: NbstoreProvider) { + super(); + } + + openStore(key: string, options: WorkerInitOptions) { + return this.nbstoreProvider.openStore(key, options); + } +} diff --git a/packages/frontend/core/src/modules/userspace/entities/user-db-engine.ts b/packages/frontend/core/src/modules/userspace/entities/user-db-engine.ts index eb927ccd6..375720bc2 100644 --- a/packages/frontend/core/src/modules/userspace/entities/user-db-engine.ts +++ b/packages/frontend/core/src/modules/userspace/entities/user-db-engine.ts @@ -1,17 +1,21 @@ -import { DocEngine, Entity } from '@toeverything/infra'; +import { IndexedDBDocStorage } from '@affine/nbstore/idb'; +import { SqliteDocStorage } from '@affine/nbstore/sqlite'; +import type { WorkerClient } from '@affine/nbstore/worker/client'; +import { Entity } from '@toeverything/infra'; -import type { WebSocketService } from '../../cloud'; -import { UserDBDocServer } from '../impls/user-db-doc-server'; -import type { UserspaceStorageProvider } from '../provider/storage'; +import type { ServerService } from '../../cloud'; +import type { NbstoreService } from '../../storage'; export class UserDBEngine extends Entity<{ userId: string; }> { private readonly userId = this.props.userId; - readonly docEngine = new DocEngine( - this.userspaceStorageProvider.getDocStorage('affine-cloud:' + this.userId), - new UserDBDocServer(this.userId, this.websocketService) - ); + readonly client: WorkerClient; + + DocStorageType = + BUILD_CONFIG.isElectron || BUILD_CONFIG.isIOS + ? SqliteDocStorage + : IndexedDBDocStorage; canGracefulStop() { // TODO(@eyhn): Implement this @@ -19,14 +23,40 @@ export class UserDBEngine extends Entity<{ } constructor( - private readonly userspaceStorageProvider: UserspaceStorageProvider, - private readonly websocketService: WebSocketService + private readonly nbstoreService: NbstoreService, + serverService: ServerService ) { super(); - this.docEngine.start(); - } - override dispose() { - this.docEngine.stop(); + const { store, dispose } = this.nbstoreService.openStore( + `userspace:${serverService.server.id},${this.userId}`, + { + local: { + doc: { + name: this.DocStorageType.identifier, + opts: { + id: `${serverService.server.id}:` + this.userId, + flavour: serverService.server.id, + type: 'userspace', + }, + }, + }, + remotes: { + cloud: { + doc: { + name: 'CloudDocStorage', + opts: { + id: this.userId, + serverBaseUrl: serverService.server.baseUrl, + type: 'userspace', + }, + }, + }, + }, + } + ); + this.client = store; + this.client.docFrontend.start(); + this.disposables.push(() => dispose()); } } diff --git a/packages/frontend/core/src/modules/userspace/entities/user-db-table.ts b/packages/frontend/core/src/modules/userspace/entities/user-db-table.ts index c8d8d5883..4fe3b47cb 100644 --- a/packages/frontend/core/src/modules/userspace/entities/user-db-table.ts +++ b/packages/frontend/core/src/modules/userspace/entities/user-db-table.ts @@ -1,8 +1,9 @@ +import type { DocFrontendDocState } from '@affine/nbstore'; import type { Table as OrmTable, TableSchemaBuilder, } from '@toeverything/infra'; -import { Entity } from '@toeverything/infra'; +import { Entity, LiveData } from '@toeverything/infra'; import type { UserDBEngine } from './user-db-engine'; @@ -12,15 +13,16 @@ export class UserDBTable extends Entity<{ engine: UserDBEngine; }> { readonly table = this.props.table; - readonly docEngine = this.props.engine.docEngine; + readonly docFrontend = this.props.engine.client.docFrontend; - isSyncing$ = this.docEngine - .docState$(this.props.storageDocId) - .map(docState => docState.syncing); + docSyncState$ = LiveData.from( + this.docFrontend.docState$(this.props.storageDocId), + null as any + ); - isLoading$ = this.docEngine - .docState$(this.props.storageDocId) - .map(docState => docState.loading); + isSyncing$ = this.docSyncState$.map(docState => docState.syncing); + + isLoaded$ = this.docSyncState$.map(docState => docState.loaded); create: typeof this.table.create = this.table.create.bind(this.table); update: typeof this.table.update = this.table.update.bind(this.table); diff --git a/packages/frontend/core/src/modules/userspace/entities/user-db.ts b/packages/frontend/core/src/modules/userspace/entities/user-db.ts index 04b5c5c7d..a436b56a7 100644 --- a/packages/frontend/core/src/modules/userspace/entities/user-db.ts +++ b/packages/frontend/core/src/modules/userspace/entities/user-db.ts @@ -19,8 +19,8 @@ export class UserDB extends Entity<{ const ydoc = new YDoc({ guid, }); - this.engine.docEngine.addDoc(ydoc, false); - this.engine.docEngine.setPriority(ydoc.guid, 50); + this.engine.client.docFrontend.connectDoc(ydoc); + this.engine.client.docFrontend.addPriority(ydoc.guid, 50); return ydoc; }, }) diff --git a/packages/frontend/core/src/modules/userspace/impls/indexeddb-storage.ts b/packages/frontend/core/src/modules/userspace/impls/indexeddb-storage.ts deleted file mode 100644 index 929bc3e7d..000000000 --- a/packages/frontend/core/src/modules/userspace/impls/indexeddb-storage.ts +++ /dev/null @@ -1,266 +0,0 @@ -import type { - ByteKV, - ByteKVBehavior, - DocEvent, - DocEventBus, - DocStorage, -} from '@toeverything/infra'; -import type { DBSchema, IDBPDatabase, IDBPObjectStore } from 'idb'; -import { openDB } from 'idb'; -import { mergeUpdates } from 'yjs'; - -class BroadcastChannelDocEventBus implements DocEventBus { - senderChannel = new BroadcastChannel('user-db:' + this.userId); - constructor(private readonly userId: string) {} - emit(event: DocEvent): void { - this.senderChannel.postMessage(event); - } - - on(cb: (event: DocEvent) => void): () => void { - const listener = (event: MessageEvent) => { - cb(event.data); - }; - const channel = new BroadcastChannel('user-db:' + this.userId); - channel.addEventListener('message', listener); - return () => { - channel.removeEventListener('message', listener); - channel.close(); - }; - } -} - -function isEmptyUpdate(binary: Uint8Array) { - return ( - binary.byteLength === 0 || - (binary.byteLength === 2 && binary[0] === 0 && binary[1] === 0) - ); -} - -export class IndexedDBUserspaceDocStorage implements DocStorage { - constructor(private readonly userId: string) {} - eventBus = new BroadcastChannelDocEventBus(this.userId); - readonly doc = new Doc(this.userId); - readonly syncMetadata = new KV(`affine-cloud:${this.userId}:sync-metadata`); - readonly serverClock = new KV(`affine-cloud:${this.userId}:server-clock`); -} - -interface DocDBSchema extends DBSchema { - userspace: { - key: string; - value: { - id: string; - updates: { - timestamp: number; - update: Uint8Array; - }[]; - }; - }; -} - -type DocType = DocStorage['doc']; -class Doc implements DocType { - dbName = 'affine-cloud:' + this.userId + ':doc'; - dbPromise: Promise> | null = null; - dbVersion = 1; - - constructor(private readonly userId: string) {} - - upgradeDB(db: IDBPDatabase) { - db.createObjectStore('userspace', { keyPath: 'id' }); - } - - getDb() { - if (this.dbPromise === null) { - this.dbPromise = openDB(this.dbName, this.dbVersion, { - upgrade: db => this.upgradeDB(db), - }); - } - return this.dbPromise; - } - - async get(docId: string): Promise { - const db = await this.getDb(); - const store = db - .transaction('userspace', 'readonly') - .objectStore('userspace'); - const data = await store.get(docId); - - if (!data) { - return null; - } - - const updates = data.updates - .map(({ update }) => update) - .filter(update => !isEmptyUpdate(update)); - const update = updates.length > 0 ? mergeUpdates(updates) : null; - - return update; - } - - async set(docId: string, data: Uint8Array) { - const db = await this.getDb(); - const store = db - .transaction('userspace', 'readwrite') - .objectStore('userspace'); - - const rows = [{ timestamp: Date.now(), update: data }]; - await store.put({ - id: docId, - updates: rows, - }); - } - - async keys() { - const db = await this.getDb(); - const store = db - .transaction('userspace', 'readonly') - .objectStore('userspace'); - - return store.getAllKeys(); - } - - clear(): void | Promise { - return; - } - - del(_key: string): void | Promise { - return; - } - - async transaction( - cb: (transaction: ByteKVBehavior) => Promise - ): Promise { - const db = await this.getDb(); - const store = db - .transaction('userspace', 'readwrite') - .objectStore('userspace'); - return await cb({ - async get(docId) { - const data = await store.get(docId); - - if (!data) { - return null; - } - - const { updates } = data; - const update = mergeUpdates(updates.map(({ update }) => update)); - - return update; - }, - keys() { - return store.getAllKeys(); - }, - async set(docId, data) { - const rows = [{ timestamp: Date.now(), update: data }]; - await store.put({ - id: docId, - updates: rows, - }); - }, - async clear() { - return await store.clear(); - }, - async del(key) { - return store.delete(key); - }, - }); - } -} - -interface KvDBSchema extends DBSchema { - kv: { - key: string; - value: { key: string; val: Uint8Array }; - }; -} - -class KV implements ByteKV { - constructor(private readonly dbName: string) {} - - dbPromise: Promise> | null = null; - dbVersion = 1; - - upgradeDB(db: IDBPDatabase) { - db.createObjectStore('kv', { keyPath: 'key' }); - } - - getDb() { - if (this.dbPromise === null) { - this.dbPromise = openDB(this.dbName, this.dbVersion, { - upgrade: db => this.upgradeDB(db), - }); - } - return this.dbPromise; - } - - async transaction( - cb: (transaction: ByteKVBehavior) => Promise - ): Promise { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - - const behavior = new KVBehavior(store); - return await cb(behavior); - } - - async get(key: string): Promise { - const db = await this.getDb(); - const store = db.transaction('kv', 'readonly').objectStore('kv'); - return new KVBehavior(store).get(key); - } - async set(key: string, value: Uint8Array): Promise { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - return new KVBehavior(store).set(key, value); - } - async keys(): Promise { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - return new KVBehavior(store).keys(); - } - async clear() { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - return new KVBehavior(store).clear(); - } - async del(key: string) { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - return new KVBehavior(store).del(key); - } -} - -class KVBehavior implements ByteKVBehavior { - constructor( - private readonly store: IDBPObjectStore - ) {} - async get(key: string): Promise { - const value = await this.store.get(key); - return value?.val ?? null; - } - async set(key: string, value: Uint8Array): Promise { - if (this.store.put === undefined) { - throw new Error('Cannot set in a readonly transaction'); - } - await this.store.put({ - key: key, - val: value, - }); - } - async keys(): Promise { - return await this.store.getAllKeys(); - } - async del(key: string) { - if (this.store.delete === undefined) { - throw new Error('Cannot set in a readonly transaction'); - } - return await this.store.delete(key); - } - - async clear() { - if (this.store.clear === undefined) { - throw new Error('Cannot set in a readonly transaction'); - } - return await this.store.clear(); - } -} diff --git a/packages/frontend/core/src/modules/userspace/impls/sqlite-storage.ts b/packages/frontend/core/src/modules/userspace/impls/sqlite-storage.ts deleted file mode 100644 index e9d5f6695..000000000 --- a/packages/frontend/core/src/modules/userspace/impls/sqlite-storage.ts +++ /dev/null @@ -1,159 +0,0 @@ -import type { - ByteKV, - ByteKVBehavior, - DocEvent, - DocEventBus, - DocStorage, -} from '@toeverything/infra'; -import { AsyncLock } from '@toeverything/infra'; - -import type { DesktopApiService } from '../../desktop-api'; - -class BroadcastChannelDocEventBus implements DocEventBus { - senderChannel = new BroadcastChannel('user-db:' + this.userId); - constructor(private readonly userId: string) {} - emit(event: DocEvent): void { - this.senderChannel.postMessage(event); - } - - on(cb: (event: DocEvent) => void): () => void { - const listener = (event: MessageEvent) => { - cb(event.data); - }; - const channel = new BroadcastChannel('user-db:' + this.userId); - channel.addEventListener('message', listener); - return () => { - channel.removeEventListener('message', listener); - channel.close(); - }; - } -} - -export class SqliteUserspaceDocStorage implements DocStorage { - constructor( - private readonly userId: string, - private readonly electronApi: DesktopApiService - ) {} - eventBus = new BroadcastChannelDocEventBus(this.userId); - readonly doc = new Doc(this.userId, this.electronApi); - readonly syncMetadata = new SyncMetadataKV(this.userId, this.electronApi); - readonly serverClock = new ServerClockKV(this.userId, this.electronApi); -} - -type DocType = DocStorage['doc']; - -class Doc implements DocType { - lock = new AsyncLock(); - apis = this.electronApi.api.handler; - - constructor( - private readonly userId: string, - private readonly electronApi: DesktopApiService - ) {} - - async transaction( - cb: (transaction: ByteKVBehavior) => Promise - ): Promise { - using _lock = await this.lock.acquire(); - return await cb(this); - } - - keys(): string[] | Promise { - return []; - } - - async get(docId: string) { - const update = await this.apis.db.getDocAsUpdates( - 'userspace', - this.userId, - docId - ); - - if (update) { - if ( - update.byteLength === 0 || - (update.byteLength === 2 && update[0] === 0 && update[1] === 0) - ) { - return null; - } - - return update; - } - - return null; - } - - async set(docId: string, data: Uint8Array) { - await this.apis.db.applyDocUpdate('userspace', this.userId, data, docId); - } - - clear(): void | Promise { - return; - } - - async del(docId: string) { - await this.apis.db.deleteDoc('userspace', this.userId, docId); - } -} - -class SyncMetadataKV implements ByteKV { - apis = this.electronApi.api.handler; - constructor( - private readonly userId: string, - private readonly electronApi: DesktopApiService - ) {} - transaction(cb: (behavior: ByteKVBehavior) => Promise): Promise { - return cb(this); - } - - get(key: string): Uint8Array | null | Promise { - return this.apis.db.getSyncMetadata('userspace', this.userId, key); - } - - set(key: string, data: Uint8Array): void | Promise { - return this.apis.db.setSyncMetadata('userspace', this.userId, key, data); - } - - keys(): string[] | Promise { - return this.apis.db.getSyncMetadataKeys('userspace', this.userId); - } - - del(key: string): void | Promise { - return this.apis.db.delSyncMetadata('userspace', this.userId, key); - } - - clear(): void | Promise { - return this.apis.db.clearSyncMetadata('userspace', this.userId); - } -} - -class ServerClockKV implements ByteKV { - apis = this.electronApi.api.handler; - constructor( - private readonly userId: string, - private readonly electronApi: DesktopApiService - ) {} - transaction(cb: (behavior: ByteKVBehavior) => Promise): Promise { - return cb(this); - } - - get(key: string): Uint8Array | null | Promise { - return this.apis.db.getServerClock('userspace', this.userId, key); - } - - set(key: string, data: Uint8Array): void | Promise { - return this.apis.db.setServerClock('userspace', this.userId, key, data); - } - - keys(): string[] | Promise { - return this.apis.db.getServerClockKeys('userspace', this.userId); - } - - del(key: string): void | Promise { - return this.apis.db.delServerClock('userspace', this.userId, key); - } - - clear(): void | Promise { - return this.apis.db.clearServerClock('userspace', this.userId); - } -} diff --git a/packages/frontend/core/src/modules/userspace/impls/user-db-doc-server.ts b/packages/frontend/core/src/modules/userspace/impls/user-db-doc-server.ts deleted file mode 100644 index 0bd4ba980..000000000 --- a/packages/frontend/core/src/modules/userspace/impls/user-db-doc-server.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { DebugLogger } from '@affine/debug'; -import { - ErrorNames, - UserFriendlyError, - type UserFriendlyErrorResponse, -} from '@affine/graphql'; -import { type DocServer, throwIfAborted } from '@toeverything/infra'; -import type { Socket } from 'socket.io-client'; - -import type { WebSocketService } from '../../cloud'; -import { - base64ToUint8Array, - uint8ArrayToBase64, -} from '../../workspace-engine/utils/base64'; - -type WebsocketResponse = { error: UserFriendlyErrorResponse } | { data: T }; -const logger = new DebugLogger('affine-cloud-doc-engine-server'); - -export class UserDBDocServer implements DocServer { - interruptCb: ((reason: string) => void) | null = null; - SEND_TIMEOUT = 30000; - - socket: Socket; - disposeSocket: () => void; - - constructor( - private readonly userId: string, - webSocketService: WebSocketService - ) { - const { socket, dispose } = webSocketService.connect(); - this.socket = socket; - this.disposeSocket = dispose; - } - - private async clientHandShake() { - await this.socket.emitWithAck('space:join', { - spaceType: 'userspace', - spaceId: this.userId, - clientVersion: BUILD_CONFIG.appVersion, - }); - } - - async pullDoc(docId: string, state: Uint8Array) { - // for testing - await (window as any)._TEST_SIMULATE_SYNC_LAG; - - const stateVector = state ? await uint8ArrayToBase64(state) : undefined; - - const response: WebsocketResponse<{ - missing: string; - state: string; - timestamp: number; - }> = await this.socket - .timeout(this.SEND_TIMEOUT) - .emitWithAck('space:load-doc', { - spaceType: 'userspace', - spaceId: this.userId, - docId: docId, - stateVector, - }); - - if ('error' in response) { - const error = new UserFriendlyError(response.error); - if (error.name === ErrorNames.DOC_NOT_FOUND) { - return null; - } else { - throw error; - } - } else { - return { - data: base64ToUint8Array(response.data.missing), - stateVector: response.data.state - ? base64ToUint8Array(response.data.state) - : undefined, - serverClock: response.data.timestamp, - }; - } - } - async pushDoc(docId: string, data: Uint8Array) { - const payload = await uint8ArrayToBase64(data); - - const response: WebsocketResponse<{ timestamp: number }> = await this.socket - .timeout(this.SEND_TIMEOUT) - .emitWithAck('space:push-doc-updates', { - spaceType: 'userspace', - spaceId: this.userId, - docId: docId, - updates: [payload], - }); - - if ('error' in response) { - logger.error('client-update-v2 error', { - userId: this.userId, - guid: docId, - response, - }); - - throw new UserFriendlyError(response.error); - } - - return { serverClock: response.data.timestamp }; - } - async loadServerClock(after: number): Promise> { - const response: WebsocketResponse> = - await this.socket - .timeout(this.SEND_TIMEOUT) - .emitWithAck('space:load-doc-timestamps', { - spaceType: 'userspace', - spaceId: this.userId, - timestamp: after, - }); - - if ('error' in response) { - logger.error('client-pre-sync error', { - workspaceId: this.userId, - response, - }); - - throw new UserFriendlyError(response.error); - } - - return new Map(Object.entries(response.data)); - } - async subscribeAllDocs( - cb: (updates: { - docId: string; - data: Uint8Array; - serverClock: number; - }) => void - ): Promise<() => void> { - const handleUpdate = async (message: { - spaceType: string; - spaceId: string; - docId: string; - updates: string[]; - timestamp: number; - }) => { - if ( - message.spaceType === 'userspace' && - message.spaceId === this.userId - ) { - message.updates.forEach(update => { - cb({ - docId: message.docId, - data: base64ToUint8Array(update), - serverClock: message.timestamp, - }); - }); - } - }; - this.socket.on('space:broadcast-doc-updates', handleUpdate); - - return () => { - this.socket.off('space:broadcast-doc-updates', handleUpdate); - }; - } - async waitForConnectingServer(signal: AbortSignal): Promise { - this.socket.on('server-version-rejected', this.handleVersionRejected); - this.socket.on('disconnect', this.handleDisconnect); - - throwIfAborted(signal); - if (this.socket.connected) { - await this.clientHandShake(); - } else { - await new Promise((resolve, reject) => { - this.socket.on('connect', () => { - resolve(); - }); - signal.addEventListener('abort', () => { - reject('aborted'); - }); - }); - throwIfAborted(signal); - await this.clientHandShake(); - } - } - disconnectServer(): void { - this.socket.emit('space:leave', { - spaceType: 'userspace', - spaceId: this.userId, - }); - this.socket.off('server-version-rejected', this.handleVersionRejected); - this.socket.off('disconnect', this.handleDisconnect); - } - onInterrupted = (cb: (reason: string) => void) => { - this.interruptCb = cb; - }; - handleInterrupted = (reason: string) => { - this.interruptCb?.(reason); - }; - handleDisconnect = (reason: Socket.DisconnectReason) => { - this.interruptCb?.(reason); - }; - handleVersionRejected = () => { - this.interruptCb?.('Client version rejected'); - }; - - dispose(): void { - this.disconnectServer(); - this.disposeSocket(); - } -} diff --git a/packages/frontend/core/src/modules/userspace/index.ts b/packages/frontend/core/src/modules/userspace/index.ts index fddc8f7fd..c199c71b5 100644 --- a/packages/frontend/core/src/modules/userspace/index.ts +++ b/packages/frontend/core/src/modules/userspace/index.ts @@ -2,16 +2,13 @@ export { UserspaceService as UserDBService } from './services/userspace'; import type { Framework } from '@toeverything/infra'; -import { AuthService, WebSocketService } from '../cloud'; +import { AuthService, ServerService } from '../cloud'; import { ServerScope } from '../cloud/scopes/server'; -import { DesktopApiService } from '../desktop-api/service/desktop-api'; +import { NbstoreService } from '../storage'; import { CurrentUserDB } from './entities/current-user-db'; import { UserDB } from './entities/user-db'; import { UserDBEngine } from './entities/user-db-engine'; import { UserDBTable } from './entities/user-db-table'; -import { IndexedDBUserspaceDocStorage } from './impls/indexeddb-storage'; -import { SqliteUserspaceDocStorage } from './impls/sqlite-storage'; -import { UserspaceStorageProvider } from './provider/storage'; import { UserspaceService } from './services/userspace'; export function configureUserspaceModule(framework: Framework) { @@ -21,23 +18,5 @@ export function configureUserspaceModule(framework: Framework) { .entity(CurrentUserDB, [UserspaceService, AuthService]) .entity(UserDB) .entity(UserDBTable) - .entity(UserDBEngine, [UserspaceStorageProvider, WebSocketService]); -} - -export function configureIndexedDBUserspaceStorageProvider( - framework: Framework -) { - framework.impl(UserspaceStorageProvider, { - getDocStorage(userId: string) { - return new IndexedDBUserspaceDocStorage(userId); - }, - }); -} - -export function configureSqliteUserspaceStorageProvider(framework: Framework) { - framework.impl(UserspaceStorageProvider, p => ({ - getDocStorage(userId: string) { - return new SqliteUserspaceDocStorage(userId, p.get(DesktopApiService)); - }, - })); + .entity(UserDBEngine, [NbstoreService, ServerService]); } diff --git a/packages/frontend/core/src/modules/userspace/provider/storage.ts b/packages/frontend/core/src/modules/userspace/provider/storage.ts deleted file mode 100644 index 5fb6c88a7..000000000 --- a/packages/frontend/core/src/modules/userspace/provider/storage.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { createIdentifier, type DocStorage } from '@toeverything/infra'; - -export interface UserspaceStorageProvider { - getDocStorage(userId: string): DocStorage; -} - -export const UserspaceStorageProvider = - createIdentifier('UserspaceStorageProvider'); diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts b/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts index 8560bbb8f..98c0f30b5 100644 --- a/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts +++ b/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts @@ -5,10 +5,29 @@ import { getWorkspaceInfoQuery, getWorkspacesQuery, } from '@affine/graphql'; +import type { BlobStorage, DocStorage } from '@affine/nbstore'; +import { CloudBlobStorage, StaticCloudDocStorage } from '@affine/nbstore/cloud'; +import { + IndexedDBBlobStorage, + IndexedDBDocStorage, + IndexedDBSyncStorage, +} from '@affine/nbstore/idb'; +import { + IndexedDBV1BlobStorage, + IndexedDBV1DocStorage, +} from '@affine/nbstore/idb/v1'; +import { + SqliteBlobStorage, + SqliteDocStorage, + SqliteSyncStorage, +} from '@affine/nbstore/sqlite'; +import { + SqliteV1BlobStorage, + SqliteV1DocStorage, +} from '@affine/nbstore/sqlite/v1'; +import type { WorkerInitOptions } from '@affine/nbstore/worker/client'; import { - type BlobStorage, catchErrorInto, - type DocStorage, effect, exhaustMapSwitchUntilChanged, fromPromise, @@ -20,35 +39,25 @@ import { } from '@toeverything/infra'; import { isEqual } from 'lodash-es'; import { EMPTY, map, mergeMap, Observable, switchMap } from 'rxjs'; -import { encodeStateAsUpdate } from 'yjs'; +import { type Doc as YDoc, encodeStateAsUpdate } from 'yjs'; import type { Server, ServersService } from '../../cloud'; import { AccountChanged, AuthService, - FetchService, GraphQLService, - WebSocketService, WorkspaceServerService, } from '../../cloud'; import type { GlobalState } from '../../storage'; import { getAFFiNEWorkspaceSchema, type Workspace, - type WorkspaceEngineProvider, type WorkspaceFlavourProvider, type WorkspaceFlavoursProvider, type WorkspaceMetadata, type WorkspaceProfileInfo, } from '../../workspace'; import { WorkspaceImpl } from '../../workspace/impls/workspace'; -import type { WorkspaceEngineStorageProvider } from '../providers/engine'; -import { BroadcastChannelAwarenessConnection } from './engine/awareness-broadcast-channel'; -import { CloudAwarenessConnection } from './engine/awareness-cloud'; -import { CloudBlobStorage } from './engine/blob-cloud'; -import { StaticBlobStorage } from './engine/blob-static'; -import { CloudDocEngineServer } from './engine/doc-cloud'; -import { CloudStaticDocStorage } from './engine/doc-cloud-static'; import { getWorkspaceProfileWorker } from './out-worker'; const getCloudWorkspaceCacheKey = (serverId: string) => { @@ -62,20 +71,14 @@ const logger = new DebugLogger('affine:cloud-workspace-flavour-provider'); class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { private readonly authService: AuthService; - private readonly webSocketService: WebSocketService; - private readonly fetchService: FetchService; private readonly graphqlService: GraphQLService; - private readonly unsubscribeAccountChanged: () => void; constructor( private readonly globalState: GlobalState, - private readonly storageProvider: WorkspaceEngineStorageProvider, private readonly server: Server ) { this.authService = server.scope.get(AuthService); - this.webSocketService = server.scope.get(WebSocketService); - this.fetchService = server.scope.get(FetchService); this.graphqlService = server.scope.get(GraphQLService); this.unsubscribeAccountChanged = this.server.scope.eventBus.on( AccountChanged, @@ -85,7 +88,30 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { ); } - flavour = this.server.id; + readonly flavour = this.server.id; + + DocStorageType = + BUILD_CONFIG.isElectron || BUILD_CONFIG.isIOS + ? SqliteDocStorage + : IndexedDBDocStorage; + DocStorageV1Type = BUILD_CONFIG.isElectron + ? SqliteV1DocStorage + : BUILD_CONFIG.isWeb || BUILD_CONFIG.isMobileWeb + ? IndexedDBV1DocStorage + : undefined; + BlobStorageType = + BUILD_CONFIG.isElectron || BUILD_CONFIG.isIOS + ? SqliteBlobStorage + : IndexedDBBlobStorage; + BlobStorageV1Type = BUILD_CONFIG.isElectron + ? SqliteV1BlobStorage + : BUILD_CONFIG.isWeb || BUILD_CONFIG.isMobileWeb + ? IndexedDBV1BlobStorage + : undefined; + SyncStorageType = + BUILD_CONFIG.isElectron || BUILD_CONFIG.isIOS + ? SqliteSyncStorage + : IndexedDBSyncStorage; async deleteWorkspace(id: string): Promise { await this.graphqlService.gql({ @@ -113,13 +139,51 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { }); // save the initial state to local storage, then sync to cloud - const blobStorage = this.storageProvider.getBlobStorage(workspaceId); - const docStorage = this.storageProvider.getDocStorage(workspaceId); + const blobStorage = new this.BlobStorageType({ + id: workspaceId, + flavour: this.flavour, + type: 'workspace', + }); + blobStorage.connection.connect(); + await blobStorage.connection.waitForConnected(); + const docStorage = new this.DocStorageType({ + id: workspaceId, + flavour: this.flavour, + type: 'workspace', + }); + docStorage.connection.connect(); + await docStorage.connection.waitForConnected(); + + const docList = new Set(); const docCollection = new WorkspaceImpl({ id: workspaceId, schema: getAFFiNEWorkspaceSchema(), - blobSource: blobStorage, + blobSource: { + get: async key => { + const record = await blobStorage.get(key); + return record ? new Blob([record.data], { type: record.mime }) : null; + }, + delete: async () => { + return; + }, + list: async () => { + return []; + }, + set: async (id, blob) => { + await blobStorage.set({ + key: id, + data: new Uint8Array(await blob.arrayBuffer()), + mime: blob.type, + }); + return id; + }, + name: 'blob', + readonly: false, + }, + onLoadDoc: doc => { + docList.add(doc); + }, }); try { @@ -127,14 +191,16 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { await initial(docCollection, blobStorage, docStorage); // save workspace to local storage, should be vary fast - await docStorage.doc.set( - workspaceId, - encodeStateAsUpdate(docCollection.doc) - ); - for (const subdocs of docCollection.doc.getSubdocs()) { - await docStorage.doc.set(subdocs.guid, encodeStateAsUpdate(subdocs)); + for (const subdocs of docList) { + await docStorage.pushDocUpdate({ + docId: subdocs.guid, + bin: encodeStateAsUpdate(subdocs), + }); } + docStorage.connection.disconnect(); + blobStorage.connection.disconnect(); + this.revalidate(); await this.waitForLoaded(); } finally { @@ -228,11 +294,23 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { // get information from both cloud and local storage // we use affine 'static' storage here, which use http protocol, no need to websocket. - const cloudStorage = new CloudStaticDocStorage(id, this.fetchService); - const docStorage = this.storageProvider.getDocStorage(id); + const cloudStorage = new StaticCloudDocStorage({ + id: id, + serverBaseUrl: this.server.serverMetadata.baseUrl, + }); + const docStorage = new this.DocStorageType({ + id: id, + flavour: this.flavour, + type: 'workspace', + readonlyMode: true, + }); + docStorage.connection.connect(); + await docStorage.connection.waitForConnected(); // download root doc - const localData = await docStorage.doc.get(id); - const cloudData = (await cloudStorage.pull(id))?.data; + const localData = (await docStorage.getDoc(id))?.bin; + const cloudData = (await cloudStorage.getDoc(id))?.bin; + + docStorage.connection.disconnect(); const info = await this.getWorkspaceInfo(id, signal); @@ -260,48 +338,27 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { }; } async getWorkspaceBlob(id: string, blob: string): Promise { - const localBlob = await this.storageProvider.getBlobStorage(id).get(blob); + const storage = new this.BlobStorageType({ + id: id, + flavour: this.flavour, + type: 'workspace', + }); + storage.connection.connect(); + await storage.connection.waitForConnected(); + const localBlob = await storage.get(blob); if (localBlob) { - return localBlob; + return new Blob([localBlob.data], { type: localBlob.mime }); } - const cloudBlob = new CloudBlobStorage( + const cloudBlob = await new CloudBlobStorage({ id, - this.fetchService, - this.graphqlService - ); - return await cloudBlob.get(blob); - } - - getEngineProvider(workspaceId: string): WorkspaceEngineProvider { - return { - getAwarenessConnections: () => { - return [ - new BroadcastChannelAwarenessConnection(workspaceId), - new CloudAwarenessConnection(workspaceId, this.webSocketService), - ]; - }, - getDocServer: () => { - return new CloudDocEngineServer(workspaceId, this.webSocketService); - }, - getDocStorage: () => { - return this.storageProvider.getDocStorage(workspaceId); - }, - getLocalBlobStorage: () => { - return this.storageProvider.getBlobStorage(workspaceId); - }, - getRemoteBlobStorages: () => { - return [ - new CloudBlobStorage( - workspaceId, - this.fetchService, - this.graphqlService - ), - new StaticBlobStorage(), - ]; - }, - }; + serverBaseUrl: this.server.serverMetadata.baseUrl, + }).get(blob); + if (!cloudBlob) { + return null; + } + return new Blob([cloudBlob.data], { type: cloudBlob.mime }); } onWorkspaceInitialized(workspace: Workspace): void { @@ -319,6 +376,90 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { }); } + getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions { + return { + local: { + doc: { + name: this.DocStorageType.identifier, + opts: { + flavour: this.flavour, + type: 'workspace', + id: workspaceId, + }, + }, + blob: { + name: this.BlobStorageType.identifier, + opts: { + flavour: this.flavour, + type: 'workspace', + id: workspaceId, + }, + }, + sync: { + name: this.SyncStorageType.identifier, + opts: { + flavour: this.flavour, + type: 'workspace', + id: workspaceId, + }, + }, + awareness: { + name: 'BroadcastChannelAwarenessStorage', + opts: { + id: `${this.flavour}:${workspaceId}`, + }, + }, + }, + remotes: { + [`cloud:${this.flavour}`]: { + doc: { + name: 'CloudDocStorage', + opts: { + type: 'workspace', + id: workspaceId, + serverBaseUrl: this.server.serverMetadata.baseUrl, + }, + }, + blob: { + name: 'CloudBlobStorage', + opts: { + id: workspaceId, + serverBaseUrl: this.server.serverMetadata.baseUrl, + }, + }, + awareness: { + name: 'CloudAwarenessStorage', + opts: { + type: 'workspace', + id: workspaceId, + serverBaseUrl: this.server.serverMetadata.baseUrl, + }, + }, + }, + v1: { + doc: this.DocStorageV1Type + ? { + name: this.DocStorageV1Type.identifier, + opts: { + id: workspaceId, + type: 'workspace', + }, + } + : undefined, + blob: this.BlobStorageV1Type + ? { + name: this.BlobStorageV1Type.identifier, + opts: { + id: workspaceId, + type: 'workspace', + }, + } + : undefined, + }, + }, + }; + } + private waitForLoaded() { return this.isRevalidating$.waitFor(loading => !loading); } @@ -335,7 +476,6 @@ export class CloudWorkspaceFlavoursProvider { constructor( private readonly globalState: GlobalState, - private readonly storageProvider: WorkspaceEngineStorageProvider, private readonly serversService: ServersService ) { super(); @@ -351,7 +491,6 @@ export class CloudWorkspaceFlavoursProvider } const provider = new CloudWorkspaceFlavourProvider( this.globalState, - this.storageProvider, server ); provider.revalidate(); diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/awareness-broadcast-channel.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/awareness-broadcast-channel.ts deleted file mode 100644 index eec1d4198..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/awareness-broadcast-channel.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { AwarenessConnection } from '@toeverything/infra'; -import type { Awareness } from 'y-protocols/awareness.js'; -import { - applyAwarenessUpdate, - encodeAwarenessUpdate, -} from 'y-protocols/awareness.js'; - -type AwarenessChanges = Record<'added' | 'updated' | 'removed', number[]>; - -type ChannelMessage = - | { type: 'connect' } - | { type: 'update'; update: Uint8Array }; - -export class BroadcastChannelAwarenessConnection - implements AwarenessConnection -{ - channel: BroadcastChannel | null = null; - awareness: Awareness | null = null; - - constructor(private readonly workspaceId: string) {} - - connect(awareness: Awareness): void { - this.awareness = awareness; - this.channel = new BroadcastChannel('awareness:' + this.workspaceId); - this.channel.postMessage({ - type: 'connect', - } satisfies ChannelMessage); - this.awareness.on('update', this.handleAwarenessUpdate); - this.channel.addEventListener('message', this.handleChannelMessage); - } - - disconnect(): void { - this.channel?.close(); - this.channel = null; - this.awareness?.off('update', this.handleAwarenessUpdate); - this.awareness = null; - } - - handleAwarenessUpdate = (changes: AwarenessChanges, origin: unknown) => { - if (this.awareness === null) { - return; - } - - if (origin === 'remote') { - return; - } - - const changedClients = Object.values(changes).reduce((res, cur) => - res.concat(cur) - ); - - const update = encodeAwarenessUpdate(this.awareness, changedClients); - this.channel?.postMessage({ - type: 'update', - update: update, - } satisfies ChannelMessage); - }; - - handleChannelMessage = (event: MessageEvent) => { - if (this.awareness === null) { - return; - } - - if (event.data.type === 'update') { - const update = event.data.update; - applyAwarenessUpdate(this.awareness, update, 'remote'); - } - if (event.data.type === 'connect') { - this.channel?.postMessage({ - type: 'update', - update: encodeAwarenessUpdate(this.awareness, [ - this.awareness.clientID, - ]), - } satisfies ChannelMessage); - } - }; -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/awareness-cloud.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/awareness-cloud.ts deleted file mode 100644 index 0d2c23a5b..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/awareness-cloud.ts +++ /dev/null @@ -1,196 +0,0 @@ -import type { WebSocketService } from '@affine/core/modules/cloud'; -import { DebugLogger } from '@affine/debug'; -import type { AwarenessConnection } from '@toeverything/infra'; -import type { Socket } from 'socket.io-client'; -import type { Awareness } from 'y-protocols/awareness'; -import { - applyAwarenessUpdate, - encodeAwarenessUpdate, - removeAwarenessStates, -} from 'y-protocols/awareness'; - -import { base64ToUint8Array, uint8ArrayToBase64 } from '../../utils/base64'; - -const logger = new DebugLogger('affine:awareness:socketio'); - -type AwarenessChanges = Record<'added' | 'updated' | 'removed', number[]>; - -export class CloudAwarenessConnection implements AwarenessConnection { - awareness: Awareness | null = null; - - socket: Socket; - disposeSocket: () => void; - - constructor( - private readonly workspaceId: string, - webSocketService: WebSocketService - ) { - const { socket, dispose } = webSocketService.connect(); - this.socket = socket; - this.disposeSocket = dispose; - } - - connect(awareness: Awareness): void { - this.socket.on('space:broadcast-awareness-update', this.awarenessBroadcast); - this.socket.on( - 'space:collect-awareness', - this.newClientAwarenessInitHandler - ); - this.awareness = awareness; - this.awareness.on('update', this.awarenessUpdate); - - window.addEventListener('beforeunload', this.windowBeforeUnloadHandler); - - this.socket.on('connect', this.handleConnect); - this.socket.on('server-version-rejected', this.handleReject); - - if (this.socket.connected) { - this.handleConnect(); - } - } - - disconnect(): void { - if (this.awareness) { - removeAwarenessStates( - this.awareness, - [this.awareness.clientID], - 'disconnect' - ); - this.awareness.off('update', this.awarenessUpdate); - } - this.awareness = null; - - this.socket.emit('space:leave-awareness', { - spaceType: 'workspace', - spaceId: this.workspaceId, - docId: this.workspaceId, - }); - this.socket.off( - 'space:broadcast-awareness-update', - this.awarenessBroadcast - ); - this.socket.off( - 'space:collect-awareness', - this.newClientAwarenessInitHandler - ); - this.socket.off('connect', this.handleConnect); - this.socket.off('server-version-rejected', this.handleReject); - window.removeEventListener('unload', this.windowBeforeUnloadHandler); - } - - awarenessBroadcast = ({ - spaceId: wsId, - spaceType, - awarenessUpdate, - }: { - spaceType: string; - spaceId: string; - docId: string; - awarenessUpdate: string; - }) => { - if (!this.awareness) { - return; - } - if (wsId !== this.workspaceId || spaceType !== 'workspace') { - return; - } - applyAwarenessUpdate( - this.awareness, - base64ToUint8Array(awarenessUpdate), - 'remote' - ); - }; - - awarenessUpdate = (changes: AwarenessChanges, origin: unknown) => { - if (!this.awareness) { - return; - } - - if (origin === 'remote') { - return; - } - - const changedClients = Object.values(changes).reduce((res, cur) => - res.concat(cur) - ); - - const update = encodeAwarenessUpdate(this.awareness, changedClients); - uint8ArrayToBase64(update) - .then(encodedUpdate => { - this.socket.emit('space:update-awareness', { - spaceType: 'workspace', - spaceId: this.workspaceId, - docId: this.workspaceId, - awarenessUpdate: encodedUpdate, - }); - }) - .catch(err => logger.error(err)); - }; - - newClientAwarenessInitHandler = () => { - if (!this.awareness) { - return; - } - - const awarenessUpdate = encodeAwarenessUpdate(this.awareness, [ - this.awareness.clientID, - ]); - uint8ArrayToBase64(awarenessUpdate) - .then(encodedAwarenessUpdate => { - this.socket.emit('space:update-awareness', { - spaceType: 'workspace', - spaceId: this.workspaceId, - docId: this.workspaceId, - awarenessUpdate: encodedAwarenessUpdate, - }); - }) - .catch(err => logger.error(err)); - }; - - windowBeforeUnloadHandler = () => { - if (!this.awareness) { - return; - } - - removeAwarenessStates( - this.awareness, - [this.awareness.clientID], - 'window unload' - ); - }; - - handleConnect = () => { - this.socket.emit( - 'space:join-awareness', - { - spaceType: 'workspace', - spaceId: this.workspaceId, - docId: this.workspaceId, - clientVersion: BUILD_CONFIG.appVersion, - }, - (res: any) => { - logger.debug('awareness handshake finished', res); - this.socket.emit( - 'space:load-awarenesses', - { - spaceType: 'workspace', - spaceId: this.workspaceId, - docId: this.workspaceId, - }, - (res: any) => { - logger.debug('awareness-init finished', res); - } - ); - } - ); - }; - - handleReject = () => { - this.socket.off('server-version-rejected', this.handleReject); - }; - - dispose() { - this.disconnect(); - this.disposeSocket(); - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts deleted file mode 100644 index e6549b0e0..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { FetchService, GraphQLService } from '@affine/core/modules/cloud'; -import { - deleteBlobMutation, - listBlobsQuery, - setBlobMutation, - UserFriendlyError, -} from '@affine/graphql'; -import type { BlobStorage } from '@toeverything/infra'; -import { BlobStorageOverCapacity } from '@toeverything/infra'; - -import { bufferToBlob } from '../../utils/buffer-to-blob'; - -export class CloudBlobStorage implements BlobStorage { - constructor( - private readonly workspaceId: string, - private readonly fetchService: FetchService, - private readonly gqlService: GraphQLService - ) {} - - name = 'cloud'; - readonly = false; - - async get(key: string) { - const suffix = key.startsWith('/') - ? key - : `/api/workspaces/${this.workspaceId}/blobs/${key}`; - - return this.fetchService - .fetch(suffix, { - cache: 'default', - headers: { - Accept: 'application/octet-stream', // this is necessary for ios native fetch to return arraybuffer - }, - }) - .then(async res => { - if (!res.ok) { - // status not in the range 200-299 - return null; - } - return bufferToBlob(await res.arrayBuffer()); - }) - .catch(() => { - return null; - }); - } - - async set(key: string, value: Blob) { - // set blob will check blob size & quota - return await this.gqlService - .gql({ - query: setBlobMutation, - variables: { - workspaceId: this.workspaceId, - blob: new File([value], key), - }, - }) - .then(res => res.setBlob) - .catch(err => { - const error = UserFriendlyError.fromAnyError(err); - if (error.status === 413) { - throw new BlobStorageOverCapacity(error); - } - - throw err; - }); - } - - async delete(key: string) { - await this.gqlService.gql({ - query: deleteBlobMutation, - variables: { - workspaceId: key, - key, - }, - }); - } - - async list() { - const result = await this.gqlService.gql({ - query: listBlobsQuery, - variables: { - workspaceId: this.workspaceId, - }, - }); - return result.workspace.blobs.map(blob => blob.key); - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-indexeddb.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-indexeddb.ts deleted file mode 100644 index d7b81db22..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-indexeddb.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { BlobStorage } from '@toeverything/infra'; -import { createStore, del, get, keys, set } from 'idb-keyval'; - -import { bufferToBlob } from '../../utils/buffer-to-blob'; - -export class IndexedDBBlobStorage implements BlobStorage { - constructor(private readonly workspaceId: string) {} - - name = 'indexeddb'; - readonly = false; - db = createStore(`${this.workspaceId}_blob`, 'blob'); - mimeTypeDb = createStore(`${this.workspaceId}_blob_mime`, 'blob_mime'); - - async get(key: string) { - const res = await get(key, this.db); - if (res) { - return bufferToBlob(res); - } - return null; - } - async set(key: string, value: Blob) { - await set(key, await value.arrayBuffer(), this.db); - await set(key, value.type, this.mimeTypeDb); - return key; - } - async delete(key: string) { - await del(key, this.db); - await del(key, this.mimeTypeDb); - } - async list() { - return keys(this.db); - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-sqlite.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-sqlite.ts deleted file mode 100644 index 27d45d624..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-sqlite.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { DesktopApiService } from '@affine/core/modules/desktop-api'; -import type { BlobStorage } from '@toeverything/infra'; - -import { bufferToBlob } from '../../utils/buffer-to-blob'; - -export class SqliteBlobStorage implements BlobStorage { - constructor( - private readonly workspaceId: string, - private readonly electronApi: DesktopApiService - ) {} - name = 'sqlite'; - readonly = false; - async get(key: string) { - const buffer = await this.electronApi.handler.db.getBlob( - 'workspace', - this.workspaceId, - key - ); - if (buffer) { - return bufferToBlob(buffer); - } - return null; - } - async set(key: string, value: Blob) { - await this.electronApi.handler.db.addBlob( - 'workspace', - this.workspaceId, - key, - new Uint8Array(await value.arrayBuffer()) - ); - return key; - } - delete(key: string) { - return this.electronApi.handler.db.deleteBlob( - 'workspace', - this.workspaceId, - key - ); - } - list() { - return this.electronApi.handler.db.getBlobKeys( - 'workspace', - this.workspaceId - ); - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-static.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-static.ts deleted file mode 100644 index d2a992436..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-static.ts +++ /dev/null @@ -1,71 +0,0 @@ -import type { BlobStorage } from '@toeverything/infra'; - -export const predefinedStaticFiles = [ - '029uztLz2CzJezK7UUhrbGiWUdZ0J7NVs_qR6RDsvb8=', - '047ebf2c9a5c7c9d8521c2ea5e6140ff7732ef9e28a9f944e9bf3ca4', - '0hjYqQd8SvwHT2gPds7qFw8W6qIEGVbZvG45uzoYjUU=', - '1326bc48553a572c6756d9ee1b30a0dfdda26222fc2d2c872b14e609', - '27f983d0765289c19d10ee0b51c00c3c7665236a1a82406370d46e0a', - '28516717d63e469cd98729ff46be6595711898bab3dc43302319a987', - '4HXJrnBZGaGPFpowNawNog0aMg3dgoVaAnNqEMeUxq0=', - '5Cfem_137WmzR35ZeIC76oTkq5SQt-eHlZwJiLy0hgU=', - '6aa785ee927547ce9dd9d7b43e01eac948337fe57571443e87bc3a60', - '8oj6ym4HlTcshT40Zn6D5DeOgaVCSOOXJvT_EyiqUw8=', - '9288be57321c8772d04e05dbb69a22742372b3534442607a2d6a9998', - '9vXwWGEX5W9v5pzwpu0eK4pf22DZ_sCloO0zCH1aVQ4=', - 'Bd5F0WRI0fLh8RK1al9PawPVT3jv7VwBrqiiBEtdV-g=', - 'CBWoKrhSDndjBJzscQKENRqiXOOZnzIA5qyiCoy4-A0=', - 'D7g-4LMqOsVWBNOD-_kGgCOvJEoc8rcpYbkfDlF2u5U=', - 'Vqc8rxFbGyc5L1QeE_Zr10XEcIai_0Xw4Qv6d3ldRPE=', - 'VuXYyM9JUv1Fv_qjg1v5Go4Zksz0r4NXFeh3Na7JkIc=', - 'bfXllFddegV9vvxPcSWnOtm-_tuzXm-0OQ59z9Su1zA=', - 'c820edeeba50006b531883903f5bb0b96bf523c9a6b3ce5868f03db5', - 'cw9XjQ-pCeSW7LKMzVREGHeCPTXWYbtE-QbZLEY3RrI=', - 'e93536e1be97e3b5206d43bf0793fdef24e60044d174f0abdefebe08', - 'f9yKnlNMgKhF-CxOgHBsXkxfViCCkC6KwTv6Uj2Fcjw=', - 'fb0SNPtMpQlzBQ90_PB7vCu34WpiSUJbNKocFkL2vIo=', - 'gZLmSgmwumNdgf0eIfOSW44emctrLyFUaZapbk8eZ6s=', - 'i39ZQ24NlUfWI0MhkbtvHTzGnWMVdr-aC2aOjvHPVg4=', - 'k07JiWnb-S7qgd9gDQNgqo-LYMe03RX8fR0TXQ-SpG4=', - 'nSEEkYxrThpZfLoPNOzMp6HWekvutAIYmADElDe1J6I=', - 'pIqdA3pM1la1gKzxOmAcpLmTh3yXBrL9mGTz_hGj5xE=', - 'qezoK6du9n3PF4dl4aq5r7LeXz_sV3xOVpFzVVgjNsE=', - 'rY96Bunn-69CnNe5X_e5CJLwgCJnN6rcbUisecs8kkQ=', - 'sNVNYDBzUDN2J9OFVJdLJlryBLzRZBLl-4MTNoPF1tA=', - 'uvpOG9DrldeqIGNaqfwjFdMw_CcfXKfiEjYf7RXdeL0=', - 'v2yF7lY2L5rtorTtTmYFsoMb9dBPKs5M1y9cUKxcI1M=', -]; - -export class StaticBlobStorage implements BlobStorage { - name = 'static'; - readonly = true; - async get(key: string) { - const isStaticResource = - predefinedStaticFiles.includes(key) || key.startsWith('/static/'); - - if (!isStaticResource) { - return null; - } - - const path = key.startsWith('/static/') ? key : `/static/${key}`; - const response = await fetch(path); - - if (response.ok) { - return await response.blob(); - } - - return null; - } - - async set(key: string) { - // ignore - return key; - } - async delete() { - // ignore - } - async list() { - // ignore - return []; - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-broadcast-channel.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-broadcast-channel.ts deleted file mode 100644 index 5f8c43e0e..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-broadcast-channel.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { DocEvent, DocEventBus } from '@toeverything/infra'; - -export class BroadcastChannelDocEventBus implements DocEventBus { - senderChannel = new BroadcastChannel('doc:' + this.workspaceId); - constructor(private readonly workspaceId: string) {} - emit(event: DocEvent): void { - this.senderChannel.postMessage(event); - } - - on(cb: (event: DocEvent) => void): () => void { - const listener = (event: MessageEvent) => { - cb(event.data); - }; - const channel = new BroadcastChannel('doc:' + this.workspaceId); - channel.addEventListener('message', listener); - return () => { - channel.removeEventListener('message', listener); - channel.close(); - }; - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-cloud-static.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-cloud-static.ts deleted file mode 100644 index 91f83385d..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-cloud-static.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { FetchService } from '@affine/core/modules/cloud'; - -export class CloudStaticDocStorage { - name = 'cloud-static'; - constructor( - private readonly workspaceId: string, - private readonly fetchService: FetchService - ) {} - - async pull( - docId: string - ): Promise<{ data: Uint8Array; state?: Uint8Array | undefined } | null> { - const response = await this.fetchService.fetch( - `/api/workspaces/${this.workspaceId}/docs/${docId}`, - { - priority: 'high', - headers: { - Accept: 'application/octet-stream', // this is necessary for ios native fetch to return arraybuffer - }, - } - ); - if (response.ok) { - const arrayBuffer = await response.arrayBuffer(); - - return { data: new Uint8Array(arrayBuffer) }; - } - - return null; - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-cloud.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-cloud.ts deleted file mode 100644 index 9d10de35a..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-cloud.ts +++ /dev/null @@ -1,204 +0,0 @@ -import type { WebSocketService } from '@affine/core/modules/cloud'; -import { DebugLogger } from '@affine/debug'; -import { - ErrorNames, - UserFriendlyError, - type UserFriendlyErrorResponse, -} from '@affine/graphql'; -import type { DocServer } from '@toeverything/infra'; -import { throwIfAborted } from '@toeverything/infra'; -import type { Socket } from 'socket.io-client'; - -import { base64ToUint8Array, uint8ArrayToBase64 } from '../../utils/base64'; - -(window as any)._TEST_SIMULATE_SYNC_LAG = Promise.resolve(); - -const logger = new DebugLogger('affine-cloud-doc-engine-server'); - -type WebsocketResponse = { error: UserFriendlyErrorResponse } | { data: T }; - -export class CloudDocEngineServer implements DocServer { - interruptCb: ((reason: string) => void) | null = null; - SEND_TIMEOUT = 30000; - - socket: Socket; - disposeSocket: () => void; - - constructor( - private readonly workspaceId: string, - webSocketService: WebSocketService - ) { - const { socket, dispose } = webSocketService.connect(); - this.socket = socket; - this.disposeSocket = dispose; - } - - private async clientHandShake() { - await this.socket.emitWithAck('space:join', { - spaceType: 'workspace', - spaceId: this.workspaceId, - clientVersion: BUILD_CONFIG.appVersion, - }); - } - - async pullDoc(docId: string, state: Uint8Array) { - // for testing - await (window as any)._TEST_SIMULATE_SYNC_LAG; - - const stateVector = state ? await uint8ArrayToBase64(state) : undefined; - - const response: WebsocketResponse<{ - missing: string; - state: string; - timestamp: number; - }> = await this.socket - .timeout(this.SEND_TIMEOUT) - .emitWithAck('space:load-doc', { - spaceType: 'workspace', - spaceId: this.workspaceId, - docId: docId, - stateVector, - }); - - if ('error' in response) { - const error = new UserFriendlyError(response.error); - if (error.name === ErrorNames.DOC_NOT_FOUND) { - return null; - } else { - throw error; - } - } else { - return { - data: base64ToUint8Array(response.data.missing), - stateVector: response.data.state - ? base64ToUint8Array(response.data.state) - : undefined, - serverClock: response.data.timestamp, - }; - } - } - async pushDoc(docId: string, data: Uint8Array) { - const payload = await uint8ArrayToBase64(data); - - const response: WebsocketResponse<{ timestamp: number }> = await this.socket - .timeout(this.SEND_TIMEOUT) - .emitWithAck('space:push-doc-updates', { - spaceType: 'workspace', - spaceId: this.workspaceId, - docId: docId, - updates: [payload], - }); - - if ('error' in response) { - logger.error('client-update-v2 error', { - workspaceId: this.workspaceId, - guid: docId, - response, - }); - - throw new UserFriendlyError(response.error); - } - - return { serverClock: response.data.timestamp }; - } - async loadServerClock(after: number): Promise> { - const response: WebsocketResponse> = - await this.socket - .timeout(this.SEND_TIMEOUT) - .emitWithAck('space:load-doc-timestamps', { - spaceType: 'workspace', - spaceId: this.workspaceId, - timestamp: after, - }); - - if ('error' in response) { - logger.error('client-pre-sync error', { - workspaceId: this.workspaceId, - response, - }); - - throw new UserFriendlyError(response.error); - } - - return new Map(Object.entries(response.data)); - } - async subscribeAllDocs( - cb: (updates: { - docId: string; - data: Uint8Array; - serverClock: number; - }) => void - ): Promise<() => void> { - const handleUpdate = async (message: { - spaceType: string; - spaceId: string; - docId: string; - updates: string[]; - timestamp: number; - }) => { - if ( - message.spaceType === 'workspace' && - message.spaceId === this.workspaceId - ) { - message.updates.forEach(update => { - cb({ - docId: message.docId, - data: base64ToUint8Array(update), - serverClock: message.timestamp, - }); - }); - } - }; - this.socket.on('space:broadcast-doc-updates', handleUpdate); - - return () => { - this.socket.off('space:broadcast-doc-updates', handleUpdate); - }; - } - async waitForConnectingServer(signal: AbortSignal): Promise { - this.socket.on('server-version-rejected', this.handleVersionRejected); - this.socket.on('disconnect', this.handleDisconnect); - - throwIfAborted(signal); - if (this.socket.connected) { - await this.clientHandShake(); - } else { - this.socket.connect(); - await new Promise((resolve, reject) => { - this.socket.on('connect', () => { - resolve(); - }); - signal.addEventListener('abort', () => { - reject('aborted'); - }); - }); - throwIfAborted(signal); - await this.clientHandShake(); - } - } - disconnectServer(): void { - this.socket.emit('space:leave', { - spaceType: 'workspace', - spaceId: this.workspaceId, - }); - this.socket.off('server-version-rejected', this.handleVersionRejected); - this.socket.off('disconnect', this.handleDisconnect); - } - onInterrupted = (cb: (reason: string) => void) => { - this.interruptCb = cb; - }; - handleInterrupted = (reason: string) => { - this.interruptCb?.(reason); - }; - handleDisconnect = (reason: Socket.DisconnectReason) => { - this.interruptCb?.(reason); - }; - handleVersionRejected = () => { - this.interruptCb?.('Client version rejected'); - }; - - dispose(): void { - this.disconnectServer(); - this.disposeSocket(); - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-indexeddb.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-indexeddb.ts deleted file mode 100644 index 4889c0133..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-indexeddb.ts +++ /dev/null @@ -1,242 +0,0 @@ -import type { ByteKV, ByteKVBehavior, DocStorage } from '@toeverything/infra'; -import type { DBSchema, IDBPDatabase, IDBPObjectStore } from 'idb'; -import { openDB } from 'idb'; -import { mergeUpdates } from 'yjs'; - -import { BroadcastChannelDocEventBus } from './doc-broadcast-channel'; - -function isEmptyUpdate(binary: Uint8Array) { - return ( - binary.byteLength === 0 || - (binary.byteLength === 2 && binary[0] === 0 && binary[1] === 0) - ); -} - -export class IndexedDBDocStorage implements DocStorage { - constructor(private readonly workspaceId: string) {} - eventBus = new BroadcastChannelDocEventBus(this.workspaceId); - readonly doc = new Doc(); - readonly syncMetadata = new KV(`${this.workspaceId}:sync-metadata`); - readonly serverClock = new KV(`${this.workspaceId}:server-clock`); -} - -interface DocDBSchema extends DBSchema { - workspace: { - key: string; - value: { - id: string; - updates: { - timestamp: number; - update: Uint8Array; - }[]; - }; - }; -} - -type DocType = DocStorage['doc']; -class Doc implements DocType { - dbName = 'affine-local'; - dbPromise: Promise> | null = null; - dbVersion = 1; - - constructor() {} - - upgradeDB(db: IDBPDatabase) { - db.createObjectStore('workspace', { keyPath: 'id' }); - } - - getDb() { - if (this.dbPromise === null) { - this.dbPromise = openDB(this.dbName, this.dbVersion, { - upgrade: db => this.upgradeDB(db), - }); - } - return this.dbPromise; - } - - async get(docId: string): Promise { - const db = await this.getDb(); - const store = db - .transaction('workspace', 'readonly') - .objectStore('workspace'); - const data = await store.get(docId); - - if (!data) { - return null; - } - - const updates = data.updates - .map(({ update }) => update) - .filter(update => !isEmptyUpdate(update)); - const update = updates.length > 0 ? mergeUpdates(updates) : null; - - return update; - } - - async set(docId: string, data: Uint8Array) { - const db = await this.getDb(); - const store = db - .transaction('workspace', 'readwrite') - .objectStore('workspace'); - - const rows = [{ timestamp: Date.now(), update: data }]; - await store.put({ - id: docId, - updates: rows, - }); - } - - async keys() { - const db = await this.getDb(); - const store = db - .transaction('workspace', 'readonly') - .objectStore('workspace'); - - return store.getAllKeys(); - } - - clear(): void | Promise { - return; - } - - del(_key: string): void | Promise { - return; - } - - async transaction( - cb: (transaction: ByteKVBehavior) => Promise - ): Promise { - const db = await this.getDb(); - const store = db - .transaction('workspace', 'readwrite') - .objectStore('workspace'); - return await cb({ - async get(docId) { - const data = await store.get(docId); - - if (!data) { - return null; - } - - const { updates } = data; - const update = mergeUpdates(updates.map(({ update }) => update)); - - return update; - }, - keys() { - return store.getAllKeys(); - }, - async set(docId, data) { - const rows = [{ timestamp: Date.now(), update: data }]; - await store.put({ - id: docId, - updates: rows, - }); - }, - async clear() { - return await store.clear(); - }, - async del(key) { - return store.delete(key); - }, - }); - } -} - -interface KvDBSchema extends DBSchema { - kv: { - key: string; - value: { key: string; val: Uint8Array }; - }; -} - -class KV implements ByteKV { - constructor(private readonly dbName: string) {} - - dbPromise: Promise> | null = null; - dbVersion = 1; - - upgradeDB(db: IDBPDatabase) { - db.createObjectStore('kv', { keyPath: 'key' }); - } - - getDb() { - if (this.dbPromise === null) { - this.dbPromise = openDB(this.dbName, this.dbVersion, { - upgrade: db => this.upgradeDB(db), - }); - } - return this.dbPromise; - } - - async transaction( - cb: (transaction: ByteKVBehavior) => Promise - ): Promise { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - - const behavior = new KVBehavior(store); - return await cb(behavior); - } - - async get(key: string): Promise { - const db = await this.getDb(); - const store = db.transaction('kv', 'readonly').objectStore('kv'); - return new KVBehavior(store).get(key); - } - async set(key: string, value: Uint8Array): Promise { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - return new KVBehavior(store).set(key, value); - } - async keys(): Promise { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - return new KVBehavior(store).keys(); - } - async clear() { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - return new KVBehavior(store).clear(); - } - async del(key: string) { - const db = await this.getDb(); - const store = db.transaction('kv', 'readwrite').objectStore('kv'); - return new KVBehavior(store).del(key); - } -} - -class KVBehavior implements ByteKVBehavior { - constructor( - private readonly store: IDBPObjectStore - ) {} - async get(key: string): Promise { - const value = await this.store.get(key); - return value?.val ?? null; - } - async set(key: string, value: Uint8Array): Promise { - if (this.store.put === undefined) { - throw new Error('Cannot set in a readonly transaction'); - } - await this.store.put({ - key: key, - val: value, - }); - } - async keys(): Promise { - return await this.store.getAllKeys(); - } - async del(key: string) { - if (this.store.delete === undefined) { - throw new Error('Cannot set in a readonly transaction'); - } - return await this.store.delete(key); - } - - async clear() { - if (this.store.clear === undefined) { - throw new Error('Cannot set in a readonly transaction'); - } - return await this.store.clear(); - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-sqlite.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-sqlite.ts deleted file mode 100644 index d9e3510c1..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/doc-sqlite.ts +++ /dev/null @@ -1,151 +0,0 @@ -import type { DesktopApiService } from '@affine/core/modules/desktop-api'; -import type { ByteKV, ByteKVBehavior, DocStorage } from '@toeverything/infra'; -import { AsyncLock } from '@toeverything/infra'; - -import { BroadcastChannelDocEventBus } from './doc-broadcast-channel'; - -export class SqliteDocStorage implements DocStorage { - constructor( - private readonly workspaceId: string, - private readonly electronApi: DesktopApiService - ) {} - eventBus = new BroadcastChannelDocEventBus(this.workspaceId); - readonly doc = new Doc(this.workspaceId, this.electronApi); - readonly syncMetadata = new SyncMetadataKV( - this.workspaceId, - this.electronApi - ); - readonly serverClock = new ServerClockKV(this.workspaceId, this.electronApi); -} - -type DocType = DocStorage['doc']; - -class Doc implements DocType { - lock = new AsyncLock(); - apis = this.electronApi.handler; - constructor( - private readonly workspaceId: string, - private readonly electronApi: DesktopApiService - ) {} - - async transaction( - cb: (transaction: ByteKVBehavior) => Promise - ): Promise { - using _lock = await this.lock.acquire(); - return await cb(this); - } - - keys(): string[] | Promise { - return []; - } - - async get(docId: string) { - const update = await this.apis.db.getDocAsUpdates( - 'workspace', - this.workspaceId, - docId - ); - - if (update) { - if ( - update.byteLength === 0 || - (update.byteLength === 2 && update[0] === 0 && update[1] === 0) - ) { - return null; - } - - return update; - } - - return null; - } - - async set(docId: string, data: Uint8Array) { - await this.apis.db.applyDocUpdate( - 'workspace', - this.workspaceId, - data, - docId - ); - } - - clear(): void | Promise { - return; - } - - async del(docId: string) { - await this.apis.db.deleteDoc('workspace', this.workspaceId, docId); - } -} - -class SyncMetadataKV implements ByteKV { - apis = this.electronApi.handler; - constructor( - private readonly workspaceId: string, - private readonly electronApi: DesktopApiService - ) {} - transaction(cb: (behavior: ByteKVBehavior) => Promise): Promise { - return cb(this); - } - - get(key: string): Uint8Array | null | Promise { - return this.apis.db.getSyncMetadata('workspace', this.workspaceId, key); - } - - set(key: string, data: Uint8Array): void | Promise { - return this.apis.db.setSyncMetadata( - 'workspace', - this.workspaceId, - key, - data - ); - } - - keys(): string[] | Promise { - return this.apis.db.getSyncMetadataKeys('workspace', this.workspaceId); - } - - del(key: string): void | Promise { - return this.apis.db.delSyncMetadata('workspace', this.workspaceId, key); - } - - clear(): void | Promise { - return this.apis.db.clearSyncMetadata('workspace', this.workspaceId); - } -} - -class ServerClockKV implements ByteKV { - apis = this.electronApi.handler; - constructor( - private readonly workspaceId: string, - private readonly electronApi: DesktopApiService - ) {} - transaction(cb: (behavior: ByteKVBehavior) => Promise): Promise { - return cb(this); - } - - get(key: string): Uint8Array | null | Promise { - return this.apis.db.getServerClock('workspace', this.workspaceId, key); - } - - set(key: string, data: Uint8Array): void | Promise { - return this.apis.db.setServerClock( - 'workspace', - this.workspaceId, - key, - data - ); - } - - keys(): string[] | Promise { - return this.apis.db.getServerClockKeys('workspace', this.workspaceId); - } - - del(key: string): void | Promise { - return this.apis.db.delServerClock('workspace', this.workspaceId, key); - } - - clear(): void | Promise { - return this.apis.db.clearServerClock('workspace', this.workspaceId); - } -} diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/in-worker.ts b/packages/frontend/core/src/modules/workspace-engine/impls/in-worker.ts index e84d32ffb..78fc68ce2 100644 --- a/packages/frontend/core/src/modules/workspace-engine/impls/in-worker.ts +++ b/packages/frontend/core/src/modules/workspace-engine/impls/in-worker.ts @@ -21,5 +21,3 @@ consumer.register('renderWorkspaceProfile', data => { avatar: typeof avatar === 'string' ? avatar : undefined, }; }); - -consumer.listen(); diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/local.ts b/packages/frontend/core/src/modules/workspace-engine/impls/local.ts index 6138138e3..90dcdf496 100644 --- a/packages/frontend/core/src/modules/workspace-engine/impls/local.ts +++ b/packages/frontend/core/src/modules/workspace-engine/impls/local.ts @@ -1,28 +1,40 @@ import { DebugLogger } from '@affine/debug'; -import type { - BlobStorage, - DocStorage, - FrameworkProvider, -} from '@toeverything/infra'; +import type { BlobStorage, DocStorage } from '@affine/nbstore'; +import { + IndexedDBBlobStorage, + IndexedDBDocStorage, + IndexedDBSyncStorage, +} from '@affine/nbstore/idb'; +import { + IndexedDBV1BlobStorage, + IndexedDBV1DocStorage, +} from '@affine/nbstore/idb/v1'; +import { + SqliteBlobStorage, + SqliteDocStorage, + SqliteSyncStorage, +} from '@affine/nbstore/sqlite'; +import { + SqliteV1BlobStorage, + SqliteV1DocStorage, +} from '@affine/nbstore/sqlite/v1'; +import type { WorkerInitOptions } from '@affine/nbstore/worker/client'; +import type { FrameworkProvider } from '@toeverything/infra'; import { LiveData, Service } from '@toeverything/infra'; import { isEqual } from 'lodash-es'; import { nanoid } from 'nanoid'; import { Observable } from 'rxjs'; -import { encodeStateAsUpdate } from 'yjs'; +import { type Doc as YDoc, encodeStateAsUpdate } from 'yjs'; import { DesktopApiService } from '../../desktop-api'; import { getAFFiNEWorkspaceSchema, - type WorkspaceEngineProvider, type WorkspaceFlavourProvider, type WorkspaceFlavoursProvider, type WorkspaceMetadata, type WorkspaceProfileInfo, } from '../../workspace'; import { WorkspaceImpl } from '../../workspace/impls/workspace'; -import type { WorkspaceEngineStorageProvider } from '../providers/engine'; -import { BroadcastChannelAwarenessConnection } from './engine/awareness-broadcast-channel'; -import { StaticBlobStorage } from './engine/blob-static'; import { getWorkspaceProfileWorker } from './out-worker'; export const LOCAL_WORKSPACE_LOCAL_STORAGE_KEY = 'affine-local-workspace'; @@ -56,16 +68,36 @@ export function setLocalWorkspaceIds( } class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider { - constructor( - private readonly storageProvider: WorkspaceEngineStorageProvider, - private readonly framework: FrameworkProvider - ) {} + constructor(private readonly framework: FrameworkProvider) {} - flavour = 'local'; - notifyChannel = new BroadcastChannel( + readonly flavour = 'local'; + readonly notifyChannel = new BroadcastChannel( LOCAL_WORKSPACE_CHANGED_BROADCAST_CHANNEL_KEY ); + DocStorageType = + BUILD_CONFIG.isElectron || BUILD_CONFIG.isIOS + ? SqliteDocStorage + : IndexedDBDocStorage; + DocStorageV1Type = BUILD_CONFIG.isElectron + ? SqliteV1DocStorage + : BUILD_CONFIG.isWeb || BUILD_CONFIG.isMobileWeb + ? IndexedDBV1DocStorage + : undefined; + BlobStorageType = + BUILD_CONFIG.isElectron || BUILD_CONFIG.isIOS + ? SqliteBlobStorage + : IndexedDBBlobStorage; + BlobStorageV1Type = BUILD_CONFIG.isElectron + ? SqliteV1BlobStorage + : BUILD_CONFIG.isWeb || BUILD_CONFIG.isMobileWeb + ? IndexedDBV1BlobStorage + : undefined; + SyncStorageType = + BUILD_CONFIG.isElectron || BUILD_CONFIG.isIOS + ? SqliteSyncStorage + : IndexedDBSyncStorage; + async deleteWorkspace(id: string): Promise { setLocalWorkspaceIds(ids => ids.filter(x => x !== id)); @@ -87,25 +119,67 @@ class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider { const id = nanoid(); // save the initial state to local storage, then sync to cloud - const blobStorage = this.storageProvider.getBlobStorage(id); - const docStorage = this.storageProvider.getDocStorage(id); + const docStorage = new this.DocStorageType({ + id: id, + flavour: this.flavour, + type: 'workspace', + }); + docStorage.connection.connect(); + await docStorage.connection.waitForConnected(); + const blobStorage = new this.BlobStorageType({ + id: id, + flavour: this.flavour, + type: 'workspace', + }); + blobStorage.connection.connect(); + await blobStorage.connection.waitForConnected(); + + const docList = new Set(); const docCollection = new WorkspaceImpl({ id: id, schema: getAFFiNEWorkspaceSchema(), - blobSource: blobStorage, + blobSource: { + get: async key => { + const record = await blobStorage.get(key); + return record ? new Blob([record.data], { type: record.mime }) : null; + }, + delete: async () => { + return; + }, + list: async () => { + return []; + }, + set: async (id, blob) => { + await blobStorage.set({ + key: id, + data: new Uint8Array(await blob.arrayBuffer()), + mime: blob.type, + }); + return id; + }, + name: 'blob', + readonly: false, + }, + onLoadDoc(doc) { + docList.add(doc); + }, }); try { // apply initial state await initial(docCollection, blobStorage, docStorage); - // save workspace to local storage, should be vary fast - await docStorage.doc.set(id, encodeStateAsUpdate(docCollection.doc)); - for (const subdocs of docCollection.doc.getSubdocs()) { - await docStorage.doc.set(subdocs.guid, encodeStateAsUpdate(subdocs)); + for (const subdocs of docList) { + await docStorage.pushDocUpdate({ + docId: subdocs.guid, + bin: encodeStateAsUpdate(subdocs), + }); } + docStorage.connection.disconnect(); + blobStorage.connection.disconnect(); + // save workspace id to local storage setLocalWorkspaceIds(ids => [...ids, id]); @@ -152,8 +226,17 @@ class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider { async getWorkspaceProfile( id: string ): Promise { - const docStorage = this.storageProvider.getDocStorage(id); - const localData = await docStorage.doc.get(id); + const docStorage = new this.DocStorageType({ + id: id, + flavour: this.flavour, + type: 'workspace', + readonlyMode: true, + }); + docStorage.connection.connect(); + await docStorage.connection.waitForConnected(); + const localData = await docStorage.getDoc(id); + + docStorage.connection.disconnect(); if (!localData) { return { @@ -165,7 +248,7 @@ class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider { const result = await client.call( 'renderWorkspaceProfile', - [localData].filter(Boolean) as Uint8Array[] + [localData.bin].filter(Boolean) as Uint8Array[] ); return { @@ -174,26 +257,74 @@ class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider { isOwner: true, }; } - getWorkspaceBlob(id: string, blob: string): Promise { - return this.storageProvider.getBlobStorage(id).get(blob); + + async getWorkspaceBlob(id: string, blobKey: string): Promise { + const storage = new this.BlobStorageType({ + id: id, + flavour: this.flavour, + type: 'workspace', + }); + storage.connection.connect(); + await storage.connection.waitForConnected(); + const blob = await storage.get(blobKey); + return blob ? new Blob([blob.data], { type: blob.mime }) : null; } - getEngineProvider(workspaceId: string): WorkspaceEngineProvider { + getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions { return { - getAwarenessConnections() { - return [new BroadcastChannelAwarenessConnection(workspaceId)]; + local: { + doc: { + name: this.DocStorageType.identifier, + opts: { + flavour: this.flavour, + type: 'workspace', + id: workspaceId, + }, + }, + blob: { + name: this.BlobStorageType.identifier, + opts: { + flavour: this.flavour, + type: 'workspace', + id: workspaceId, + }, + }, + sync: { + name: this.SyncStorageType.identifier, + opts: { + flavour: this.flavour, + type: 'workspace', + id: workspaceId, + }, + }, + awareness: { + name: 'BroadcastChannelAwarenessStorage', + opts: { + id: workspaceId, + }, + }, }, - getDocServer() { - return null; - }, - getDocStorage: () => { - return this.storageProvider.getDocStorage(workspaceId); - }, - getLocalBlobStorage: () => { - return this.storageProvider.getBlobStorage(workspaceId); - }, - getRemoteBlobStorages() { - return [new StaticBlobStorage()]; + remotes: { + v1: { + doc: this.DocStorageV1Type + ? { + name: this.DocStorageV1Type.identifier, + opts: { + id: workspaceId, + type: 'workspace', + }, + } + : undefined, + blob: this.BlobStorageV1Type + ? { + name: this.BlobStorageV1Type.identifier, + opts: { + id: workspaceId, + type: 'workspace', + }, + } + : undefined, + }, }, }; } @@ -203,13 +334,11 @@ export class LocalWorkspaceFlavoursProvider extends Service implements WorkspaceFlavoursProvider { - constructor( - private readonly storageProvider: WorkspaceEngineStorageProvider - ) { + constructor() { super(); } workspaceFlavours$ = new LiveData([ - new LocalWorkspaceFlavourProvider(this.storageProvider, this.framework), + new LocalWorkspaceFlavourProvider(this.framework), ]); } diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/out-worker.ts b/packages/frontend/core/src/modules/workspace-engine/impls/out-worker.ts index 46a10b18a..9a1f1c3b3 100644 --- a/packages/frontend/core/src/modules/workspace-engine/impls/out-worker.ts +++ b/packages/frontend/core/src/modules/workspace-engine/impls/out-worker.ts @@ -17,6 +17,5 @@ export function getWorkspaceProfileWorker() { ); worker = new OpClient(rawWorker); - worker.listen(); return worker; } diff --git a/packages/frontend/core/src/modules/workspace-engine/index.ts b/packages/frontend/core/src/modules/workspace-engine/index.ts index aa634f354..be09cedd6 100644 --- a/packages/frontend/core/src/modules/workspace-engine/index.ts +++ b/packages/frontend/core/src/modules/workspace-engine/index.ts @@ -1,64 +1,25 @@ import { type Framework } from '@toeverything/infra'; import { ServersService } from '../cloud/services/servers'; -import { DesktopApiService } from '../desktop-api'; import { GlobalState } from '../storage'; import { WorkspaceFlavoursProvider } from '../workspace'; import { CloudWorkspaceFlavoursProvider } from './impls/cloud'; -import { IndexedDBBlobStorage } from './impls/engine/blob-indexeddb'; -import { SqliteBlobStorage } from './impls/engine/blob-sqlite'; -import { IndexedDBDocStorage } from './impls/engine/doc-indexeddb'; -import { SqliteDocStorage } from './impls/engine/doc-sqlite'; import { LOCAL_WORKSPACE_LOCAL_STORAGE_KEY, LocalWorkspaceFlavoursProvider, } from './impls/local'; -import { WorkspaceEngineStorageProvider } from './providers/engine'; -export { CloudBlobStorage } from './impls/engine/blob-cloud'; export { base64ToUint8Array, uint8ArrayToBase64 } from './utils/base64'; export function configureBrowserWorkspaceFlavours(framework: Framework) { framework - .impl(WorkspaceFlavoursProvider('LOCAL'), LocalWorkspaceFlavoursProvider, [ - WorkspaceEngineStorageProvider, - ]) + .impl(WorkspaceFlavoursProvider('LOCAL'), LocalWorkspaceFlavoursProvider) .impl(WorkspaceFlavoursProvider('CLOUD'), CloudWorkspaceFlavoursProvider, [ GlobalState, - WorkspaceEngineStorageProvider, ServersService, ]); } -export function configureIndexedDBWorkspaceEngineStorageProvider( - framework: Framework -) { - framework.impl(WorkspaceEngineStorageProvider, { - getDocStorage(workspaceId: string) { - return new IndexedDBDocStorage(workspaceId); - }, - getBlobStorage(workspaceId: string) { - return new IndexedDBBlobStorage(workspaceId); - }, - }); -} - -export function configureSqliteWorkspaceEngineStorageProvider( - framework: Framework -) { - framework.impl(WorkspaceEngineStorageProvider, p => { - const electronApi = p.get(DesktopApiService); - return { - getDocStorage(workspaceId: string) { - return new SqliteDocStorage(workspaceId, electronApi); - }, - getBlobStorage(workspaceId: string) { - return new SqliteBlobStorage(workspaceId, electronApi); - }, - }; - }); -} - /** * a hack for directly add local workspace to workspace list * Used after copying sqlite database file to appdata folder diff --git a/packages/frontend/core/src/modules/workspace-engine/providers/engine.ts b/packages/frontend/core/src/modules/workspace-engine/providers/engine.ts deleted file mode 100644 index c85ad1bf0..000000000 --- a/packages/frontend/core/src/modules/workspace-engine/providers/engine.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { - type BlobStorage, - createIdentifier, - type DocStorage, -} from '@toeverything/infra'; - -export interface WorkspaceEngineStorageProvider { - getDocStorage(workspaceId: string): DocStorage; - getBlobStorage(workspaceId: string): BlobStorage; -} - -export const WorkspaceEngineStorageProvider = - createIdentifier( - 'WorkspaceEngineStorageProvider' - ); diff --git a/packages/frontend/core/src/modules/workspace/entities/engine.ts b/packages/frontend/core/src/modules/workspace/entities/engine.ts index 8433cf6c5..324cbf62e 100644 --- a/packages/frontend/core/src/modules/workspace/entities/engine.ts +++ b/packages/frontend/core/src/modules/workspace/entities/engine.ts @@ -1,83 +1,67 @@ -import { - AwarenessEngine, - BlobEngine, - DocEngine, - Entity, - throwIfAborted, -} from '@toeverything/infra'; -import type { Doc as YDoc } from 'yjs'; +import type { + WorkerClient, + WorkerInitOptions, +} from '@affine/nbstore/worker/client'; +import { Entity } from '@toeverything/infra'; +import type { NbstoreService } from '../../storage'; import { WorkspaceEngineBeforeStart } from '../events'; -import type { WorkspaceEngineProvider } from '../providers/flavour'; import type { WorkspaceService } from '../services/workspace'; export class WorkspaceEngine extends Entity<{ - engineProvider: WorkspaceEngineProvider; + isSharedMode?: boolean; + engineWorkerInitOptions: WorkerInitOptions; }> { - doc = new DocEngine( - this.props.engineProvider.getDocStorage(), - this.props.engineProvider.getDocServer() - ); + worker?: WorkerClient; + started = false; - blob = new BlobEngine( - this.props.engineProvider.getLocalBlobStorage(), - this.props.engineProvider.getRemoteBlobStorages() - ); - - awareness = new AwarenessEngine( - this.props.engineProvider.getAwarenessConnections() - ); - - constructor(private readonly workspaceService: WorkspaceService) { + constructor( + private readonly workspaceService: WorkspaceService, + private readonly nbstoreService: NbstoreService + ) { super(); } - setRootDoc(yDoc: YDoc) { - this.doc.setPriority(yDoc.guid, 100); - this.doc.addDoc(yDoc); + get doc() { + if (!this.worker) { + throw new Error('Engine is not initialized'); + } + return this.worker.docFrontend; + } + + get blob() { + if (!this.worker) { + throw new Error('Engine is not initialized'); + } + return this.worker.blobFrontend; + } + + get awareness() { + if (!this.worker) { + throw new Error('Engine is not initialized'); + } + return this.worker.awarenessFrontend; } start() { - this.eventBus.emit(WorkspaceEngineBeforeStart, this); - this.doc.start(); - this.awareness.connect(this.workspaceService.workspace.awareness); - if (!BUILD_CONFIG.isMobileEdition) { - // currently, blob synchronization consumes a lot of memory and is temporarily disabled on mobile devices. - this.blob.start(); + if (this.started) { + throw new Error('Engine is already started'); } - } + this.started = true; - canGracefulStop() { - return this.doc.engineState$.value.saving === 0; - } + const { store, dispose } = this.nbstoreService.openStore( + (this.props.isSharedMode ? 'shared:' : '') + + `workspace:${this.workspaceService.workspace.flavour}:${this.workspaceService.workspace.id}`, + this.props.engineWorkerInitOptions + ); + this.worker = store; + this.disposables.push(dispose); + this.eventBus.emit(WorkspaceEngineBeforeStart, this); - async waitForGracefulStop(abort?: AbortSignal) { - await this.doc.waitForSaved(); - throwIfAborted(abort); - this.forceStop(); - } - - forceStop() { - this.doc.stop(); - this.awareness.disconnect(); - this.blob.stop(); - } - - docEngineState$ = this.doc.engineState$; - - rootDocState$ = this.doc.docState$(this.workspaceService.workspace.id); - - waitForDocSynced() { - return this.doc.waitForSynced(); - } - - waitForRootDocReady() { - return this.doc.waitForReady(this.workspaceService.workspace.id); - } - - override dispose(): void { - this.forceStop(); - this.doc.dispose(); - this.awareness.dispose(); + const rootDoc = this.workspaceService.workspace.docCollection.doc; + // priority load root doc + this.doc.addPriority(rootDoc.guid, 100); + this.doc.start(); + this.disposables.push(() => this.doc.stop()); } } diff --git a/packages/frontend/core/src/modules/workspace/entities/workspace.ts b/packages/frontend/core/src/modules/workspace/entities/workspace.ts index 6ef752e3e..6dea85e09 100644 --- a/packages/frontend/core/src/modules/workspace/entities/workspace.ts +++ b/packages/frontend/core/src/modules/workspace/entities/workspace.ts @@ -3,7 +3,6 @@ import { Entity, LiveData } from '@toeverything/infra'; import { Observable } from 'rxjs'; import type { Awareness } from 'y-protocols/awareness.js'; -import { WorkspaceDBService } from '../../db'; import { getAFFiNEWorkspaceSchema } from '../global-schema'; import { WorkspaceImpl } from '../impls/workspace'; import type { WorkspaceScope } from '../scopes/workspace'; @@ -28,20 +27,39 @@ export class Workspace extends Entity { if (!this._docCollection) { this._docCollection = new WorkspaceImpl({ id: this.openOptions.metadata.id, - blobSource: this.engine.blob, + blobSource: { + get: async key => { + const record = await this.engine.blob.get(key); + return record + ? new Blob([record.data], { type: record.mime }) + : null; + }, + delete: async () => { + return; + }, + list: async () => { + return []; + }, + set: async (id, blob) => { + await this.engine.blob.set({ + key: id, + data: new Uint8Array(await blob.arrayBuffer()), + mime: blob.type, + }); + return id; + }, + name: 'blob', + readonly: false, + }, schema: getAFFiNEWorkspaceSchema(), - }); - this._docCollection.slots.docCreated.on(id => { - this.engine.doc.markAsReady(id); + onLoadDoc: doc => this.engine.doc.connectDoc(doc), + onLoadAwareness: awareness => + this.engine.awareness.connectAwareness(awareness), }); } return this._docCollection; } - get db() { - return this.framework.get(WorkspaceDBService).db; - } - get awareness() { return this.docCollection.awarenessStore.awareness as Awareness; } diff --git a/packages/frontend/core/src/modules/workspace/impls/doc.ts b/packages/frontend/core/src/modules/workspace/impls/doc.ts index 1316309f6..3c347829b 100644 --- a/packages/frontend/core/src/modules/workspace/impls/doc.ts +++ b/packages/frontend/core/src/modules/workspace/impls/doc.ts @@ -45,42 +45,32 @@ export class DocImpl implements Doc { }; private readonly _initSubDoc = () => { - let subDoc = this.rootDoc.getMap('spaces').get(this.id); - if (!subDoc) { - subDoc = new Y.Doc({ - guid: this.id, - }); - this.rootDoc.getMap('spaces').set(this.id, subDoc); - this._loaded = true; - this._onLoadSlot.emit(); - } else { - this._loaded = false; - this.rootDoc.on('subdocs', this._onSubdocEvent); + { + // This is a piece of old version compatible code. The old version relies on the subdoc instance on `spaces`. + // So if there is no subdoc on spaces, we will create it. + // new version no longer needs subdoc on `spaces`. + let subDoc = this.rootDoc.getMap('spaces').get(this.id); + if (!subDoc) { + subDoc = new Y.Doc({ + guid: this.id, + }); + this.rootDoc.getMap('spaces').set(this.id, subDoc); + } } - return subDoc; + const spaceDoc = new Y.Doc({ + guid: this.id, + }); + spaceDoc.clientID = this.rootDoc.clientID; + this._loaded = false; + + return spaceDoc; }; private _loaded!: boolean; private readonly _onLoadSlot = new Slot(); - private readonly _onSubdocEvent = ({ - loaded, - }: { - loaded: Set; - }): void => { - const result = Array.from(loaded).find( - doc => doc.guid === this._ySpaceDoc.guid - ); - if (!result) { - return; - } - this.rootDoc.off('subdocs', this._onSubdocEvent); - this._loaded = true; - this._onLoadSlot.emit(); - }; - /** Indicate whether the block tree is ready */ private _ready = false; @@ -301,7 +291,8 @@ export class DocImpl implements Doc { return this; } - this._ySpaceDoc.load(); + this.spaceDoc.load(); + this.workspace.onLoadDoc?.(this.spaceDoc); if ((this.workspace.meta.docs?.length ?? 0) <= 1) { this._handleVersion(); @@ -315,6 +306,7 @@ export class DocImpl implements Doc { initFn?.(); + this._loaded = true; this._ready = true; return this; diff --git a/packages/frontend/core/src/modules/workspace/impls/workspace.ts b/packages/frontend/core/src/modules/workspace/impls/workspace.ts index 9b3e6a1c4..3ed3d17ca 100644 --- a/packages/frontend/core/src/modules/workspace/impls/workspace.ts +++ b/packages/frontend/core/src/modules/workspace/impls/workspace.ts @@ -30,6 +30,8 @@ type WorkspaceOptions = { id?: string; schema: Schema; blobSource?: BlobSource; + onLoadDoc?: (doc: Y.Doc) => void; + onLoadAwareness?: (awareness: Awareness) => void; }; export class WorkspaceImpl implements Workspace { @@ -63,12 +65,25 @@ export class WorkspaceImpl implements Workspace { return this._schema; } - constructor({ id, schema, blobSource }: WorkspaceOptions) { + readonly onLoadDoc?: (doc: Y.Doc) => void; + readonly onLoadAwareness?: (awareness: Awareness) => void; + + constructor({ + id, + schema, + blobSource, + onLoadDoc, + onLoadAwareness, + }: WorkspaceOptions) { this._schema = schema; this.id = id || ''; this.doc = new Y.Doc({ guid: id }); this.awarenessStore = new AwarenessStore(new Awareness(this.doc)); + this.onLoadDoc = onLoadDoc; + this.onLoadAwareness = onLoadAwareness; + this.onLoadDoc?.(this.doc); + this.onLoadAwareness?.(this.awarenessStore.awareness); blobSource = blobSource ?? new MemoryBlobSource(); const logger = new NoopLogger(); diff --git a/packages/frontend/core/src/modules/workspace/index.ts b/packages/frontend/core/src/modules/workspace/index.ts index b0730d141..f41f5666d 100644 --- a/packages/frontend/core/src/modules/workspace/index.ts +++ b/packages/frontend/core/src/modules/workspace/index.ts @@ -4,7 +4,6 @@ export { WorkspaceEngineBeforeStart, WorkspaceInitialized } from './events'; export { getAFFiNEWorkspaceSchema } from './global-schema'; export type { WorkspaceMetadata } from './metadata'; export type { WorkspaceOpenOptions } from './open-options'; -export type { WorkspaceEngineProvider } from './providers/flavour'; export type { WorkspaceFlavourProvider } from './providers/flavour'; export { WorkspaceFlavoursProvider } from './providers/flavour'; export { WorkspaceLocalCache, WorkspaceLocalState } from './providers/storage'; @@ -14,7 +13,7 @@ export { WorkspacesService } from './services/workspaces'; import type { Framework } from '@toeverything/infra'; -import { GlobalCache, GlobalState } from '../storage'; +import { GlobalCache, GlobalState, NbstoreService } from '../storage'; import { WorkspaceEngine } from './entities/engine'; import { WorkspaceList } from './entities/list'; import { WorkspaceProfile } from './entities/profile'; @@ -73,7 +72,7 @@ export function configureWorkspaceModule(framework: Framework) { .service(WorkspaceService) .entity(Workspace, [WorkspaceScope]) .service(WorkspaceEngineService, [WorkspaceScope]) - .entity(WorkspaceEngine, [WorkspaceService]) + .entity(WorkspaceEngine, [WorkspaceService, NbstoreService]) .impl(WorkspaceLocalState, WorkspaceLocalStateImpl, [ WorkspaceService, GlobalState, diff --git a/packages/frontend/core/src/modules/workspace/providers/flavour.ts b/packages/frontend/core/src/modules/workspace/providers/flavour.ts index d7f704a98..aef6f46bc 100644 --- a/packages/frontend/core/src/modules/workspace/providers/flavour.ts +++ b/packages/frontend/core/src/modules/workspace/providers/flavour.ts @@ -1,25 +1,12 @@ +import type { BlobStorage, DocStorage } from '@affine/nbstore'; +import type { WorkerInitOptions } from '@affine/nbstore/worker/client'; import type { Workspace as BSWorkspace } from '@blocksuite/affine/store'; -import { - type AwarenessConnection, - type BlobStorage, - createIdentifier, - type DocServer, - type DocStorage, - type LiveData, -} from '@toeverything/infra'; +import { createIdentifier, type LiveData } from '@toeverything/infra'; import type { WorkspaceProfileInfo } from '../entities/profile'; import type { Workspace } from '../entities/workspace'; import type { WorkspaceMetadata } from '../metadata'; -export interface WorkspaceEngineProvider { - getDocServer(): DocServer | null; - getDocStorage(): DocStorage; - getLocalBlobStorage(): BlobStorage; - getRemoteBlobStorages(): BlobStorage[]; - getAwarenessConnections(): AwarenessConnection[]; -} - export interface WorkspaceFlavourProvider { flavour: string; @@ -54,7 +41,7 @@ export interface WorkspaceFlavourProvider { getWorkspaceBlob(id: string, blob: string): Promise; - getEngineProvider(workspaceId: string): WorkspaceEngineProvider; + getEngineWorkerInitOptions(workspaceId: string): WorkerInitOptions; onWorkspaceInitialized?(workspace: Workspace): void; } diff --git a/packages/frontend/core/src/modules/workspace/scopes/workspace.ts b/packages/frontend/core/src/modules/workspace/scopes/workspace.ts index a174e9a27..b06736811 100644 --- a/packages/frontend/core/src/modules/workspace/scopes/workspace.ts +++ b/packages/frontend/core/src/modules/workspace/scopes/workspace.ts @@ -1,9 +1,9 @@ +import type { WorkerInitOptions } from '@affine/nbstore/worker/client'; import { Scope } from '@toeverything/infra'; import type { WorkspaceOpenOptions } from '../open-options'; -import type { WorkspaceEngineProvider } from '../providers/flavour'; export class WorkspaceScope extends Scope<{ openOptions: WorkspaceOpenOptions; - engineProvider: WorkspaceEngineProvider; + engineWorkerInitOptions: WorkerInitOptions; }> {} diff --git a/packages/frontend/core/src/modules/workspace/services/engine.ts b/packages/frontend/core/src/modules/workspace/services/engine.ts index 10907d6e8..1b8acc025 100644 --- a/packages/frontend/core/src/modules/workspace/services/engine.ts +++ b/packages/frontend/core/src/modules/workspace/services/engine.ts @@ -8,7 +8,9 @@ export class WorkspaceEngineService extends Service { get engine() { if (!this._engine) { this._engine = this.framework.createEntity(WorkspaceEngine, { - engineProvider: this.workspaceScope.props.engineProvider, + isSharedMode: this.workspaceScope.props.openOptions.isSharedMode, + engineWorkerInitOptions: + this.workspaceScope.props.engineWorkerInitOptions, }); } return this._engine; diff --git a/packages/frontend/core/src/modules/workspace/services/factory.ts b/packages/frontend/core/src/modules/workspace/services/factory.ts index a4f952cbd..343f7886b 100644 --- a/packages/frontend/core/src/modules/workspace/services/factory.ts +++ b/packages/frontend/core/src/modules/workspace/services/factory.ts @@ -1,9 +1,6 @@ +import type { BlobStorage, DocStorage } from '@affine/nbstore'; import type { Workspace } from '@blocksuite/affine/store'; -import { - type BlobStorage, - type DocStorage, - Service, -} from '@toeverything/infra'; +import { Service } from '@toeverything/infra'; import type { WorkspaceFlavoursService } from './flavours'; @@ -22,8 +19,8 @@ export class WorkspaceFactoryService extends Service { flavour: string, initial: ( docCollection: Workspace, - blobStorage: BlobStorage, - docStorage: DocStorage + blobFrontend: BlobStorage, + docFrontend: DocStorage ) => Promise = () => Promise.resolve() ) => { const provider = this.flavoursService.flavours$.value.find( diff --git a/packages/frontend/core/src/modules/workspace/services/repo.ts b/packages/frontend/core/src/modules/workspace/services/repo.ts index 7f803e0d8..0c9c94a18 100644 --- a/packages/frontend/core/src/modules/workspace/services/repo.ts +++ b/packages/frontend/core/src/modules/workspace/services/repo.ts @@ -1,10 +1,10 @@ import { DebugLogger } from '@affine/debug'; +import type { WorkerInitOptions } from '@affine/nbstore/worker/client'; import { ObjectPool, Service } from '@toeverything/infra'; import type { Workspace } from '../entities/workspace'; import { WorkspaceInitialized } from '../events'; import type { WorkspaceOpenOptions } from '../open-options'; -import type { WorkspaceEngineProvider } from '../providers/flavour'; import { WorkspaceScope } from '../scopes/workspace'; import type { WorkspaceFlavoursService } from './flavours'; import type { WorkspaceListService } from './list'; @@ -40,13 +40,16 @@ export class WorkspaceRepositoryService extends Service { */ open = ( options: WorkspaceOpenOptions, - customProvider?: WorkspaceEngineProvider + customEngineWorkerInitOptions?: WorkerInitOptions ): { workspace: Workspace; dispose: () => void; } => { if (options.isSharedMode) { - const workspace = this.instantiate(options, customProvider); + const workspace = this.instantiate( + options, + customEngineWorkerInitOptions + ); return { workspace, dispose: () => { @@ -63,9 +66,7 @@ export class WorkspaceRepositoryService extends Service { }; } - const workspace = this.instantiate(options, customProvider); - // sync information with workspace list, when workspace's avatar and name changed, information will be updated - // this.list.getInformation(metadata).syncWithWorkspace(workspace); + const workspace = this.instantiate(options, customEngineWorkerInitOptions); const ref = this.pool.put(workspace.meta.id, workspace); @@ -83,7 +84,7 @@ export class WorkspaceRepositoryService extends Service { instantiate( openOptions: WorkspaceOpenOptions, - customProvider?: WorkspaceEngineProvider + customEngineWorkerInitOptions?: WorkerInitOptions ) { logger.info( `open workspace [${openOptions.metadata.flavour}] ${openOptions.metadata.id} ` @@ -91,10 +92,10 @@ export class WorkspaceRepositoryService extends Service { const flavourProvider = this.flavoursService.flavours$.value.find( p => p.flavour === openOptions.metadata.flavour ); - const provider = - customProvider ?? - flavourProvider?.getEngineProvider(openOptions.metadata.id); - if (!provider) { + const engineWorkerInitOptions = + customEngineWorkerInitOptions ?? + flavourProvider?.getEngineWorkerInitOptions(openOptions.metadata.id); + if (!engineWorkerInitOptions) { throw new Error( `Unknown workspace flavour: ${openOptions.metadata.flavour}` ); @@ -102,12 +103,11 @@ export class WorkspaceRepositoryService extends Service { const workspaceScope = this.framework.createScope(WorkspaceScope, { openOptions, - engineProvider: provider, + engineWorkerInitOptions, }); const workspace = workspaceScope.get(WorkspaceService).workspace; - workspace.engine.setRootDoc(workspace.docCollection.doc); workspace.engine.start(); this.framework.emitEvent(WorkspaceInitialized, workspace); diff --git a/packages/frontend/core/src/modules/workspace/services/transform.ts b/packages/frontend/core/src/modules/workspace/services/transform.ts index 27d97cfac..358896339 100644 --- a/packages/frontend/core/src/modules/workspace/services/transform.ts +++ b/packages/frontend/core/src/modules/workspace/services/transform.ts @@ -28,23 +28,29 @@ export class WorkspaceTransformService extends Service { ): Promise => { assertEquals(local.flavour, 'local'); - const localDocStorage = local.engine.doc.storage.behavior; + const localDocStorage = local.engine.doc.storage; + const localDocList = Array.from(local.docCollection.docs.keys()); const newMetadata = await this.factory.create( flavour, async (docCollection, blobStorage, docStorage) => { - const rootDocBinary = await localDocStorage.doc.get( - local.docCollection.doc.guid - ); + const rootDocBinary = ( + await localDocStorage.getDoc(local.docCollection.doc.guid) + )?.bin; if (rootDocBinary) { applyUpdate(docCollection.doc, rootDocBinary); } - for (const subdoc of docCollection.doc.getSubdocs()) { - const subdocBinary = await localDocStorage.doc.get(subdoc.guid); + for (const subdocId of localDocList) { + const subdocBinary = (await localDocStorage.getDoc(subdocId))?.bin; if (subdocBinary) { - applyUpdate(subdoc, subdocBinary); + const doc = docCollection.getDoc(subdocId); + if (doc) { + const spaceDoc = doc.spaceDoc; + doc.load(); + applyUpdate(spaceDoc, subdocBinary); + } } } @@ -57,12 +63,12 @@ export class WorkspaceTransformService extends Service { accountId ); - const blobList = await local.engine.blob.list(); + const blobList = await local.engine.blob.storage.list(); - for (const blobKey of blobList) { - const blob = await local.engine.blob.get(blobKey); + for (const { key } of blobList) { + const blob = await local.engine.blob.storage.get(key); if (blob) { - await blobStorage.set(blobKey, blob); + await blobStorage.set(blob); } } } diff --git a/packages/frontend/core/src/modules/workspace/services/workspaces.ts b/packages/frontend/core/src/modules/workspace/services/workspaces.ts index 274c6f30b..b478f66cd 100644 --- a/packages/frontend/core/src/modules/workspace/services/workspaces.ts +++ b/packages/frontend/core/src/modules/workspace/services/workspaces.ts @@ -55,4 +55,10 @@ export class WorkspacesService extends Service { .find(x => x.flavour === meta.flavour) ?.getWorkspaceBlob(meta.id, blob); } + + getWorkspaceFlavourProvider(meta: WorkspaceMetadata) { + return this.flavoursService.flavours$.value.find( + x => x.flavour === meta.flavour + ); + } } diff --git a/packages/frontend/core/src/utils/first-app-data.ts b/packages/frontend/core/src/utils/first-app-data.ts index 368bd7acf..329e81073 100644 --- a/packages/frontend/core/src/utils/first-app-data.ts +++ b/packages/frontend/core/src/utils/first-app-data.ts @@ -21,7 +21,7 @@ export async function buildShowcaseWorkspace( const { workspace, dispose } = workspacesService.open({ metadata: meta }); - await workspace.engine.waitForRootDocReady(); + await workspace.engine.doc.waitForDocReady(workspace.id); const docsService = workspace.scope.get(DocsService); diff --git a/packages/frontend/core/tsconfig.json b/packages/frontend/core/tsconfig.json index a70c21049..6e4743c91 100644 --- a/packages/frontend/core/tsconfig.json +++ b/packages/frontend/core/tsconfig.json @@ -13,6 +13,7 @@ { "path": "../../common/env" }, { "path": "../graphql" }, { "path": "../i18n" }, + { "path": "../../common/nbstore" }, { "path": "../track" }, { "path": "../../../blocksuite/affine/all" }, { "path": "../../../blocksuite/affine/components" }, diff --git a/packages/frontend/electron-api/src/web-worker.ts b/packages/frontend/electron-api/src/web-worker.ts deleted file mode 100644 index ed986e46f..000000000 --- a/packages/frontend/electron-api/src/web-worker.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { AsyncCall, type EventBasedChannel } from 'async-call-rpc'; - -import type { ClientHandler } from '.'; - -const WORKER_PORT_MESSAGE_TYPE = 'electron-api-port'; - -// connect web worker to preload, so that the web worker can use the electron APIs -export function connectWebWorker(worker: Worker) { - const { portId, cleanup } = (globalThis as any).__requestWebWorkerPort(); - - const portMessageListener = (event: MessageEvent) => { - if ( - event.data.type === 'electron:request-api-port' && - event.data.portId === portId - ) { - const [port] = event.data.ports as MessagePort[]; - - // worker should be ready to receive message - worker.postMessage( - { - type: WORKER_PORT_MESSAGE_TYPE, - ports: [port], - }, - [port] - ); - } - }; - - window.addEventListener('message', portMessageListener); - - return () => { - window.removeEventListener('message', portMessageListener); - cleanup(); - }; -} - -const createMessagePortChannel = (port: MessagePort): EventBasedChannel => { - return { - on(listener) { - port.onmessage = e => { - listener(e.data); - }; - port.start(); - return () => { - port.onmessage = null; - try { - port.close(); - } catch (err) { - console.error('[worker] close port error', err); - } - }; - }, - send(data) { - port.postMessage(data); - }, - }; -}; - -// get the electron APIs for the web worker (should be called in the web worker) -export function getElectronAPIs(): ClientHandler { - const { promise, resolve } = Promise.withResolvers(); - globalThis.addEventListener('message', event => { - if (event.data.type === WORKER_PORT_MESSAGE_TYPE) { - const [port] = event.ports; - resolve(port); - } - }); - - const rpc = AsyncCall>(null, { - channel: promise.then(p => createMessagePortChannel(p)), - log: false, - }); - - return new Proxy(rpc as any, { - get(_, namespace: string) { - return new Proxy(rpc as any, { - get(_, method: string) { - return rpc[`${namespace}:${method}`]; - }, - }); - }, - }); -} diff --git a/packages/frontend/i18n/src/resources/en.json b/packages/frontend/i18n/src/resources/en.json index 262436000..ee71ad105 100644 --- a/packages/frontend/i18n/src/resources/en.json +++ b/packages/frontend/i18n/src/resources/en.json @@ -1049,8 +1049,6 @@ "com.affine.peek-view-controls.open-doc-in-center-peek": "Open in center peek", "com.affine.quicksearch.group.creation": "New", "com.affine.quicksearch.group.searchfor": "Search for \"{{query}}\"", - "com.affine.resetSyncStatus.button": "Reset sync", - "com.affine.resetSyncStatus.description": "This operation may fix some synchronization issues.", "com.affine.rootAppSidebar.collections": "Collections", "com.affine.rootAppSidebar.doc.link-doc-only": "Only doc can be placed on here", "com.affine.rootAppSidebar.docs.no-subdoc": "No linked docs", diff --git a/packages/frontend/mobile-native/src/lib.rs b/packages/frontend/mobile-native/src/lib.rs index 59163f5b6..ff7be8c80 100644 --- a/packages/frontend/mobile-native/src/lib.rs +++ b/packages/frontend/mobile-native/src/lib.rs @@ -437,14 +437,14 @@ impl DocStoragePool { universal_id: String, peer: String, doc_id: String, - ) -> Result { + ) -> Result> { Ok( self .inner .ensure_storage(universal_id)? .get_peer_remote_clock(peer, doc_id) .await? - .into(), + .map(Into::into), ) } @@ -492,14 +492,14 @@ impl DocStoragePool { universal_id: String, peer: String, doc_id: String, - ) -> Result { + ) -> Result> { Ok( self .inner .ensure_storage(universal_id)? .get_peer_pulled_remote_clock(peer, doc_id) .await? - .into(), + .map(Into::into), ) } @@ -525,6 +525,22 @@ impl DocStoragePool { ) } + pub async fn get_peer_pushed_clock( + &self, + universal_id: String, + peer: String, + doc_id: String, + ) -> Result> { + Ok( + self + .inner + .ensure_storage(universal_id)? + .get_peer_pushed_clock(peer, doc_id) + .await? + .map(Into::into), + ) + } + pub async fn get_peer_pushed_clocks( &self, universal_id: String, diff --git a/packages/frontend/native/index.d.ts b/packages/frontend/native/index.d.ts index 1b3b3904c..1ef0c9bd3 100644 --- a/packages/frontend/native/index.d.ts +++ b/packages/frontend/native/index.d.ts @@ -20,13 +20,13 @@ export declare class DocStoragePool { releaseBlobs(universalId: string): Promise listBlobs(universalId: string): Promise> getPeerRemoteClocks(universalId: string, peer: string): Promise> - getPeerRemoteClock(universalId: string, peer: string, docId: string): Promise + getPeerRemoteClock(universalId: string, peer: string, docId: string): Promise setPeerRemoteClock(universalId: string, peer: string, docId: string, clock: Date): Promise getPeerPulledRemoteClocks(universalId: string, peer: string): Promise> - getPeerPulledRemoteClock(universalId: string, peer: string, docId: string): Promise + getPeerPulledRemoteClock(universalId: string, peer: string, docId: string): Promise setPeerPulledRemoteClock(universalId: string, peer: string, docId: string, clock: Date): Promise getPeerPushedClocks(universalId: string, peer: string): Promise> - getPeerPushedClock(universalId: string, peer: string, docId: string): Promise + getPeerPushedClock(universalId: string, peer: string, docId: string): Promise setPeerPushedClock(universalId: string, peer: string, docId: string, clock: Date): Promise clearClocks(universalId: string): Promise } diff --git a/packages/frontend/native/nbstore/src/doc.rs b/packages/frontend/native/nbstore/src/doc.rs index d9bfade80..0eeead284 100644 --- a/packages/frontend/native/nbstore/src/doc.rs +++ b/packages/frontend/native/nbstore/src/doc.rs @@ -324,6 +324,7 @@ mod tests { let clocks = storage .get_peer_pulled_remote_clock("remote".to_string(), "new_id".to_string()) .await + .unwrap() .unwrap(); assert_eq!(clocks.doc_id, "new_id"); diff --git a/packages/frontend/native/nbstore/src/lib.rs b/packages/frontend/native/nbstore/src/lib.rs index b9d6b0df1..fd676e0a1 100644 --- a/packages/frontend/native/nbstore/src/lib.rs +++ b/packages/frontend/native/nbstore/src/lib.rs @@ -300,7 +300,7 @@ impl DocStoragePool { universal_id: String, peer: String, doc_id: String, - ) -> Result { + ) -> Result> { Ok( self .pool @@ -347,7 +347,7 @@ impl DocStoragePool { universal_id: String, peer: String, doc_id: String, - ) -> Result { + ) -> Result> { Ok( self .pool @@ -394,7 +394,7 @@ impl DocStoragePool { universal_id: String, peer: String, doc_id: String, - ) -> Result { + ) -> Result> { Ok( self .pool diff --git a/packages/frontend/native/nbstore/src/sync.rs b/packages/frontend/native/nbstore/src/sync.rs index 42f5f9a62..3c12423b4 100644 --- a/packages/frontend/native/nbstore/src/sync.rs +++ b/packages/frontend/native/nbstore/src/sync.rs @@ -1,7 +1,6 @@ use chrono::NaiveDateTime; -use super::DocClock; -use super::{error::Result, storage::SqliteDocStorage}; +use super::{error::Result, storage::SqliteDocStorage, DocClock}; impl SqliteDocStorage { pub async fn get_peer_remote_clocks(&self, peer: String) -> Result> { @@ -16,14 +15,18 @@ impl SqliteDocStorage { Ok(result) } - pub async fn get_peer_remote_clock(&self, peer: String, doc_id: String) -> Result { + pub async fn get_peer_remote_clock( + &self, + peer: String, + doc_id: String, + ) -> Result> { let result = sqlx::query_as!( DocClock, "SELECT doc_id, remote_clock as timestamp FROM peer_clocks WHERE peer = ? AND doc_id = ?", peer, doc_id ) - .fetch_one(&self.pool) + .fetch_optional(&self.pool) .await?; Ok(result) @@ -67,14 +70,14 @@ impl SqliteDocStorage { &self, peer: String, doc_id: String, - ) -> Result { + ) -> Result> { let result = sqlx::query_as!( DocClock, - "SELECT doc_id, pulled_remote_clock as timestamp FROM peer_clocks WHERE peer = ? AND doc_id = ?", + r#"SELECT doc_id, pulled_remote_clock as timestamp FROM peer_clocks WHERE peer = ? AND doc_id = ?"#, peer, doc_id ) - .fetch_one(&self.pool) + .fetch_optional(&self.pool) .await?; Ok(result) @@ -114,14 +117,18 @@ impl SqliteDocStorage { Ok(result) } - pub async fn get_peer_pushed_clock(&self, peer: String, doc_id: String) -> Result { + pub async fn get_peer_pushed_clock( + &self, + peer: String, + doc_id: String, + ) -> Result> { let result = sqlx::query_as!( DocClock, "SELECT doc_id, pushed_clock as timestamp FROM peer_clocks WHERE peer = ? AND doc_id = ?", peer, doc_id ) - .fetch_one(&self.pool) + .fetch_optional(&self.pool) .await?; Ok(result) diff --git a/tests/affine-cloud/e2e/collaboration.spec.ts b/tests/affine-cloud/e2e/collaboration.spec.ts index aa791b7a9..f29a5f6df 100644 --- a/tests/affine-cloud/e2e/collaboration.spec.ts +++ b/tests/affine-cloud/e2e/collaboration.spec.ts @@ -11,10 +11,7 @@ import { waitForEditorLoad, } from '@affine-test/kit/utils/page-logic'; import { clickUserInfoCard } from '@affine-test/kit/utils/setting'; -import { - clickSideBarCurrentWorkspaceBanner, - clickSideBarSettingButton, -} from '@affine-test/kit/utils/sidebar'; +import { clickSideBarSettingButton } from '@affine-test/kit/utils/sidebar'; import { createLocalWorkspace } from '@affine-test/kit/utils/workspace'; import { expect } from '@playwright/test'; @@ -211,72 +208,3 @@ test('can sync svg between different browsers', async ({ page, browser }) => { expect(svg2).toEqual(svg1); } }); - -test('When the first sync is not completed, should always show loading', async ({ - page, - browser, -}) => { - await page.reload(); - await waitForEditorLoad(page); - await createLocalWorkspace( - { - name: 'test', - }, - page - ); - await enableCloudWorkspace(page); - await clickNewPageButton(page); - await waitForEditorLoad(page); - const title = getBlockSuiteEditorTitle(page); - await title.pressSequentially('TEST TITLE', { - delay: 50, - }); - - const context = await browser.newContext(); - await skipOnboarding(context); - const page2 = await context.newPage(); - await loginUser(page2, user); - - // simulate sync stuck - await page2.evaluate(() => { - (window as any)._TEST_SIMULATE_SYNC_LAG = new Promise(() => {}); - }); - const localWorkspaceUrl = page2.url(); - await clickSideBarCurrentWorkspaceBanner(page2); - await page2.getByTestId('workspace-card').getByText('test').click(); // enter "test" workspace - - await page2.waitForTimeout(1000); - - await expect( - page2.getByTestId('page-list-item').getByText('TEST TITLE') - ).not.toBeVisible(); // should be loading - - // Simulate user refresh and re-enter workspace, should still be loading - await page2.goto(localWorkspaceUrl); - - // setup sync lag - await page2.evaluate(() => { - (window as any).resolveSyncLag = null; - (window as any)._TEST_SIMULATE_SYNC_LAG = new Promise(resolve => { - (window as any).resolveSyncLag = resolve; - }); - }); - await clickSideBarCurrentWorkspaceBanner(page2); - await page2.getByTestId('workspace-card').getByText('test').click(); // enter "test" workspace - - await page2.waitForTimeout(1000); - - await expect( - page2.getByTestId('page-list-item').getByText('TEST TITLE') - ).not.toBeVisible(); // should be loading - - await page2.evaluate(() => { - (window as any).resolveSyncLag(); - }); // start syncing - await page2.getByTestId('page-list-item').getByText('TEST TITLE').click(); // should be able to click page - await waitForEditorLoad(page2); - - expect(await getBlockSuiteEditorTitle(page2).innerText()).toContain( - 'TEST TITLE' - ); -}); diff --git a/tests/affine-cloud/e2e/share-page.spec.ts b/tests/affine-cloud/e2e/share-page.spec.ts index 2d8d4552f..7407605ae 100644 --- a/tests/affine-cloud/e2e/share-page.spec.ts +++ b/tests/affine-cloud/e2e/share-page.spec.ts @@ -22,11 +22,8 @@ let user: { password: string; }; -test.beforeEach(async () => { - user = await createRandomUser(); -}); - test.beforeEach(async ({ page }) => { + user = await createRandomUser(); await loginUser(page, user); }); diff --git a/tests/affine-desktop/e2e/workspace.spec.ts b/tests/affine-desktop/e2e/workspace.spec.ts index fee552a76..adbd08aa0 100644 --- a/tests/affine-desktop/e2e/workspace.spec.ts +++ b/tests/affine-desktop/e2e/workspace.spec.ts @@ -28,7 +28,8 @@ test('check workspace has a DB file', async ({ appInfo, workspace }) => { expect(await fs.exists(dbPath)).toBe(true); }); -test('export then add', async ({ page, appInfo, workspace }) => { +// TODO(@eyhn): fix this +test.skip('export then add', async ({ page, appInfo, workspace }) => { await clickNewPageButton(page); const w = await workspace.current(); diff --git a/tools/cli/src/webpack/html-plugin.ts b/tools/cli/src/webpack/html-plugin.ts index 76cf91398..f47090432 100644 --- a/tools/cli/src/webpack/html-plugin.ts +++ b/tools/cli/src/webpack/html-plugin.ts @@ -99,6 +99,19 @@ export function createShellHTMLPlugin( }); } +export function createBackgroundWorkerHTMLPlugin( + flags: BuildFlags, + BUILD_CONFIG: BUILD_CONFIG_TYPE +) { + const htmlPluginOptions = getHTMLPluginOptions(flags, BUILD_CONFIG); + + return new HTMLPlugin({ + ...htmlPluginOptions, + chunks: ['backgroundWorker'], + filename: `background-worker.html`, + }); +} + export function createHTMLPlugins( flags: BuildFlags, BUILD_CONFIG: BUILD_CONFIG_TYPE diff --git a/tools/cli/src/webpack/index.ts b/tools/cli/src/webpack/index.ts index ee8df27dc..e4ff37ca7 100644 --- a/tools/cli/src/webpack/index.ts +++ b/tools/cli/src/webpack/index.ts @@ -14,7 +14,11 @@ import webpack from 'webpack'; import type { Configuration as DevServerConfiguration } from 'webpack-dev-server'; import { productionCacheGroups } from './cache-group.js'; -import { createHTMLPlugins, createShellHTMLPlugin } from './html-plugin.js'; +import { + createBackgroundWorkerHTMLPlugin, + createHTMLPlugins, + createShellHTMLPlugin, +} from './html-plugin.js'; import { WebpackS3Plugin } from './s3-plugin.js'; import type { BuildFlags } from './types'; @@ -426,6 +430,7 @@ export function createWebpackConfig( if (buildConfig.isElectron) { config.plugins.push(createShellHTMLPlugin(flags, buildConfig)); + config.plugins.push(createBackgroundWorkerHTMLPlugin(flags, buildConfig)); } return config; diff --git a/tools/utils/src/workspace.gen.ts b/tools/utils/src/workspace.gen.ts index 2cbd2e659..5a48bffa1 100644 --- a/tools/utils/src/workspace.gen.ts +++ b/tools/utils/src/workspace.gen.ts @@ -527,6 +527,7 @@ export const PackageList = [ 'packages/frontend/core', 'packages/frontend/electron-api', 'packages/frontend/i18n', + 'packages/common/nbstore', 'packages/common/infra', 'tools/utils', ], @@ -552,6 +553,7 @@ export const PackageList = [ 'packages/frontend/component', 'packages/frontend/core', 'packages/frontend/i18n', + 'packages/common/nbstore', 'blocksuite/affine/all', 'packages/common/infra', ], @@ -563,6 +565,7 @@ export const PackageList = [ 'packages/frontend/component', 'packages/frontend/core', 'packages/frontend/i18n', + 'packages/common/nbstore', 'packages/common/infra', ], }, @@ -587,6 +590,7 @@ export const PackageList = [ 'packages/common/env', 'packages/frontend/graphql', 'packages/frontend/i18n', + 'packages/common/nbstore', 'packages/frontend/templates', 'packages/frontend/track', 'blocksuite/affine/all', diff --git a/yarn.lock b/yarn.lock index 59ea58754..83f691ada 100644 --- a/yarn.lock +++ b/yarn.lock @@ -356,6 +356,7 @@ __metadata: "@affine/env": "workspace:*" "@affine/graphql": "workspace:*" "@affine/i18n": "workspace:*" + "@affine/nbstore": "workspace:*" "@affine/templates": "workspace:*" "@affine/track": "workspace:*" "@blocksuite/affine": "workspace:*" @@ -475,6 +476,7 @@ __metadata: "@affine/core": "workspace:*" "@affine/electron-api": "workspace:*" "@affine/i18n": "workspace:*" + "@affine/nbstore": "workspace:*" "@emotion/react": "npm:^11.14.0" "@sentry/react": "npm:^8.44.0" "@toeverything/infra": "workspace:*" @@ -482,6 +484,7 @@ __metadata: "@types/react": "npm:^19.0.1" "@types/react-dom": "npm:^19.0.2" "@vanilla-extract/css": "npm:^1.16.1" + async-call-rpc: "npm:^6.4.2" cross-env: "npm:^7.0.3" next-themes: "npm:^0.4.4" react: "npm:^19.0.0" @@ -618,6 +621,7 @@ __metadata: "@toeverything/infra": "workspace:^" "@types/react": "npm:^19.0.1" "@types/react-dom": "npm:^19.0.2" + async-call-rpc: "npm:^6.4.2" cross-env: "npm:^7.0.3" next-themes: "npm:^0.4.4" react: "npm:^19.0.0" @@ -635,6 +639,7 @@ __metadata: "@affine/component": "workspace:*" "@affine/core": "workspace:*" "@affine/i18n": "workspace:*" + "@affine/nbstore": "workspace:*" "@blocksuite/affine": "workspace:*" "@blocksuite/icons": "npm:2.2.2" "@sentry/react": "npm:^8.44.0" @@ -889,6 +894,7 @@ __metadata: "@affine/component": "workspace:*" "@affine/core": "workspace:*" "@affine/i18n": "workspace:*" + "@affine/nbstore": "workspace:*" "@emotion/react": "npm:^11.14.0" "@sentry/react": "npm:^8.44.0" "@toeverything/infra": "workspace:*"