refactor(infra): directory structure (#4615)

packages/common/y-indexeddb/.gitignore

@@ -0,0 +1 @@
+lib

packages/common/y-indexeddb/README.md

@@ -0,0 +1,38 @@
+# @toeverything/y-indexeddb
+
+## Features
+
+- persistence data in indexeddb
+- sub-documents support
+- fully TypeScript
+
+## Usage
+
+```ts
+import { createIndexedDBProvider, downloadBinary } from '@toeverything/y-indexeddb';
+import * as Y from 'yjs';
+
+const yDoc = new Y.Doc({
+  // we use `guid` as unique key
+  guid: 'my-doc',
+});
+
+// sync yDoc with indexedDB
+const provider = createIndexedDBProvider(yDoc);
+provider.connect();
+await provider.whenSynced.then(() => {
+  console.log('synced');
+  provider.disconnect();
+});
+
+// dowload binary data from indexedDB for once
+downloadBinary(yDoc.guid).then(blob => {
+  if (blob !== false) {
+    Y.applyUpdate(yDoc, blob);
+  }
+});
+```
+
+## LICENSE
+
+[MIT](https://github.com/toeverything/AFFiNE/blob/master/LICENSE-MIT)

packages/common/y-indexeddb/benchmark/README.md

@@ -0,0 +1 @@
+This benchmark is outdated because our new API for IndexedDB has no direct parity with the official provider.

packages/common/y-indexeddb/benchmark/index.ts

@@ -0,0 +1,113 @@
+#!/usr/bin/env ts-node-esm
+import 'fake-indexeddb/auto';
+
+const map = new Map();
+const localStorage = {
+  getItem: (key: string) => map.get(key),
+  setItem: (key: string, value: string) => map.set(key, value),
+  clear: () => map.clear(),
+};
+
+// @ts-expect-error
+globalThis.localStorage = localStorage;
+
+import { Workspace } from '@blocksuite/store';
+import { IndexeddbPersistence } from 'y-indexeddb';
+
+const Y = Workspace.Y;
+
+// eslint-disable-next-line @typescript-eslint/no-restricted-imports
+import { createIndexedDBProvider } from '../dist/index.js';
+
+// @ts-expect-error
+globalThis.window = {
+  addEventListener: () => void 0,
+  removeEventListener: () => void 0,
+};
+
+async function yjs_create_persistence(n = 1e3) {
+  for (let i = 0; i < n; i++) {
+    const yDoc = new Y.Doc();
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+}
+
+async function yjs_single_persistence(n = 1e5) {
+  const yDoc = new Y.Doc();
+  const map = yDoc.getMap();
+  for (let i = 0; i < n; i++) {
+    map.set(`${i}`, i);
+  }
+  {
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+  {
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+}
+
+async function toeverything_create_provider(n = 1e3) {
+  for (let i = 0; i < n; i++) {
+    const yDoc = new Y.Doc({
+      guid: 'test',
+    });
+    const provider = createIndexedDBProvider(yDoc);
+    provider.connect();
+    await provider.whenSynced;
+    provider.disconnect();
+  }
+}
+async function toeverything_single_persistence(n = 1e5) {
+  const yDoc = new Y.Doc({
+    guid: 'test',
+  });
+  const map = yDoc.getMap();
+  for (let i = 0; i < n; i++) {
+    map.set(`${i}`, i);
+  }
+  const provider = createIndexedDBProvider(yDoc, 'test');
+  provider.connect();
+  await provider.whenSynced;
+  provider.disconnect();
+  provider.connect();
+  await provider.whenSynced;
+  provider.disconnect();
+}
+
+async function main() {
+  console.log('create many persistence');
+  performance.mark('start');
+  await yjs_create_persistence();
+  performance.mark('end');
+  performance.measure('yjs', 'start', 'end');
+  indexedDB.deleteDatabase('test');
+  performance.mark('start');
+  await toeverything_create_provider();
+  performance.mark('end');
+  performance.measure('toeverything', 'start', 'end');
+  console.log(performance.getEntriesByType('measure'));
+  indexedDB.deleteDatabase('test');
+  performance.clearMarks();
+  performance.clearMeasures();
+  localStorage.clear();
+
+  console.log('single persistence with huge updates');
+  performance.mark('start');
+  await yjs_single_persistence();
+  performance.mark('end');
+  performance.measure('yjs', 'start', 'end');
+  indexedDB.deleteDatabase('test');
+  performance.mark('start');
+  await toeverything_single_persistence();
+  performance.mark('end');
+  performance.measure('toeverything', 'start', 'end');
+  console.log(performance.getEntriesByType('measure'));
+}
+
+main().then();

packages/common/y-indexeddb/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@toeverything/y-indexeddb",
+  "type": "module",
+  "version": "0.10.0-canary.1",
+  "description": "IndexedDB database adapter for Yjs",
+  "repository": "toeverything/AFFiNE",
+  "author": "toeverything",
+  "license": "MIT",
+  "keywords": [
+    "indexeddb",
+    "yjs",
+    "yjs-adapter"
+  ],
+  "scripts": {
+    "build": "vite build"
+  },
+  "files": [
+    "dist"
+  ],
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "publishConfig": {
+    "access": "public",
+    "exports": {
+      ".": {
+        "types": "./dist/index.d.ts",
+        "import": "./dist/index.js",
+        "require": "./dist/index.cjs",
+        "default": "./dist/index.umd.cjs"
+      }
+    }
+  },
+  "dependencies": {
+    "idb": "^7.1.1",
+    "nanoid": "^5.0.1",
+    "y-provider": "workspace:*"
+  },
+  "devDependencies": {
+    "@blocksuite/blocks": "0.0.0-20231018100009-361737d3-nightly",
+    "@blocksuite/store": "0.0.0-20231018100009-361737d3-nightly",
+    "fake-indexeddb": "^5.0.0",
+    "vite": "^4.4.11",
+    "vite-plugin-dts": "3.6.0",
+    "vitest": "0.34.6",
+    "y-indexeddb": "^9.0.11",
+    "yjs": "^13.6.8"
+  },
+  "peerDependencies": {
+    "yjs": "^13"
+  }
+}

packages/common/y-indexeddb/project.json

@@ -0,0 +1,21 @@
+{
+  "name": "y-indexeddb",
+  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+  "projectType": "library",
+  "sourceRoot": "packages/common/y-indexeddb/src",
+  "targets": {
+    "build": {
+      "executor": "@nx/vite:build",
+      "options": {
+        "outputPath": "packages/common/y-indexeddb/dist"
+      }
+    },
+    "serve": {
+      "executor": "@nx/vite:build",
+      "options": {
+        "outputPath": "packages/common/y-indexeddb/dist",
+        "watch": true
+      }
+    }
+  }
+}

packages/common/y-indexeddb/src/__tests__/index.spec.ts

@@ -0,0 +1,448 @@
+/**
+ * @vitest-environment happy-dom
+ */
+import 'fake-indexeddb/auto';
+
+import { setTimeout } from 'node:timers/promises';
+
+import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
+import { assertExists } from '@blocksuite/global/utils';
+import type { Page } from '@blocksuite/store';
+import { Schema, Workspace } from '@blocksuite/store';
+import { openDB } from 'idb';
+import { nanoid } from 'nanoid';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+import { applyUpdate, Doc, encodeStateAsUpdate } from 'yjs';
+
+import type { WorkspacePersist } from '../index';
+import {
+  createIndexedDBProvider,
+  dbVersion,
+  DEFAULT_DB_NAME,
+  downloadBinary,
+  getMilestones,
+  markMilestone,
+  overwriteBinary,
+  revertUpdate,
+  setMergeCount,
+} from '../index';
+
+function initEmptyPage(page: Page) {
+  const pageBlockId = page.addBlock('affine:page', {
+    title: new page.Text(''),
+  });
+  const surfaceBlockId = page.addBlock('affine:surface', {}, pageBlockId);
+  const frameBlockId = page.addBlock('affine:note', {}, pageBlockId);
+  const paragraphBlockId = page.addBlock('affine:paragraph', {}, frameBlockId);
+  return {
+    pageBlockId,
+    surfaceBlockId,
+    frameBlockId,
+    paragraphBlockId,
+  };
+}
+
+async function getUpdates(id: string): Promise<Uint8Array[]> {
+  const db = await openDB(rootDBName, dbVersion);
+  const store = db
+    .transaction('workspace', 'readonly')
+    .objectStore('workspace');
+  const data = (await store.get(id)) as WorkspacePersist | undefined;
+  assertExists(data, 'data should not be undefined');
+  expect(data.id).toBe(id);
+  return data.updates.map(({ update }) => update);
+}
+
+let id: string;
+let workspace: Workspace;
+const rootDBName = DEFAULT_DB_NAME;
+
+const schema = new Schema();
+
+schema.register(AffineSchemas).register(__unstableSchemas);
+
+beforeEach(() => {
+  id = nanoid();
+  workspace = new Workspace({
+    id,
+    isSSR: true,
+    schema,
+  });
+});
+
+afterEach(() => {
+  indexedDB.deleteDatabase('affine-local');
+  localStorage.clear();
+});
+
+describe('indexeddb provider', () => {
+  test('connect', async () => {
+    const provider = createIndexedDBProvider(workspace.doc);
+    provider.connect();
+
+    // todo: has a better way to know when data is synced
+    await setTimeout(200);
+
+    const db = await openDB(rootDBName, dbVersion);
+    {
+      const store = db
+        .transaction('workspace', 'readonly')
+        .objectStore('workspace');
+      const data = await store.get(id);
+      expect(data).toEqual({
+        id,
+        updates: [
+          {
+            timestamp: expect.any(Number),
+            update: encodeStateAsUpdate(workspace.doc),
+          },
+        ],
+      });
+      const page = workspace.createPage({ id: 'page0' });
+      await page.waitForLoaded();
+      const pageBlockId = page.addBlock('affine:page', { title: '' });
+      const frameId = page.addBlock('affine:note', {}, pageBlockId);
+      page.addBlock('affine:paragraph', {}, frameId);
+    }
+    await setTimeout(200);
+    {
+      const store = db
+        .transaction('workspace', 'readonly')
+        .objectStore('workspace');
+      const data = (await store.get(id)) as WorkspacePersist | undefined;
+      assertExists(data);
+      expect(data.id).toBe(id);
+      const testWorkspace = new Workspace({
+        id: 'test',
+        schema,
+      });
+      // data should only contain updates for the root doc
+      data.updates.forEach(({ update }) => {
+        Workspace.Y.applyUpdate(testWorkspace.doc, update);
+      });
+      const subPage = testWorkspace.doc.spaces.get('page0');
+      {
+        assertExists(subPage);
+        await store.get(subPage.guid);
+        const data = (await store.get(subPage.guid)) as
+          | WorkspacePersist
+          | undefined;
+        assertExists(data);
+        await testWorkspace.getPage('page0')?.waitForLoaded();
+        data.updates.forEach(({ update }) => {
+          Workspace.Y.applyUpdate(subPage, update);
+        });
+      }
+      expect(workspace.doc.toJSON()).toEqual(testWorkspace.doc.toJSON());
+    }
+  });
+
+  test('connect and disconnect', async () => {
+    const provider = createIndexedDBProvider(workspace.doc, rootDBName);
+    provider.connect();
+    expect(provider.connected).toBe(true);
+    await setTimeout(200);
+    const snapshot = encodeStateAsUpdate(workspace.doc);
+    provider.disconnect();
+    expect(provider.connected).toBe(false);
+    {
+      const page = workspace.createPage({ id: 'page0' });
+      await page.waitForLoaded();
+      const pageBlockId = page.addBlock('affine:page', { title: '' });
+      const frameId = page.addBlock('affine:note', {}, pageBlockId);
+      page.addBlock('affine:paragraph', {}, frameId);
+    }
+    {
+      const updates = await getUpdates(workspace.id);
+      expect(updates.length).toBe(1);
+      expect(updates[0]).toEqual(snapshot);
+    }
+    expect(provider.connected).toBe(false);
+    provider.connect();
+    expect(provider.connected).toBe(true);
+    await setTimeout(200);
+    {
+      const updates = await getUpdates(workspace.id);
+      expect(updates).not.toEqual([]);
+    }
+    expect(provider.connected).toBe(true);
+    provider.disconnect();
+    expect(provider.connected).toBe(false);
+  });
+
+  test('cleanup', async () => {
+    const provider = createIndexedDBProvider(workspace.doc);
+    provider.connect();
+    await setTimeout(200);
+    const db = await openDB(rootDBName, dbVersion);
+
+    {
+      const store = db
+        .transaction('workspace', 'readonly')
+        .objectStore('workspace');
+      const keys = await store.getAllKeys();
+      expect(keys).contain(workspace.id);
+    }
+
+    await provider.cleanup();
+    provider.disconnect();
+
+    {
+      const store = db
+        .transaction('workspace', 'readonly')
+        .objectStore('workspace');
+      const keys = await store.getAllKeys();
+      expect(keys).not.contain(workspace.id);
+    }
+  });
+
+  test('merge', async () => {
+    setMergeCount(5);
+    const provider = createIndexedDBProvider(workspace.doc, rootDBName);
+    provider.connect();
+    {
+      const page = workspace.createPage({ id: 'page0' });
+      await page.waitForLoaded();
+      const pageBlockId = page.addBlock('affine:page', { title: '' });
+      const frameId = page.addBlock('affine:note', {}, pageBlockId);
+      for (let i = 0; i < 99; i++) {
+        page.addBlock('affine:paragraph', {}, frameId);
+      }
+    }
+    await setTimeout(200);
+    {
+      const updates = await getUpdates(id);
+      expect(updates.length).lessThanOrEqual(5);
+    }
+  });
+
+  test("data won't be lost", async () => {
+    const doc = new Workspace.Y.Doc();
+    const map = doc.getMap('map');
+    for (let i = 0; i < 100; i++) {
+      map.set(`${i}`, i);
+    }
+    {
+      const provider = createIndexedDBProvider(doc, rootDBName);
+      provider.connect();
+      provider.disconnect();
+    }
+    {
+      const newDoc = new Workspace.Y.Doc();
+      const provider = createIndexedDBProvider(newDoc, rootDBName);
+      provider.connect();
+      provider.disconnect();
+      newDoc.getMap('map').forEach((value, key) => {
+        expect(value).toBe(parseInt(key));
+      });
+    }
+  });
+
+  test('beforeunload', async () => {
+    const oldAddEventListener = window.addEventListener;
+    window.addEventListener = vi.fn((event: string, fn, options) => {
+      expect(event).toBe('beforeunload');
+      return oldAddEventListener(event, fn, options);
+    });
+    const oldRemoveEventListener = window.removeEventListener;
+    window.removeEventListener = vi.fn((event: string, fn, options) => {
+      expect(event).toBe('beforeunload');
+      return oldRemoveEventListener(event, fn, options);
+    });
+    const doc = new Doc({
+      guid: '1',
+    });
+    const provider = createIndexedDBProvider(doc);
+    const map = doc.getMap('map');
+    map.set('1', 1);
+    provider.connect();
+
+    await setTimeout(200);
+
+    expect(window.addEventListener).toBeCalledTimes(1);
+    expect(window.removeEventListener).toBeCalledTimes(1);
+
+    window.addEventListener = oldAddEventListener;
+    window.removeEventListener = oldRemoveEventListener;
+  });
+});
+
+describe('milestone', () => {
+  test('milestone', async () => {
+    const doc = new Doc();
+    const map = doc.getMap('map');
+    const array = doc.getArray('array');
+    map.set('1', 1);
+    array.push([1]);
+    await markMilestone('1', doc, 'test1');
+    const milestones = await getMilestones('1');
+    assertExists(milestones);
+    expect(milestones).toBeDefined();
+    expect(Object.keys(milestones).length).toBe(1);
+    expect(milestones.test1).toBeInstanceOf(Uint8Array);
+    const snapshot = new Doc();
+    applyUpdate(snapshot, milestones.test1);
+    {
+      const map = snapshot.getMap('map');
+      expect(map.get('1')).toBe(1);
+    }
+    map.set('1', 2);
+    {
+      const map = snapshot.getMap('map');
+      expect(map.get('1')).toBe(1);
+    }
+    revertUpdate(doc, milestones.test1, key =>
+      key === 'map' ? 'Map' : 'Array'
+    );
+    {
+      const map = doc.getMap('map');
+      expect(map.get('1')).toBe(1);
+    }
+
+    const fn = vi.fn(() => true);
+    doc.gcFilter = fn;
+    expect(fn).toBeCalledTimes(0);
+
+    for (let i = 0; i < 1e5; i++) {
+      map.set(`${i}`, i + 1);
+    }
+    for (let i = 0; i < 1e5; i++) {
+      map.delete(`${i}`);
+    }
+    for (let i = 0; i < 1e5; i++) {
+      map.set(`${i}`, i - 1);
+    }
+
+    expect(fn).toBeCalled();
+
+    const doc2 = new Doc();
+    applyUpdate(doc2, encodeStateAsUpdate(doc));
+
+    revertUpdate(doc2, milestones.test1, key =>
+      key === 'map' ? 'Map' : 'Array'
+    );
+    {
+      const map = doc2.getMap('map');
+      expect(map.get('1')).toBe(1);
+    }
+  });
+});
+
+describe('subDoc', () => {
+  test('basic', async () => {
+    let json1: any, json2: any;
+    {
+      const doc = new Doc({
+        guid: 'test',
+      });
+      const map = doc.getMap();
+      const subDoc = new Doc();
+      subDoc.load();
+      map.set('1', subDoc);
+      map.set('2', 'test');
+      const provider = createIndexedDBProvider(doc);
+      provider.connect();
+      await setTimeout(200);
+      provider.disconnect();
+      json1 = doc.toJSON();
+    }
+    {
+      const doc = new Doc({
+        guid: 'test',
+      });
+      const provider = createIndexedDBProvider(doc);
+      provider.connect();
+      await setTimeout(200);
+      const map = doc.getMap();
+      const subDoc = map.get('1') as Doc;
+      subDoc.load();
+      provider.disconnect();
+      json2 = doc.toJSON();
+    }
+    // the following line compares {} with {}
+    expect(json1['']['1'].toJSON()).toEqual(json2['']['1'].toJSON());
+    expect(json1['']['2']).toEqual(json2['']['2']);
+  });
+
+  test('blocksuite', async () => {
+    const page0 = workspace.createPage({
+      id: 'page0',
+    });
+    await page0.waitForLoaded();
+    const { paragraphBlockId: paragraphBlockIdPage1 } = initEmptyPage(page0);
+    const provider = createIndexedDBProvider(workspace.doc, rootDBName);
+    provider.connect();
+    const page1 = workspace.createPage({
+      id: 'page1',
+    });
+    await page1.waitForLoaded();
+    const { paragraphBlockId: paragraphBlockIdPage2 } = initEmptyPage(page1);
+    await setTimeout(200);
+    provider.disconnect();
+    {
+      const newWorkspace = new Workspace({
+        id,
+        isSSR: true,
+        schema,
+      });
+      const provider = createIndexedDBProvider(newWorkspace.doc, rootDBName);
+      provider.connect();
+      await setTimeout(200);
+      const page0 = newWorkspace.getPage('page0') as Page;
+      await page0.waitForLoaded();
+      await setTimeout(200);
+      {
+        const block = page0.getBlockById(paragraphBlockIdPage1);
+        assertExists(block);
+      }
+      const page1 = newWorkspace.getPage('page1') as Page;
+      await page1.waitForLoaded();
+      await setTimeout(200);
+      {
+        const block = page1.getBlockById(paragraphBlockIdPage2);
+        assertExists(block);
+      }
+    }
+  });
+});
+
+describe('utils', () => {
+  test('download binary', async () => {
+    const page = workspace.createPage({ id: 'page0' });
+    await page.waitForLoaded();
+    initEmptyPage(page);
+    const provider = createIndexedDBProvider(workspace.doc, rootDBName);
+    provider.connect();
+    await setTimeout(200);
+    provider.disconnect();
+    const update = (await downloadBinary(
+      workspace.id,
+      rootDBName
+    )) as Uint8Array;
+    expect(update).toBeInstanceOf(Uint8Array);
+    const newWorkspace = new Workspace({
+      id,
+      isSSR: true,
+      schema,
+    });
+    applyUpdate(newWorkspace.doc, update);
+    await setTimeout();
+    expect(workspace.doc.toJSON()['meta']).toEqual(
+      newWorkspace.doc.toJSON()['meta']
+    );
+    expect(Object.keys(workspace.doc.toJSON()['spaces'])).toEqual(
+      Object.keys(newWorkspace.doc.toJSON()['spaces'])
+    );
+  });
+
+  test('overwrite binary', async () => {
+    const doc = new Doc();
+    const map = doc.getMap();
+    map.set('1', 1);
+    await overwriteBinary('test', new Uint8Array(encodeStateAsUpdate(doc)));
+    {
+      const binary = await downloadBinary('test');
+      expect(binary).toEqual(new Uint8Array(encodeStateAsUpdate(doc)));
+    }
+  });
+});

packages/common/y-indexeddb/src/index.ts

@@ -0,0 +1,134 @@
+import { openDB } from 'idb';
+import {
+  applyUpdate,
+  Doc,
+  encodeStateAsUpdate,
+  encodeStateVector,
+  UndoManager,
+} from 'yjs';
+
+import type { BlockSuiteBinaryDB, WorkspaceMilestone } from './shared';
+import { dbVersion, DEFAULT_DB_NAME, upgradeDB } from './shared';
+
+const snapshotOrigin = 'snapshot-origin';
+
+/**
+ * @internal
+ */
+const saveAlert = (event: BeforeUnloadEvent) => {
+  event.preventDefault();
+  return (event.returnValue =
+    'Data is not saved. Are you sure you want to leave?');
+};
+
+export const writeOperation = async (op: Promise<unknown>) => {
+  window.addEventListener('beforeunload', saveAlert, {
+    capture: true,
+  });
+  await op;
+  window.removeEventListener('beforeunload', saveAlert, {
+    capture: true,
+  });
+};
+
+export function revertUpdate(
+  doc: Doc,
+  snapshotUpdate: Uint8Array,
+  getMetadata: (key: string) => 'Text' | 'Map' | 'Array'
+) {
+  const snapshotDoc = new Doc();
+  applyUpdate(snapshotDoc, snapshotUpdate, snapshotOrigin);
+
+  const currentStateVector = encodeStateVector(doc);
+  const snapshotStateVector = encodeStateVector(snapshotDoc);
+
+  const changesSinceSnapshotUpdate = encodeStateAsUpdate(
+    doc,
+    snapshotStateVector
+  );
+  const undoManager = new UndoManager(
+    [...snapshotDoc.share.keys()].map(key => {
+      const type = getMetadata(key);
+      if (type === 'Text') {
+        return snapshotDoc.getText(key);
+      } else if (type === 'Map') {
+        return snapshotDoc.getMap(key);
+      } else if (type === 'Array') {
+        return snapshotDoc.getArray(key);
+      }
+      throw new Error('Unknown type');
+    }),
+    {
+      trackedOrigins: new Set([snapshotOrigin]),
+    }
+  );
+  applyUpdate(snapshotDoc, changesSinceSnapshotUpdate, snapshotOrigin);
+  undoManager.undo();
+  const revertChangesSinceSnapshotUpdate = encodeStateAsUpdate(
+    snapshotDoc,
+    currentStateVector
+  );
+  applyUpdate(doc, revertChangesSinceSnapshotUpdate, snapshotOrigin);
+}
+
+export class EarlyDisconnectError extends Error {
+  constructor() {
+    super('Early disconnect');
+  }
+}
+
+export class CleanupWhenConnectingError extends Error {
+  constructor() {
+    super('Cleanup when connecting');
+  }
+}
+
+export const markMilestone = async (
+  id: string,
+  doc: Doc,
+  name: string,
+  dbName = DEFAULT_DB_NAME
+): Promise<void> => {
+  const dbPromise = openDB<BlockSuiteBinaryDB>(dbName, dbVersion, {
+    upgrade: upgradeDB,
+  });
+  const db = await dbPromise;
+  const store = db
+    .transaction('milestone', 'readwrite')
+    .objectStore('milestone');
+  const milestone = await store.get('id');
+  const binary = encodeStateAsUpdate(doc);
+  if (!milestone) {
+    await store.put({
+      id,
+      milestone: {
+        [name]: binary,
+      },
+    });
+  } else {
+    milestone.milestone[name] = binary;
+    await store.put(milestone);
+  }
+};
+
+export const getMilestones = async (
+  id: string,
+  dbName: string = DEFAULT_DB_NAME
+): Promise<null | WorkspaceMilestone['milestone']> => {
+  const dbPromise = openDB<BlockSuiteBinaryDB>(dbName, dbVersion, {
+    upgrade: upgradeDB,
+  });
+  const db = await dbPromise;
+  const store = db
+    .transaction('milestone', 'readonly')
+    .objectStore('milestone');
+  const milestone = await store.get(id);
+  if (!milestone) {
+    return null;
+  }
+  return milestone.milestone;
+};
+
+export * from './provider';
+export * from './shared';
+export * from './utils';

packages/common/y-indexeddb/src/provider.ts

@@ -0,0 +1,162 @@
+import { assertExists } from '@blocksuite/global/utils';
+import type { IDBPDatabase } from 'idb';
+import { openDB } from 'idb';
+import {
+  createLazyProvider,
+  type DocDataSource,
+  writeOperation,
+} from 'y-provider';
+import type { Doc } from 'yjs';
+import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs';
+
+import {
+  type BlockSuiteBinaryDB,
+  dbVersion,
+  DEFAULT_DB_NAME,
+  type IndexedDBProvider,
+  type UpdateMessage,
+  upgradeDB,
+} from './shared';
+import { mergeUpdates } from './utils';
+
+let mergeCount = 500;
+
+export function setMergeCount(count: number) {
+  mergeCount = count;
+}
+
+export const createIndexedDBDatasource = ({
+  dbName = DEFAULT_DB_NAME,
+  mergeCount,
+}: {
+  dbName?: string;
+  mergeCount?: number;
+}) => {
+  let dbPromise: Promise<IDBPDatabase<BlockSuiteBinaryDB>> | null = null;
+  const getDb = async () => {
+    if (dbPromise === null) {
+      dbPromise = openDB<BlockSuiteBinaryDB>(dbName, dbVersion, {
+        upgrade: upgradeDB,
+      });
+    }
+    return dbPromise;
+  };
+
+  const adapter = {
+    queryDocState: async (guid, options) => {
+      try {
+        const db = await getDb();
+        const store = db
+          .transaction('workspace', 'readonly')
+          .objectStore('workspace');
+        const data = await store.get(guid);
+
+        if (!data) {
+          return false;
+        }
+
+        const { updates } = data;
+        const update = mergeUpdates(updates.map(({ update }) => update));
+
+        const missing = options?.stateVector
+          ? diffUpdate(update, options?.stateVector)
+          : update;
+
+        return { missing, state: encodeStateVectorFromUpdate(update) };
+      } catch (err: any) {
+        if (!err.message?.includes('The database connection is closing.')) {
+          throw err;
+        }
+        return false;
+      }
+    },
+    sendDocUpdate: async (guid, update) => {
+      try {
+        const db = await getDb();
+        const store = db
+          .transaction('workspace', 'readwrite')
+          .objectStore('workspace');
+
+        // TODO: maybe we do not need to get data every time
+        const { updates } = (await store.get(guid)) ?? { updates: [] };
+        let rows: UpdateMessage[] = [
+          ...updates,
+          { timestamp: Date.now(), update },
+        ];
+        if (mergeCount && rows.length >= mergeCount) {
+          const merged = mergeUpdates(rows.map(({ update }) => update));
+          rows = [{ timestamp: Date.now(), update: merged }];
+        }
+        await writeOperation(
+          store.put({
+            id: guid,
+            updates: rows,
+          })
+        );
+      } catch (err: any) {
+        if (!err.message?.includes('The database connection is closing.')) {
+          throw err;
+        }
+      }
+    },
+  } satisfies DocDataSource;
+
+  return {
+    ...adapter,
+    disconnect: () => {
+      getDb()
+        .then(db => db.close())
+        .then(() => {
+          dbPromise = null;
+        })
+        .catch(console.error);
+    },
+    cleanup: async () => {
+      const db = await getDb();
+      await db.clear('workspace');
+    },
+  };
+};
+
+/**
+ * We use `doc.guid` as the unique key, please make sure it not changes.
+ */
+export const createIndexedDBProvider = (
+  doc: Doc,
+  dbName: string = DEFAULT_DB_NAME
+): IndexedDBProvider => {
+  const datasource = createIndexedDBDatasource({ dbName, mergeCount });
+  let provider: ReturnType<typeof createLazyProvider> | null = null;
+
+  const apis = {
+    get status() {
+      assertExists(provider);
+      return provider.status;
+    },
+    subscribeStatusChange(onStatusChange) {
+      assertExists(provider);
+      return provider.subscribeStatusChange(onStatusChange);
+    },
+    connect: () => {
+      if (apis.connected) {
+        apis.disconnect();
+      }
+      provider = createLazyProvider(doc, datasource, { origin: 'idb' });
+      provider.connect();
+    },
+    disconnect: () => {
+      datasource?.disconnect();
+      provider?.disconnect();
+      provider = null;
+    },
+    cleanup: async () => {
+      await datasource?.cleanup();
+    },
+    get connected() {
+      return provider?.connected || false;
+    },
+    datasource,
+  } satisfies IndexedDBProvider;
+
+  return apis;
+};

packages/common/y-indexeddb/src/shared.ts

@@ -0,0 +1,50 @@
+import type { DBSchema, IDBPDatabase } from 'idb';
+import type { DataSourceAdapter } from 'y-provider';
+
+export const dbVersion = 1;
+export const DEFAULT_DB_NAME = 'affine-local';
+
+export function upgradeDB(db: IDBPDatabase<BlockSuiteBinaryDB>) {
+  db.createObjectStore('workspace', { keyPath: 'id' });
+  db.createObjectStore('milestone', { keyPath: 'id' });
+}
+
+export interface IndexedDBProvider extends DataSourceAdapter {
+  connect: () => void;
+  disconnect: () => void;
+  cleanup: () => Promise<void>;
+  readonly connected: boolean;
+}
+
+export type UpdateMessage = {
+  timestamp: number;
+  update: Uint8Array;
+};
+
+export type WorkspacePersist = {
+  id: string;
+  updates: UpdateMessage[];
+};
+
+export type WorkspaceMilestone = {
+  id: string;
+  milestone: Record<string, Uint8Array>;
+};
+
+export interface BlockSuiteBinaryDB extends DBSchema {
+  workspace: {
+    key: string;
+    value: WorkspacePersist;
+  };
+  milestone: {
+    key: string;
+    value: WorkspaceMilestone;
+  };
+}
+
+export interface OldYjsDB extends DBSchema {
+  updates: {
+    key: number;
+    value: Uint8Array;
+  };
+}

packages/common/y-indexeddb/src/utils.ts

@@ -0,0 +1,205 @@
+import type { IDBPDatabase } from 'idb';
+import { openDB } from 'idb';
+import { applyUpdate, Doc, encodeStateAsUpdate } from 'yjs';
+
+import type { BlockSuiteBinaryDB, OldYjsDB, UpdateMessage } from './shared';
+import { dbVersion, DEFAULT_DB_NAME, upgradeDB } from './shared';
+
+let allDb: IDBDatabaseInfo[];
+
+export function mergeUpdates(updates: Uint8Array[]) {
+  const doc = new Doc();
+  updates.forEach(update => {
+    applyUpdate(doc, update);
+  });
+  return encodeStateAsUpdate(doc);
+}
+
+async function databaseExists(name: string): Promise<boolean> {
+  return new Promise(resolve => {
+    const req = indexedDB.open(name);
+    let existed = true;
+    req.onsuccess = function () {
+      req.result.close();
+      if (!existed) {
+        indexedDB.deleteDatabase(name);
+      }
+      resolve(existed);
+    };
+    req.onupgradeneeded = function () {
+      existed = false;
+    };
+  });
+}
+
+/**
+ * try to migrate the old database to the new database
+ * this function will be removed in the future
+ * since we don't need to support the old database
+ */
+export async function tryMigrate(
+  db: IDBPDatabase<BlockSuiteBinaryDB>,
+  id: string,
+  dbName = DEFAULT_DB_NAME
+) {
+  do {
+    if (!allDb || localStorage.getItem(`${dbName}-migration`) !== 'true') {
+      try {
+        allDb = await indexedDB.databases();
+      } catch {
+        // in firefox, `indexedDB.databases` is not existed
+        if (await databaseExists(id)) {
+          await openDB<IDBPDatabase<OldYjsDB>>(id, 1).then(async oldDB => {
+            if (!oldDB.objectStoreNames.contains('updates')) {
+              return;
+            }
+            const t = oldDB
+              .transaction('updates', 'readonly')
+              .objectStore('updates');
+            const updates = await t.getAll();
+            if (
+              !Array.isArray(updates) ||
+              !updates.every(update => update instanceof Uint8Array)
+            ) {
+              return;
+            }
+            const update = mergeUpdates(updates);
+            const workspaceTransaction = db
+              .transaction('workspace', 'readwrite')
+              .objectStore('workspace');
+            const data = await workspaceTransaction.get(id);
+            if (!data) {
+              console.log('upgrading the database');
+              await workspaceTransaction.put({
+                id,
+                updates: [
+                  {
+                    timestamp: Date.now(),
+                    update,
+                  },
+                ],
+              });
+            }
+          });
+          break;
+        }
+      }
+      // run the migration
+      await Promise.all(
+        allDb &&
+          allDb.map(meta => {
+            if (meta.name && meta.version === 1) {
+              const name = meta.name;
+              const version = meta.version;
+              return openDB<IDBPDatabase<OldYjsDB>>(name, version).then(
+                async oldDB => {
+                  if (!oldDB.objectStoreNames.contains('updates')) {
+                    return;
+                  }
+                  const t = oldDB
+                    .transaction('updates', 'readonly')
+                    .objectStore('updates');
+                  const updates = await t.getAll();
+                  if (
+                    !Array.isArray(updates) ||
+                    !updates.every(update => update instanceof Uint8Array)
+                  ) {
+                    return;
+                  }
+                  const update = mergeUpdates(updates);
+                  const workspaceTransaction = db
+                    .transaction('workspace', 'readwrite')
+                    .objectStore('workspace');
+                  const data = await workspaceTransaction.get(name);
+                  if (!data) {
+                    console.log('upgrading the database');
+                    await workspaceTransaction.put({
+                      id: name,
+                      updates: [
+                        {
+                          timestamp: Date.now(),
+                          update,
+                        },
+                      ],
+                    });
+                  }
+                }
+              );
+            }
+            return void 0;
+          })
+      );
+      localStorage.setItem(`${dbName}-migration`, 'true');
+      break;
+    }
+    // eslint-disable-next-line no-constant-condition
+  } while (false);
+}
+
+export async function downloadBinary(
+  guid: string,
+  dbName = DEFAULT_DB_NAME
+): Promise<UpdateMessage['update'] | false> {
+  const dbPromise = openDB<BlockSuiteBinaryDB>(dbName, dbVersion, {
+    upgrade: upgradeDB,
+  });
+  const db = await dbPromise;
+  const t = db.transaction('workspace', 'readonly').objectStore('workspace');
+  const doc = await t.get(guid);
+  if (!doc) {
+    return false;
+  } else {
+    return mergeUpdates(doc.updates.map(({ update }) => update));
+  }
+}
+
+export async function overwriteBinary(
+  guid: string,
+  update: UpdateMessage['update'],
+  dbName = DEFAULT_DB_NAME
+) {
+  const dbPromise = openDB<BlockSuiteBinaryDB>(dbName, dbVersion, {
+    upgrade: upgradeDB,
+  });
+  const db = await dbPromise;
+  const t = db.transaction('workspace', 'readwrite').objectStore('workspace');
+  await t.put({
+    id: guid,
+    updates: [
+      {
+        timestamp: Date.now(),
+        update,
+      },
+    ],
+  });
+}
+
+export async function pushBinary(
+  guid: string,
+  update: UpdateMessage['update'],
+  dbName = DEFAULT_DB_NAME
+) {
+  const dbPromise = openDB<BlockSuiteBinaryDB>(dbName, dbVersion, {
+    upgrade: upgradeDB,
+  });
+  const db = await dbPromise;
+  const t = db.transaction('workspace', 'readwrite').objectStore('workspace');
+  const doc = await t.get(guid);
+  if (!doc) {
+    await t.put({
+      id: guid,
+      updates: [
+        {
+          timestamp: Date.now(),
+          update,
+        },
+      ],
+    });
+  } else {
+    doc.updates.push({
+      timestamp: Date.now(),
+      update,
+    });
+    await t.put(doc);
+  }
+}

packages/common/y-indexeddb/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "extends": "../../../tsconfig.json",
+  "include": ["./src"],
+  "compilerOptions": {
+    "composite": true,
+    "noEmit": false,
+    "outDir": "lib"
+  },
+  "references": [
+    {
+      "path": "./tsconfig.node.json"
+    },
+    {
+      "path": "../y-provider"
+    }
+  ]
+}

packages/common/y-indexeddb/tsconfig.node.json

@@ -0,0 +1,11 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "composite": true,
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "allowSyntheticDefaultImports": true,
+    "outDir": "lib"
+  },
+  "include": ["vite.config.ts"]
+}

packages/common/y-indexeddb/vite.config.ts

@@ -0,0 +1,35 @@
+import { resolve } from 'node:path';
+
+import { fileURLToPath } from 'url';
+import { defineConfig } from 'vite';
+import dts from 'vite-plugin-dts';
+
+const __dirname = fileURLToPath(new URL('.', import.meta.url));
+
+export default defineConfig({
+  build: {
+    minify: 'esbuild',
+    sourcemap: true,
+    lib: {
+      entry: resolve(__dirname, 'src/index.ts'),
+      fileName: 'index',
+      name: 'ToEverythingIndexedDBProvider',
+      formats: ['es', 'cjs', 'umd'],
+    },
+    rollupOptions: {
+      output: {
+        globals: {
+          idb: 'idb',
+          yjs: 'yjs',
+          'y-provider': 'yProvider',
+        },
+      },
+      external: ['idb', 'yjs', 'y-provider'],
+    },
+  },
+  plugins: [
+    dts({
+      entryRoot: resolve(__dirname, 'src'),
+    }),
+  ],
+});