perf(y-indexeddb): improve boost and loading time (#1879)

apps/electron/yarn.lock

@@ -5662,6 +5662,7 @@ __metadata:
     idb: ^7.1.1
     vite: ^4.2.1
     vite-plugin-dts: ^2.2.0
+    y-indexeddb: ^9.0.10
   peerDependencies:
     yjs: ^13.5.51
   languageName: unknown
@@ -12383,7 +12384,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lib0@npm:^0.2.42, lib0@npm:^0.2.68, lib0@npm:^0.2.72, lib0@npm:^0.2.73":
+"lib0@npm:^0.2.35, lib0@npm:^0.2.42, lib0@npm:^0.2.68, lib0@npm:^0.2.72, lib0@npm:^0.2.73":
   version: 0.2.73
   resolution: "lib0@npm:0.2.73"
   dependencies:
@@ -17474,6 +17475,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"y-indexeddb@npm:^9.0.10":
+  version: 9.0.10
+  resolution: "y-indexeddb@npm:9.0.10"
+  dependencies:
+    lib0: ^0.2.35
+  peerDependencies:
+    yjs: ^13.0.0
+  checksum: 6a57825b599cdf77da7c9857b1acc0f782492fc41531618bd7392bdfbcf11c783ff1a30b82ae080b050a5ebafd54754a978de7a6ac42144ec59eb1fbdebd090b
+  languageName: node
+  linkType: hard
+
 "y-protocols@npm:^1.0.5":
   version: 1.0.5
   resolution: "y-protocols@npm:1.0.5"

packages/y-indexeddb/benchmark/index.ts

@@ -0,0 +1,103 @@
+#!/usr/bin/env ts-node-esm
+import 'fake-indexeddb/auto';
+
+const map = new Map();
+const localStorage = {
+  getItem: (key: string) => map.get(key),
+  setItem: (key: string, value: string) => map.set(key, value),
+  clear: () => map.clear(),
+};
+
+// @ts-expect-error
+globalThis.localStorage = localStorage;
+
+import { Workspace } from '@blocksuite/store';
+import { IndexeddbPersistence } from 'y-indexeddb';
+
+const Y = Workspace.Y;
+
+// eslint-disable-next-line @typescript-eslint/no-restricted-imports
+import { createIndexedDBProvider } from '../src/index.js';
+
+async function yjs_create_persistence(n = 1e3) {
+  for (let i = 0; i < n; i++) {
+    const yDoc = new Y.Doc();
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+}
+
+async function yjs_single_persistence(n = 1e5) {
+  const yDoc = new Y.Doc();
+  const map = yDoc.getMap();
+  for (let i = 0; i < n; i++) {
+    map.set(`${i}`, i);
+  }
+  {
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+  {
+    const persistence = new IndexeddbPersistence('test', yDoc);
+    await persistence.whenSynced;
+    persistence.destroy();
+  }
+}
+
+async function toeverything_create_provider(n = 1e3) {
+  for (let i = 0; i < n; i++) {
+    const yDoc = new Y.Doc();
+    const provider = createIndexedDBProvider('test', yDoc);
+    provider.connect();
+    await provider.whenSynced;
+    provider.disconnect();
+  }
+}
+async function toeverything_single_persistence(n = 1e5) {
+  const yDoc = new Y.Doc();
+  const map = yDoc.getMap();
+  for (let i = 0; i < n; i++) {
+    map.set(`${i}`, i);
+  }
+  const provider = createIndexedDBProvider('test', yDoc, 'test');
+  provider.connect();
+  await provider.whenSynced;
+  provider.disconnect();
+  provider.connect();
+  await provider.whenSynced;
+  provider.disconnect();
+}
+
+async function main() {
+  console.log('create many persistence');
+  performance.mark('start');
+  await yjs_create_persistence();
+  performance.mark('end');
+  performance.measure('yjs', 'start', 'end');
+  indexedDB.deleteDatabase('test');
+  performance.mark('start');
+  await toeverything_create_provider();
+  performance.mark('end');
+  performance.measure('toeverything', 'start', 'end');
+  console.log(performance.getEntriesByType('measure'));
+  indexedDB.deleteDatabase('test');
+  performance.clearMarks();
+  performance.clearMeasures();
+  localStorage.clear();
+
+  console.log('single persistence with huge updates');
+  performance.mark('start');
+  await yjs_single_persistence();
+  performance.mark('end');
+  performance.measure('yjs', 'start', 'end');
+  indexedDB.deleteDatabase('test');
+  performance.mark('start');
+  await toeverything_single_persistence();
+  performance.mark('end');
+  performance.measure('toeverything', 'start', 'end');
+  console.log(performance.getEntriesByType('measure'));
+}
+
+main().then();

packages/y-indexeddb/package.json

@@ -28,7 +28,8 @@
     "@blocksuite/blocks": "0.0.0-20230411141436-ec6b051d-nightly",
     "@blocksuite/store": "0.0.0-20230411141436-ec6b051d-nightly",
     "vite": "^4.2.1",
-    "vite-plugin-dts": "^2.2.0"
+    "vite-plugin-dts": "^2.2.0",
+    "y-indexeddb": "^9.0.10"
   },
   "peerDependencies": {
     "yjs": "^13.5.51"

packages/y-indexeddb/src/__tests__/index.spec.ts

@@ -6,7 +6,7 @@ import 'fake-indexeddb/auto';
 import { __unstableSchemas, AffineSchemas } from '@blocksuite/blocks/models';
 import { assertExists, uuidv4, Workspace } from '@blocksuite/store';
 import { openDB } from 'idb';
-import { beforeEach, describe, expect, test, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
 import { applyUpdate, Doc, encodeStateAsUpdate } from 'yjs';
 
 import type { WorkspacePersist } from '../index';
@@ -20,7 +20,7 @@ import {
 } from '../index';
 
 async function getUpdates(id: string): Promise<ArrayBuffer[]> {
-  const db = await openDB('affine-local', dbVersion);
+  const db = await openDB(rootDBName, dbVersion);
   const store = await db
     .transaction('workspace', 'readonly')
     .objectStore('workspace');
@@ -32,6 +32,7 @@ async function getUpdates(id: string): Promise<ArrayBuffer[]> {
 
 let id: string;
 let workspace: Workspace;
+const rootDBName = 'affine-local';
 
 beforeEach(() => {
   id = uuidv4();
@@ -42,12 +43,16 @@ beforeEach(() => {
   workspace.register(AffineSchemas).register(__unstableSchemas);
 });
 
+afterEach(() => {
+  indexedDB.deleteDatabase('affine-local');
+});
+
 describe('indexeddb provider', () => {
   test('connect', async () => {
     const provider = createIndexedDBProvider(workspace.id, workspace.doc);
     provider.connect();
     await provider.whenSynced;
-    const db = await openDB('affine-local', dbVersion);
+    const db = await openDB(rootDBName, dbVersion);
     {
       const store = await db
         .transaction('workspace', 'readonly')
@@ -89,7 +94,8 @@ describe('indexeddb provider', () => {
       .register(__unstableSchemas);
     const provider2 = createIndexedDBProvider(
       secondWorkspace.id,
-      secondWorkspace.doc
+      secondWorkspace.doc,
+      rootDBName
     );
     provider2.connect();
     await provider2.whenSynced;
@@ -99,7 +105,11 @@ describe('indexeddb provider', () => {
   });
 
   test('disconnect suddenly', async () => {
-    const provider = createIndexedDBProvider(workspace.id, workspace.doc);
+    const provider = createIndexedDBProvider(
+      workspace.id,
+      workspace.doc,
+      rootDBName
+    );
     const fn = vi.fn();
     provider.connect();
     provider.disconnect();
@@ -109,10 +119,14 @@ describe('indexeddb provider', () => {
   });
 
   test('connect and disconnect', async () => {
-    const provider = createIndexedDBProvider(workspace.id, workspace.doc);
+    const provider = createIndexedDBProvider(
+      workspace.id,
+      workspace.doc,
+      rootDBName
+    );
     provider.connect();
     const p1 = provider.whenSynced;
-    await provider.whenSynced;
+    await p1;
     provider.disconnect();
     {
       const page = workspace.createPage('page0');
@@ -126,7 +140,7 @@ describe('indexeddb provider', () => {
     }
     provider.connect();
     const p2 = provider.whenSynced;
-    await provider.whenSynced;
+    await p2;
     {
       const updates = await getUpdates(workspace.id);
       expect(updates).not.toEqual([]);
@@ -137,7 +151,11 @@ describe('indexeddb provider', () => {
 
   test('merge', async () => {
     setMergeCount(5);
-    const provider = createIndexedDBProvider(workspace.id, workspace.doc);
+    const provider = createIndexedDBProvider(
+      workspace.id,
+      workspace.doc,
+      rootDBName
+    );
     provider.connect();
     {
       const page = workspace.createPage('page0');
@@ -153,6 +171,31 @@ describe('indexeddb provider', () => {
       expect(updates.length).lessThanOrEqual(5);
     }
   });
+
+  test("data won't be lost", async () => {
+    const id = uuidv4();
+    const doc = new Workspace.Y.Doc();
+    const map = doc.getMap('map');
+    for (let i = 0; i < 100; i++) {
+      map.set(`${i}`, i);
+    }
+    {
+      const provider = createIndexedDBProvider(id, doc, rootDBName);
+      provider.connect();
+      await provider.whenSynced;
+      provider.disconnect();
+    }
+    {
+      const newDoc = new Workspace.Y.Doc();
+      const provider = createIndexedDBProvider(id, newDoc, rootDBName);
+      provider.connect();
+      await provider.whenSynced;
+      provider.disconnect();
+      newDoc.getMap('map').forEach((value, key) => {
+        expect(value).toBe(parseInt(key));
+      });
+    }
+  });
 });
 
 describe('milestone', () => {

packages/y-indexeddb/src/index.ts

@@ -299,8 +299,16 @@ export const createIndexedDBProvider = (
         });
       } else {
         const updates = data.updates.map(({ update }) => update);
-        const update = mergeUpdates(updates);
+        const fakeDoc = new Doc();
-        const newUpdate = diffUpdate(encodeStateAsUpdate(doc), update);
+        fakeDoc.transact(() => {
+          updates.forEach(update => {
+            applyUpdate(fakeDoc, update);
+          });
+        }, indexeddbOrigin);
+        const newUpdate = diffUpdate(
+          encodeStateAsUpdate(doc),
+          encodeStateAsUpdate(fakeDoc)
+        );
         await store.put({
           ...data,
           updates: [

tests/libs/utils.ts

@@ -1,3 +1,6 @@
+import { ok } from 'node:assert';
+import { resolve } from 'node:path';
+
 import type { PageMeta } from '@blocksuite/store';
 import { faker } from '@faker-js/faker';
 import type { Page } from '@playwright/test';
@@ -7,6 +10,13 @@ const user1 = require('@affine-test/fixtures/built-in-user1.json');
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const user2 = require('@affine-test/fixtures/built-in-user2.json');
 
+export const rootDir = resolve(__dirname, '..', '..');
+// assert that the rootDir is the root of the project
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+ok(require(resolve(rootDir, 'package.json')).name.toLowerCase() === 'affine');
+
+export const testResultDir = resolve(rootDir, 'test-results');
+
 export async function getBuiltInUser() {
   return Promise.all([
     fetch('http://localhost:3000/api/user/token', {

tests/parallels/local-first-setting-page.spec.ts

@@ -1,9 +1,12 @@
+import { resolve } from 'node:path';
+
 import { expect } from '@playwright/test';
 
 import { openHomePage } from '../libs/load-page';
 import { waitMarkdownImported } from '../libs/page-logic';
 import { test } from '../libs/playwright';
 import { clickSideBarSettingButton } from '../libs/sidebar';
+import { testResultDir } from '../libs/utils';
 
 test.describe('Local first setting page', () => {
   test('Should highlight the setting page menu when selected', async ({
@@ -14,13 +17,20 @@ test.describe('Local first setting page', () => {
     const element = await page.getByTestId(
       'slider-bar-workspace-setting-button'
     );
-    const prevColor = await element.evaluate(
+    const prev = await element.screenshot({
-      element => window.getComputedStyle(element).color
+      path: resolve(
-    );
+        testResultDir,
+        'slider-bar-workspace-setting-button-prev.png'
+      ),
+    });
     await clickSideBarSettingButton(page);
-    const currentColor = await element.evaluate(
+    await page.waitForTimeout(50);
-      element => window.getComputedStyle(element).color
+    const after = await element.screenshot({
-    );
+      path: resolve(
-    expect(prevColor).not.toBe(currentColor);
+        testResultDir,
+        'slider-bar-workspace-setting-button-after.png'
+      ),
+    });
+    expect(prev).not.toEqual(after);
   });
 });

yarn.lock

@@ -6219,6 +6219,7 @@ __metadata:
     idb: ^7.1.1
     vite: ^4.2.1
     vite-plugin-dts: ^2.2.0
+    y-indexeddb: ^9.0.10
   peerDependencies:
     yjs: ^13.5.51
   languageName: unknown
@@ -14229,7 +14230,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"lib0@npm:^0.2.42, lib0@npm:^0.2.68, lib0@npm:^0.2.72, lib0@npm:^0.2.73":
+"lib0@npm:^0.2.35, lib0@npm:^0.2.42, lib0@npm:^0.2.68, lib0@npm:^0.2.72, lib0@npm:^0.2.73":
   version: 0.2.73
   resolution: "lib0@npm:0.2.73"
   dependencies:
@@ -20163,6 +20164,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"y-indexeddb@npm:^9.0.10":
+  version: 9.0.10
+  resolution: "y-indexeddb@npm:9.0.10"
+  dependencies:
+    lib0: ^0.2.35
+  peerDependencies:
+    yjs: ^13.0.0
+  checksum: 6a57825b599cdf77da7c9857b1acc0f782492fc41531618bd7392bdfbcf11c783ff1a30b82ae080b050a5ebafd54754a978de7a6ac42144ec59eb1fbdebd090b
+  languageName: node
+  linkType: hard
+
 "y-protocols@npm:^1.0.5":
   version: 1.0.5
   resolution: "y-protocols@npm:1.0.5"