feat: add subdoc migration script (#2820)

Co-authored-by: himself65 <himself65@outlook.com>

packages/env/src/__tests__/subdoc-migration.spec.ts

@@ -0,0 +1,76 @@
+import { readFileSync } from 'fs';
+import { dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
+import { describe, expect, test } from 'vitest';
+import * as Y from 'yjs';
+
+import { migrateToSubdoc } from '../blocksuite';
+
+const fixturePath = resolve(
+  dirname(fileURLToPath(import.meta.url)),
+  'workspace.ydoc'
+);
+const yDocBuffer = readFileSync(fixturePath);
+const doc = new Y.Doc();
+Y.applyUpdate(doc, new Uint8Array(yDocBuffer));
+const migratedDoc = migrateToSubdoc(doc);
+
+describe('subdoc', () => {
+  test('Migration to subdoc', async () => {
+    const { default: json } = await import('@affine-test/fixtures/output.json');
+    const length = Object.keys(json).length;
+    const binary = new Uint8Array(length);
+    for (let i = 0; i < length; i++) {
+      binary[i] = (json as any)[i];
+    }
+    const doc = new Y.Doc();
+    Y.applyUpdate(doc, binary);
+    {
+      // invoke data
+      doc.getMap('space:hello-world');
+      doc.getMap('space:meta');
+    }
+    const blocks = doc.getMap('space:hello-world').toJSON();
+    const newDoc = migrateToSubdoc(doc);
+    const subDoc = newDoc.getMap('spaces').get('space:hello-world') as Y.Doc;
+    const data = (subDoc.toJSON() as any).blocks;
+    Object.keys(data).forEach(id => {
+      if (id === 'xyWNqindHH') {
+        return;
+      }
+      expect(data[id]).toEqual(blocks[id]);
+    });
+  });
+
+  test('Test fixture should be set correctly', () => {
+    const meta = doc.getMap('space:meta');
+    const versions = meta.get('versions') as Y.Map<unknown>;
+    expect(versions.get('affine:code')).toBeTypeOf('number');
+  });
+
+  test('Meta data should be migrated correctly', () => {
+    const originalMeta = doc.getMap('space:meta');
+    const originalVersions = originalMeta.get('versions') as Y.Map<unknown>;
+
+    const meta = migratedDoc.getMap('meta');
+    const blockVersions = meta.get('blockVersions') as Y.Map<unknown>;
+
+    expect(meta.get('workspaceVersion')).toBe(1);
+    expect(blockVersions.get('affine:code')).toBe(
+      originalVersions.get('affine:code')
+    );
+    expect((meta.get('pages') as Y.Array<unknown>).length).toBe(
+      (originalMeta.get('pages') as Y.Array<unknown>).length
+    );
+
+    expect(blockVersions.get('affine:embed')).toBeUndefined();
+    expect(blockVersions.get('affine:image')).toBe(
+      originalVersions.get('affine:embed')
+    );
+
+    expect(blockVersions.get('affine:frame')).toBeUndefined();
+    expect(blockVersions.get('affine:note')).toBe(
+      originalVersions.get('affine:frame')
+    );
+  });
+});

packages/env/src/blocksuite/index.ts

@@ -14,3 +14,5 @@ export function initEmptyPage(page: Page): void {
   const frameId = page.addBlock('affine:frame', {}, pageBlockId);
   page.addBlock('affine:paragraph', {}, frameId);
 }
+
+export * from './subdoc-migration';

packages/env/src/blocksuite/subdoc-migration.ts

@@ -0,0 +1,94 @@
+import * as Y from 'yjs';
+
+function runBlockMigration(
+  flavour: string,
+  data: Y.Map<unknown>,
+  version: number
+) {
+  if (flavour === 'affine:frame') {
+    data.set('sys:flavour', 'affine:note');
+    return;
+  }
+  if (flavour === 'affine:surface' && version <= 3 && data.has('elements')) {
+    const elements = data.get('elements') as Y.Map<unknown>;
+    data.set('prop:elements', elements.clone());
+    data.delete('elements');
+  }
+  if (flavour === 'affine:embed') {
+    data.set('sys:flavour', 'affine:image');
+    data.delete('prop:type');
+  }
+}
+
+function updateBlockVersions(versions: Y.Map<number>) {
+  const frameVersion = versions.get('affine:frame');
+  if (frameVersion !== undefined) {
+    versions.set('affine:note', frameVersion);
+    versions.delete('affine:frame');
+  }
+  const embedVersion = versions.get('affine:embed');
+  if (embedVersion !== undefined) {
+    versions.set('affine:image', embedVersion);
+    versions.delete('affine:embed');
+  }
+}
+
+function migrateMeta(oldDoc: Y.Doc, newDoc: Y.Doc) {
+  const originalMeta = oldDoc.getMap('space:meta');
+  const originalVersions = originalMeta.get('versions') as Y.Map<number>;
+  const originalPages = originalMeta.get('pages') as Y.Array<Y.Map<unknown>>;
+  const meta = newDoc.getMap('meta');
+  const pages = new Y.Array();
+  const blockVersions = originalVersions.clone();
+
+  meta.set('workspaceVersion', 1);
+  meta.set('blockVersions', blockVersions);
+  meta.set('pages', pages);
+
+  updateBlockVersions(blockVersions);
+  const mapList = originalPages.map(page => {
+    const map = new Y.Map();
+    Array.from(page.entries())
+      .filter(([key]) => key !== 'subpageIds')
+      .forEach(([key, value]) => {
+        map.set(key, value);
+      });
+    return map;
+  });
+  pages.push(mapList);
+}
+
+function migrateBlocks(oldDoc: Y.Doc, newDoc: Y.Doc) {
+  const spaces = newDoc.getMap('spaces');
+  const originalMeta = oldDoc.getMap('space:meta');
+  const originalVersions = originalMeta.get('versions') as Y.Map<number>;
+  const originalPages = originalMeta.get('pages') as Y.Array<Y.Map<unknown>>;
+  originalPages.forEach(page => {
+    const id = page.get('id') as string;
+    const spaceId = id.startsWith('space:') ? id : `space:${id}`;
+    const originalBlocks = oldDoc.getMap(spaceId) as Y.Map<unknown>;
+    const subdoc = new Y.Doc();
+    spaces.set(spaceId, subdoc);
+    const blocks = subdoc.getMap('blocks');
+    Array.from(originalBlocks.entries()).forEach(([key, value]) => {
+      const blockData = value.clone();
+      blocks.set(key, blockData);
+      const flavour = blockData.get('sys:flavour') as string;
+      const version = originalVersions.get(flavour);
+      if (version !== undefined) {
+        runBlockMigration(flavour, blockData, version);
+      }
+    });
+  });
+}
+
+export function migrateToSubdoc(doc: Y.Doc): Y.Doc {
+  const needMigration = Array.from(doc.getMap('space:meta').keys()).length > 0;
+  if (!needMigration) {
+    return doc;
+  }
+  const output = new Y.Doc();
+  migrateMeta(doc, output);
+  migrateBlocks(doc, output);
+  return output;
+}

packages/env/tsconfig.json

@@ -9,6 +9,9 @@
   "references": [
     {
       "path": "../infra"
+    },
+    {
+      "path": "../../tests/fixtures"
     }
   ]
 }