feat(core): use zip snapshot for onboarding page (#6495)

packages/common/infra/src/initialization/index.ts

@@ -1,17 +1,4 @@
-import type { WorkspaceFlavour } from '@affine/env/workspace';
+import type { Doc } from '@blocksuite/store';
-import type {
-  CollectionInfoSnapshot,
-  Doc,
-  DocSnapshot,
-  JobMiddleware,
-} from '@blocksuite/store';
-import { Job } from '@blocksuite/store';
-import { Map as YMap } from 'yjs';
-
-import { getLatestVersions } from '../blocksuite/migration/blocksuite';
-import { PageRecordList } from '../page';
-import type { WorkspaceManager } from '../workspace';
-import { replaceIdMiddleware } from './middleware';
 
 export function initEmptyPage(page: Doc, title?: string) {
   page.load(() => {
@@ -38,108 +25,3 @@ export function initEmptyPage(page: Doc, title?: string) {
     );
   });
 }
-
-/**
- * FIXME: Use exported json data to instead of building data.
- */
-export async function buildShowcaseWorkspace(
-  workspaceManager: WorkspaceManager,
-  flavour: WorkspaceFlavour,
-  workspaceName: string
-) {
-  const meta = await workspaceManager.createWorkspace(
-    flavour,
-    async (docCollection, blobStorage) => {
-      docCollection.meta.setName(workspaceName);
-      const { onboarding } = await import('@affine/templates');
-
-      const info = onboarding['info.json'] as CollectionInfoSnapshot;
-      const blob = onboarding['blob.json'] as { [key: string]: string };
-
-      const migrationMiddleware: JobMiddleware = ({ slots, collection }) => {
-        slots.afterImport.on(payload => {
-          if (payload.type === 'page') {
-            collection.schema.upgradeDoc(
-              info?.pageVersion ?? 0,
-              {},
-              payload.page.spaceDoc
-            );
-          }
-        });
-      };
-
-      const job = new Job({
-        collection: docCollection,
-        middlewares: [replaceIdMiddleware, migrationMiddleware],
-      });
-
-      job.snapshotToCollectionInfo(info);
-
-      // for now all onboarding assets are considered served via CDN
-      // hack assets so that every blob exists
-      // @ts-expect-error - rethinking API
-      job._assetsManager.writeToBlob = async () => {};
-
-      const docSnapshots: DocSnapshot[] = Object.entries(onboarding)
-        .filter(([key]) => {
-          return key.endsWith('snapshot.json');
-        })
-        .map(([_, value]) => value as unknown as DocSnapshot);
-
-      await Promise.all(
-        docSnapshots.map(snapshot => {
-          return job.snapshotToDoc(snapshot);
-        })
-      );
-
-      const newVersions = getLatestVersions(docCollection.schema);
-      docCollection.doc
-        .getMap('meta')
-        .set('blockVersions', new YMap(Object.entries(newVersions)));
-
-      for (const [key, base64] of Object.entries(blob)) {
-        await blobStorage.set(key, new Blob([base64ToUint8Array(base64)]));
-      }
-    }
-  );
-
-  const { workspace, release } = workspaceManager.open(meta);
-
-  await workspace.engine.waitForRootDocReady();
-
-  const pageRecordList = workspace.services.get(PageRecordList);
-
-  // todo: find better way to do the following
-  // perhaps put them into middleware?
-  {
-    // the "Write, Draw, Plan all at Once." page should be set to edgeless mode
-    const edgelessPage1 = pageRecordList.records$.value.find(
-      p => p.title$.value === 'Write, Draw, Plan all at Once.'
-    );
-
-    if (edgelessPage1) {
-      edgelessPage1.setMode('edgeless');
-    }
-
-    // should jump to "Write, Draw, Plan all at Once." by default
-    const defaultPage = pageRecordList.records$.value.find(p =>
-      p.title$.value.startsWith('Write, Draw, Plan all at Once.')
-    );
-
-    if (defaultPage) {
-      defaultPage.setMeta({
-        jumpOnce: true,
-      });
-    }
-  }
-  release();
-  return meta;
-}
-
-function base64ToUint8Array(base64: string) {
-  const binaryString = atob(base64);
-  const binaryArray = binaryString.split('').map(function (char) {
-    return char.charCodeAt(0);
-  });
-  return new Uint8Array(binaryArray);
-}

packages/common/infra/src/initialization/middleware.ts

@@ -1,142 +0,0 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment */
-/* eslint-disable @typescript-eslint/no-restricted-imports */
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
-// @ts-nocheck
-// TODO: remove this file after blocksuite exposed it
-import type {
-  DatabaseBlockModel,
-  ListBlockModel,
-  ParagraphBlockModel,
-} from '@blocksuite/blocks/dist/models.js';
-import { assertExists } from '@blocksuite/global/utils';
-import type { DeltaOperation, JobMiddleware } from '@blocksuite/store';
-
-export const replaceIdMiddleware: JobMiddleware = ({ slots, collection }) => {
-  const idMap = new Map<string, string>();
-  slots.afterImport.on(payload => {
-    if (
-      payload.type === 'block' &&
-      payload.snapshot.flavour === 'affine:database'
-    ) {
-      const model = payload.model as DatabaseBlockModel;
-      Object.keys(model.cells).forEach(cellId => {
-        if (idMap.has(cellId)) {
-          model.cells[idMap.get(cellId)!] = model.cells[cellId];
-          delete model.cells[cellId];
-        }
-      });
-    }
-
-    // replace LinkedPage pageId with new id in paragraph blocks
-    if (
-      payload.type === 'block' &&
-      ['affine:paragraph', 'affine:list'].includes(payload.snapshot.flavour)
-    ) {
-      const model = payload.model as ParagraphBlockModel | ListBlockModel;
-      let prev = 0;
-      const delta: DeltaOperation[] = [];
-      for (const d of model.text.toDelta()) {
-        if (d.attributes?.reference?.pageId) {
-          if (prev > 0) {
-            delta.push({ retain: prev });
-          }
-          delta.push({
-            retain: d.insert.length,
-            attributes: {
-              reference: {
-                ...d.attributes.reference,
-                pageId: idMap.get(d.attributes.reference.pageId)!,
-              },
-            },
-          });
-          prev = 0;
-        } else {
-          prev += d.insert.length;
-        }
-      }
-      if (delta.length > 0) {
-        model.text.applyDelta(delta);
-      }
-    }
-  });
-  slots.beforeImport.on(payload => {
-    if (payload.type === 'page') {
-      const newId = collection.idGenerator('page');
-      idMap.set(payload.snapshot.meta.id, newId);
-      payload.snapshot.meta.id = newId;
-      return;
-    }
-
-    if (payload.type === 'block') {
-      const { snapshot } = payload;
-      if (snapshot.flavour === 'affine:page') {
-        const index = snapshot.children.findIndex(
-          c => c.flavour === 'affine:surface'
-        );
-        if (index !== -1) {
-          const [surface] = snapshot.children.splice(index, 1);
-          snapshot.children.push(surface);
-        }
-      }
-
-      const original = snapshot.id;
-      let newId: string;
-      if (idMap.has(original)) {
-        newId = idMap.get(original)!;
-      } else {
-        newId = collection.idGenerator('block');
-        idMap.set(original, newId);
-      }
-      snapshot.id = newId;
-
-      if (snapshot.flavour === 'affine:surface') {
-        // Generate new IDs for images and frames in advance.
-        snapshot.children.forEach(child => {
-          const original = child.id;
-          if (idMap.has(original)) {
-            newId = idMap.get(original)!;
-          } else {
-            newId = collection.idGenerator('block');
-            idMap.set(original, newId);
-          }
-        });
-
-        Object.entries(
-          snapshot.props.elements as Record<string, Record<string, unknown>>
-        ).forEach(([_, value]) => {
-          switch (value.type) {
-            case 'connector': {
-              let connection = value.source as Record<string, string>;
-              if (idMap.has(connection.id)) {
-                const newId = idMap.get(connection.id);
-                assertExists(newId, 'reference id must exist');
-                connection.id = newId;
-              }
-              connection = value.target as Record<string, string>;
-              if (idMap.has(connection.id)) {
-                const newId = idMap.get(connection.id);
-                assertExists(newId, 'reference id must exist');
-                connection.id = newId;
-              }
-              break;
-            }
-            case 'group': {
-              const json = value.children.json as Record<string, unknown>;
-              Object.entries(json).forEach(([key, value]) => {
-                if (idMap.has(key)) {
-                  delete json[key];
-                  const newKey = idMap.get(key);
-                  assertExists(newKey, 'reference id must exist');
-                  json[newKey] = value;
-                }
-              });
-              break;
-            }
-            default:
-              break;
-          }
-        });
-      }
-    }
-  });
-};

packages/common/infra/src/workspace/testing.ts

@@ -57,6 +57,13 @@ export class TestingLocalWorkspaceListProvider
       id: id,
       idGenerator: () => nanoid(),
       schema: globalBlockSuiteSchema,
+      blobStorages: [
+        () => {
+          return {
+            crud: blobStorage,
+          };
+        },
+      ],
     });
 
     // apply initial state

packages/frontend/core/src/bootstrap/first-app-data.ts

@@ -1,8 +1,61 @@
 import { DebugLogger } from '@affine/debug';
 import { DEFAULT_WORKSPACE_NAME } from '@affine/env/constant';
 import { WorkspaceFlavour } from '@affine/env/workspace';
-import type { WorkspaceManager } from '@toeverything/infra';
+import onboardingUrl from '@affine/templates/onboarding.zip';
-import { buildShowcaseWorkspace, initEmptyPage } from '@toeverything/infra';
+import { ZipTransformer } from '@blocksuite/blocks';
+import {
+  initEmptyPage,
+  PageRecordList,
+  type WorkspaceManager,
+} from '@toeverything/infra';
+
+export async function buildShowcaseWorkspace(
+  workspaceManager: WorkspaceManager,
+  flavour: WorkspaceFlavour,
+  workspaceName: string
+) {
+  const meta = await workspaceManager.createWorkspace(
+    flavour,
+    async docCollection => {
+      docCollection.meta.setName(workspaceName);
+      const blob = await (await fetch(onboardingUrl)).blob();
+
+      await ZipTransformer.importDocs(docCollection, blob);
+    }
+  );
+
+  const { workspace, release } = workspaceManager.open(meta);
+
+  await workspace.engine.waitForRootDocReady();
+
+  const pageRecordList = workspace.services.get(PageRecordList);
+
+  // todo: find better way to do the following
+  // perhaps put them into middleware?
+  {
+    // the "Write, Draw, Plan all at Once." page should be set to edgeless mode
+    const edgelessPage1 = pageRecordList.records$.value.find(
+      p => p.title$.value === 'Write, Draw, Plan all at Once.'
+    );
+
+    if (edgelessPage1) {
+      edgelessPage1.setMode('edgeless');
+    }
+
+    // should jump to "Write, Draw, Plan all at Once." by default
+    const defaultPage = pageRecordList.records$.value.find(p =>
+      p.title$.value.startsWith('Write, Draw, Plan all at Once.')
+    );
+
+    if (defaultPage) {
+      defaultPage.setMeta({
+        jumpOnce: true,
+      });
+    }
+  }
+  release();
+  return meta;
+}
 
 const logger = new DebugLogger('createFirstAppData');
 

packages/frontend/core/src/components/affine/create-workspace-modal/index.tsx

@@ -10,7 +10,6 @@ import { WorkspaceFlavour } from '@affine/env/workspace';
 import { useAFFiNEI18N } from '@affine/i18n/hooks';
 import { _addLocalWorkspace } from '@affine/workspace-impl';
 import {
-  buildShowcaseWorkspace,
   initEmptyPage,
   useService,
   WorkspaceManager,
@@ -19,6 +18,7 @@ import { useSetAtom } from 'jotai';
 import type { KeyboardEvent } from 'react';
 import { useCallback, useLayoutEffect, useState } from 'react';
 
+import { buildShowcaseWorkspace } from '../../../bootstrap/first-app-data';
 import { mixpanel } from '../../../utils';
 import { CloudSvg } from '../share-page-modal/cloud-svg';
 import * as styles from './index.css';

packages/frontend/core/src/pages/workspace/index.tsx

@@ -1,4 +1,5 @@
 import { useWorkspace } from '@affine/core/hooks/use-workspace';
+import { ZipTransformer } from '@blocksuite/blocks';
 import type { Workspace } from '@toeverything/infra';
 import {
   ServiceProviderContext,
@@ -28,6 +29,8 @@ declare global {
    */
   // eslint-disable-next-line no-var
   var currentWorkspace: Workspace | undefined;
+  // eslint-disable-next-line no-var
+  var exportWorkspaceSnapshot: () => Promise<void>;
   interface WindowEventMap {
     'affine:workspace:change': CustomEvent<{ id: string }>;
   }
@@ -60,6 +63,19 @@ export const Component = (): ReactElement => {
 
     // for debug purpose
     window.currentWorkspace = workspace;
+    window.exportWorkspaceSnapshot = async () => {
+      const zip = await ZipTransformer.exportDocs(
+        workspace.docCollection,
+        Array.from(workspace.docCollection.docs.values())
+      );
+      const url = URL.createObjectURL(zip);
+      // download url
+      const a = document.createElement('a');
+      a.href = url;
+      a.download = `${workspace.docCollection.meta.name}.zip`;
+      a.click();
+      URL.revokeObjectURL(url);
+    };
     window.dispatchEvent(
       new CustomEvent('affine:workspace:change', {
         detail: {

packages/frontend/core/src/types/types.d.ts

@@ -14,6 +14,11 @@ declare module '*.assets.svg' {
   export default url;
 }
 
+declare module '*.zip' {
+  const url: string;
+  export default url;
+}
+
 declare module '*.png' {
   const url: string;
   export default url;

packages/frontend/templates/build.mjs

@@ -1,32 +0,0 @@
-import fs from 'node:fs';
-import path, { join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-import * as glob from 'glob';
-
-// purpose: bundle all json files into one json file in onboarding folder
-const __dirname = join(fileURLToPath(import.meta.url), '..');
-
-const jsonFiles = glob.sync('./*.json', {
-  cwd: path.join(__dirname, 'onboarding'),
-});
-
-const imports = jsonFiles
-  .map(
-    (fileName, index) => `import json_${index} from './onboarding/${fileName}';`
-  )
-  .join('\n');
-
-const exports = `export const onboarding = {
-${jsonFiles
-  .map((fileName, index) => {
-    return `  '${fileName}': json_${index}`;
-  })
-  .join(',\n')}
-}`;
-
-const template = `/* eslint-disable simple-import-sort/imports */
-// Auto generated, do not edit manually
-${imports}\n\n${exports}`;
-
-fs.writeFileSync(path.join(__dirname, 'templates.gen.ts'), template);

packages/frontend/templates/onboarding/info.json

@@ -1,21 +0,0 @@
-{
-  "type": "info",
-  "id": "dc61f2e3-a973-432f-a463-164a15cfc778",
-  "pageVersion": 2,
-  "workspaceVersion": 2,
-  "properties": {
-    "tags": {
-      "options": []
-    }
-  },
-  "pages": [
-    {
-      "id": "W-d9_llZ6rE-qoTiHKTk4",
-      "createDate": 1706862386590,
-      "tags": [],
-      "favorite": false,
-      "title": "Write, Draw, Plan all at Once.",
-      "updatedDate": 1709110332309
-    }
-  ]
-}

packages/frontend/templates/package.json

@@ -4,11 +4,11 @@
   "sideEffect": false,
   "version": "0.14.0",
   "scripts": {
-    "postinstall": "node ./build.mjs && node ./build-edgeless.mjs"
+    "postinstall": "node ./build-edgeless.mjs"
   },
   "type": "module",
   "exports": {
-    ".": "./templates.gen.ts",
+    "./onboarding.zip": "./onboarding/onboarding.zip",
     "./edgeless": "./edgeless-templates.gen.ts",
     "./build-edgeless": "./build-edgeless.mjs"
   },

packages/frontend/templates/templates.gen.ts

@@ -1,11 +0,0 @@
-/* eslint-disable simple-import-sort/imports */
-// Auto generated, do not edit manually
-import json_0 from './onboarding/info.json';
-import json_1 from './onboarding/blob.json';
-import json_2 from './onboarding/W-d9_llZ6rE-qoTiHKTk4.snapshot.json';
-
-export const onboarding = {
-  'info.json': json_0,
-  'blob.json': json_1,
-  'W-d9_llZ6rE-qoTiHKTk4.snapshot.json': json_2
-}

packages/frontend/workspace-impl/src/cloud/list.ts

@@ -76,12 +76,6 @@ export class CloudWorkspaceListProvider implements WorkspaceListProvider {
   ): Promise<WorkspaceMetadata> {
     const tempId = nanoid();
 
-    const docCollection = new DocCollection({
-      id: tempId,
-      idGenerator: () => nanoid(),
-      schema: globalBlockSuiteSchema,
-    });
-
     // create workspace on cloud, get workspace id
     const {
       createWorkspace: { id: workspaceId },
@@ -97,6 +91,19 @@ export class CloudWorkspaceListProvider implements WorkspaceListProvider {
       ? new SqliteDocStorage(workspaceId)
       : new IndexedDBDocStorage(workspaceId);
 
+    const docCollection = new DocCollection({
+      id: tempId,
+      idGenerator: () => nanoid(),
+      schema: globalBlockSuiteSchema,
+      blobStorages: [
+        () => {
+          return {
+            crud: blobStorage,
+          };
+        },
+      ],
+    });
+
     // apply initial state
     await initial(docCollection, blobStorage);
 

packages/frontend/workspace-impl/src/local/list.ts

@@ -70,6 +70,13 @@ export class LocalWorkspaceListProvider implements WorkspaceListProvider {
       id: id,
       idGenerator: () => nanoid(),
       schema: globalBlockSuiteSchema,
+      blobStorages: [
+        () => {
+          return {
+            crud: blobStorage,
+          };
+        },
+      ],
     });
 
     // apply initial state

tools/cli/src/webpack/config.ts

@@ -280,7 +280,7 @@ export const createConfiguration: (
               },
             },
             {
-              test: /\.(png|jpg|gif|svg|webp|mp4)$/,
+              test: /\.(png|jpg|gif|svg|webp|mp4|zip)$/,
               type: 'asset/resource',
             },
             {