feat: sqlite subdocument (#2816)

Co-authored-by: Alex Yang <himself65@outlook.com>

packages/component/src/components/image-preview-modal/hooks/use-zoom.tsx

@@ -181,7 +181,6 @@ export const useZoomControls = ({
     }
   }, [imageRef]);
 
-
   useEffect(() => {
     const handleScroll = (event: WheelEvent) => {
       const { deltaY } = event;

packages/component/src/components/image-preview-modal/index.css.ts

@@ -62,7 +62,6 @@ export const imagePreviewModalCloseButtonStyle = style({
   zIndex: 1,
   marginTop: '38px',
   marginRight: '38px',
-
 });
 
 export const imagePreviewModalGoStyle = style({

packages/component/src/components/page-list/all-page.tsx

@@ -5,7 +5,14 @@ import { useMediaQuery, useTheme } from '@mui/material';
 import type React from 'react';
 import { type CSSProperties } from 'react';
 
-import { ScrollableContainer, Table, TableBody, TableCell, TableHead, TableHeadRow } from '../..';
+import {
+  ScrollableContainer,
+  Table,
+  TableBody,
+  TableCell,
+  TableHead,
+  TableHeadRow,
+} from '../..';
 import { TableBodyRow } from '../../ui/table';
 import { useHasScrollTop } from '../app-sidebar/sidebar-containers/use-has-scroll-top';
 import { AllPagesBody } from './all-pages-body';
@@ -141,12 +148,9 @@ export const PageList = ({
       ? DEFAULT_SORT_KEY
       : undefined;
 
-  return (
-    sorter.data.length === 0 && fallback ?
-    <StyledTableContainer>
-     {fallback}
-    </StyledTableContainer>
-   :
+  return sorter.data.length === 0 && fallback ? (
+    <StyledTableContainer>{fallback}</StyledTableContainer>
+  ) : (
     <ScrollableContainer inTableView>
       <StyledTableContainer ref={ref}>
         <Table showBorder={hasScrollTop} style={{ maxHeight: '100%' }}>
@@ -158,7 +162,6 @@ export const PageList = ({
             importFile={onImportFile}
           />
           <AllPagesBody
-            
             isPublicWorkspace={isPublicWorkspace}
             groupKey={groupKey}
             data={sorter.data}
@@ -246,12 +249,9 @@ export const PageListTrashView: React.FC<{
     }
   );
 
-  return (
-    list.length === 0 && fallback ?
-    <StyledTableContainer>
-    {fallback}
-    </StyledTableContainer>
-  :
+  return list.length === 0 && fallback ? (
+    <StyledTableContainer>{fallback}</StyledTableContainer>
+  ) : (
     <ScrollableContainer inTableView>
       <StyledTableContainer ref={ref}>
         <Table showBorder={hasScrollTop}>

packages/component/src/ui/scrollbar/index.ts

@@ -1 +1 @@
-export * from './scrollbar';
+export * from './scrollbar';

packages/component/src/ui/scrollbar/scrollbar.tsx

@@ -15,29 +15,27 @@ export const ScrollableContainer = ({
   showScrollTopBorder = false,
   inTableView = false,
 }: PropsWithChildren<ScrollableContainerProps>) => {
-    const [hasScrollTop, ref] = useHasScrollTop();
-    return (
-      <ScrollArea.Root className={styles.scrollableContainerRoot}>
-        <div
-          data-has-scroll-top={hasScrollTop}
-          className={clsx({[styles.scrollTopBorder]:showScrollTopBorder})}
-        />
-        <ScrollArea.Viewport
-          className={clsx([styles.scrollableViewport])}
-          ref={ref}
-        >
-          <div className={styles.scrollableContainer}>
-          {children}
-          </div>
-         
-        </ScrollArea.Viewport>
-        <ScrollArea.Scrollbar
-          
-          orientation="vertical"
-          className={clsx(styles.scrollbar,{[styles.TableScrollbar]:inTableView})}
-        >
-          <ScrollArea.Thumb className={styles.scrollbarThumb} />
-        </ScrollArea.Scrollbar>
-      </ScrollArea.Root>
-    );
-  }
+  const [hasScrollTop, ref] = useHasScrollTop();
+  return (
+    <ScrollArea.Root className={styles.scrollableContainerRoot}>
+      <div
+        data-has-scroll-top={hasScrollTop}
+        className={clsx({ [styles.scrollTopBorder]: showScrollTopBorder })}
+      />
+      <ScrollArea.Viewport
+        className={clsx([styles.scrollableViewport])}
+        ref={ref}
+      >
+        <div className={styles.scrollableContainer}>{children}</div>
+      </ScrollArea.Viewport>
+      <ScrollArea.Scrollbar
+        orientation="vertical"
+        className={clsx(styles.scrollbar, {
+          [styles.TableScrollbar]: inTableView,
+        })}
+      >
+        <ScrollArea.Thumb className={styles.scrollbarThumb} />
+      </ScrollArea.Scrollbar>
+    </ScrollArea.Root>
+  );
+};

packages/hooks/src/__tests__/index.spec.ts

@@ -25,7 +25,7 @@ beforeEach(async () => {
     .register(AffineSchemas)
     .register(__unstableSchemas);
   const initPage = async (page: Page) => {
-    await page.waitForLoaded()
+    await page.waitForLoaded();
     expect(page).not.toBeNull();
     assertExists(page);
     const pageBlockId = page.addBlock('affine:page', {

packages/hooks/src/use-block-suite-workspace-page.ts

@@ -51,5 +51,6 @@ export function useBlockSuiteWorkspacePage(
 ): Page | null {
   const pageAtom = getAtom(blockSuiteWorkspace, pageId);
   assertExists(pageAtom);
-  return useAtomValue(pageAtom);
+  const page = useAtomValue(pageAtom);
+  return page;
 }

packages/infra/src/handler.ts

@@ -17,8 +17,15 @@ export abstract class HandlerManager<
 }
 
 type DBHandlers = {
-  getDocAsUpdates: (id: string) => Promise<Uint8Array>;
-  applyDocUpdate: (id: string, update: Uint8Array) => Promise<void>;
+  getDocAsUpdates: (
+    workspaceId: string,
+    subdocId?: string
+  ) => Promise<Uint8Array>;
+  applyDocUpdate: (
+    id: string,
+    update: Uint8Array,
+    subdocId?: string
+  ) => Promise<void>;
   addBlob: (
     workspaceId: string,
     key: string,

packages/native/build.rs

@@ -1,12 +1,22 @@
 use sqlx::sqlite::SqliteConnectOptions;
+use std::fs;
 
 #[tokio::main]
 async fn main() -> Result<(), std::io::Error> {
   dotenv::dotenv().ok();
 
+  // always start with a fresh database to have
+  // latest db schema
+  let db_path = "../../affine.db";
+
+  // check if db exists and then remove file
+  if fs::metadata(db_path).is_ok() {
+    fs::remove_file(db_path)?;
+  }
+
   napi_build::setup();
   let options = SqliteConnectOptions::new()
-    .filename("../../affine.db")
+    .filename(db_path)
     .journal_mode(sqlx::sqlite::SqliteJournalMode::Off)
     .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive)
     .create_if_missing(true);

packages/native/index.d.ts

@@ -32,6 +32,11 @@ export interface UpdateRow {
   id: number;
   timestamp: Date;
   data: Buffer;
+  docId?: string;
+}
+export interface InsertRow {
+  docId?: string;
+  data: Uint8Array;
 }
 export class Subscription {
   toString(): string;
@@ -56,8 +61,14 @@ export class SqliteConnection {
   getBlob(key: string): Promise<BlobRow | null>;
   deleteBlob(key: string): Promise<void>;
   getBlobKeys(): Promise<Array<string>>;
-  getUpdates(): Promise<Array<UpdateRow>>;
-  insertUpdates(updates: Array<Uint8Array>): Promise<void>;
+  getUpdates(docId?: string | undefined | null): Promise<Array<UpdateRow>>;
+  getUpdatesCount(docId?: string | undefined | null): Promise<number>;
+  getAllUpdates(): Promise<Array<UpdateRow>>;
+  insertUpdates(updates: Array<InsertRow>): Promise<void>;
+  replaceUpdates(
+    docId: string | undefined | null,
+    updates: Array<InsertRow>
+  ): Promise<void>;
   close(): Promise<void>;
   get isClose(): boolean;
   static validate(path: string): Promise<boolean>;

packages/native/schema/src/lib.rs

@@ -4,7 +4,8 @@
 pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   data BLOB NOT NULL,
-  timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
+  timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
+  doc_id TEXT
 );
 CREATE TABLE IF NOT EXISTS "blobs" (
   key TEXT PRIMARY KEY NOT NULL,

packages/native/src/sqlite/mod.rs

@@ -19,6 +19,13 @@ pub struct UpdateRow {
   pub id: i64,
   pub timestamp: NaiveDateTime,
   pub data: Buffer,
+  pub doc_id: Option<String>,
+}
+
+#[napi(object)]
+pub struct InsertRow {
+  pub doc_id: Option<String>,
+  pub data: Uint8Array,
 }
 
 #[napi]
@@ -53,6 +60,7 @@ impl SqliteConnection {
       .execute(connection.as_mut())
       .await
       .map_err(anyhow::Error::from)?;
+    self.migrate_add_doc_id().await?;
     connection.detach();
     Ok(())
   }
@@ -74,10 +82,14 @@ impl SqliteConnection {
 
   #[napi]
   pub async fn get_blob(&self, key: String) -> Option<BlobRow> {
-    sqlx::query_as!(BlobRow, "SELECT * FROM blobs WHERE key = ?", key)
-      .fetch_one(&self.pool)
-      .await
-      .ok()
+    sqlx::query_as!(
+      BlobRow,
+      "SELECT key, data, timestamp FROM blobs WHERE key = ?",
+      key
+    )
+    .fetch_one(&self.pool)
+    .await
+    .ok()
   }
 
   #[napi]
@@ -100,8 +112,54 @@ impl SqliteConnection {
   }
 
   #[napi]
-  pub async fn get_updates(&self) -> napi::Result<Vec<UpdateRow>> {
-    let updates = sqlx::query_as!(UpdateRow, "SELECT * FROM updates")
+  pub async fn get_updates(&self, doc_id: Option<String>) -> napi::Result<Vec<UpdateRow>> {
+    let updates = match doc_id {
+      Some(doc_id) => sqlx::query_as!(
+        UpdateRow,
+        "SELECT id, timestamp, data, doc_id FROM updates WHERE doc_id = ?",
+        doc_id
+      )
+      .fetch_all(&self.pool)
+      .await
+      .map_err(anyhow::Error::from)?,
+      None => sqlx::query_as!(
+        UpdateRow,
+        "SELECT id, timestamp, data, doc_id FROM updates WHERE doc_id is NULL",
+      )
+      .fetch_all(&self.pool)
+      .await
+      .map_err(anyhow::Error::from)?,
+    };
+    Ok(updates)
+  }
+
+  #[napi]
+  pub async fn get_updates_count(&self, doc_id: Option<String>) -> napi::Result<i32> {
+    let count = match doc_id {
+      Some(doc_id) => {
+        sqlx::query!(
+          "SELECT COUNT(*) as count FROM updates WHERE doc_id = ?",
+          doc_id
+        )
+        .fetch_one(&self.pool)
+        .await
+        .map_err(anyhow::Error::from)?
+        .count
+      }
+      None => {
+        sqlx::query!("SELECT COUNT(*) as count FROM updates WHERE doc_id is NULL")
+          .fetch_one(&self.pool)
+          .await
+          .map_err(anyhow::Error::from)?
+          .count
+      }
+    };
+    Ok(count)
+  }
+
+  #[napi]
+  pub async fn get_all_updates(&self) -> napi::Result<Vec<UpdateRow>> {
+    let updates = sqlx::query_as!(UpdateRow, "SELECT id, timestamp, data, doc_id FROM updates")
       .fetch_all(&self.pool)
       .await
       .map_err(anyhow::Error::from)?;
@@ -109,14 +167,54 @@ impl SqliteConnection {
   }
 
   #[napi]
-  pub async fn insert_updates(&self, updates: Vec<Uint8Array>) -> napi::Result<()> {
+  pub async fn insert_updates(&self, updates: Vec<InsertRow>) -> napi::Result<()> {
     let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
-    for update in updates.into_iter() {
-      let update = update.as_ref();
-      sqlx::query_as!(UpdateRow, "INSERT INTO updates (data) VALUES ($1)", update)
+    for InsertRow { data, doc_id } in updates {
+      let update = data.as_ref();
+      sqlx::query_as!(
+        UpdateRow,
+        "INSERT INTO updates (data, doc_id) VALUES ($1, $2)",
+        update,
+        doc_id
+      )
+      .execute(&mut *transaction)
+      .await
+      .map_err(anyhow::Error::from)?;
+    }
+    transaction.commit().await.map_err(anyhow::Error::from)?;
+    Ok(())
+  }
+
+  #[napi]
+  pub async fn replace_updates(
+    &self,
+    doc_id: Option<String>,
+    updates: Vec<InsertRow>,
+  ) -> napi::Result<()> {
+    let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
+
+    match doc_id {
+      Some(doc_id) => sqlx::query!("DELETE FROM updates where doc_id = ?", doc_id)
         .execute(&mut *transaction)
         .await
-        .map_err(anyhow::Error::from)?;
+        .map_err(anyhow::Error::from)?,
+      None => sqlx::query!("DELETE FROM updates where doc_id is NULL",)
+        .execute(&mut *transaction)
+        .await
+        .map_err(anyhow::Error::from)?,
+    };
+
+    for InsertRow { data, doc_id } in updates {
+      let update = data.as_ref();
+      sqlx::query_as!(
+        UpdateRow,
+        "INSERT INTO updates (data, doc_id) VALUES ($1, $2)",
+        update,
+        doc_id
+      )
+      .execute(&mut *transaction)
+      .await
+      .map_err(anyhow::Error::from)?;
     }
     transaction.commit().await.map_err(anyhow::Error::from)?;
     Ok(())
@@ -158,4 +256,22 @@ impl SqliteConnection {
       false
     }
   }
+
+  // todo: have a better way to handle migration
+  async fn migrate_add_doc_id(&self) -> Result<(), anyhow::Error> {
+    // ignore errors
+    match sqlx::query("ALTER TABLE updates ADD COLUMN doc_id TEXT")
+      .execute(&self.pool)
+      .await
+    {
+      Ok(_) => Ok(()),
+      Err(err) => {
+        if err.to_string().contains("duplicate column name") {
+          Ok(()) // Ignore error if it's due to duplicate column
+        } else {
+          Err(anyhow::Error::from(err)) // Propagate other errors
+        }
+      }
+    }
+  }
 }

packages/workspace/src/affine/__tests__/sync.spec.ts

@@ -140,7 +140,7 @@ describe('ydoc sync', () => {
 
       const pageId = uuidv4();
       const page1 = workspace1.createPage({ id: pageId });
-      await page1.waitForLoaded()
+      await page1.waitForLoaded();
       const pageBlockId = page1.addBlock('affine:page', {
         title: new page1.Text(''),
       });
@@ -153,7 +153,7 @@ describe('ydoc sync', () => {
         workspace1.doc.getMap(`space:${pageId}`).toJSON()
       );
       const page2 = workspace2.getPage(pageId) as Page;
-      await page2.waitForLoaded()
+      await page2.waitForLoaded();
       page1.updateBlock(
         page1.getBlockById(paragraphId) as ParagraphBlockModel,
         {

packages/workspace/src/providers/__tests__/sqlite-provider.spec.ts

@@ -7,7 +7,10 @@ import type { Y as YType } from '@blocksuite/store';
 import { uuidv4, Workspace } from '@blocksuite/store';
 import { beforeEach, describe, expect, test, vi } from 'vitest';
 
-import { createSQLiteDBDownloadProvider, createSQLiteProvider } from '../index';
+import {
+  createSQLiteDBDownloadProvider,
+  createSQLiteProvider,
+} from '../sqlite-providers';
 
 const Y = Workspace.Y;
 
@@ -148,15 +151,21 @@ describe('SQLite download provider', () => {
   test('disconnect handlers', async () => {
     const offHandler = vi.fn();
     let handleUpdate = () => {};
-    workspace.doc.on = (_: string, fn: () => void) => {
-      handleUpdate = fn;
+    let handleSubdocs = () => {};
+    workspace.doc.on = (event: string, fn: () => void) => {
+      if (event === 'update') {
+        handleUpdate = fn;
+      } else if (event === 'subdocs') {
+        handleSubdocs = fn;
+      }
     };
     workspace.doc.off = offHandler;
-    await provider.connect();
+    provider.connect();
 
     provider.disconnect();
 
     expect(triggerDBUpdate).toBe(null);
     expect(offHandler).toBeCalledWith('update', handleUpdate);
+    expect(offHandler).toBeCalledWith('subdocs', handleSubdocs);
   });
 });

packages/workspace/src/providers/index.ts

@@ -3,8 +3,6 @@ import type {
   AffineWebSocketProvider,
   LocalIndexedDBBackgroundProvider,
   LocalIndexedDBDownloadProvider,
-  SQLiteDBDownloadProvider,
-  SQLiteProvider,
 } from '@affine/env/workspace';
 import type { Disposable, DocProviderCreator } from '@blocksuite/store';
 import { assertExists, Workspace } from '@blocksuite/store';
@@ -21,6 +19,10 @@ import { getLoginStorage, storageChangeSlot } from '../affine/login';
 import { CallbackSet } from '../utils';
 import { createAffineDownloadProvider } from './affine-download';
 import { localProviderLogger as logger } from './logger';
+import {
+  createSQLiteDBDownloadProvider,
+  createSQLiteProvider,
+} from './sqlite-providers';
 
 const Y = Workspace.Y;
 
@@ -151,151 +153,6 @@ const createIndexedDBDownloadProvider: DocProviderCreator = (
   };
 };
 
-const sqliteOrigin = Symbol('sqlite-provider-origin');
-
-const createSQLiteProvider: DocProviderCreator = (id, doc): SQLiteProvider => {
-  const { apis, events } = window;
-  // make sure it is being used in Electron with APIs
-  assertExists(apis);
-  assertExists(events);
-
-  function handleUpdate(update: Uint8Array, origin: unknown) {
-    if (origin === sqliteOrigin) {
-      return;
-    }
-    apis.db.applyDocUpdate(id, update).catch(err => {
-      console.error(err);
-    });
-  }
-
-  let unsubscribe = () => {};
-  let connected = false;
-
-  const connect = () => {
-    logger.info('connecting sqlite provider', id);
-    doc.on('update', handleUpdate);
-    unsubscribe = events.db.onExternalUpdate(
-      ({
-        update,
-        workspaceId,
-      }: {
-        workspaceId: string;
-        update: Uint8Array;
-      }) => {
-        if (workspaceId === id) {
-          Y.applyUpdate(doc, update, sqliteOrigin);
-        }
-      }
-    );
-    connected = true;
-    logger.info('connecting sqlite done', id);
-  };
-
-  const cleanup = () => {
-    logger.info('disconnecting sqlite provider', id);
-    unsubscribe();
-    doc.off('update', handleUpdate);
-    connected = false;
-  };
-
-  return {
-    flavour: 'sqlite',
-    passive: true,
-    get connected(): boolean {
-      return connected;
-    },
-    cleanup,
-    connect,
-    disconnect: cleanup,
-  };
-};
-
-const createSQLiteDBDownloadProvider: DocProviderCreator = (
-  id,
-  doc
-): SQLiteDBDownloadProvider => {
-  const { apis } = window;
-  let disconnected = false;
-
-  let _resolve: () => void;
-  let _reject: (error: unknown) => void;
-  const promise = new Promise<void>((resolve, reject) => {
-    _resolve = resolve;
-    _reject = reject;
-  });
-
-  async function syncUpdates() {
-    logger.info('syncing updates from sqlite', id);
-    const updates = await apis.db.getDocAsUpdates(id);
-
-    if (disconnected) {
-      return;
-    }
-
-    if (updates) {
-      Y.applyUpdate(doc, updates, sqliteOrigin);
-    }
-
-    const diff = Y.encodeStateAsUpdate(doc, updates);
-
-    // also apply updates to sqlite
-    await apis.db.applyDocUpdate(id, diff);
-  }
-
-  // fixme(pengx17): should n't sync blob in doc provider
-  // async function _syncBlobIntoSQLite(bs: BlobManager) {
-  //   const persistedKeys = await apis.db.getBlobKeys(id);
-  //
-  //   if (disconnected) {
-  //     return;
-  //   }
-  //
-  //   const allKeys = await bs.list().catch(() => []);
-  //   const keysToPersist = allKeys.filter(k => !persistedKeys.includes(k));
-  //
-  //   logger.info('persisting blobs', keysToPersist, 'to sqlite');
-  //   return Promise.all(
-  //     keysToPersist.map(async k => {
-  //       const blob = await bs.get(k);
-  //       if (!blob) {
-  //         logger.warn('blob not found for', k);
-  //         return;
-  //       }
-  //
-  //       if (disconnected) {
-  //         return;
-  //       }
-  //
-  //       return apis?.db.addBlob(
-  //         id,
-  //         k,
-  //         new Uint8Array(await blob.arrayBuffer())
-  //       );
-  //     })
-  //   );
-  // }
-
-  return {
-    flavour: 'sqlite-download',
-    active: true,
-    get whenReady() {
-      return promise;
-    },
-    cleanup: () => {
-      disconnected = true;
-    },
-    sync: async () => {
-      logger.info('connect indexeddb provider', id);
-      try {
-        await syncUpdates();
-        _resolve();
-      } catch (error) {
-        _reject(error);
-      }
-    },
-  };
-};
-
 export {
   createAffineDownloadProvider,
   createAffineWebSocketProvider,

packages/workspace/src/providers/sqlite-providers.ts

@@ -0,0 +1,212 @@
+import type {
+  SQLiteDBDownloadProvider,
+  SQLiteProvider,
+} from '@affine/env/workspace';
+import type { DocProviderCreator } from '@blocksuite/store';
+import {
+  assertExists,
+  Workspace as BlockSuiteWorkspace,
+} from '@blocksuite/store';
+import type { Doc } from 'yjs';
+
+import { localProviderLogger as logger } from './logger';
+
+const Y = BlockSuiteWorkspace.Y;
+
+const sqliteOrigin = Symbol('sqlite-provider-origin');
+
+type SubDocsEvent = {
+  added: Set<Doc>;
+  removed: Set<Doc>;
+  loaded: Set<Doc>;
+};
+
+/**
+ * A provider that is responsible for syncing updates the workspace with the local SQLite database.
+ */
+export const createSQLiteProvider: DocProviderCreator = (
+  id,
+  rootDoc
+): SQLiteProvider => {
+  const { apis, events } = window;
+  // make sure it is being used in Electron with APIs
+  assertExists(apis);
+  assertExists(events);
+
+  const updateHandlerMap = new WeakMap<
+    Doc,
+    (update: Uint8Array, origin: unknown) => void
+  >();
+  const subDocsHandlerMap = new WeakMap<Doc, (event: SubDocsEvent) => void>();
+
+  const createOrHandleUpdate = (doc: Doc) => {
+    if (updateHandlerMap.has(doc)) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      return updateHandlerMap.get(doc)!;
+    }
+
+    function handleUpdate(update: Uint8Array, origin: unknown) {
+      if (origin === sqliteOrigin) {
+        return;
+      }
+      const subdocId = doc.guid === id ? undefined : doc.guid;
+      apis.db.applyDocUpdate(id, update, subdocId).catch(err => {
+        logger.error(err);
+      });
+    }
+    updateHandlerMap.set(doc, handleUpdate);
+    return handleUpdate;
+  };
+
+  const createOrGetHandleSubDocs = (doc: Doc) => {
+    if (subDocsHandlerMap.has(doc)) {
+      // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+      return subDocsHandlerMap.get(doc)!;
+    }
+    function handleSubdocs(event: SubDocsEvent) {
+      event.removed.forEach(doc => {
+        untrackDoc(doc);
+      });
+      event.loaded.forEach(doc => {
+        trackDoc(doc);
+      });
+    }
+    subDocsHandlerMap.set(doc, handleSubdocs);
+    return handleSubdocs;
+  };
+
+  function trackDoc(doc: Doc) {
+    doc.on('update', createOrHandleUpdate(doc));
+    doc.on('subdocs', createOrGetHandleSubDocs(doc));
+    doc.subdocs.forEach(doc => {
+      trackDoc(doc);
+    });
+  }
+
+  function untrackDoc(doc: Doc) {
+    doc.subdocs.forEach(doc => {
+      untrackDoc(doc);
+    });
+    doc.off('update', createOrHandleUpdate(doc));
+    doc.off('subdocs', createOrGetHandleSubDocs(doc));
+  }
+
+  let unsubscribe = () => {};
+  let connected = false;
+
+  const connect = () => {
+    logger.info('connecting sqlite provider', id);
+    trackDoc(rootDoc);
+
+    unsubscribe = events.db.onExternalUpdate(
+      ({
+        update,
+        workspaceId,
+        docId,
+      }: {
+        workspaceId: string;
+        update: Uint8Array;
+        docId?: string;
+      }) => {
+        if (workspaceId === id) {
+          if (docId) {
+            for (const doc of rootDoc.subdocs) {
+              if (doc.guid === docId) {
+                Y.applyUpdate(doc, update, sqliteOrigin);
+                return;
+              }
+            }
+          } else {
+            Y.applyUpdate(rootDoc, update, sqliteOrigin);
+          }
+        }
+      }
+    );
+    connected = true;
+    logger.info('connecting sqlite done', id);
+  };
+
+  const cleanup = () => {
+    logger.info('disconnecting sqlite provider', id);
+    unsubscribe();
+    untrackDoc(rootDoc);
+    connected = false;
+  };
+
+  return {
+    flavour: 'sqlite',
+    passive: true,
+    get connected(): boolean {
+      return connected;
+    },
+    cleanup,
+    connect,
+    disconnect: cleanup,
+  };
+};
+
+/**
+ * A provider that is responsible for DOWNLOADING updates from the local SQLite database.
+ */
+export const createSQLiteDBDownloadProvider: DocProviderCreator = (
+  id,
+  rootDoc
+): SQLiteDBDownloadProvider => {
+  const { apis } = window;
+  let disconnected = false;
+
+  let _resolve: () => void;
+  let _reject: (error: unknown) => void;
+  const promise = new Promise<void>((resolve, reject) => {
+    _resolve = resolve;
+    _reject = reject;
+  });
+
+  async function syncUpdates(doc: Doc) {
+    logger.info('syncing updates from sqlite', id);
+    const subdocId = doc.guid === id ? undefined : doc.guid;
+    const updates = await apis.db.getDocAsUpdates(id, subdocId);
+
+    if (disconnected) {
+      return false;
+    }
+
+    if (updates) {
+      Y.applyUpdate(doc, updates, sqliteOrigin);
+    }
+
+    const diff = Y.encodeStateAsUpdate(doc, updates);
+
+    // also apply updates to sqlite
+    await apis.db.applyDocUpdate(id, diff, subdocId);
+
+    return true;
+  }
+
+  async function syncAllUpdates(doc: Doc) {
+    if (await syncUpdates(doc)) {
+      const subdocs = Array.from(doc.subdocs).filter(d => d.shouldLoad);
+      await Promise.all(subdocs.map(syncAllUpdates));
+    }
+  }
+
+  return {
+    flavour: 'sqlite-download',
+    active: true,
+    get whenReady() {
+      return promise;
+    },
+    cleanup: () => {
+      disconnected = true;
+    },
+    sync: async () => {
+      logger.info('connect indexeddb provider', id);
+      try {
+        await syncAllUpdates(rootDoc);
+        _resolve();
+      } catch (error) {
+        _reject(error);
+      }
+    },
+  };
+};

packages/y-indexeddb/src/__tests__/index.spec.ts

@@ -107,6 +107,7 @@ describe('indexeddb provider', () => {
       })
         .register(AffineSchemas)
         .register(__unstableSchemas);
+      // data should only contain updates for the root doc
       data.updates.forEach(({ update }) => {
         Workspace.Y.applyUpdate(testWorkspace.doc, update);
       });
@@ -125,19 +126,6 @@ describe('indexeddb provider', () => {
       }
       expect(workspace.doc.toJSON()).toEqual(testWorkspace.doc.toJSON());
     }
-
-    const secondWorkspace = new Workspace({
-      id,
-    })
-      .register(AffineSchemas)
-      .register(__unstableSchemas);
-    const provider2 = createIndexedDBProvider(secondWorkspace.doc, rootDBName);
-    provider2.connect();
-    await provider2.whenSynced;
-    const page = secondWorkspace.getPage('page0');
-    assertExists(page);
-    await page.waitForLoaded();
-    expect(workspace.doc.toJSON()).toEqual(secondWorkspace.doc.toJSON());
   });
 
   test('disconnect suddenly', async () => {
@@ -423,6 +411,7 @@ describe('subDoc', () => {
       provider.disconnect();
       json2 = doc.toJSON();
     }
+    // the following line compares {} with {}
     expect(json1['']['1'].toJSON()).toEqual(json2['']['1'].toJSON());
     expect(json1['']['2']).toEqual(json2['']['2']);
   });