feat(native): move sqlite operation into Rust (#2497)

Co-authored-by: Peng Xiao <pengxiao@outlook.com>

.github/actions/build-rust/action.yml

@@ -29,21 +29,23 @@ runs:
       if: ${{ inputs.target != 'x86_64-unknown-linux-gnu' && inputs.target != 'aarch64-unknown-linux-gnu' }}
       shell: bash
       run: yarn workspace @affine/native build --target ${{ inputs.target }}
-      env:
-        CARGO_BUILD_INCREMENTAL: 'false'
 
     - name: Build
       if: ${{ inputs.target == 'x86_64-unknown-linux-gnu' }}
       uses: addnab/docker-run-action@v3
       with:
         image: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
-        options: --user 0:0 -e CARGO_BUILD_INCREMENTAL=false -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
+        options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
-        run: yarn workspace @affine/native build --target ${{ inputs.target }}
+        run: >-
+          export CC=x86_64-unknown-linux-gnu-gcc &&
+          export CC_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-gcc &&
+          yarn workspace @affine/native build --target ${{ inputs.target }}
 
     - name: Build
       if: ${{ inputs.target == 'aarch64-unknown-linux-gnu' }}
       uses: addnab/docker-run-action@v3
       with:
         image: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
-        options: --user 0:0 -e CARGO_BUILD_INCREMENTAL=false -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
+        options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
-        run: yarn workspace @affine/native build --target ${{ inputs.target }}
+        run: >-
+          yarn workspace @affine/native build --target ${{ inputs.target }}

.github/workflows/build.yml

@@ -118,8 +118,7 @@ jobs:
           COVERAGE: true
 
       - name: Export static resources
-        run: yarn export
+        run: yarn workspace @affine/web export
-        working-directory: apps/web
 
       - name: Upload static resources artifact
         uses: actions/upload-artifact@v3
@@ -327,10 +326,7 @@ jobs:
       - name: Run unit tests
         if: ${{ matrix.spec.test }}
         shell: bash
-        run: |
+        run: yarn test:unit
-          rm -rf apps/electron/node_modules/better-sqlite3/build
-          yarn --cwd apps/electron/node_modules/better-sqlite3 run install
-          yarn test:unit
         env:
           NATIVE_TEST: 'true'
       - name: Build layers
@@ -342,12 +338,6 @@ jobs:
           name: next-js-static
           path: ./apps/electron/resources/web-static
 
-      - name: Rebuild Electron dependences
-        shell: bash
-        run: |
-          rm -rf apps/electron/node_modules/better-sqlite3/build
-          yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }}
-
       - name: Run desktop tests
         if: ${{ matrix.spec.test && matrix.spec.os == 'ubuntu-latest' }}
         run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- yarn workspace @affine/electron test

.github/workflows/nightly-build.yml

@@ -128,11 +128,6 @@ jobs:
         with:
           name: before-make-web-static
           path: apps/electron/resources/web-static
-      - name: Rebuild Electron dependences
-        shell: bash
-        run: |
-          rm -rf apps/electron/node_modules/better-sqlite3/build
-          yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }}
 
       - name: Build layers
         run: yarn workspace @affine/electron build-layers

.github/workflows/release-desktop-app.yml

@@ -126,12 +126,6 @@ jobs:
           name: before-make-web-static
           path: apps/electron/resources/web-static
 
-      - name: Rebuild Electron dependences
-        shell: bash
-        run: |
-          rm -rf apps/electron/node_modules/better-sqlite3/build
-          yarn workspace @affine/electron rebuild:for-electron --arch=${{ matrix.spec.arch }}
-
       - name: Build layers
         run: yarn workspace @affine/electron build-layers
 

.gitignore

@@ -73,3 +73,4 @@ target
 *.node
 tsconfig.node.tsbuildinfo
 lib
+affine.db

.vscode/settings.template.json

@@ -38,5 +38,7 @@
     "tests/unit/**/*.spec.ts",
     "tests/unit/**/*.spec.tsx"
   ],
-  "deepscan.enable": true
+  "rust-analyzer.check.extraEnv": {
+    "DATABASE_URL": "sqlite:affine.db"
+  }
 }

Cargo.toml

@@ -1,5 +1,8 @@
 [workspace]
-members = ["./packages/native"]
+members = ["./packages/native", "./packages/native/schema"]
+
+[profile.dev.package.sqlx-macros]
+opt-level = 3
 
 [profile.release]
 lto = true

apps/electron/README.md

@@ -17,22 +17,6 @@ yarn dev # or yarn prod for production build
 
 ## Troubleshooting
 
-### better-sqlite3 error
-
-When running tests or starting electron, you may encounter the following error:
-
-> Error: The module 'apps/electron/node_modules/better-sqlite3/build/Release/better_sqlite3.node'
-
-This is due to the fact that the `better-sqlite3` package is built for the Node.js version in Electron & in your machine. To fix this, run the following command based on different cases:
-
-```sh
-# for running unit tests, we are not using Electron's node:
-yarn rebuild better-sqlite3
-
-# for running Electron, we are using Electron's node:
-yarn postinstall
-```
-
 ## Credits
 
 Most of the boilerplate code is generously borrowed from the following

apps/electron/layers/main/src/__tests__/integration.spec.ts

@@ -1,5 +1,6 @@
 import assert from 'node:assert';
 import path from 'node:path';
+import { setTimeout } from 'node:timers/promises';
 
 import fs from 'fs-extra';
 import { v4 } from 'uuid';
@@ -13,8 +14,6 @@ const registeredHandlers = new Map<
   ((...args: any[]) => Promise<any>)[]
 >();
 
-const delay = (ms: number) => new Promise(r => setTimeout(r, ms));
-
 type WithoutFirstParameter<T> = T extends (_: any, ...args: infer P) => infer R
   ? (...args: P) => R
   : T;
@@ -72,11 +71,14 @@ const nativeTheme = {
   themeSource: 'light',
 };
 
-function compareBuffer(a: Uint8Array | null, b: Uint8Array | null) {
+function compareBuffer(
+  a: Uint8Array | null | undefined,
+  b: Uint8Array | null | undefined
+) {
   if (
-    (a === null && b === null) ||
+    (a == null && b == null) ||
-    a === null ||
+    a == null ||
-    b === null ||
+    b == null ||
     a.length !== b.length
   ) {
     return false;
@@ -105,11 +107,11 @@ const electronModule = {
       handlers.push(callback);
       registeredHandlers.set(name, handlers);
     },
-    addEventListener: (...args: any[]) => {
+    addListener: (...args: any[]) => {
       // @ts-ignore
       electronModule.app.on(...args);
     },
-    removeEventListener: () => {},
+    removeListener: () => {},
   },
   BrowserWindow: {
     getAllWindows: () => {
@@ -135,7 +137,6 @@ beforeEach(async () => {
   const { registerEvents } = await import('../events');
   registerEvents();
   await fs.mkdirp(SESSION_DATA_PATH);
-  await import('../db/ensure-db');
 
   registeredHandlers.get('ready')?.forEach(fn => fn());
 });
@@ -143,7 +144,10 @@ beforeEach(async () => {
 afterEach(async () => {
   // reset registered handlers
   registeredHandlers.get('before-quit')?.forEach(fn => fn());
-
+  // wait for the db to be closed on Windows
+  if (process.platform === 'win32') {
+    await setTimeout(200);
+  }
   await fs.remove(SESSION_DATA_PATH);
 });
 
@@ -175,7 +179,7 @@ describe('ensureSQLiteDB', () => {
     const fileExists = await fs.pathExists(file);
     expect(fileExists).toBe(true);
     registeredHandlers.get('before-quit')?.forEach(fn => fn());
-    await delay(100);
+    await setTimeout(100);
     expect(workspaceDB.db).toBe(null);
   });
 });
@@ -254,7 +258,7 @@ describe('db handlers', () => {
   test('get non existent blob', async () => {
     const workspaceId = v4();
     const bin = await dispatch('db', 'getBlob', workspaceId, 'non-existent-id');
-    expect(bin).toBeNull();
+    expect(bin).toBeUndefined();
   });
 
   test('list blobs (empty)', async () => {

apps/electron/layers/main/src/db/__tests__/ensure-db.spec.ts

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import { setTimeout } from 'node:timers/promises';
 
 import fs from 'fs-extra';
 import { v4 } from 'uuid';
@@ -30,18 +31,20 @@ const electronModule = {
       handlers.push(callback);
       registeredHandlers.set(name, handlers);
     },
-    addEventListener: (...args: any[]) => {
+    addListener: (...args: any[]) => {
       // @ts-ignore
       electronModule.app.on(...args);
     },
-    removeEventListener: () => {},
+    removeListener: () => {},
   },
   shell: {} as Partial<Electron.Shell>,
   dialog: {} as Partial<Electron.Dialog>,
 };
 
-const runHandler = (key: string) => {
+const runHandler = async (key: string) => {
-  registeredHandlers.get(key)?.forEach(handler => handler());
+  await Promise.all(
+    (registeredHandlers.get(key) ?? []).map(handler => handler())
+  );
 };
 
 // dynamically import handlers so that we can inject local variables to mocks
@@ -51,6 +54,7 @@ vi.doMock('electron', () => {
 
 const constructorStub = vi.fn();
 const destroyStub = vi.fn();
+destroyStub.mockReturnValue(Promise.resolve());
 
 vi.doMock('../secondary-db', () => {
   return {
@@ -59,6 +63,10 @@ vi.doMock('../secondary-db', () => {
         constructorStub(...args);
       }
 
+      connectIfNeeded = () => Promise.resolve();
+
+      pull = () => Promise.resolve();
+
       destroy = destroyStub;
     },
   };
@@ -69,7 +77,7 @@ beforeEach(() => {
 });
 
 afterEach(async () => {
-  runHandler('before-quit');
+  await runHandler('before-quit');
   await fs.remove(tmpDir);
   vi.useRealTimers();
 });
@@ -98,12 +106,24 @@ test('db should be destroyed when app quits', async () => {
   expect(db0.db).not.toBeNull();
   expect(db1.db).not.toBeNull();
 
-  runHandler('before-quit');
+  await runHandler('before-quit');
+
+  // wait the async `db.destroy()` to be called
+  await setTimeout(100);
 
   expect(db0.db).toBeNull();
   expect(db1.db).toBeNull();
 });
 
+test('db should be removed in db$Map after destroyed', async () => {
+  const { ensureSQLiteDB, db$Map } = await import('../ensure-db');
+  const workspaceId = v4();
+  const db = await ensureSQLiteDB(workspaceId);
+  await db.destroy();
+  await setTimeout(100);
+  expect(db$Map.has(workspaceId)).toBe(false);
+});
+
 test('if db has a secondary db path, we should also poll that', async () => {
   const { ensureSQLiteDB } = await import('../ensure-db');
   const { appContext } = await import('../../context');
@@ -115,10 +135,7 @@ test('if db has a secondary db path, we should also poll that', async () => {
 
   const db = await ensureSQLiteDB(workspaceId);
 
-  await vi.advanceTimersByTimeAsync(1500);
+  await setTimeout(10);
-
-  // not sure why but we still need to wait with real timer
-  await new Promise(resolve => setTimeout(resolve, 100));
 
   expect(constructorStub).toBeCalledTimes(1);
   expect(constructorStub).toBeCalledWith(path.join(tmpDir, 'secondary.db'), db);
@@ -128,7 +145,8 @@ test('if db has a secondary db path, we should also poll that', async () => {
     secondaryDBPath: path.join(tmpDir, 'secondary2.db'),
   });
 
-  await vi.advanceTimersByTimeAsync(1500);
+  // wait the async `db.destroy()` to be called
+  await setTimeout(100);
   expect(constructorStub).toBeCalledTimes(2);
   expect(destroyStub).toBeCalledTimes(1);
 
@@ -141,7 +159,7 @@ test('if db has a secondary db path, we should also poll that', async () => {
   expect(destroyStub).toBeCalledTimes(1);
 
   // if primary is destroyed, secondary should also be destroyed
-  db.destroy();
+  await db.destroy();
-  await new Promise(resolve => setTimeout(resolve, 100));
+  await setTimeout(100);
   expect(destroyStub).toBeCalledTimes(2);
 });

apps/electron/layers/main/src/db/__tests__/workspace-db-adapter.spec.ts

@@ -16,10 +16,7 @@ const testAppContext: AppContext = {
 };
 
 afterEach(async () => {
-  if (process.platform !== 'win32') {
+  await fs.remove(tmpDir);
-    // hmmm ....
-    await fs.remove(tmpDir);
-  }
 });
 
 function getTestUpdates() {
@@ -40,7 +37,7 @@ test('can create new db file if not exists', async () => {
     `storage.db`
   );
   expect(await fs.exists(dbPath)).toBe(true);
-  db.destroy();
+  await db.destroy();
 });
 
 test('on applyUpdate (from self), will not trigger update', async () => {
@@ -52,7 +49,7 @@ test('on applyUpdate (from self), will not trigger update', async () => {
   db.update$.subscribe(onUpdate);
   db.applyUpdate(getTestUpdates(), 'self');
   expect(onUpdate).not.toHaveBeenCalled();
-  db.destroy();
+  await db.destroy();
 });
 
 test('on applyUpdate (from renderer), will trigger update', async () => {
@@ -67,7 +64,7 @@ test('on applyUpdate (from renderer), will trigger update', async () => {
   db.applyUpdate(getTestUpdates(), 'renderer');
   expect(onUpdate).toHaveBeenCalled(); // not yet updated
   sub.unsubscribe();
-  db.destroy();
+  await db.destroy();
 });
 
 test('on applyUpdate (from external), will trigger update & send external update event', async () => {
@@ -83,7 +80,7 @@ test('on applyUpdate (from external), will trigger update & send external update
   expect(onUpdate).toHaveBeenCalled();
   expect(onExternalUpdate).toHaveBeenCalled();
   sub.unsubscribe();
-  db.destroy();
+  await db.destroy();
 });
 
 test('on destroy, check if resources have been released', async () => {
@@ -95,7 +92,7 @@ test('on destroy, check if resources have been released', async () => {
     next: vi.fn(),
   };
   db.update$ = updateSub as any;
-  db.destroy();
+  await db.destroy();
   expect(db.db).toBe(null);
   expect(updateSub.complete).toHaveBeenCalled();
 });

apps/electron/layers/main/src/db/base-db-adapter.ts

@@ -1,120 +1,76 @@
+import { SqliteConnection } from '@affine/native';
 import assert from 'assert';
-import type { Database } from 'better-sqlite3';
-import sqlite from 'better-sqlite3';
 
 import { logger } from '../logger';
 
-const schemas = [
-  `CREATE TABLE IF NOT EXISTS "updates" (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  data BLOB NOT NULL,
-  timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
-)`,
-  `CREATE TABLE IF NOT EXISTS "blobs" (
-  key TEXT PRIMARY KEY NOT NULL,
-  data BLOB NOT NULL,
-  timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
-)`,
-];
-
-interface UpdateRow {
-  id: number;
-  data: Buffer;
-  timestamp: string;
-}
-
-interface BlobRow {
-  key: string;
-  data: Buffer;
-  timestamp: string;
-}
-
 /**
  * A base class for SQLite DB adapter that provides basic methods around updates & blobs
  */
 export abstract class BaseSQLiteAdapter {
-  db: Database | null = null;
+  db: SqliteConnection | null = null;
   abstract role: string;
 
-  constructor(public path: string) {}
+  constructor(public readonly path: string) {
-
+    logger.info(`[SQLiteAdapter]`, 'path:', path);
-  ensureTables() {
-    assert(this.db, 'db is not connected');
-    this.db.exec(schemas.join(';'));
   }
 
-  // todo: what if SQLite DB wrapper later is not sync?
+  async connectIfNeeded() {
-  connect(): Database | undefined {
+    if (!this.db) {
-    if (this.db) {
+      this.db = new SqliteConnection(this.path);
-      return this.db;
+      await this.db.connect();
     }
-    logger.log(`[SQLiteAdapter][${this.role}] open db`, this.path);
+    return this.db;
-    const db = (this.db = sqlite(this.path));
-    this.ensureTables();
-    return db;
   }
 
-  destroy() {
+  async destroy() {
-    this.db?.close();
+    const { db } = this;
     this.db = null;
+    await db?.close();
   }
 
-  addBlob(key: string, data: Uint8Array) {
+  async addBlob(key: string, data: Uint8Array) {
     try {
-      assert(this.db, 'db is not connected');
+      assert(this.db, `${this.path} is not connected`);
-      const statement = this.db.prepare(
+      await this.db.addBlob(key, data);
-        'INSERT INTO blobs (key, data) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET data = ?'
-      );
-      statement.run(key, data, data);
-      return key;
     } catch (error) {
       logger.error('addBlob', error);
     }
   }
 
-  getBlob(key: string) {
+  async getBlob(key: string) {
     try {
-      assert(this.db, 'db is not connected');
+      assert(this.db, `${this.path} is not connected`);
-      const statement = this.db.prepare('SELECT data FROM blobs WHERE key = ?');
+      const blob = await this.db.getBlob(key);
-      const row = statement.get(key) as BlobRow;
+      return blob?.data;
-      if (!row) {
-        return null;
-      }
-      return row.data;
     } catch (error) {
       logger.error('getBlob', error);
       return null;
     }
   }
 
-  deleteBlob(key: string) {
+  async deleteBlob(key: string) {
     try {
-      assert(this.db, 'db is not connected');
+      assert(this.db, `${this.path} is not connected`);
-      const statement = this.db.prepare('DELETE FROM blobs WHERE key = ?');
+      await this.db.deleteBlob(key);
-      statement.run(key);
     } catch (error) {
-      logger.error('deleteBlob', error);
+      logger.error(`${this.path} delete blob failed`, error);
     }
   }
 
-  getBlobKeys() {
+  async getBlobKeys() {
     try {
-      assert(this.db, 'db is not connected');
+      assert(this.db, `${this.path} is not connected`);
-      const statement = this.db.prepare('SELECT key FROM blobs');
+      return await this.db.getBlobKeys();
-      const rows = statement.all() as BlobRow[];
-      return rows.map(row => row.key);
     } catch (error) {
-      logger.error('getBlobKeys', error);
+      logger.error(`getBlobKeys failed`, error);
       return [];
     }
   }
 
-  getUpdates() {
+  async getUpdates() {
     try {
-      assert(this.db, 'db is not connected');
+      assert(this.db, `${this.path} is not connected`);
-      const statement = this.db.prepare('SELECT * FROM updates');
+      return await this.db.getUpdates();
-      const rows = statement.all() as UpdateRow[];
-      return rows;
     } catch (error) {
       logger.error('getUpdates', error);
       return [];
@@ -122,22 +78,12 @@ export abstract class BaseSQLiteAdapter {
   }
 
   // add a single update to SQLite
-  addUpdateToSQLite(updates: Uint8Array[]) {
+  async addUpdateToSQLite(db: SqliteConnection, updates: Uint8Array[]) {
     // batch write instead write per key stroke?
     try {
-      assert(this.db, 'db is not connected');
       const start = performance.now();
-      const statement = this.db.prepare(
+      await db.connect();
-        'INSERT INTO updates (data) VALUES (?)'
+      await db.insertUpdates(updates);
-      );
-      const insertMany = this.db.transaction(updates => {
-        for (const d of updates) {
-          statement.run(d);
-        }
-      });
-
-      insertMany(updates);
-
       logger.debug(
         `[SQLiteAdapter][${this.role}] addUpdateToSQLite`,
         'length:',

apps/electron/layers/main/src/db/ensure-db.ts

@@ -1,12 +1,14 @@
 import { app } from 'electron';
+import type { Subject } from 'rxjs';
+import { Observable } from 'rxjs';
 import {
+  concat,
   defer,
-  firstValueFrom,
   from,
   fromEvent,
   interval,
+  lastValueFrom,
   merge,
-  Observable,
 } from 'rxjs';
 import {
   distinctUntilChanged,
@@ -17,41 +19,83 @@ import {
   shareReplay,
   startWith,
   switchMap,
+  take,
   takeUntil,
   tap,
 } from 'rxjs/operators';
 
 import { appContext } from '../context';
 import { logger } from '../logger';
-import { getWorkspaceMeta$ } from '../workspace';
+import { getWorkspaceMeta, workspaceSubjects } from '../workspace';
 import { SecondaryWorkspaceSQLiteDB } from './secondary-db';
 import type { WorkspaceSQLiteDB } from './workspace-db-adapter';
 import { openWorkspaceDatabase } from './workspace-db-adapter';
 
-const db$Map = new Map<string, Observable<WorkspaceSQLiteDB>>();
+// export for testing
+export const db$Map = new Map<string, Observable<WorkspaceSQLiteDB>>();
 
+// use defer to prevent `app` is undefined while running tests
 const beforeQuit$ = defer(() => fromEvent(app, 'before-quit'));
 
+// return a stream that emit a single event when the subject completes
+function completed<T>(subject: Subject<T>) {
+  return new Observable(subscriber => {
+    const sub = subject.subscribe({
+      complete: () => {
+        subscriber.next();
+        subscriber.complete();
+      },
+    });
+    return () => sub.unsubscribe();
+  });
+}
+
 function getWorkspaceDB$(id: string) {
   if (!db$Map.has(id)) {
     db$Map.set(
       id,
       from(openWorkspaceDatabase(appContext, id)).pipe(
-        shareReplay(1),
+        tap({
-        switchMap(db => {
+          next: db => {
-          return startPollingSecondaryDB(db).pipe(
+            logger.info(
-            ignoreElements(),
+              '[ensureSQLiteDB] db connection established',
-            startWith(db),
+              db.workspaceId
-            takeUntil(beforeQuit$),
+            );
-            tap({
+          },
-              complete: () => {
-                logger.info('[ensureSQLiteDB] close db connection');
-                db.destroy();
-                db$Map.delete(id);
-              },
-            })
-          );
         }),
+        switchMap(db =>
+          // takeUntil the polling stream, and then destroy the db
+          concat(
+            startPollingSecondaryDB(db).pipe(
+              ignoreElements(),
+              startWith(db),
+              takeUntil(merge(beforeQuit$, completed(db.update$))),
+              last(),
+              tap({
+                next() {
+                  logger.info(
+                    '[ensureSQLiteDB] polling secondary db complete',
+                    db.workspaceId
+                  );
+                },
+              })
+            ),
+            defer(async () => {
+              try {
+                await db.destroy();
+                db$Map.delete(id);
+                logger.info(
+                  '[ensureSQLiteDB] db connection destroyed',
+                  db.workspaceId
+                );
+                return db;
+              } catch (err) {
+                logger.error('[ensureSQLiteDB] destroy db failed', err);
+                throw err;
+              }
+            })
+          ).pipe(startWith(db))
+        ),
         shareReplay(1)
       )
     );
@@ -60,51 +104,43 @@ function getWorkspaceDB$(id: string) {
 }
 
 function startPollingSecondaryDB(db: WorkspaceSQLiteDB) {
-  const meta$ = getWorkspaceMeta$(db.workspaceId);
+  return merge(
-  const secondaryDB$ = meta$.pipe(
+    getWorkspaceMeta(appContext, db.workspaceId),
+    workspaceSubjects.meta.pipe(
+      map(({ meta }) => meta),
+      filter(meta => meta.id === db.workspaceId)
+    )
+  ).pipe(
     map(meta => meta?.secondaryDBPath),
-    distinctUntilChanged(),
     filter((p): p is string => !!p),
+    distinctUntilChanged(),
     switchMap(path => {
-      return new Observable<SecondaryWorkspaceSQLiteDB>(observer => {
+      // on secondary db path change, destroy the old db and create a new one
-        const secondaryDB = new SecondaryWorkspaceSQLiteDB(path, db);
+      const secondaryDB = new SecondaryWorkspaceSQLiteDB(path, db);
-        observer.next(secondaryDB);
+      return new Observable<SecondaryWorkspaceSQLiteDB>(subscriber => {
-        return () => {
+        subscriber.next(secondaryDB);
-          logger.info(
+        return () => secondaryDB.destroy();
-            '[ensureSQLiteDB] close secondary db connection',
-            secondaryDB.path
-          );
-          secondaryDB.destroy();
-        };
       });
     }),
-    takeUntil(db.update$.pipe(last())),
+    switchMap(secondaryDB => {
-    shareReplay(1)
+      return interval(300000).pipe(
+        startWith(0),
+        tap({
+          next: () => {
+            secondaryDB.pull();
+          },
+          error: err => {
+            logger.error(`[ensureSQLiteDB] polling secondary db error`, err);
+          },
+          complete: () => {
+            logger.info('[ensureSQLiteDB] polling secondary db complete');
+          },
+        })
+      );
+    })
   );
-
-  const firstDelayedTick$ = defer(() => {
-    return new Promise<number>(resolve =>
-      setTimeout(() => {
-        resolve(0);
-      }, 1000)
-    );
-  });
-
-  // pull every 30 seconds
-  const poll$ = merge(firstDelayedTick$, interval(30000)).pipe(
-    switchMap(() => secondaryDB$),
-    tap({
-      next: secondaryDB => {
-        secondaryDB.pull();
-      },
-    }),
-    takeUntil(db.update$.pipe(last())),
-    shareReplay(1)
-  );
-
-  return poll$;
 }
 
 export function ensureSQLiteDB(id: string) {
-  return firstValueFrom(getWorkspaceDB$(id));
+  return lastValueFrom(getWorkspaceDB$(id).pipe(take(1)));
 }

apps/electron/layers/main/src/db/helper.ts

@@ -1,38 +0,0 @@
-import type { Database } from 'better-sqlite3';
-import sqlite from 'better-sqlite3';
-
-import { logger } from '../logger';
-
-export function isValidateDB(db: Database) {
-  // check if db has two tables, one for updates and one for blobs
-  const statement = db.prepare(
-    `SELECT name FROM sqlite_schema WHERE type='table'`
-  );
-  const rows = statement.all() as { name: string }[];
-  const tableNames = rows.map(row => row.name);
-  if (!tableNames.includes('updates') || !tableNames.includes('blobs')) {
-    return false;
-  }
-}
-
-export function isValidDBFile(path: string) {
-  let db: Database | null = null;
-  try {
-    db = sqlite(path);
-    // check if db has two tables, one for updates and one for blobs
-    const statement = db.prepare(
-      `SELECT name FROM sqlite_schema WHERE type='table'`
-    );
-    const rows = statement.all() as { name: string }[];
-    const tableNames = rows.map(row => row.name);
-    if (!tableNames.includes('updates') || !tableNames.includes('blobs')) {
-      return false;
-    }
-    return true;
-  } catch (error) {
-    logger.error('isValidDBFile', error);
-    return false;
-  } finally {
-    db?.close();
-  }
-}

apps/electron/layers/main/src/db/secondary-db.ts

@@ -1,3 +1,6 @@
+import assert from 'node:assert';
+
+import type { SqliteConnection } from '@affine/native';
 import { debounce } from 'lodash-es';
 import * as Y from 'yjs';
 
@@ -30,16 +33,12 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
     logger.debug('[SecondaryWorkspaceSQLiteDB] created', this.workspaceId);
   }
 
-  close() {
+  override async destroy() {
-    this.db?.close();
+    const { db } = this;
-    this.db = null;
+    await this.flushUpdateQueue(db);
-  }
-
-  override destroy() {
-    this.flushUpdateQueue();
     this.unsubscribers.forEach(unsub => unsub());
-    this.db?.close();
     this.yDoc.destroy();
+    await super.destroy();
   }
 
   get workspaceId() {
@@ -48,12 +47,15 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
 
   // do not update db immediately, instead, push to a queue
   // and flush the queue in a future time
-  addUpdateToUpdateQueue(update: Uint8Array) {
+  async addUpdateToUpdateQueue(db: SqliteConnection, update: Uint8Array) {
     this.updateQueue.push(update);
-    this.debouncedFlush();
+    await this.debouncedFlush(db);
   }
 
-  flushUpdateQueue() {
+  async flushUpdateQueue(db = this.db) {
+    if (!db) {
+      return; // skip if db is not connected
+    }
     logger.debug(
       'flushUpdateQueue',
       this.workspaceId,
@@ -62,9 +64,8 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
     );
     const updates = [...this.updateQueue];
     this.updateQueue = [];
-    this.connect();
+    await db.connect();
-    this.addUpdateToSQLite(updates);
+    await this.addUpdateToSQLite(db, updates);
-    this.close();
   }
 
   // flush after 5s, but will not wait for more than 10s
@@ -75,29 +76,31 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
   runCounter = 0;
 
   // wrap the fn with connect and close
-  // it only works for sync functions
+  async run<T extends (...args: any[]) => any>(
-  run = <T extends (...args: any[]) => any>(fn: T) => {
+    fn: T
+  ): Promise<
+    (T extends (...args: any[]) => infer U ? Awaited<U> : unknown) | undefined
+  > {
     try {
-      if (this.runCounter === 0) {
+      await this.connectIfNeeded();
-        this.connect();
-      }
       this.runCounter++;
-      return fn();
+      return await fn();
     } catch (err) {
       logger.error(err);
     } finally {
       this.runCounter--;
       if (this.runCounter === 0) {
-        this.close();
+        await super.destroy();
       }
     }
-  };
+  }
 
   setupAndListen() {
     if (this.firstConnected) {
       return;
     }
     this.firstConnected = true;
+    const { db } = this;
 
     const onUpstreamUpdate = (update: Uint8Array, origin: YOrigin) => {
       if (origin === 'renderer') {
@@ -109,7 +112,7 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
     const onSelfUpdate = (update: Uint8Array, origin: YOrigin) => {
       // for self update from upstream, we need to push it to external DB
       if (origin === 'upstream') {
-        this.addUpdateToUpdateQueue(update);
+        this.addUpdateToUpdateQueue(db!, update);
       }
 
       if (origin === 'self') {
@@ -126,13 +129,11 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
       this.yDoc.off('update', onSelfUpdate);
     });
 
-    this.run(() => {
+    this.run(async () => {
       // apply all updates from upstream
       const upstreamUpdate = this.upstream.getDocAsUpdates();
       // to initialize the yDoc, we need to apply all updates from the db
       this.applyUpdate(upstreamUpdate, 'upstream');
-
-      this.pull();
     });
   }
 
@@ -141,17 +142,17 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
   };
 
   // TODO: have a better solution to handle blobs
-  syncBlobs() {
+  async syncBlobs() {
-    this.run(() => {
+    await this.run(async () => {
       // pull blobs
-      const blobsKeys = this.getBlobKeys();
+      const blobsKeys = await this.getBlobKeys();
-      const upstreamBlobsKeys = this.upstream.getBlobKeys();
+      const upstreamBlobsKeys = await this.upstream.getBlobKeys();
       // put every missing blob to upstream
       for (const key of blobsKeys) {
         if (!upstreamBlobsKeys.includes(key)) {
-          const blob = this.getBlob(key);
+          const blob = await this.getBlob(key);
           if (blob) {
-            this.upstream.addBlob(key, blob);
+            await this.upstream.addBlob(key, blob);
             logger.debug('syncBlobs', this.workspaceId, key);
           }
         }
@@ -170,12 +171,17 @@ export class SecondaryWorkspaceSQLiteDB extends BaseSQLiteAdapter {
    */
   async pull() {
     const start = performance.now();
-    const updates = this.run(() => {
+    assert(this.upstream.db, 'upstream db should be connected');
+    const updates = await this.run(async () => {
       // TODO: no need to get all updates, just get the latest ones (using a cursor, etc)?
-      this.syncBlobs();
+      await this.syncBlobs();
-      return this.getUpdates().map(update => update.data);
+      return (await this.getUpdates()).map(update => update.data);
     });
 
+    if (!updates) {
+      return;
+    }
+
     const merged = await mergeUpdateWorker(updates);
     this.applyUpdate(merged, 'self');
 

apps/electron/layers/main/src/db/workspace-db-adapter.ts

@@ -1,4 +1,4 @@
-import type { Database } from 'better-sqlite3';
+import type { SqliteConnection } from '@affine/native';
 import { Subject } from 'rxjs';
 import * as Y from 'yjs';
 
@@ -21,38 +21,38 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
     super(path);
   }
 
-  override destroy() {
+  override async destroy() {
-    this.db?.close();
+    await super.destroy();
-    this.db = null;
     this.yDoc.destroy();
 
     // when db is closed, we can safely remove it from ensure-db list
     this.update$.complete();
+    this.firstConnected = false;
   }
 
   getWorkspaceName = () => {
     return this.yDoc.getMap('space:meta').get('name') as string;
   };
 
-  async init(): Promise<Database | undefined> {
+  async init() {
-    const db = super.connect();
+    const db = await super.connectIfNeeded();
 
     if (!this.firstConnected) {
-      this.yDoc.on('update', (update: Uint8Array, origin: YOrigin) => {
+      this.yDoc.on('update', async (update: Uint8Array, origin: YOrigin) => {
         if (origin === 'renderer') {
-          this.addUpdateToSQLite([update]);
+          await this.addUpdateToSQLite(db, [update]);
         } else if (origin === 'external') {
-          this.addUpdateToSQLite([update]);
-          logger.debug('external update', this.workspaceId);
           dbSubjects.externalUpdate.next({
             workspaceId: this.workspaceId,
             update,
           });
+          await this.addUpdateToSQLite(db, [update]);
+          logger.debug('external update', this.workspaceId);
         }
       });
     }
 
-    const updates = this.getUpdates();
+    const updates = await this.getUpdates();
     const merged = await mergeUpdateWorker(updates.map(update => update.data));
 
     // to initialize the yDoc, we need to apply all updates from the db
@@ -78,19 +78,19 @@ export class WorkspaceSQLiteDB extends BaseSQLiteAdapter {
     Y.applyUpdate(this.yDoc, data, origin);
   };
 
-  override addBlob(key: string, value: Uint8Array) {
+  override async addBlob(key: string, value: Uint8Array) {
-    const res = super.addBlob(key, value);
+    const res = await super.addBlob(key, value);
     this.update$.next();
     return res;
   }
 
-  override deleteBlob(key: string) {
+  override async deleteBlob(key: string) {
     super.deleteBlob(key);
     this.update$.next();
   }
 
-  override addUpdateToSQLite(data: Uint8Array[]) {
+  override async addUpdateToSQLite(db: SqliteConnection, data: Uint8Array[]) {
-    super.addUpdateToSQLite(data);
+    super.addUpdateToSQLite(db, data);
     this.update$.next();
   }
 }
@@ -102,5 +102,6 @@ export async function openWorkspaceDatabase(
   const meta = await getWorkspaceMeta(context, workspaceId);
   const db = new WorkspaceSQLiteDB(meta.mainDBPath, workspaceId);
   await db.init();
+  logger.info(`openWorkspaceDatabase [${workspaceId}]`);
   return db;
 }

apps/electron/layers/main/src/dialog/dialog.ts

@@ -7,7 +7,6 @@ import { nanoid } from 'nanoid';
 
 import { appContext } from '../context';
 import { ensureSQLiteDB } from '../db/ensure-db';
-import { isValidDBFile } from '../db/helper';
 import type { WorkspaceSQLiteDB } from '../db/workspace-db-adapter';
 import { logger } from '../logger';
 import {
@@ -208,7 +207,9 @@ export async function loadDBFile(): Promise<LoadDBFileResult> {
       return { error: 'DB_FILE_ALREADY_LOADED' };
     }
 
-    if (!isValidDBFile(filePath)) {
+    const { SqliteConnection } = await import('@affine/native');
+
+    if (!(await SqliteConnection.validate(filePath))) {
       // TODO: report invalid db file error?
       return { error: 'DB_FILE_INVALID' }; // invalid db file
     }
@@ -305,7 +306,9 @@ export async function moveDBFile(
 
     // remove the old db file, but we don't care if it fails
     if (meta.secondaryDBPath) {
-      fs.remove(meta.secondaryDBPath);
+      fs.remove(meta.secondaryDBPath).catch(err => {
+        logger.error(`[moveDBFile] remove ${meta.secondaryDBPath} failed`, err);
+      });
     }
 
     // update meta

apps/electron/layers/main/src/workspace/__tests__/handlers.spec.ts

@@ -172,37 +172,3 @@ test('storeWorkspaceMeta', async () => {
     secondaryDBPath: path.join(tmpDir, 'test.db'),
   });
 });
-
-test('getWorkspaceMeta observable', async () => {
-  const { storeWorkspaceMeta } = await import('../handlers');
-  const { getWorkspaceMeta$ } = await import('../index');
-
-  const workspaceId = v4();
-  const workspacePath = path.join(
-    testAppContext.appDataPath,
-    'workspaces',
-    workspaceId
-  );
-
-  const metaChange = vi.fn();
-
-  const meta$ = getWorkspaceMeta$(workspaceId);
-
-  meta$.subscribe(metaChange);
-  await new Promise(resolve => setTimeout(resolve, 100));
-
-  expect(metaChange).toHaveBeenCalledWith({
-    id: workspaceId,
-    mainDBPath: path.join(workspacePath, 'storage.db'),
-  });
-
-  await storeWorkspaceMeta(testAppContext, workspaceId, {
-    secondaryDBPath: path.join(tmpDir, 'test.db'),
-  });
-
-  expect(metaChange).toHaveBeenCalledWith({
-    id: workspaceId,
-    mainDBPath: path.join(workspacePath, 'storage.db'),
-    secondaryDBPath: path.join(tmpDir, 'test.db'),
-  });
-});

apps/electron/layers/main/src/workspace/handlers.ts

@@ -41,7 +41,7 @@ export async function deleteWorkspace(context: AppContext, id: string) {
   );
   try {
     const db = await ensureSQLiteDB(id);
-    db.destroy();
+    await db.destroy();
     return await fs.move(basePath, movedPath, {
       overwrite: true,
     });

apps/electron/layers/main/src/workspace/index.ts

@@ -1,6 +1,3 @@
-import { merge } from 'rxjs';
-import { filter, map } from 'rxjs/operators';
-
 import { appContext } from '../context';
 import type {
   MainEventListener,
@@ -31,14 +28,3 @@ export const workspaceHandlers = {
     return getWorkspaceMeta(appContext, id);
   },
 } satisfies NamespaceHandlers;
-
-// used internally. Get a stream of workspace id -> meta
-export const getWorkspaceMeta$ = (workspaceId: string) => {
-  return merge(
-    getWorkspaceMeta(appContext, workspaceId),
-    workspaceSubjects.meta.pipe(
-      map(meta => meta.meta),
-      filter(meta => meta.id === workspaceId)
-    )
-  );
-};

apps/electron/package.json

@@ -10,15 +10,13 @@
   "description": "AFFiNE App",
   "homepage": "https://github.com/toeverything/AFFiNE",
   "scripts": {
-    "dev": "yarn electron-rebuild && yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs",
+    "dev": "yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs",
-    "watch": "yarn electron-rebuild && yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs --watch",
+    "watch": "yarn cross-env DEV_SERVER_URL=http://localhost:8080 node scripts/dev.mjs --watch",
-    "prod": "yarn electron-rebuild && yarn node scripts/dev.mjs",
+    "prod": "yarn node scripts/dev.mjs",
     "build-layers": "zx scripts/build-layers.mjs",
     "generate-assets": "zx scripts/generate-assets.mjs",
     "package": "electron-forge package",
     "make": "electron-forge make",
-    "rebuild:for-unit-test": "yarn rebuild better-sqlite3",
-    "rebuild:for-electron": "yarn electron-rebuild",
     "test": "playwright test"
   },
   "config": {
@@ -36,9 +34,7 @@
     "@electron-forge/maker-squirrel": "^6.1.1",
     "@electron-forge/maker-zip": "^6.1.1",
     "@electron-forge/shared-types": "^6.1.1",
-    "@electron/rebuild": "^3.2.13",
     "@electron/remote": "2.0.9",
-    "@types/better-sqlite3": "^7.6.4",
     "@types/fs-extra": "^11.0.1",
     "@types/uuid": "^9.0.1",
     "cross-env": "7.0.3",
@@ -55,8 +51,7 @@
     "zx": "^7.2.2"
   },
   "dependencies": {
-    "better-sqlite3": "^8.4.0",
+    "cheerio": "^1.0.0-rc.12",
-    "chokidar": "^3.5.3",
     "electron-updater": "^5.3.0",
     "lodash-es": "^4.17.21",
     "nanoid": "^4.0.2",

apps/electron/scripts/common.mjs

@@ -46,7 +46,7 @@ export const config = () => {
       bundle: true,
       target: `node${NODE_MAJOR_VERSION}`,
       platform: 'node',
-      external: ['electron', 'yjs', 'better-sqlite3', 'electron-updater'],
+      external: ['electron', 'yjs', 'electron-updater'],
       define: define,
       format: 'cjs',
       loader: {

packages/native/.env

@@ -0,0 +1 @@
+DATABASE_URL="sqlite:affine.db"

packages/native/.gitignore

@@ -1 +1,2 @@
 *.fixture
+lib

packages/native/Cargo.toml

@@ -7,13 +7,15 @@ version = "0.0.0"
 crate-type = ["cdylib"]
 
 [dependencies]
+affine_schema = { path = "./schema" }
 anyhow = "1"
-# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
+chrono = "0.4"
 napi = { version = "2", default-features = false, features = [
-  "napi4",
+  "napi5",
   "tokio_rt",
   "serde-json",
   "error_anyhow",
+  "chrono_date",
 ] }
 napi-derive = "2"
 notify = { version = "6", features = ["serde"] }
@@ -21,6 +23,13 @@ once_cell = "1"
 parking_lot = "0.12"
 serde = "1"
 serde_json = "1"
+sqlx = { version = "0.7.0-alpha.3", default-features = false, features = [
+  "sqlite",
+  "runtime-tokio",
+  "tls-rustls",
+  "chrono",
+  "macros",
+] }
 tokio = { version = "1", features = ["full"] }
 uuid = { version = "1", default-features = false, features = [
   "serde",
@@ -29,4 +38,16 @@ uuid = { version = "1", default-features = false, features = [
 ] }
 
 [build-dependencies]
+affine_schema = { path = "./schema" }
+dotenv = "0.15"
 napi-build = "2"
+sqlx = { version = "0.7.0-alpha.3", default-features = false, features = [
+  "sqlite",
+  "runtime-tokio",
+  "tls-rustls",
+  "chrono",
+  "macros",
+  "migrate",
+  "json",
+] }
+tokio = { version = "1", features = ["full"] }

packages/native/affine.db

@@ -0,0 +1 @@
+../../affine.db

packages/native/build.rs

@@ -1,6 +1,23 @@
-extern crate napi_build;
+use sqlx::sqlite::SqliteConnectOptions;
+
+#[tokio::main]
+async fn main() -> Result<(), std::io::Error> {
+  dotenv::dotenv().ok();
 
-fn main() -> Result<(), std::io::Error> {
   napi_build::setup();
+  let options = SqliteConnectOptions::new()
+    .filename("../../affine.db")
+    .journal_mode(sqlx::sqlite::SqliteJournalMode::Off)
+    .locking_mode(sqlx::sqlite::SqliteLockingMode::Exclusive)
+    .create_if_missing(true);
+  let pool = sqlx::sqlite::SqlitePoolOptions::new()
+    .max_connections(1)
+    .connect_with(options)
+    .await
+    .unwrap();
+  sqlx::query(affine_schema::SCHEMA)
+    .execute(&pool)
+    .await
+    .unwrap();
   Ok(())
 }

packages/native/index.d.ts

@@ -7,7 +7,7 @@ export interface WatchOptions {
   recursive?: boolean;
 }
 /** Watcher kind enumeration */
-export enum WatcherKind {
+export const enum WatcherKind {
   /** inotify backend (linux) */
   Inotify = 'Inotify',
   /** FS-Event backend (mac) */
@@ -23,6 +23,16 @@ export enum WatcherKind {
   Unknown = 'Unknown',
 }
 export function moveFile(src: string, dst: string): Promise<void>;
+export interface BlobRow {
+  key: string;
+  data: Buffer;
+  timestamp: Date;
+}
+export interface UpdateRow {
+  id: number;
+  timestamp: Date;
+  data: Buffer;
+}
 export class Subscription {
   toString(): string;
   unsubscribe(): void;
@@ -39,3 +49,16 @@ export class FsWatcher {
   static unwatch(p: string): void;
   static close(): void;
 }
+export class SqliteConnection {
+  constructor(path: string);
+  connect(): Promise<void>;
+  addBlob(key: string, blob: Uint8Array): Promise<void>;
+  getBlob(key: string): Promise<BlobRow | null>;
+  deleteBlob(key: string): Promise<void>;
+  getBlobKeys(): Promise<Array<string>>;
+  getUpdates(): Promise<Array<UpdateRow>>;
+  insertUpdates(updates: Array<Uint8Array>): Promise<void>;
+  close(): Promise<void>;
+  get isClose(): boolean;
+  static validate(path: string): Promise<boolean>;
+}

packages/native/index.js

@@ -263,9 +263,11 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`);
 }
 
-const { WatcherKind, Subscription, FsWatcher, moveFile } = nativeBinding;
+const { WatcherKind, Subscription, FsWatcher, moveFile, SqliteConnection } =
+  nativeBinding;
 
 module.exports.WatcherKind = WatcherKind;
 module.exports.Subscription = Subscription;
 module.exports.FsWatcher = FsWatcher;
 module.exports.moveFile = moveFile;
+module.exports.SqliteConnection = SqliteConnection;

packages/native/schema/Cargo.toml

@@ -0,0 +1,4 @@
+[package]
+edition = "2021"
+name = "affine_schema"
+version = "0.0.0"

packages/native/schema/README.md

@@ -0,0 +1 @@
+A temporary crate to share the schema between AFFiNE native and `build.rs` in the AFFiNE native.

packages/native/schema/src/lib.rs

@@ -0,0 +1,13 @@
+// TODO
+// dynamic create it from JavaScript side
+// and remove this crate then.
+pub const SCHEMA: &str = r#"CREATE TABLE IF NOT EXISTS "updates" (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  data BLOB NOT NULL,
+  timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
+CREATE TABLE IF NOT EXISTS "blobs" (
+  key TEXT PRIMARY KEY NOT NULL,
+  data BLOB NOT NULL,
+  timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL
+);"#;

packages/native/src/lib.rs

@@ -1 +1,2 @@
 pub mod fs;
+pub mod sqlite;

packages/native/src/sqlite/mod.rs

@@ -0,0 +1,161 @@
+use chrono::NaiveDateTime;
+use napi::bindgen_prelude::{Buffer, Uint8Array};
+use napi_derive::napi;
+use sqlx::{
+  migrate::MigrateDatabase,
+  sqlite::{Sqlite, SqliteConnectOptions, SqlitePoolOptions},
+  Pool, Row,
+};
+
+#[napi(object)]
+pub struct BlobRow {
+  pub key: String,
+  pub data: Buffer,
+  pub timestamp: NaiveDateTime,
+}
+
+#[napi(object)]
+pub struct UpdateRow {
+  pub id: i64,
+  pub timestamp: NaiveDateTime,
+  pub data: Buffer,
+}
+
+#[napi]
+pub struct SqliteConnection {
+  pool: Pool<Sqlite>,
+  path: String,
+}
+
+#[napi]
+impl SqliteConnection {
+  #[napi(constructor)]
+  pub fn new(path: String) -> napi::Result<Self> {
+    let sqlite_options = SqliteConnectOptions::new()
+      .filename(&path)
+      .foreign_keys(false)
+      .journal_mode(sqlx::sqlite::SqliteJournalMode::Off);
+    let pool = SqlitePoolOptions::new()
+      .max_connections(4)
+      .connect_lazy_with(sqlite_options);
+    Ok(Self { pool, path })
+  }
+
+  #[napi]
+  pub async fn connect(&self) -> napi::Result<()> {
+    if !Sqlite::database_exists(&self.path).await.unwrap_or(false) {
+      Sqlite::create_database(&self.path)
+        .await
+        .map_err(anyhow::Error::from)?;
+    };
+    let mut connection = self.pool.acquire().await.map_err(anyhow::Error::from)?;
+    sqlx::query(affine_schema::SCHEMA)
+      .execute(connection.as_mut())
+      .await
+      .map_err(anyhow::Error::from)?;
+    connection.detach();
+    Ok(())
+  }
+
+  #[napi]
+  pub async fn add_blob(&self, key: String, blob: Uint8Array) -> napi::Result<()> {
+    let blob = blob.as_ref();
+    sqlx::query_as!(
+      BlobRow,
+      "INSERT INTO blobs (key, data) VALUES ($1, $2) ON CONFLICT(key) DO UPDATE SET data = excluded.data",
+      key,
+      blob,
+    )
+    .execute(&self.pool)
+    .await
+    .map_err(anyhow::Error::from)?;
+    Ok(())
+  }
+
+  #[napi]
+  pub async fn get_blob(&self, key: String) -> Option<BlobRow> {
+    sqlx::query_as!(BlobRow, "SELECT * FROM blobs WHERE key = ?", key)
+      .fetch_one(&self.pool)
+      .await
+      .ok()
+  }
+
+  #[napi]
+  pub async fn delete_blob(&self, key: String) -> napi::Result<()> {
+    sqlx::query!("DELETE FROM blobs WHERE key = ?", key)
+      .execute(&self.pool)
+      .await
+      .map_err(anyhow::Error::from)?;
+    Ok(())
+  }
+
+  #[napi]
+  pub async fn get_blob_keys(&self) -> napi::Result<Vec<String>> {
+    let keys = sqlx::query!("SELECT key FROM blobs")
+      .fetch_all(&self.pool)
+      .await
+      .map(|rows| rows.into_iter().map(|row| row.key).collect())
+      .map_err(anyhow::Error::from)?;
+    Ok(keys)
+  }
+
+  #[napi]
+  pub async fn get_updates(&self) -> napi::Result<Vec<UpdateRow>> {
+    let updates = sqlx::query_as!(UpdateRow, "SELECT * FROM updates")
+      .fetch_all(&self.pool)
+      .await
+      .map_err(anyhow::Error::from)?;
+    Ok(updates)
+  }
+
+  #[napi]
+  pub async fn insert_updates(&self, updates: Vec<Uint8Array>) -> napi::Result<()> {
+    let mut transaction = self.pool.begin().await.map_err(anyhow::Error::from)?;
+    for update in updates.into_iter() {
+      let update = update.as_ref();
+      sqlx::query_as!(UpdateRow, "INSERT INTO updates (data) VALUES ($1)", update)
+        .execute(&mut *transaction)
+        .await
+        .map_err(anyhow::Error::from)?;
+    }
+    transaction.commit().await.map_err(anyhow::Error::from)?;
+    Ok(())
+  }
+
+  #[napi]
+  pub async fn close(&self) {
+    self.pool.close().await;
+  }
+
+  #[napi(getter)]
+  pub fn is_close(&self) -> bool {
+    self.pool.is_closed()
+  }
+
+  #[napi]
+  pub async fn validate(path: String) -> bool {
+    if let Ok(pool) = SqlitePoolOptions::new()
+      .max_connections(1)
+      .connect(&path)
+      .await
+    {
+      if let Ok(res) = sqlx::query("SELECT name FROM sqlite_master WHERE type='table'")
+        .fetch_all(&pool)
+        .await
+      {
+        let names = res.iter().map(|row| row.get(0));
+        names.fold(0, |acc, cur: String| {
+          if cur == "updates" || cur == "blobs" {
+            acc + 1
+          } else {
+            acc
+          }
+        }) == 2
+      } else {
+        false
+      }
+    } else {
+      false
+    }
+  }
+}

yarn.lock

@@ -178,13 +178,10 @@ __metadata:
     "@electron-forge/maker-squirrel": ^6.1.1
     "@electron-forge/maker-zip": ^6.1.1
     "@electron-forge/shared-types": ^6.1.1
-    "@electron/rebuild": ^3.2.13
     "@electron/remote": 2.0.9
-    "@types/better-sqlite3": ^7.6.4
     "@types/fs-extra": ^11.0.1
     "@types/uuid": ^9.0.1
-    better-sqlite3: ^8.4.0
+    cheerio: ^1.0.0-rc.12
-    chokidar: ^3.5.3
     cross-env: 7.0.3
     electron: 25.0.1
     electron-log: ^5.0.0-beta.24
@@ -3317,7 +3314,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@electron/rebuild@npm:^3.2.10, @electron/rebuild@npm:^3.2.13":
+"@electron/rebuild@npm:^3.2.10":
   version: 3.2.13
   resolution: "@electron/rebuild@npm:3.2.13"
   dependencies:
@@ -9233,15 +9230,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/better-sqlite3@npm:^7.6.4":
-  version: 7.6.4
-  resolution: "@types/better-sqlite3@npm:7.6.4"
-  dependencies:
-    "@types/node": "*"
-  checksum: 75ab00d31b56437cc65fe15ff673cf8d1609edca52628083921bcbab1cbd828d135a2859fb4e68af8ef5a4801705ba99d54b96499f997bce65dd306ade3dbe58
-  languageName: node
-  linkType: hard
-
 "@types/body-parser@npm:*":
   version: 1.19.2
   resolution: "@types/body-parser@npm:1.19.2"
@@ -11689,17 +11677,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"better-sqlite3@npm:^8.4.0":
-  version: 8.4.0
-  resolution: "better-sqlite3@npm:8.4.0"
-  dependencies:
-    bindings: ^1.5.0
-    node-gyp: latest
-    prebuild-install: ^7.1.0
-  checksum: f8b180c26428a2d381482e83b4519d0d81a918e00f92cafd255f42eb9583f49b2fed1015121ad49ebe1f3f790cc8c6f4697d1e805193d65e70dacf2e8abbd6af
-  languageName: node
-  linkType: hard
-
 "big-integer@npm:^1.6.44":
   version: 1.6.51
   resolution: "big-integer@npm:1.6.51"
@@ -11728,15 +11705,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"bindings@npm:^1.5.0":
-  version: 1.5.0
-  resolution: "bindings@npm:1.5.0"
-  dependencies:
-    file-uri-to-path: 1.0.0
-  checksum: 65b6b48095717c2e6105a021a7da4ea435aa8d3d3cd085cb9e85bcb6e5773cf318c4745c3f7c504412855940b585bdf9b918236612a1c7a7942491de176f1ae7
-  languageName: node
-  linkType: hard
-
 "bl@npm:^4.0.3, bl@npm:^4.1.0":
   version: 4.1.0
   resolution: "bl@npm:4.1.0"
@@ -13708,7 +13676,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"detect-libc@npm:^2.0.0, detect-libc@npm:^2.0.1":
+"detect-libc@npm:^2.0.1":
   version: 2.0.1
   resolution: "detect-libc@npm:2.0.1"
   checksum: ccb05fcabbb555beb544d48080179c18523a343face9ee4e1a86605a8715b4169f94d663c21a03c310ac824592f2ba9a5270218819bb411ad7be578a527593d7
@@ -15412,13 +15380,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"expand-template@npm:^2.0.3":
-  version: 2.0.3
-  resolution: "expand-template@npm:2.0.3"
-  checksum: 588c19847216421ed92befb521767b7018dc88f88b0576df98cb242f20961425e96a92cbece525ef28cc5becceae5d544ae0f5b9b5e2aa05acb13716ca5b3099
-  languageName: node
-  linkType: hard
-
 "expand-tilde@npm:^1.2.2":
   version: 1.2.2
   resolution: "expand-tilde@npm:1.2.2"
@@ -15773,13 +15734,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"file-uri-to-path@npm:1.0.0":
-  version: 1.0.0
-  resolution: "file-uri-to-path@npm:1.0.0"
-  checksum: b648580bdd893a008c92c7ecc96c3ee57a5e7b6c4c18a9a09b44fb5d36d79146f8e442578bc0e173dc027adf3987e254ba1dfd6e3ec998b7c282873010502144
-  languageName: node
-  linkType: hard
-
 "filelist@npm:^1.0.4":
   version: 1.0.4
   resolution: "filelist@npm:1.0.4"
@@ -16633,13 +16587,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"github-from-package@npm:0.0.0":
-  version: 0.0.0
-  resolution: "github-from-package@npm:0.0.0"
-  checksum: 14e448192a35c1e42efee94c9d01a10f42fe790375891a24b25261246ce9336ab9df5d274585aedd4568f7922246c2a78b8a8cd2571bfe99c693a9718e7dd0e3
-  languageName: node
-  linkType: hard
-
 "github-slugger@npm:^1.0.0":
   version: 1.5.0
   resolution: "github-slugger@npm:1.5.0"
@@ -20892,7 +20839,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"minimist@npm:^1.1.1, minimist@npm:^1.1.3, minimist@npm:^1.2.0, minimist@npm:^1.2.3, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8, minimist@npm:~1.2.5":
+"minimist@npm:^1.1.1, minimist@npm:^1.1.3, minimist@npm:^1.2.0, minimist@npm:^1.2.5, minimist@npm:^1.2.6, minimist@npm:^1.2.7, minimist@npm:^1.2.8, minimist@npm:~1.2.5":
   version: 1.2.8
   resolution: "minimist@npm:1.2.8"
   checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0
@@ -20976,7 +20923,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"mkdirp-classic@npm:^0.5.2, mkdirp-classic@npm:^0.5.3":
+"mkdirp-classic@npm:^0.5.2":
   version: 0.5.3
   resolution: "mkdirp-classic@npm:0.5.3"
   checksum: 3f4e088208270bbcc148d53b73e9a5bd9eef05ad2cbf3b3d0ff8795278d50dd1d11a8ef1875ff5aea3fa888931f95bfcb2ad5b7c1061cfefd6284d199e6776ac
@@ -21255,13 +21202,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"napi-build-utils@npm:^1.0.1":
-  version: 1.0.2
-  resolution: "napi-build-utils@npm:1.0.2"
-  checksum: 06c14271ee966e108d55ae109f340976a9556c8603e888037145d6522726aebe89dd0c861b4b83947feaf6d39e79e08817559e8693deedc2c94e82c5cbd090c7
-  languageName: node
-  linkType: hard
-
 "natural-compare-lite@npm:^1.4.0":
   version: 1.4.0
   resolution: "natural-compare-lite@npm:1.4.0"
@@ -21396,7 +21336,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"node-abi@npm:^3.0.0, node-abi@npm:^3.3.0":
+"node-abi@npm:^3.0.0":
   version: 3.40.0
   resolution: "node-abi@npm:3.40.0"
   dependencies:
@@ -22767,28 +22707,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"prebuild-install@npm:^7.1.0":
-  version: 7.1.1
-  resolution: "prebuild-install@npm:7.1.1"
-  dependencies:
-    detect-libc: ^2.0.0
-    expand-template: ^2.0.3
-    github-from-package: 0.0.0
-    minimist: ^1.2.3
-    mkdirp-classic: ^0.5.3
-    napi-build-utils: ^1.0.1
-    node-abi: ^3.3.0
-    pump: ^3.0.0
-    rc: ^1.2.7
-    simple-get: ^4.0.0
-    tar-fs: ^2.0.0
-    tunnel-agent: ^0.6.0
-  bin:
-    prebuild-install: bin.js
-  checksum: dbf96d0146b6b5827fc8f67f72074d2e19c69628b9a7a0a17d0fad1bf37e9f06922896972e074197fc00a52eae912993e6ef5a0d471652f561df5cb516f3f467
-  languageName: node
-  linkType: hard
-
 "precinct@npm:^8.1.0":
   version: 8.3.1
   resolution: "precinct@npm:8.3.1"
@@ -24884,24 +24802,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"simple-concat@npm:^1.0.0":
-  version: 1.0.1
-  resolution: "simple-concat@npm:1.0.1"
-  checksum: 4d211042cc3d73a718c21ac6c4e7d7a0363e184be6a5ad25c8a1502e49df6d0a0253979e3d50dbdd3f60ef6c6c58d756b5d66ac1e05cda9cacd2e9fc59e3876a
-  languageName: node
-  linkType: hard
-
-"simple-get@npm:^4.0.0":
-  version: 4.0.1
-  resolution: "simple-get@npm:4.0.1"
-  dependencies:
-    decompress-response: ^6.0.0
-    once: ^1.3.1
-    simple-concat: ^1.0.0
-  checksum: e4132fd27cf7af230d853fa45c1b8ce900cb430dd0a3c6d3829649fe4f2b26574c803698076c4006450efb0fad2ba8c5455fbb5755d4b0a5ec42d4f12b31d27e
-  languageName: node
-  linkType: hard
-
 "simple-git@npm:^3.15.0":
   version: 3.19.0
   resolution: "simple-git@npm:3.19.0"
@@ -25841,7 +25741,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"tar-fs@npm:^2.0.0, tar-fs@npm:^2.1.1":
+"tar-fs@npm:^2.1.1":
   version: 2.1.1
   resolution: "tar-fs@npm:2.1.1"
   dependencies:
@@ -26354,15 +26254,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"tunnel-agent@npm:^0.6.0":
-  version: 0.6.0
-  resolution: "tunnel-agent@npm:0.6.0"
-  dependencies:
-    safe-buffer: ^5.0.1
-  checksum: 05f6510358f8afc62a057b8b692f05d70c1782b70db86d6a1e0d5e28a32389e52fa6e7707b6c5ecccacc031462e4bc35af85ecfe4bbc341767917b7cf6965711
-  languageName: node
-  linkType: hard
-
 "turndown@npm:^7.1.1":
   version: 7.1.2
   resolution: "turndown@npm:7.1.2"