feat(server): use job system (#10218)

packages/backend/server/src/core/doc/adapters/workspace.ts

@@ -8,6 +8,7 @@ import {
   EventBus,
   FailedToSaveUpdates,
   FailedToUpsertSnapshot,
+  JobQueue,
   metrics,
   Mutex,
 } from '../../../base';
@@ -50,7 +51,8 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
     private readonly mutex: Mutex,
     private readonly cache: Cache,
     private readonly event: EventBus,
-    protected override readonly options: DocStorageOptions
+    protected override readonly options: DocStorageOptions,
+    private readonly queue: JobQueue
   ) {
     super(options);
   }
@@ -95,9 +97,19 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
               };
             })
           );
+          await this.queue.add(
+            'doc.mergePendingDocUpdates',
+            {
+              workspaceId,
+              docId,
+            },
+            {
+              // keep it simple to let all update merged in one job
+              jobId: `doc:merge-pending-updates:${workspaceId}:${docId}`,
+            }
+          );
           turn++;
           done += batch.length;
-          await this.updateCachedUpdatesCount(workspaceId, docId, batch.length);
         }
       });
 
@@ -146,14 +158,11 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
     docId: string,
     updates: DocUpdate[]
   ) {
-    const count = await this.models.doc.deleteUpdates(
+    return await this.models.doc.deleteUpdates(
       workspaceId,
       docId,
       updates.map(u => u.timestamp)
     );
-
-    await this.updateCachedUpdatesCount(workspaceId, docId, -count);
-    return count;
   }
 
   async listDocHistories(
@@ -395,23 +404,4 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {
 
     return null;
   }
-
-  private async updateCachedUpdatesCount(
-    workspaceId: string,
-    guid: string,
-    count: number
-  ) {
-    const result = await this.cache.mapIncrease(
-      UPDATES_QUEUE_CACHE_KEY,
-      `${workspaceId}::${guid}`,
-      count
-    );
-
-    if (result <= 0) {
-      await this.cache.mapDelete(
-        UPDATES_QUEUE_CACHE_KEY,
-        `${workspaceId}::${guid}`
-      );
-    }
-  }
 }

packages/backend/server/src/core/doc/config.ts

@@ -5,28 +5,6 @@ import {
 } from '../../base/config';
 
 interface DocStartupConfigurations {
-  manager: {
-    /**
-     * Whether auto merge updates into doc snapshot.
-     */
-    enableUpdateAutoMerging: boolean;
-
-    /**
-     * How often the [DocManager] will start a new turn of merging pending updates into doc snapshot.
-     *
-     * This is not the latency a new joint client will take to see the latest doc,
-     * but the buffer time we introduced to reduce the load of our service.
-     *
-     * in {ms}
-     */
-    updatePollInterval: number;
-
-    /**
-     * The maximum number of updates that will be pulled from the server at once.
-     * Existing for avoiding the server to be overloaded when there are too many updates for one doc.
-     */
-    maxUpdatesPullCount: number;
-  };
   history: {
     /**
      * How long the buffer time of creating a new history snapshot when doc get updated.
@@ -53,11 +31,6 @@ declare module '../../base/config' {
 }
 
 defineStartupConfig('doc', {
-  manager: {
-    enableUpdateAutoMerging: true,
-    updatePollInterval: 3000,
-    maxUpdatesPullCount: 500,
-  },
   history: {
     interval: 1000 * 60 * 10 /* 10 mins */,
   },

packages/backend/server/src/core/doc/job.ts

@@ -1,27 +1,38 @@
 import { Injectable } from '@nestjs/common';
 import { Cron, CronExpression } from '@nestjs/schedule';
 
-import { metrics, OnEvent } from '../../base';
+import { JobQueue, OnEvent, OnJob } from '../../base';
 import { Models } from '../../models';
 import { PgWorkspaceDocStorageAdapter } from './adapters/workspace';
 
+declare global {
+  interface Jobs {
+    'nightly.cleanExpiredHistories': {};
+  }
+}
+
 @Injectable()
 export class DocStorageCronJob {
   constructor(
     private readonly models: Models,
-    private readonly workspace: PgWorkspaceDocStorageAdapter
+    private readonly workspace: PgWorkspaceDocStorageAdapter,
+    private readonly queue: JobQueue
   ) {}
 
-  @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT /* everyday at 12am */)
-  async cleanupExpiredHistory() {
-    await this.models.doc.deleteExpiredHistories();
+  @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
+  async nightlyJob() {
+    await this.queue.add(
+      'nightly.cleanExpiredHistories',
+      {},
+      {
+        jobId: 'nightly-doc-clean-expired-histories',
+      }
+    );
   }
 
-  @Cron(CronExpression.EVERY_MINUTE)
-  async reportUpdatesQueueCount() {
-    metrics.doc
-      .gauge('updates_queue_count')
-      .record(await this.models.doc.getGlobalUpdateCount());
+  @OnJob('nightly.cleanExpiredHistories')
+  async cleanExpiredHistories() {
+    await this.models.doc.deleteExpiredHistories();
   }
 
   @OnEvent('user.deleted')