chore(server): add job backoff strategies (#12499)

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **New Features**
  - Implemented an exponential backoff retry strategy for job queues, resulting in progressively longer wait times between retry attempts.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.docker/selfhost/schema.json

@@ -31,9 +31,13 @@
       "properties": {
         "queue": {
           "type": "object",
-          "description": "The config for job queues\n@default {\"attempts\":5,\"removeOnComplete\":true,\"removeOnFail\":{\"age\":86400,\"count\":500}}\n@link https://api.docs.bullmq.io/interfaces/v5.QueueOptions.html",
+          "description": "The config for job queues\n@default {\"attempts\":5,\"backoff\":{\"type\":\"exponential\",\"delay\":1000},\"removeOnComplete\":true,\"removeOnFail\":{\"age\":86400,\"count\":500}}\n@link https://api.docs.bullmq.io/interfaces/v5.QueueOptions.html",
           "default": {
             "attempts": 5,
+            "backoff": {
+              "type": "exponential",
+              "delay": 1000
+            },
             "removeOnComplete": true,
             "removeOnFail": {
               "age": 86400,

packages/backend/server/src/base/job/queue/config.ts

@@ -29,6 +29,8 @@ defineModuleConfig('job', {
     desc: 'The config for job queues',
     default: {
       attempts: 5,
+      // retry after 2 ^ (attempts - 1) * delay milliseconds
+      backoff: { type: 'exponential', delay: 1000 },
       // should remove job after it's completed, because we will add a new job with the same job id
       removeOnComplete: true,
       removeOnFail: {