refactor(server): object storages (#5405)

packages/backend/server/src/affine.config.ts

@@ -0,0 +1,21 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+// Custom configurations
+
+const env = process.env;
+const node = AFFiNE.node;
+
+// TODO: may be separate config overring in `affine.[env].config`?
+if (node.prod && env.R2_OBJECT_STORAGE_ACCOUNT_ID) {
+  AFFiNE.storage.providers.r2 = {
+    accountId: env.R2_OBJECT_STORAGE_ACCOUNT_ID,
+    credentials: {
+      accessKeyId: env.R2_OBJECT_STORAGE_ACCESS_KEY_ID!,
+      secretAccessKey: env.R2_OBJECT_STORAGE_SECRET_ACCESS_KEY!,
+    },
+  };
+  AFFiNE.storage.storages.avatar.provider = 'r2';
+  AFFiNE.storage.storages.avatar.bucket = 'account-avatar';
+
+  AFFiNE.storage.storages.blob.provider = 'r2';
+  AFFiNE.storage.storages.blob.bucket = 'workspace-blobs';
+}

packages/backend/server/src/affine.ts

@@ -0,0 +1,3 @@
+import { getDefaultAFFiNEConfig } from './config/default';
+
+globalThis.AFFiNE = getDefaultAFFiNEConfig();

packages/backend/server/src/config/def.ts

@@ -1,6 +1,7 @@
 import type { ApolloDriverConfig } from '@nestjs/apollo';
 
 import type { LeafPaths } from '../utils/types';
+import type { AFFiNEStorageConfig } from './storage';
 
 declare global {
   // eslint-disable-next-line @typescript-eslint/no-namespace
@@ -165,11 +166,18 @@ export interface AFFiNEConfig {
   featureFlags: {
     earlyAccessPreview: boolean;
   };
+
+  /**
+   * Configuration for Object Storage, which defines how blobs and avatar assets are stored.
+   */
+  storage: AFFiNEStorageConfig;
+
   /**
    * object storage Config
    *
    * all artifacts and logs will be stored on instance disk,
    * and can not shared between instances if not configured
+   * @deprecated use `storage` instead
    */
   objectStorage: {
     /**

packages/backend/server/src/config/default.ts

@@ -9,6 +9,7 @@ import parse from 'parse-duration';
 import pkg from '../../package.json' assert { type: 'json' };
 import type { AFFiNEConfig, ServerFlavor } from './def';
 import { applyEnvToConfig } from './env';
+import { getDefaultAFFiNEStorageConfig } from './storage';
 
 export const SERVER_FLAVOR = (process.env.SERVER_FLAVOR ??
   'allinone') as ServerFlavor;
@@ -59,11 +60,6 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
       AFFINE_SERVER_SUB_PATH: 'path',
       AFFINE_ENV: 'affineEnv',
       DATABASE_URL: 'db.url',
-      ENABLE_R2_OBJECT_STORAGE: ['objectStorage.r2.enabled', 'boolean'],
-      R2_OBJECT_STORAGE_ACCOUNT_ID: 'objectStorage.r2.accountId',
-      R2_OBJECT_STORAGE_ACCESS_KEY_ID: 'objectStorage.r2.accessKeyId',
-      R2_OBJECT_STORAGE_SECRET_ACCESS_KEY: 'objectStorage.r2.secretAccessKey',
-      R2_OBJECT_STORAGE_BUCKET: 'objectStorage.r2.bucket',
       ENABLE_CAPTCHA: ['auth.captcha.enable', 'boolean'],
       CAPTCHA_TURNSTILE_SECRET: ['auth.captcha.turnstile.secret', 'string'],
       OAUTH_GOOGLE_ENABLED: ['auth.oauthProviders.google.enabled', 'boolean'],
@@ -180,6 +176,7 @@ export const getDefaultAFFiNEConfig: () => AFFiNEConfig = () => {
         password: '',
       },
     },
+    storage: getDefaultAFFiNEStorageConfig(),
     objectStorage: {
       r2: {
         enabled: false,

packages/backend/server/src/config/index.ts

@@ -74,3 +74,4 @@ export class ConfigModule {
 
 export type { AFFiNEConfig } from './def';
 export { SERVER_FLAVOR } from './default';
+export * from './storage';

packages/backend/server/src/config/storage/index.ts

@@ -0,0 +1,58 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+import { S3ClientConfigType } from '@aws-sdk/client-s3';
+
+export type StorageProviderType = 'fs' | 'r2' | 's3';
+export interface FsStorageConfig {
+  path: string;
+}
+export type R2StorageConfig = S3ClientConfigType & {
+  accountId: string;
+};
+export type S3StorageConfig = S3ClientConfigType;
+
+export type StorageTargetConfig = {
+  provider: StorageProviderType;
+  bucket: string;
+};
+
+export interface AFFiNEStorageConfig {
+  /**
+   * All providers for object storage
+   *
+   * Support different providers for different usage at the same time.
+   */
+  providers: {
+    fs?: FsStorageConfig;
+    s3?: S3StorageConfig;
+    r2?: R2StorageConfig;
+  };
+  storages: {
+    avatar: StorageTargetConfig;
+    blob: StorageTargetConfig;
+  };
+}
+
+export type StorageProviders = AFFiNEStorageConfig['providers'];
+export type Storages = keyof AFFiNEStorageConfig['storages'];
+
+export function getDefaultAFFiNEStorageConfig(): AFFiNEStorageConfig {
+  return {
+    providers: {
+      fs: {
+        path: join(homedir(), '.affine/storage'),
+      },
+    },
+    storages: {
+      avatar: {
+        provider: 'fs',
+        bucket: 'avatars',
+      },
+      blob: {
+        provider: 'fs',
+        bucket: 'blobs',
+      },
+    },
+  };
+}

packages/backend/server/src/prelude.ts

@@ -1,9 +1,7 @@
 import 'reflect-metadata';
 import 'dotenv/config';
-
+import './affine';
-import { getDefaultAFFiNEConfig } from './config/default';
+import './affine.config';
-
-globalThis.AFFiNE = getDefaultAFFiNEConfig();
 
 if (process.env.NODE_ENV === 'development') {
   console.log('AFFiNE Config:', globalThis.AFFiNE);