diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts index f8e18a297..18a6aae11 100644 --- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts @@ -64,16 +64,16 @@ test.serial.before(async t => { copilot: { providers: { openai: { - apiKey: process.env.COPILOT_OPENAI_API_KEY, + apiKey: process.env.COPILOT_OPENAI_API_KEY || '', }, fal: { - apiKey: process.env.COPILOT_FAL_API_KEY, + apiKey: process.env.COPILOT_FAL_API_KEY || '', }, perplexity: { - apiKey: process.env.COPILOT_PERPLEXITY_API_KEY, + apiKey: process.env.COPILOT_PERPLEXITY_API_KEY || '', }, gemini: { - apiKey: process.env.COPILOT_GOOGLE_API_KEY, + apiKey: process.env.COPILOT_GOOGLE_API_KEY || '', }, }, }, diff --git a/packages/backend/server/src/base/config/__tests__/config.spec.ts b/packages/backend/server/src/base/config/__tests__/config.spec.ts index ac0cc706a..20001c4b7 100644 --- a/packages/backend/server/src/base/config/__tests__/config.spec.ts +++ b/packages/backend/server/src/base/config/__tests__/config.spec.ts @@ -3,6 +3,7 @@ import test from 'ava'; import { createModule } from '../../../__tests__/create-module'; import { ConfigFactory, ConfigModule } from '..'; import { Config } from '../config'; +import { override } from '../register'; const module = await createModule(); test.after.always(async () => { @@ -49,13 +50,14 @@ test('should override config', async t => { auth: { passwordRequirements: { max: 10, + min: 1, }, }, }); t.deepEqual(config.auth.passwordRequirements, { max: 10, - min: 6, + min: 1, }); }); @@ -88,3 +90,87 @@ Error: Minimum length of password must be less than maximum length`, } ); }); + +test('should override correctly', t => { + const config = { + auth: { + // object config + passwordRequirements: { + max: 10, + min: 6, + }, + allowSignup: false, + // keyed config + // 'session.ttl', 'session.ttr' + session: { + ttl: 2000, + ttr: 1000, + }, + }, + storages: { + avatar: { + // keyed config + // "avatar.publicPath: String" + publicPath: '/', + // object config + // "avatar.storage => Object { }" + storage: { + provider: 'fs', + config: { + path: '/path/to/avatar', + }, + }, + }, + }, + } as AppConfig; + + override(config, { + auth: { + passwordRequirements: { + max: 20, + }, + allowSignup: true, + session: { + ttl: 3000, + }, + }, + storages: { + avatar: { + storage: { + provider: 'aws-s3', + config: { + credentials: { + accessKeyId: '1', + accessKeySecret: '1', + }, + }, + }, + }, + }, + }); + + // simple value override + t.deepEqual(config.auth.allowSignup, true); + + // right covered left + t.deepEqual(config.auth.passwordRequirements, { + max: 20, + }); + + // right merged to left + t.deepEqual(config.auth.session, { + ttl: 3000, + ttr: 1000, + }); + + // right covered left + t.deepEqual(config.storages.avatar.storage, { + provider: 'aws-s3', + config: { + credentials: { + accessKeyId: '1', + accessKeySecret: '1', + }, + }, + }); +}); diff --git a/packages/backend/server/src/base/config/factory.ts b/packages/backend/server/src/base/config/factory.ts index ad6c7a887..82f2673d2 100644 --- a/packages/backend/server/src/base/config/factory.ts +++ b/packages/backend/server/src/base/config/factory.ts @@ -1,14 +1,13 @@ import { Inject, Injectable, Optional } from '@nestjs/common'; -import { merge } from 'lodash-es'; import { InvalidAppConfig } from '../error'; -import { APP_CONFIG_DESCRIPTORS, getDefaultConfig } from './register'; +import { APP_CONFIG_DESCRIPTORS, getDefaultConfig, override } from './register'; export const OVERRIDE_CONFIG_TOKEN = Symbol('OVERRIDE_CONFIG_TOKEN'); @Injectable() export class ConfigFactory { - readonly #config: DeepReadonly; + readonly #config: AppConfig; constructor( @Inject(OVERRIDE_CONFIG_TOKEN) @@ -22,8 +21,12 @@ export class ConfigFactory { return this.#config; } + clone() { + return structuredClone(this.#config); + } + override(updates: DeepPartial) { - merge(this.#config, updates); + override(this.#config, updates); } validate(updates: Array<{ module: string; key: string; value: any }>) { @@ -53,8 +56,9 @@ Error: ${issue.message}`); } } - private loadDefault(): DeepReadonly { + private loadDefault() { const config = getDefaultConfig(); - return merge(config, this.overrides); + override(config, this.overrides); + return config; } } diff --git a/packages/backend/server/src/base/config/register.ts b/packages/backend/server/src/base/config/register.ts index 9d203c5a4..28a329804 100644 --- a/packages/backend/server/src/base/config/register.ts +++ b/packages/backend/server/src/base/config/register.ts @@ -1,7 +1,8 @@ import { existsSync, readFileSync } from 'node:fs'; +import { homedir } from 'node:os'; import { join } from 'node:path'; -import { merge, once, set } from 'lodash-es'; +import { mergeWith, once, set } from 'lodash-es'; import { z } from 'zod'; import { type EnvConfigType, parseEnvValue } from './env'; @@ -208,14 +209,15 @@ export function defineModuleConfig( }; } -function readConfigJSONOverrides() { - if (existsSync(join(env.projectRoot, 'config.json'))) { +const CONFIG_JSON_PATHS = [ + join(env.projectRoot, 'config.json'), + `${homedir()}/.affine/config/config.json`, +]; +function readConfigJSONOverrides(path: string) { + const overrides: DeepPartial = {}; + if (existsSync(path)) { try { - const config = JSON.parse( - readFileSync(join(env.projectRoot, 'config.json'), 'utf-8') - ) as AppConfig; - - const overrides = {}; + const config = JSON.parse(readFileSync(path, 'utf-8')) as AppConfig; Object.entries(config).forEach(([key, value]) => { if (key === '$schema') { @@ -226,18 +228,41 @@ function readConfigJSONOverrides() { set(overrides, `${key}.${k}`, v); }); }); - - return overrides; } catch (e) { console.error('Invalid json config file', e); } } - return {}; + return overrides; } -export function getDefaultConfig(): AppConfigSchema { - const config: Record = {}; +export function override(config: AppConfig, update: DeepPartial) { + Object.keys(update).forEach(module => { + const moduleDescriptors = APP_CONFIG_DESCRIPTORS[module]; + const configKeys = new Set(Object.keys(moduleDescriptors)); + + const moduleConfig = config[module as keyof AppConfig]; + const moduleOverrides = update[module as keyof AppConfig]; + + const merge = (left: any, right: any, path: string = '') => { + // if we found the key in the config keys + // we should use the override object instead of merge it with left + if (configKeys.has(path)) { + return right; + } + + // go deeper + return mergeWith(left, right, (left, right, key) => { + return merge(left, right, path === '' ? key : `${path}.${key}`); + }); + }; + + config[module as keyof AppConfig] = merge(moduleConfig, moduleOverrides); + }); +} + +export function getDefaultConfig(): AppConfig { + const config = {} as AppConfig; const envs = process.env; for (const [module, defs] of Object.entries(APP_CONFIG_DESCRIPTORS)) { @@ -271,12 +296,14 @@ Error: ${issue.message}`; set(modulizedConfig, key, defaultValue); } + // @ts-expect-error all keys are known config[module] = modulizedConfig; } - const fileOverrides = readConfigJSONOverrides(); - - merge(config, fileOverrides); + CONFIG_JSON_PATHS.forEach(path => { + const overrides = readConfigJSONOverrides(path); + override(config, overrides); + }); return config as AppConfigSchema; } diff --git a/packages/backend/server/src/base/event/eventbus.ts b/packages/backend/server/src/base/event/eventbus.ts index a3ea945e4..af0c33a01 100644 --- a/packages/backend/server/src/base/event/eventbus.ts +++ b/packages/backend/server/src/base/event/eventbus.ts @@ -155,7 +155,7 @@ export class EventBus add.call(this.emitter, event, handler as any, opts); - this.logger.verbose(`Event handler registered ${signature}`); + this.logger.log(`Event handler registered ${signature}`); return () => { this.emitter.off(event, handler as any); diff --git a/packages/backend/server/src/base/job/queue/scanner.ts b/packages/backend/server/src/base/job/queue/scanner.ts index cf53a053d..52ecec578 100644 --- a/packages/backend/server/src/base/job/queue/scanner.ts +++ b/packages/backend/server/src/base/job/queue/scanner.ts @@ -1,4 +1,4 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { ModuleScanner } from '../../nestjs'; import { getJobHandlerMetadata, JOB_SIGNAL } from './def'; @@ -11,6 +11,7 @@ interface JobHandler { @Injectable() export class JobHandlerScanner implements OnModuleInit { private readonly handlers: Record = {}; + private readonly logger = new Logger(JobHandlerScanner.name); constructor(private readonly scanner: ModuleScanner) {} @@ -70,6 +71,8 @@ export class JobHandlerScanner implements OnModuleInit { return instance[method].bind(instance)(payload); }, }; + + this.logger.log(`Job handler registered [${jobName}] (${signature})`); }); }); }); diff --git a/packages/backend/server/src/core/config/__tests__/service.spec.ts b/packages/backend/server/src/core/config/__tests__/service.spec.ts index 5c0bcb415..694711313 100644 --- a/packages/backend/server/src/core/config/__tests__/service.spec.ts +++ b/packages/backend/server/src/core/config/__tests__/service.spec.ts @@ -23,7 +23,7 @@ test.after.always(async () => { }); test('should update config', async t => { - const oldValue = service.config.server.externalUrl; + const oldValue = service.getConfig().server.externalUrl; const newValue = faker.internet.url(); await service.updateConfig(user.id, [ { @@ -33,8 +33,8 @@ test('should update config', async t => { }, ]); - t.not(service.config.server.externalUrl, oldValue); - t.is(service.config.server.externalUrl, newValue); + t.not(service.getConfig().server.externalUrl, oldValue); + t.is(service.getConfig().server.externalUrl, newValue); }); test('should validate config before update', async t => { @@ -53,7 +53,7 @@ Error: Invalid url`, } ); - t.not(service.config.server.externalUrl, 'invalid-url'); + t.not(service.getConfig().server.externalUrl, 'invalid-url'); await t.throwsAsync( service.updateConfig(user.id, [ @@ -68,8 +68,11 @@ Error: Invalid url`, } ); - // @ts-expect-error allow - t.is(service.config.auth['unknown-key'], undefined); + t.is( + // @ts-expect-error testing + service.getConfig().auth['unknown-key'], + undefined + ); }); test('should emit config.init event', async t => { @@ -77,12 +80,12 @@ test('should emit config.init event', async t => { const event = module.event.last('config.init'); t.is(event.name, 'config.init'); t.deepEqual(event.payload, { - config: service.config, + config: service.getConfig(), }); }); test('should revalidate config', async t => { - const outdatedValue = service.config.server.externalUrl; + const outdatedValue = service.getConfig().server.externalUrl; const newValue = faker.internet.url(); await models.appConfig.save(user.id, [ @@ -94,8 +97,8 @@ test('should revalidate config', async t => { await service.revalidateConfig(); - t.not(service.config.server.externalUrl, outdatedValue); - t.is(service.config.server.externalUrl, newValue); + t.not(service.getConfig().server.externalUrl, outdatedValue); + t.is(service.getConfig().server.externalUrl, newValue); }); test('should emit config changed event', async t => { diff --git a/packages/backend/server/src/core/config/resolver.ts b/packages/backend/server/src/core/config/resolver.ts index a76fa0bb3..05ea8356d 100644 --- a/packages/backend/server/src/core/config/resolver.ts +++ b/packages/backend/server/src/core/config/resolver.ts @@ -191,7 +191,7 @@ export class AppConfigResolver { description: 'get the whole app configuration', }) appConfig() { - return this.service.config; + return this.service.getConfig(); } @Mutation(() => GraphQLJSONObject, { diff --git a/packages/backend/server/src/core/config/service.ts b/packages/backend/server/src/core/config/service.ts index 3fa993875..b8634b0bb 100644 --- a/packages/backend/server/src/core/config/service.ts +++ b/packages/backend/server/src/core/config/service.ts @@ -56,8 +56,8 @@ export class ServerService implements OnApplicationBootstrap { this.#features.delete(feature); } - get config() { - return this.configFactory.config; + getConfig() { + return this.configFactory.clone(); } async updateConfig(