From 760d900f99aa14a39887fdaa36e8660e089fc6d1 Mon Sep 17 00:00:00 2001 From: liuyi Date: Wed, 3 Jan 2024 10:56:57 +0000 Subject: [PATCH] feat(server): blob data migration (#5461) --- .github/actions/deploy/deploy.mjs | 2 - .../charts/graphql/templates/deployment.yaml | 5 --- .../charts/graphql/templates/migration.yaml | 17 +++++++++ .../charts/graphql/templates/r2-secret.yaml | 1 - .../helm/affine/charts/graphql/values.yaml | 1 - .github/workflows/build-test.yml | 35 ++++++++--------- .github/workflows/deploy.yml | 1 - packages/backend/server/src/data/app.ts | 20 ++++++++-- .../backend/server/src/data/commands/run.ts | 21 ++++++---- .../1703828796699-workspace-blobs.ts | 38 +++++++++++++++++++ .../src/modules/storage/wrappers/blob.ts | 7 +++- 11 files changed, 109 insertions(+), 39 deletions(-) create mode 100644 packages/backend/server/src/data/migrations/1703828796699-workspace-blobs.ts diff --git a/.github/actions/deploy/deploy.mjs b/.github/actions/deploy/deploy.mjs index 204d8471a..ea8ab78e1 100644 --- a/.github/actions/deploy/deploy.mjs +++ b/.github/actions/deploy/deploy.mjs @@ -13,7 +13,6 @@ const { R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, - R2_BUCKET, ENABLE_CAPTCHA, CAPTCHA_TURNSTILE_SECRET, OAUTH_EMAIL_SENDER, @@ -96,7 +95,6 @@ const createHelmCommand = ({ isDryRun }) => { `--set-string graphql.app.objectStorage.r2.accountId="${R2_ACCOUNT_ID}"`, `--set-string graphql.app.objectStorage.r2.accessKeyId="${R2_ACCESS_KEY_ID}"`, `--set-string graphql.app.objectStorage.r2.secretAccessKey="${R2_SECRET_ACCESS_KEY}"`, - `--set-string graphql.app.objectStorage.r2.bucket="${R2_BUCKET}"`, `--set-string graphql.app.oauth.email.sender="${OAUTH_EMAIL_SENDER}"`, `--set-string graphql.app.oauth.email.login="${OAUTH_EMAIL_LOGIN}"`, `--set-string graphql.app.oauth.email.password="${OAUTH_EMAIL_PASSWORD}"`, diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml index be1473098..38e53c342 100644 --- a/.github/helm/affine/charts/graphql/templates/deployment.yaml +++ b/.github/helm/affine/charts/graphql/templates/deployment.yaml @@ -136,11 +136,6 @@ spec: secretKeyRef: name: "{{ .Values.app.objectStorage.r2.secretName }}" key: secretAccessKey - - name: R2_OBJECT_STORAGE_BUCKET - valueFrom: - secretKeyRef: - name: "{{ .Values.app.objectStorage.r2.secretName }}" - key: bucket {{ end }} {{ if .Values.app.captcha.enabled }} - name: CAPTCHA_TURNSTILE_SECRET diff --git a/.github/helm/affine/charts/graphql/templates/migration.yaml b/.github/helm/affine/charts/graphql/templates/migration.yaml index 658a7f3ad..8d95457ff 100644 --- a/.github/helm/affine/charts/graphql/templates/migration.yaml +++ b/.github/helm/affine/charts/graphql/templates/migration.yaml @@ -35,6 +35,23 @@ spec: - name: DATABASE_URL value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.gcloud.cloudSqlInternal }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }} {{ end }} + {{ if .Values.app.objectStorage.r2.enabled }} + - name: R2_OBJECT_STORAGE_ACCOUNT_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.app.objectStorage.r2.secretName }}" + key: accountId + - name: R2_OBJECT_STORAGE_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: "{{ .Values.app.objectStorage.r2.secretName }}" + key: accessKeyId + - name: R2_OBJECT_STORAGE_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: "{{ .Values.app.objectStorage.r2.secretName }}" + key: secretAccessKey + {{ end }} resources: requests: cpu: '100m' diff --git a/.github/helm/affine/charts/graphql/templates/r2-secret.yaml b/.github/helm/affine/charts/graphql/templates/r2-secret.yaml index d4a655b94..a521c2713 100644 --- a/.github/helm/affine/charts/graphql/templates/r2-secret.yaml +++ b/.github/helm/affine/charts/graphql/templates/r2-secret.yaml @@ -8,5 +8,4 @@ data: accountId: {{ .Values.app.objectStorage.r2.accountId | b64enc }} accessKeyId: {{ .Values.app.objectStorage.r2.accessKeyId | b64enc }} secretAccessKey: {{ .Values.app.objectStorage.r2.secretAccessKey | b64enc }} - bucket: {{ .Values.app.objectStorage.r2.bucket | b64enc }} {{- end }} diff --git a/.github/helm/affine/charts/graphql/values.yaml b/.github/helm/affine/charts/graphql/values.yaml index 2353c3939..629997382 100644 --- a/.github/helm/affine/charts/graphql/values.yaml +++ b/.github/helm/affine/charts/graphql/values.yaml @@ -34,7 +34,6 @@ app: accountId: '' accessKeyId: '' secretAccessKey: '' - bucket: '' oauth: email: secretName: 'oauth-email' diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index c439d3572..003f2614c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -315,6 +315,12 @@ jobs: electron-install: false full-cache: true + - name: Download storage.node + uses: actions/download-artifact@v4 + with: + name: storage.node + path: ./packages/backend/server + - name: Initialize database run: | psql -h localhost -U postgres -c "CREATE DATABASE affine;" @@ -337,12 +343,6 @@ jobs: env: DATABASE_URL: postgresql://affine:affine@localhost:5432/affine - - name: Download storage.node - uses: actions/download-artifact@v4 - with: - name: storage.node - path: ./packages/backend/server - - name: Run server tests run: yarn workspace @affine/server test:coverage env: @@ -407,6 +407,18 @@ jobs: playwright-install: true hard-link-nm: false + - name: Download storage.node + uses: actions/download-artifact@v4 + with: + name: storage.node + path: ./packages/backend/server + + - name: Download affine.linux-x64-gnu.node + uses: actions/download-artifact@v4 + with: + name: affine.linux-x64-gnu.node + path: ./packages/frontend/native + - name: Initialize database run: | psql -h localhost -U postgres -c "CREATE DATABASE affine;" @@ -426,17 +438,6 @@ jobs: run: | yarn workspace @affine/server data-migration run yarn workspace @affine/server exec node --loader ts-node/esm/transpile-only ./scripts/init-db.ts - - name: Download storage.node - uses: actions/download-artifact@v4 - with: - name: storage.node - path: ./packages/backend/server - - - name: Download affine.linux-x64-gnu.node - uses: actions/download-artifact@v4 - with: - name: affine.linux-x64-gnu.node - path: ./packages/frontend/native - name: ${{ matrix.tests.name }} run: | diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8606e23e2..39185531b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -240,7 +240,6 @@ jobs: R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - R2_BUCKET: ${{ secrets.R2_BUCKET }} ENABLE_CAPTCHA: true CAPTCHA_TURNSTILE_SECRET: ${{ secrets.CAPTCHA_TURNSTILE_SECRET }} OAUTH_EMAIL_SENDER: ${{ secrets.OAUTH_EMAIL_SENDER }} diff --git a/packages/backend/server/src/data/app.ts b/packages/backend/server/src/data/app.ts index 1369bad62..5287e74fd 100644 --- a/packages/backend/server/src/data/app.ts +++ b/packages/backend/server/src/data/app.ts @@ -1,18 +1,32 @@ import { Logger, Module } from '@nestjs/common'; import { CommandFactory } from 'nest-commander'; -import { PrismaModule } from '../prisma'; +import { AppModule as BusinessAppModule } from '../app'; +import { ConfigModule } from '../config'; import { CreateCommand, NameQuestion } from './commands/create'; import { RevertCommand, RunCommand } from './commands/run'; @Module({ - imports: [PrismaModule], + imports: [ + ConfigModule.forRoot({ + doc: { + manager: { + enableUpdateAutoMerging: false, + }, + }, + }), + BusinessAppModule, + ], providers: [NameQuestion, CreateCommand, RunCommand, RevertCommand], }) class AppModule {} async function bootstrap() { - await CommandFactory.run(AppModule, new Logger()); + await CommandFactory.run(AppModule, new Logger()).catch(e => { + console.error(e); + process.exit(1); + }); + process.exit(0); } await bootstrap(); diff --git a/packages/backend/server/src/data/commands/run.ts b/packages/backend/server/src/data/commands/run.ts index 97b3c3098..b8123cd8d 100644 --- a/packages/backend/server/src/data/commands/run.ts +++ b/packages/backend/server/src/data/commands/run.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { Logger } from '@nestjs/common'; +import { ModuleRef } from '@nestjs/core'; import { Command, CommandRunner } from 'nest-commander'; import { PrismaService } from '../../prisma'; @@ -10,8 +11,8 @@ import { PrismaService } from '../../prisma'; interface Migration { file: string; name: string; - up: (db: PrismaService) => Promise; - down: (db: PrismaService) => Promise; + up: (db: PrismaService, injector: ModuleRef) => Promise; + down: (db: PrismaService, injector: ModuleRef) => Promise; } export async function collectMigrations(): Promise { @@ -46,7 +47,10 @@ export async function collectMigrations(): Promise { }) export class RunCommand extends CommandRunner { logger = new Logger(RunCommand.name); - constructor(private readonly db: PrismaService) { + constructor( + private readonly db: PrismaService, + private readonly injector: ModuleRef + ) { super(); } @@ -103,14 +107,14 @@ export class RunCommand extends CommandRunner { }); try { - await migration.up(this.db); + await migration.up(this.db, this.injector); } catch (e) { await this.db.dataMigration.delete({ where: { id: record.id, }, }); - await migration.down(this.db); + await migration.down(this.db, this.injector); this.logger.error('Failed to run data migration', e); process.exit(1); } @@ -134,7 +138,10 @@ export class RunCommand extends CommandRunner { export class RevertCommand extends CommandRunner { logger = new Logger(RevertCommand.name); - constructor(private readonly db: PrismaService) { + constructor( + private readonly db: PrismaService, + private readonly injector: ModuleRef + ) { super(); } @@ -168,7 +175,7 @@ export class RevertCommand extends CommandRunner { try { this.logger.log(`Reverting ${name}...`); - await migration.down(this.db); + await migration.down(this.db, this.injector); this.logger.log('Done reverting'); } catch (e) { this.logger.error(`Failed to revert data migration ${name}`, e); diff --git a/packages/backend/server/src/data/migrations/1703828796699-workspace-blobs.ts b/packages/backend/server/src/data/migrations/1703828796699-workspace-blobs.ts new file mode 100644 index 000000000..6bbeff1c3 --- /dev/null +++ b/packages/backend/server/src/data/migrations/1703828796699-workspace-blobs.ts @@ -0,0 +1,38 @@ +import { ModuleRef } from '@nestjs/core'; + +import { WorkspaceBlobStorage } from '../../modules/storage'; +import { PrismaService } from '../../prisma'; + +export class WorkspaceBlobs1703828796699 { + // do the migration + static async up(db: PrismaService, injector: ModuleRef) { + const blobStorage = injector.get(WorkspaceBlobStorage, { strict: false }); + let hasMore = true; + let turn = 0; + const eachTurnCount = 50; + + while (hasMore) { + const blobs = await db.blob.findMany({ + skip: turn * eachTurnCount, + take: eachTurnCount, + orderBy: { + createdAt: 'asc', + }, + }); + + hasMore = blobs.length === eachTurnCount; + turn += 1; + + await Promise.all( + blobs.map(async ({ workspaceId, hash, blob }) => + blobStorage.put(workspaceId, hash, blob) + ) + ); + } + } + + // revert the migration + static async down(_db: PrismaService) { + // old data kept, no need to downgrade the migration + } +} diff --git a/packages/backend/server/src/modules/storage/wrappers/blob.ts b/packages/backend/server/src/modules/storage/wrappers/blob.ts index 254a2801a..04028a947 100644 --- a/packages/backend/server/src/modules/storage/wrappers/blob.ts +++ b/packages/backend/server/src/modules/storage/wrappers/blob.ts @@ -37,8 +37,11 @@ export class WorkspaceBlobStorage { return blobs; } - async delete(workspaceId: string, key: string) { - return this.provider.delete(`${workspaceId}/${key}`); + /** + * we won't really delete the blobs until the doc blobs manager is implemented sounded + */ + async delete(_workspaceId: string, _key: string) { + // return this.provider.delete(`${workspaceId}/${key}`); } async totalSize(workspaceId: string) {