diff --git a/tools/cli/src/bundle.ts b/tools/cli/src/bundle.ts index 186573df2..182a56b57 100644 --- a/tools/cli/src/bundle.ts +++ b/tools/cli/src/bundle.ts @@ -31,6 +31,10 @@ import { createNodeTargetConfig as createWebpackNodeTargetConfig, createWorkerTargetConfig as createWebpackWorkerTargetConfig, } from './webpack'; +import { + shouldUploadReleaseAssets, + uploadDistAssetsToS3, +} from './webpack/s3-plugin.js'; type WorkerConfig = { name: string }; type CreateWorkerTargetConfig = (pkg: Package, entry: string) => WorkerConfig; @@ -39,6 +43,21 @@ function assertRspackSupportedPackage(pkg: Package) { assertRspackSupportedPackageName(pkg.name); } +function shouldUploadAssetsForPackage(pkg: Package): boolean { + return ( + !!process.env.R2_SECRET_ACCESS_KEY && shouldUploadReleaseAssets(pkg.name) + ); +} + +async function uploadAssetsForPackage(pkg: Package, logger: Logger) { + if (!shouldUploadAssetsForPackage(pkg)) { + return; + } + logger.info('Uploading dist assets to R2...'); + await uploadDistAssetsToS3(pkg.distPath.value); + logger.info('Uploaded dist assets to R2.'); +} + function getBaseWorkerConfigs( pkg: Package, createWorkerTargetConfig: CreateWorkerTargetConfig @@ -253,20 +272,34 @@ export class BundleCommand extends PackageCommand { throw new Error('Failed to create webpack compiler'); } - compiler.run((error, stats) => { - if (error) { - console.error(error); - process.exit(1); - } - if (stats) { - if (stats.hasErrors()) { - console.error(stats.toString('errors-only')); - process.exit(1); - } else { - console.log(stats.toString('minimal')); + try { + const stats = await new Promise( + (resolve, reject) => { + compiler.run((error, stats) => { + if (error) { + reject(error); + return; + } + if (!stats) { + reject(new Error('Failed to get webpack stats')); + return; + } + resolve(stats); + }); } + ); + if (stats.hasErrors()) { + console.error(stats.toString('errors-only')); + process.exit(1); + return; } - }); + console.log(stats.toString('minimal')); + await uploadAssetsForPackage(pkg, logger); + } catch (error) { + console.error(error); + process.exit(1); + return; + } } static async dev( @@ -338,20 +371,32 @@ export class BundleCommand extends PackageCommand { throw new Error('Failed to create rspack compiler'); } - compiler.run((error, stats) => { - if (error) { - console.error(error); + try { + const stats = await new Promise((resolve, reject) => { + compiler.run((error, stats) => { + if (error) { + reject(error); + return; + } + if (!stats) { + reject(new Error('Failed to get rspack stats')); + return; + } + resolve(stats); + }); + }); + if (stats.hasErrors()) { + console.error(stats.toString('errors-only')); process.exit(1); + return; } - if (stats) { - if (stats.hasErrors()) { - console.error(stats.toString('errors-only')); - process.exit(1); - } else { - console.log(stats.toString('minimal')); - } - } - }); + console.log(stats.toString('minimal')); + await uploadAssetsForPackage(pkg, logger); + } catch (error) { + console.error(error); + process.exit(1); + return; + } } static async devWithRspack( diff --git a/tools/cli/src/rspack/index.ts b/tools/cli/src/rspack/index.ts index 112c54fb4..e1c3cffcc 100644 --- a/tools/cli/src/rspack/index.ts +++ b/tools/cli/src/rspack/index.ts @@ -17,7 +17,6 @@ import { type CreateHTMLPluginConfig, createHTMLPlugins as createWebpackCompatibleHTMLPlugins, } from '../webpack/html-plugin.js'; -import { WebpackS3Plugin } from '../webpack/s3-plugin.js'; const require = createRequire(import.meta.url); @@ -277,10 +276,6 @@ export function createHTMLTargetConfig( }, ], }), - !buildConfig.debug && - (buildConfig.isWeb || buildConfig.isMobileWeb || buildConfig.isAdmin) && - process.env.R2_SECRET_ACCESS_KEY && - new WebpackS3Plugin(), process.env.SENTRY_AUTH_TOKEN && process.env.SENTRY_ORG && process.env.SENTRY_PROJECT && diff --git a/tools/cli/src/webpack/index.ts b/tools/cli/src/webpack/index.ts index 65ac6057a..a28300b50 100644 --- a/tools/cli/src/webpack/index.ts +++ b/tools/cli/src/webpack/index.ts @@ -18,7 +18,6 @@ import { type CreateHTMLPluginConfig, createHTMLPlugins, } from './html-plugin.js'; -import { WebpackS3Plugin } from './s3-plugin.js'; const require = createRequire(import.meta.url); const cssnano = require('cssnano'); @@ -279,10 +278,6 @@ export function createHTMLTargetConfig( }, ], }), - !buildConfig.debug && - (buildConfig.isWeb || buildConfig.isMobileWeb || buildConfig.isAdmin) && - process.env.R2_SECRET_ACCESS_KEY && - new WebpackS3Plugin(), !buildConfig.debug && process.env.PERFSEE_TOKEN && new PerfseePlugin({ project: 'affine-toeverything' }), diff --git a/tools/cli/src/webpack/s3-plugin.ts b/tools/cli/src/webpack/s3-plugin.ts index 4e5459d7a..2c562a634 100644 --- a/tools/cli/src/webpack/s3-plugin.ts +++ b/tools/cli/src/webpack/s3-plugin.ts @@ -1,43 +1,141 @@ -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { readdir, readFile } from 'node:fs/promises'; +import { join, relative, sep } from 'node:path'; import { createS3CompatClient } from '@affine/s3-compat'; import { lookup } from 'mime-types'; -import type { Compiler, WebpackPluginInstance } from 'webpack'; export const R2_BUCKET = process.env.R2_BUCKET ?? (process.env.BUILD_TYPE === 'canary' ? 'assets-dev' : 'assets-prod'); -export class WebpackS3Plugin implements WebpackPluginInstance { - private readonly s3 = createS3CompatClient( +const S3_UPLOAD_PACKAGE_NAMES = new Set([ + '@affine/web', + '@affine/mobile', + '@affine/admin', +]); +const MAX_UPLOAD_RETRIES = 3; +const UPLOAD_RETRY_BASE_DELAY_MS = 500; + +function createR2Client() { + const { R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY } = process.env; + if (!R2_ACCOUNT_ID || !R2_ACCESS_KEY_ID || !R2_SECRET_ACCESS_KEY) { + throw new Error('Missing R2 credentials for uploading release assets'); + } + + return createS3CompatClient( { region: 'auto', bucket: R2_BUCKET, forcePathStyle: true, - endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, + endpoint: `https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com`, }, { - accessKeyId: process.env.R2_ACCESS_KEY_ID!, - secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!, + accessKeyId: R2_ACCESS_KEY_ID, + secretAccessKey: R2_SECRET_ACCESS_KEY, + } + ); +} + +async function collectFiles(dir: string): Promise { + const dirs = [dir]; + const files: string[] = []; + + while (dirs.length > 0) { + const current = dirs.pop()!; + const entries = await readdir(current, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = join(current, entry.name); + if (entry.isDirectory()) { + dirs.push(fullPath); + } else if (entry.isFile()) { + files.push(fullPath); + } + } + } + + return files; +} + +function toAssetKey(outputPath: string, filePath: string): string { + return relative(outputPath, filePath).split(sep).join('/'); +} + +function sleep(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function putObjectWithRetry( + s3: ReturnType, + asset: string, + assetSource: Buffer, + contentType: string | false | undefined +) { + let retries = 0; + while (true) { + try { + await s3.putObject(asset, assetSource, { + contentType: contentType || undefined, + contentLength: assetSource.byteLength, + }); + return; + } catch (error) { + if (retries >= MAX_UPLOAD_RETRIES) { + throw error; + } + retries += 1; + const delay = UPLOAD_RETRY_BASE_DELAY_MS * 2 ** (retries - 1); + const errorMessage = + error instanceof Error ? error.message : String(error); + console.warn( + `[s3-upload] Retry ${retries}/${MAX_UPLOAD_RETRIES} for ${asset}: ${errorMessage}` + ); + await sleep(delay); + } + } +} + +async function runInParallel( + values: T[], + worker: (value: T) => Promise, + concurrency = 16 +) { + if (values.length === 0) { + return; + } + let nextIndex = 0; + const workers = Array.from( + { length: Math.min(concurrency, values.length) }, + async () => { + while (true) { + const index = nextIndex++; + if (index >= values.length) { + return; + } + await worker(values[index]!); + } } ); - apply(compiler: Compiler) { - compiler.hooks.assetEmitted.tapPromise( - 'WebpackS3Plugin', - async (asset, { outputPath }) => { - if (asset.endsWith('.html')) { - return; - } - const assetPath = join(outputPath, asset); - const assetSource = await readFile(assetPath); - const contentType = lookup(asset) || undefined; - await this.s3.putObject(asset, assetSource, { - contentType, - contentLength: assetSource.byteLength, - }); - } - ); - } + await Promise.all(workers); +} + +export function shouldUploadReleaseAssets(pkgName: string): boolean { + return S3_UPLOAD_PACKAGE_NAMES.has(pkgName); +} + +export async function uploadDistAssetsToS3(outputPath: string) { + const allFiles = await collectFiles(outputPath); + const uploadFiles = allFiles.filter(file => !file.endsWith('.html')); + + if (uploadFiles.length === 0) { + return; + } + + const s3 = createR2Client(); + await runInParallel(uploadFiles, async filePath => { + const asset = toAssetKey(outputPath, filePath); + const assetSource = await readFile(filePath); + const contentType = lookup(asset); + await putObjectWithRetry(s3, asset, assetSource, contentType); + }); }