feat: asset upload with retry

tools/cli/src/bundle.ts

@@ -31,6 +31,10 @@ import {
   createNodeTargetConfig as createWebpackNodeTargetConfig,
   createWorkerTargetConfig as createWebpackWorkerTargetConfig,
 } from './webpack';
+import {
+  shouldUploadReleaseAssets,
+  uploadDistAssetsToS3,
+} from './webpack/s3-plugin.js';
 
 type WorkerConfig = { name: string };
 type CreateWorkerTargetConfig = (pkg: Package, entry: string) => WorkerConfig;
@@ -39,6 +43,21 @@ function assertRspackSupportedPackage(pkg: Package) {
   assertRspackSupportedPackageName(pkg.name);
 }
 
+function shouldUploadAssetsForPackage(pkg: Package): boolean {
+  return (
+    !!process.env.R2_SECRET_ACCESS_KEY && shouldUploadReleaseAssets(pkg.name)
+  );
+}
+
+async function uploadAssetsForPackage(pkg: Package, logger: Logger) {
+  if (!shouldUploadAssetsForPackage(pkg)) {
+    return;
+  }
+  logger.info('Uploading dist assets to R2...');
+  await uploadDistAssetsToS3(pkg.distPath.value);
+  logger.info('Uploaded dist assets to R2.');
+}
+
 function getBaseWorkerConfigs(
   pkg: Package,
   createWorkerTargetConfig: CreateWorkerTargetConfig
@@ -253,21 +272,35 @@ export class BundleCommand extends PackageCommand {
       throw new Error('Failed to create webpack compiler');
     }
 
+    try {
+      const stats = await new Promise<webpack.Stats | webpack.MultiStats>(
+        (resolve, reject) => {
           compiler.run((error, stats) => {
             if (error) {
-        console.error(error);
+              reject(error);
-        process.exit(1);
+              return;
             }
-      if (stats) {
+            if (!stats) {
+              reject(new Error('Failed to get webpack stats'));
+              return;
+            }
+            resolve(stats);
+          });
+        }
+      );
       if (stats.hasErrors()) {
         console.error(stats.toString('errors-only'));
         process.exit(1);
-        } else {
+        return;
+      }
       console.log(stats.toString('minimal'));
+      await uploadAssetsForPackage(pkg, logger);
+    } catch (error) {
+      console.error(error);
+      process.exit(1);
+      return;
     }
   }
-    });
-  }
 
   static async dev(
     pkg: Package,
@@ -338,21 +371,33 @@ export class BundleCommand extends PackageCommand {
       throw new Error('Failed to create rspack compiler');
     }
 
+    try {
+      const stats = await new Promise<any>((resolve, reject) => {
         compiler.run((error, stats) => {
           if (error) {
-        console.error(error);
+            reject(error);
-        process.exit(1);
+            return;
           }
-      if (stats) {
+          if (!stats) {
+            reject(new Error('Failed to get rspack stats'));
+            return;
+          }
+          resolve(stats);
+        });
+      });
       if (stats.hasErrors()) {
         console.error(stats.toString('errors-only'));
         process.exit(1);
-        } else {
+        return;
+      }
       console.log(stats.toString('minimal'));
+      await uploadAssetsForPackage(pkg, logger);
+    } catch (error) {
+      console.error(error);
+      process.exit(1);
+      return;
     }
   }
-    });
-  }
 
   static async devWithRspack(
     pkg: Package,

tools/cli/src/rspack/index.ts

@@ -17,7 +17,6 @@ import {
   type CreateHTMLPluginConfig,
   createHTMLPlugins as createWebpackCompatibleHTMLPlugins,
 } from '../webpack/html-plugin.js';
-import { WebpackS3Plugin } from '../webpack/s3-plugin.js';
 
 const require = createRequire(import.meta.url);
 
@@ -277,10 +276,6 @@ export function createHTMLTargetConfig(
             },
           ],
         }),
-      !buildConfig.debug &&
-        (buildConfig.isWeb || buildConfig.isMobileWeb || buildConfig.isAdmin) &&
-        process.env.R2_SECRET_ACCESS_KEY &&
-        new WebpackS3Plugin(),
       process.env.SENTRY_AUTH_TOKEN &&
         process.env.SENTRY_ORG &&
         process.env.SENTRY_PROJECT &&

tools/cli/src/webpack/index.ts

@@ -18,7 +18,6 @@ import {
   type CreateHTMLPluginConfig,
   createHTMLPlugins,
 } from './html-plugin.js';
-import { WebpackS3Plugin } from './s3-plugin.js';
 
 const require = createRequire(import.meta.url);
 const cssnano = require('cssnano');
@@ -279,10 +278,6 @@ export function createHTMLTargetConfig(
             },
           ],
         }),
-      !buildConfig.debug &&
-        (buildConfig.isWeb || buildConfig.isMobileWeb || buildConfig.isAdmin) &&
-        process.env.R2_SECRET_ACCESS_KEY &&
-        new WebpackS3Plugin(),
       !buildConfig.debug &&
         process.env.PERFSEE_TOKEN &&
         new PerfseePlugin({ project: 'affine-toeverything' }),

tools/cli/src/webpack/s3-plugin.ts

@@ -1,43 +1,141 @@
-import { readFile } from 'node:fs/promises';
+import { readdir, readFile } from 'node:fs/promises';
-import { join } from 'node:path';
+import { join, relative, sep } from 'node:path';
 
 import { createS3CompatClient } from '@affine/s3-compat';
 import { lookup } from 'mime-types';
-import type { Compiler, WebpackPluginInstance } from 'webpack';
 
 export const R2_BUCKET =
   process.env.R2_BUCKET ??
   (process.env.BUILD_TYPE === 'canary' ? 'assets-dev' : 'assets-prod');
 
-export class WebpackS3Plugin implements WebpackPluginInstance {
+const S3_UPLOAD_PACKAGE_NAMES = new Set([
-  private readonly s3 = createS3CompatClient(
+  '@affine/web',
+  '@affine/mobile',
+  '@affine/admin',
+]);
+const MAX_UPLOAD_RETRIES = 3;
+const UPLOAD_RETRY_BASE_DELAY_MS = 500;
+
+function createR2Client() {
+  const { R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY } = process.env;
+  if (!R2_ACCOUNT_ID || !R2_ACCESS_KEY_ID || !R2_SECRET_ACCESS_KEY) {
+    throw new Error('Missing R2 credentials for uploading release assets');
+  }
+
+  return createS3CompatClient(
     {
       region: 'auto',
       bucket: R2_BUCKET,
       forcePathStyle: true,
-      endpoint: `https://${process.env.R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
+      endpoint: `https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com`,
     },
     {
-      accessKeyId: process.env.R2_ACCESS_KEY_ID!,
+      accessKeyId: R2_ACCESS_KEY_ID,
-      secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
+      secretAccessKey: R2_SECRET_ACCESS_KEY,
+    }
+  );
+}
+
+async function collectFiles(dir: string): Promise<string[]> {
+  const dirs = [dir];
+  const files: string[] = [];
+
+  while (dirs.length > 0) {
+    const current = dirs.pop()!;
+    const entries = await readdir(current, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = join(current, entry.name);
+      if (entry.isDirectory()) {
+        dirs.push(fullPath);
+      } else if (entry.isFile()) {
+        files.push(fullPath);
+      }
+    }
+  }
+
+  return files;
+}
+
+function toAssetKey(outputPath: string, filePath: string): string {
+  return relative(outputPath, filePath).split(sep).join('/');
+}
+
+function sleep(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function putObjectWithRetry(
+  s3: ReturnType<typeof createR2Client>,
+  asset: string,
+  assetSource: Buffer,
+  contentType: string | false | undefined
+) {
+  let retries = 0;
+  while (true) {
+    try {
+      await s3.putObject(asset, assetSource, {
+        contentType: contentType || undefined,
+        contentLength: assetSource.byteLength,
+      });
+      return;
+    } catch (error) {
+      if (retries >= MAX_UPLOAD_RETRIES) {
+        throw error;
+      }
+      retries += 1;
+      const delay = UPLOAD_RETRY_BASE_DELAY_MS * 2 ** (retries - 1);
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      console.warn(
+        `[s3-upload] Retry ${retries}/${MAX_UPLOAD_RETRIES} for ${asset}: ${errorMessage}`
+      );
+      await sleep(delay);
+    }
+  }
+}
+
+async function runInParallel<T>(
+  values: T[],
+  worker: (value: T) => Promise<void>,
+  concurrency = 16
+) {
+  if (values.length === 0) {
+    return;
+  }
+  let nextIndex = 0;
+  const workers = Array.from(
+    { length: Math.min(concurrency, values.length) },
+    async () => {
+      while (true) {
+        const index = nextIndex++;
+        if (index >= values.length) {
+          return;
+        }
+        await worker(values[index]!);
+      }
     }
   );
 
-  apply(compiler: Compiler) {
+  await Promise.all(workers);
-    compiler.hooks.assetEmitted.tapPromise(
+}
-      'WebpackS3Plugin',
+
-      async (asset, { outputPath }) => {
+export function shouldUploadReleaseAssets(pkgName: string): boolean {
-        if (asset.endsWith('.html')) {
+  return S3_UPLOAD_PACKAGE_NAMES.has(pkgName);
+}
+
+export async function uploadDistAssetsToS3(outputPath: string) {
+  const allFiles = await collectFiles(outputPath);
+  const uploadFiles = allFiles.filter(file => !file.endsWith('.html'));
+
+  if (uploadFiles.length === 0) {
     return;
   }
-        const assetPath = join(outputPath, asset);
+
-        const assetSource = await readFile(assetPath);
+  const s3 = createR2Client();
-        const contentType = lookup(asset) || undefined;
+  await runInParallel(uploadFiles, async filePath => {
-        await this.s3.putObject(asset, assetSource, {
+    const asset = toAssetKey(outputPath, filePath);
-          contentType,
+    const assetSource = await readFile(filePath);
-          contentLength: assetSource.byteLength,
+    const contentType = lookup(asset);
+    await putObjectWithRetry(s3, asset, assetSource, contentType);
   });
 }
-    );
-  }
-}