feat(server): lightweight s3 client (#14348)

#### PR Dependency Tree


* **PR #14348** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added a dedicated S3-compatible client package and expanded
S3-compatible storage config (endpoint, region, forcePathStyle,
requestTimeoutMs, minPartSize, presign options, sessionToken).
* Document sync now broadcasts batched/compressed doc updates for more
efficient real-time syncing.

* **Tests**
* New unit and benchmark tests for base64 utilities and S3 multipart
listing; updated storage-related tests to match new formats.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

packages/frontend/admin/src/modules/settings/config.ts

@@ -112,7 +112,7 @@ export const KNOWN_CONFIG_GROUPS = [
         key: 'blob.storage',
         sub: 'config',
         type: 'JSON',
-        desc: 'The config passed directly to the storage provider(e.g. aws-sdk)',
+        desc: 'The S3 compatible config for the storage provider (endpoint/region/credentials).',
       },
       {
         key: 'avatar.storage',
@@ -131,7 +131,7 @@ export const KNOWN_CONFIG_GROUPS = [
         key: 'avatar.storage',
         sub: 'config',
         type: 'JSON',
-        desc: 'The config passed directly to the storage provider(e.g. aws-sdk)',
+        desc: 'The S3 compatible config for the storage provider (endpoint/region/credentials).',
       },
       {
         key: 'avatar.publicPath',
@@ -175,7 +175,7 @@ export const KNOWN_CONFIG_GROUPS = [
         key: 'storage',
         sub: 'config',
         type: 'JSON',
-        desc: 'The config passed directly to the storage provider(e.g. aws-sdk)',
+        desc: 'The S3 compatible config for the storage provider (endpoint/region/credentials).',
       },
     ],
   } as ConfigGroup<'copilot'>,

packages/frontend/core/src/modules/workspace-engine/utils/__tests__/base64.spec.ts

@@ -0,0 +1,20 @@
+import { describe, expect, test } from 'vitest';
+
+import { base64ToUint8Array, uint8ArrayToBase64 } from '../base64';
+
+function makeSample(size: number) {
+  const data = new Uint8Array(size);
+  for (let i = 0; i < size; i++) {
+    data[i] = (i * 13) % 251;
+  }
+  return data;
+}
+
+describe('base64 helpers', () => {
+  test('roundtrip preserves data', async () => {
+    const input = makeSample(2048);
+    const encoded = await uint8ArrayToBase64(input);
+    const decoded = base64ToUint8Array(encoded);
+    expect(decoded).toEqual(input);
+  });
+});

packages/frontend/core/src/modules/workspace-engine/utils/base64.ts

@@ -1,28 +1,37 @@
-export function uint8ArrayToBase64(array: Uint8Array): Promise<string> {
-  return new Promise<string>(resolve => {
-    // Create a blob from the Uint8Array
-    const blob = new Blob([array]);
+type BufferConstructorLike = {
+  from(
+    data: Uint8Array | string,
+    encoding?: string
+  ): Uint8Array & {
+    toString(encoding: string): string;
+  };
+};
 
-    const reader = new FileReader();
-    reader.onload = function () {
-      const dataUrl = reader.result as string | null;
-      if (!dataUrl) {
-        resolve('');
-        return;
-      }
-      // The result includes the `data:` URL prefix and the MIME type. We only want the Base64 data
-      const base64 = dataUrl.split(',')[1];
-      resolve(base64);
-    };
+const BufferCtor = (globalThis as { Buffer?: BufferConstructorLike }).Buffer;
+const CHUNK_SIZE = 0x8000;
 
-    reader.readAsDataURL(blob);
-  });
+export async function uint8ArrayToBase64(array: Uint8Array): Promise<string> {
+  if (BufferCtor) {
+    return BufferCtor.from(array).toString('base64');
+  }
+
+  let binary = '';
+  for (let i = 0; i < array.length; i += CHUNK_SIZE) {
+    const chunk = array.subarray(i, i + CHUNK_SIZE);
+    binary += String.fromCharCode(...chunk);
+  }
+  return btoa(binary);
 }
 
 export function base64ToUint8Array(base64: string) {
+  if (BufferCtor) {
+    return new Uint8Array(BufferCtor.from(base64, 'base64'));
+  }
+
   const binaryString = atob(base64);
-  const binaryArray = [...binaryString].map(function (char) {
-    return char.charCodeAt(0);
-  });
-  return new Uint8Array(binaryArray);
+  const bytes = new Uint8Array(binaryString.length);
+  for (let i = 0; i < binaryString.length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+  return bytes;
 }

packages/frontend/mobile-native/Cargo.toml

@@ -12,6 +12,7 @@ name = "uniffi-bindgen"
 path = "uniffi-bindgen.rs"
 
 [features]
+default    = ["use-as-lib"]
 use-as-lib = ["affine_nbstore/use-as-lib"]
 
 [dependencies]

packages/frontend/mobile-native/src/lib.rs

@@ -89,11 +89,57 @@ impl TryFrom<DocUpdate> for affine_nbstore::DocUpdate {
       timestamp: chrono::DateTime::<chrono::Utc>::from_timestamp_millis(update.timestamp)
         .ok_or(UniffiError::TimestampDecodingError)?
         .naive_utc(),
-      bin: update.bin.into(),
+      bin: Into::<Data>::into(
+        base64_simd::STANDARD
+          .decode_to_vec(update.bin)
+          .map_err(|e| UniffiError::Base64DecodingError(e.to_string()))?,
+      ),
     })
   }
 }
 
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn doc_update_roundtrip_base64() {
+    let timestamp = chrono::DateTime::<chrono::Utc>::from_timestamp_millis(1_700_000_000_000)
+      .unwrap()
+      .naive_utc();
+    let original = affine_nbstore::DocUpdate {
+      doc_id: "doc-1".to_string(),
+      timestamp,
+      bin: vec![1, 2, 3, 4, 5],
+    };
+
+    let encoded: DocUpdate = original.into();
+    let decoded = affine_nbstore::DocUpdate::try_from(encoded).unwrap();
+
+    assert_eq!(decoded.doc_id, "doc-1");
+    assert_eq!(decoded.timestamp, timestamp);
+    assert_eq!(decoded.bin, vec![1, 2, 3, 4, 5]);
+  }
+
+  #[test]
+  fn doc_update_rejects_invalid_base64() {
+    let update = DocUpdate {
+      doc_id: "doc-2".to_string(),
+      timestamp: 0,
+      bin: "not-base64!!".to_string(),
+    };
+
+    let err = match affine_nbstore::DocUpdate::try_from(update) {
+      Ok(_) => panic!("expected base64 decode error"),
+      Err(err) => err,
+    };
+    match err {
+      UniffiError::Base64DecodingError(_) => {}
+      other => panic!("unexpected error: {other:?}"),
+    }
+  }
+}
+
 #[derive(uniffi::Record)]
 pub struct DocClock {
   pub doc_id: String,

packages/frontend/native/Cargo.toml

@@ -9,7 +9,7 @@ crate-type = ["cdylib", "rlib"]
 [dependencies]
 affine_common = { workspace = true, features = ["hashcash"] }
 affine_media_capture = { path = "./media_capture" }
-affine_nbstore = { path = "./nbstore" }
+affine_nbstore = { workspace = true, features = ["napi"] }
 affine_sqlite_v1 = { path = "./sqlite_v1" }
 napi = { workspace = true }
 napi-derive = { workspace = true }
@@ -25,6 +25,12 @@ sqlx = { workspace = true, default-features = false, features = [
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 
+[target.'cfg(not(target_os = "linux"))'.dependencies]
+mimalloc = { workspace = true }
+
+[target.'cfg(all(target_os = "linux", not(target_arch = "arm")))'.dependencies]
+mimalloc = { workspace = true, features = ["local_dynamic_tls"] }
+
 [dev-dependencies]
 chrono     = { workspace = true }
 serde_json = { workspace = true }

packages/frontend/native/index.d.ts

@@ -19,10 +19,10 @@ export declare class ApplicationStateChangedSubscriber {
 }
 
 export declare class AudioCaptureSession {
+  stop(): void
   get sampleRate(): number
   get channels(): number
   get actualSampleRate(): number
-  stop(): void
 }
 
 export declare class ShareableContent {
@@ -31,9 +31,9 @@ export declare class ShareableContent {
   constructor()
   static applications(): Array<ApplicationInfo>
   static applicationWithProcessId(processId: number): ApplicationInfo | null
+  static isUsingMicrophone(processId: number): boolean
   static tapAudio(processId: number, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession
   static tapGlobalAudio(excludedProcesses: Array<ApplicationInfo> | undefined | null, audioStreamCallback: ((err: Error | null, arg: Float32Array) => void)): AudioCaptureSession
-  static isUsingMicrophone(processId: number): boolean
 }
 
 export declare function decodeAudio(buf: Uint8Array, destSampleRate?: number | undefined | null, filename?: string | undefined | null, signal?: AbortSignal | undefined | null): Promise<Float32Array>

packages/frontend/native/nbstore/Cargo.toml

@@ -7,6 +7,8 @@ version = "0.0.0"
 crate-type = ["cdylib", "rlib"]
 
 [features]
+default    = []
+napi       = ["affine_common/napi"]
 use-as-lib = ["napi-derive/noop", "napi/noop"]
 
 [dependencies]

packages/frontend/native/nbstore/src/lib.rs

@@ -8,6 +8,8 @@ pub mod indexer_sync;
 pub mod pool;
 pub mod storage;
 
+#[cfg(not(feature = "use-as-lib"))]
+use affine_common::napi_utils::to_napi_error;
 use chrono::NaiveDateTime;
 use napi::bindgen_prelude::*;
 use napi_derive::napi;
@@ -23,7 +25,7 @@ type Result<T> = napi::Result<T>;
 #[cfg(not(feature = "use-as-lib"))]
 impl From<error::Error> for napi::Error {
   fn from(err: error::Error) -> Self {
-    napi::Error::new(napi::Status::GenericFailure, err.to_string())
+    to_napi_error(err, napi::Status::GenericFailure)
   }
 }
 
@@ -491,3 +493,15 @@ impl DocStorage {
     Ok(())
   }
 }
+
+#[cfg(all(test, not(feature = "use-as-lib")))]
+mod tests {
+  use super::error;
+
+  #[test]
+  fn napi_error_mapping_preserves_reason() {
+    let err: napi::Error = error::Error::InvalidOperation.into();
+    assert_eq!(err.status, napi::Status::GenericFailure);
+    assert!(err.reason.contains("Invalid operation"));
+  }
+}

packages/frontend/native/src/hashcash.rs

@@ -64,3 +64,27 @@ impl Task for AsyncMintChallengeResponse {
 pub fn mint_challenge_response(resource: String, bits: Option<u32>) -> AsyncTask<AsyncMintChallengeResponse> {
   AsyncTask::new(AsyncMintChallengeResponse { bits, resource })
 }
+
+#[cfg(test)]
+mod tests {
+  use napi::Task;
+
+  use super::*;
+
+  #[test]
+  fn hashcash_roundtrip() {
+    let resource = "test-resource".to_string();
+    let mut mint = AsyncMintChallengeResponse {
+      bits: Some(8),
+      resource: resource.clone(),
+    };
+    let stamp = mint.compute().unwrap();
+
+    let mut verify = AsyncVerifyChallengeResponse {
+      response: stamp,
+      bits: 8,
+      resource,
+    };
+    assert!(verify.compute().unwrap());
+  }
+}

packages/frontend/native/src/lib.rs

@@ -1,5 +1,9 @@
 pub mod hashcash;
 
+#[cfg(not(target_arch = "arm"))]
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
 #[allow(unused_imports)]
 pub use affine_media_capture::*;
 pub use affine_nbstore::*;