fix(server): storage prefix url (#15477)

#### PR Dependency Tree


* **PR #15477** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added support for custom signed download URLs with configurable CDN
origins, object paths, expiration times, and security signatures.
* Custom download signing is available when both the URL prefix and
signing key are configured.

* **Bug Fixes**
* Preserved custom URL and signing settings when generating download
links.
  * Added validation for invalid signing configurations.
* Standard S3 storage continues using its existing presigned URL
behavior.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

.docker/selfhost/schema.json

@@ -494,11 +494,11 @@
                         },
                         "urlPrefix": {
                           "type": "string",
-                          "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
+                          "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured."
                         },
                         "signKey": {
                           "type": "string",
-                          "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
+                          "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix."
                         }
                       }
                     }
@@ -577,11 +577,11 @@
                         },
                         "urlPrefix": {
                           "type": "string",
-                          "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
+                          "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured."
                         },
                         "signKey": {
                           "type": "string",
-                          "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
+                          "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix."
                         }
                       }
                     },
@@ -741,11 +741,11 @@
                         },
                         "urlPrefix": {
                           "type": "string",
-                          "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
+                          "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured."
                         },
                         "signKey": {
                           "type": "string",
-                          "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
+                          "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix."
                         }
                       }
                     }
@@ -824,11 +824,11 @@
                         },
                         "urlPrefix": {
                           "type": "string",
-                          "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
+                          "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured."
                         },
                         "signKey": {
                           "type": "string",
-                          "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
+                          "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix."
                         }
                       }
                     },
@@ -1294,11 +1294,11 @@
                         },
                         "urlPrefix": {
                           "type": "string",
-                          "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
+                          "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured."
                         },
                         "signKey": {
                           "type": "string",
-                          "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
+                          "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix."
                         }
                       }
                     }
@@ -1377,11 +1377,11 @@
                         },
                         "urlPrefix": {
                           "type": "string",
-                          "description": "Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured."
+                          "description": "Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured."
                         },
                         "signKey": {
                           "type": "string",
-                          "description": "Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin."
+                          "description": "Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix."
                         }
                       }
                     },

packages/backend/native/src/runtime/object_storage/config.rs

@@ -1,13 +1,20 @@
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use base64::{Engine as _, engine::general_purpose::STANDARD};
+use hmac::{Hmac, KeyInit, Mac};
 use rusty_s3::{Bucket, Credentials, UrlStyle};
 use serde::Deserialize;
+use sha2::Sha256;
 use url::Url;
 
 use super::{
   client::ObjectStorageClient,
   error::{ObjectStorageError, ObjectStorageResult},
-  types::StorageProviderConfig,
+  types::{ObjectKey, PresignedObjectRequest, StorageProviderConfig},
 };
 
+type HmacSha256 = Hmac<Sha256>;
+
 #[derive(Clone, Debug)]
 pub(crate) struct ObjectStorageConfig {
   pub(crate) provider: String,
@@ -24,6 +31,8 @@ pub(crate) struct ObjectStorageConfig {
   pub(crate) presign_sign_content_type_for_put: Option<bool>,
   pub(crate) use_presigned_url: bool,
   pub(crate) proxy_upload: bool,
+  pub(crate) custom_get_url_prefix: Option<String>,
+  pub(crate) custom_get_sign_key: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -124,6 +133,8 @@ impl ObjectStorageConfig {
       presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put),
       use_presigned_url: config.use_presigned_url.map(|v| v.enabled).unwrap_or(false),
       proxy_upload: false,
+      custom_get_url_prefix: None,
+      custom_get_sign_key: None,
     }))
   }
 
@@ -135,17 +146,25 @@ impl ObjectStorageConfig {
       Some(R2Jurisdiction::Default) | None => config.account_id,
     };
     let credentials = config.credentials.unwrap_or_default();
-    let (use_presigned_url, proxy_upload) = config
+    let (use_presigned_url, proxy_upload, custom_get_url_prefix, custom_get_sign_key) = config
       .use_presigned_url
       .map(|value| {
+        let url_prefix = value.url_prefix.filter(|prefix| !prefix.is_empty());
+        let sign_key = value.sign_key.filter(|key| !key.is_empty());
+        let custom_get_enabled = value.enabled && url_prefix.is_some() && sign_key.is_some();
+        let (custom_get_url_prefix, custom_get_sign_key) = if custom_get_enabled {
+          (url_prefix, sign_key)
+        } else {
+          (None, None)
+        };
         (
           value.enabled,
-          value.enabled
+          custom_get_enabled,
-            && value.url_prefix.as_ref().is_some_and(|prefix| !prefix.is_empty())
+          custom_get_url_prefix,
-            && value.sign_key.as_ref().is_some_and(|key| !key.is_empty()),
+          custom_get_sign_key,
         )
       })
-      .unwrap_or((false, false));
+      .unwrap_or((false, false, None, None));
 
     Ok(Some(Self {
       provider: storage.provider,
@@ -162,6 +181,57 @@ impl ObjectStorageConfig {
       presign_sign_content_type_for_put: config.presign.as_ref().and_then(|v| v.sign_content_type_for_put),
       use_presigned_url,
       proxy_upload,
+      custom_get_url_prefix,
+      custom_get_sign_key,
+    }))
+  }
+
+  pub(crate) fn custom_presign_get(&self, key: &ObjectKey) -> ObjectStorageResult<Option<PresignedObjectRequest>> {
+    let timestamp = SystemTime::now()
+      .duration_since(UNIX_EPOCH)
+      .map_err(|err| ObjectStorageError::Config(format!("system time before unix epoch: {err}")))?
+      .as_secs();
+    self.custom_presign_get_at(key, timestamp)
+  }
+
+  pub(super) fn custom_presign_get_at(
+    &self,
+    key: &ObjectKey,
+    timestamp: u64,
+  ) -> ObjectStorageResult<Option<PresignedObjectRequest>> {
+    let (Some(prefix), Some(sign_key)) = (
+      self.custom_get_url_prefix.as_deref(),
+      self.custom_get_sign_key.as_deref(),
+    ) else {
+      return Ok(None);
+    };
+    let mut url = Url::parse(prefix)
+      .map_err(|err| ObjectStorageError::Config(format!("invalid object storage URL prefix: {err}")))?;
+    if !matches!(url.scheme(), "http" | "https") || url.query().is_some() || url.fragment().is_some() {
+      return Err(ObjectStorageError::Config(
+        "object storage URL prefix must be an HTTP(S) URL without query or fragment".to_string(),
+      ));
+    }
+    url
+      .path_segments_mut()
+      .map_err(|_| ObjectStorageError::Config("object storage URL prefix cannot be a base URL".to_string()))?
+      .pop_if_empty()
+      .extend(key.as_str().split('/'));
+    let payload = format!("{}{timestamp}", url.path());
+    let mut mac = HmacSha256::new_from_slice(sign_key.as_bytes())
+      .map_err(|err| ObjectStorageError::Config(format!("invalid object storage signing key: {err}")))?;
+    mac.update(payload.as_bytes());
+    let signature = STANDARD.encode(mac.finalize().into_bytes());
+    url
+      .query_pairs_mut()
+      .append_pair("sign", &format!("{timestamp}-{signature}"));
+
+    Ok(Some(PresignedObjectRequest {
+      url: url.to_string(),
+      headers: Default::default(),
+      expires_at_ms: i64::try_from(timestamp.saturating_add(self.presign_expires_in_seconds.unwrap_or(60)))
+        .unwrap_or(i64::MAX)
+        .saturating_mul(1000),
     }))
   }
 

packages/backend/native/src/runtime/object_storage/service.rs

@@ -182,12 +182,17 @@ impl ObjectStorageService {
   ) -> RuntimeResult<Option<PresignedObjectRequest>> {
     match self.backend_for_scope(locator.scope)? {
       StorageBackendConfig::Fs(_) | StorageBackendConfig::Assetpack(_) => Ok(None),
-      StorageBackendConfig::S3(config) => config
+      StorageBackendConfig::S3(config) => {
-        .build_client()?
+        if let Some(request) = config.custom_presign_get(&locator.key)? {
-        .presign_get(&locator.key)
+          return Ok(Some(request));
-        .await
+        }
-        .map(Some)
+        config
-        .map_err(Into::into),
+          .build_client()?
+          .presign_get(&locator.key)
+          .await
+          .map(Some)
+          .map_err(Into::into)
+      }
     }
   }
 

packages/backend/native/src/runtime/object_storage/tests.rs

@@ -261,7 +261,7 @@ fn resolves_r2_proxy_upload_capability_from_config_json_shape() {
       },
       "usePresignedURL": {
         "enabled": true,
-        "urlPrefix": "https://cdn.example.com",
+        "urlPrefix": "https://cdn.example.com/storage/",
         "signKey": "secret"
       }
     }),
@@ -271,6 +271,50 @@ fn resolves_r2_proxy_upload_capability_from_config_json_shape() {
 
   assert!(config.use_presigned_url);
   assert!(config.proxy_upload);
+  let request = config
+    .custom_presign_get_at(&ObjectKey::new("workspace/blob.m4a").unwrap(), 1_700_000_000)
+    .unwrap()
+    .unwrap();
+  let url = url::Url::parse(&request.url).unwrap();
+  assert_eq!(url.origin().ascii_serialization(), "https://cdn.example.com");
+  assert_eq!(url.path(), "/storage/workspace/blob.m4a");
+  assert_eq!(
+    url.query_pairs().find(|(key, _)| key == "sign").unwrap().1,
+    "1700000000-01IngHvoE2trslxVyYUzfWkhgdlYdpcRXcpSYqZ9gkc="
+  );
+
+  for use_presigned_url in [
+    serde_json::json!({
+      "enabled": true,
+      "urlPrefix": "https://cdn.example.com"
+    }),
+    serde_json::json!({
+      "enabled": true,
+      "urlPrefix": "https://cdn.example.com",
+      "signKey": ""
+    }),
+  ] {
+    let storage = StorageProviderConfig {
+      provider: "cloudflare-r2".to_string(),
+      bucket: "workspace-blobs".to_string(),
+      config: serde_json::json!({
+        "accountId": "account",
+        "credentials": {
+          "accessKeyId": "key",
+          "secretAccessKey": "secret"
+        },
+        "usePresignedURL": use_presigned_url
+      }),
+    };
+    let config = ObjectStorageConfig::from_r2_config(storage).unwrap().unwrap();
+    assert!(!config.proxy_upload);
+    assert!(
+      config
+        .custom_presign_get_at(&ObjectKey::new("workspace/blob.m4a").unwrap(), 1_700_000_000)
+        .unwrap()
+        .is_none()
+    );
+  }
 }
 
 #[test]

packages/backend/native/src/runtime/storage_runtime/capabilities.rs

@@ -96,6 +96,8 @@ mod tests {
       presign_sign_content_type_for_put: Some(true),
       use_presigned_url: true,
       proxy_upload: false,
+      custom_get_url_prefix: None,
+      custom_get_sign_key: None,
     }));
 
     assert!(capabilities.presign_put);
@@ -121,6 +123,8 @@ mod tests {
       presign_sign_content_type_for_put: Some(true),
       use_presigned_url: true,
       proxy_upload: true,
+      custom_get_url_prefix: None,
+      custom_get_sign_key: None,
     }));
 
     assert!(capabilities.proxy_upload);

packages/backend/server/src/base/storage/providers/index.ts

@@ -127,12 +127,12 @@ const S3ConfigSchema: JSONSchema = {
         urlPrefix: {
           type: 'string',
           description:
-            'Optional custom origin for browser upload URLs. Provider presigned URLs also use this origin when signKey is not configured.',
+            'Optional base URL for proxied uploads and signed custom GET URLs. Provider presigned uploads require an origin-only value when signKey is not configured.',
         },
         signKey: {
           type: 'string',
           description:
-            'Optional HMAC key for signed upload URLs. Without urlPrefix, upload URLs use the server origin.',
+            'Optional HMAC key for signed upload and custom GET URLs. Custom GET URLs require both signKey and urlPrefix.',
         },
       },
     },