fix(server): transcript key & bump models (#15485)
#### PR Dependency Tree * **PR #15485** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Upgraded built-in AI actions and chat model selection to Gemini 3.7 Flash. * Transcript attachments now retain storage keys and support secure presigned URLs during processing. * Completed transcript results preserve attachment metadata for reliable access. * **Bug Fixes** * Improved transcript handling across uploads, retries, and native processing. * Added safeguards for invalid or missing attachment URLs. * **Maintenance** * Added a migration to backfill storage keys for existing transcript records without altering their URLs. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
6
Cargo.lock
generated
6
Cargo.lock
generated
@@ -2963,7 +2963,7 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
"log",
|
"log",
|
||||||
"rustversion",
|
"rustversion",
|
||||||
"windows-link 0.2.1",
|
"windows-link 0.1.3",
|
||||||
"windows-result 0.4.1",
|
"windows-result 0.4.1",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -4756,9 +4756,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "llm_adapter"
|
name = "llm_adapter"
|
||||||
version = "0.2.18"
|
version = "0.2.19"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "49df35c253da1563c33f0733ef3a46da3194299807cbd7e03c48dfa12a89a72f"
|
checksum = "c17ef06aac72e31a4fae7937998029267c4d923a0671bf1a4062c96a1ab5fa70"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64",
|
||||||
"jsonschema",
|
"jsonschema",
|
||||||
|
|||||||
@@ -3,8 +3,8 @@
|
|||||||
"name": "Transcript audio",
|
"name": "Transcript audio",
|
||||||
"action": "Transcript audio",
|
"action": "Transcript audio",
|
||||||
"managedRoute": {
|
"managedRoute": {
|
||||||
"targets": ["gemini-3.5-flash-lite"],
|
"targets": ["gemini-3.7-flash"],
|
||||||
"premiumTargets": ["gemini-3.6-flash"]
|
"premiumTargets": ["gemini-3.7-flash"]
|
||||||
},
|
},
|
||||||
"config": {
|
"config": {
|
||||||
"requireContent": false,
|
"requireContent": false,
|
||||||
@@ -17,8 +17,8 @@
|
|||||||
"name": "Transcript audio structured",
|
"name": "Transcript audio structured",
|
||||||
"action": "Transcript audio structured",
|
"action": "Transcript audio structured",
|
||||||
"managedRoute": {
|
"managedRoute": {
|
||||||
"targets": ["gemini-3.5-flash-lite"],
|
"targets": ["gemini-3.7-flash"],
|
||||||
"premiumTargets": ["gemini-3.6-flash"]
|
"premiumTargets": ["gemini-3.7-flash"]
|
||||||
},
|
},
|
||||||
"config": {
|
"config": {
|
||||||
"requireContent": false,
|
"requireContent": false,
|
||||||
@@ -39,7 +39,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Generate a caption",
|
"name": "Generate a caption",
|
||||||
"action": "Generate a caption",
|
"action": "Generate a caption",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"config": {
|
"config": {
|
||||||
"requireContent": false,
|
"requireContent": false,
|
||||||
"requireAttachment": true
|
"requireAttachment": true
|
||||||
@@ -129,7 +129,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Explain this image",
|
"name": "Explain this image",
|
||||||
"action": "Explain this image",
|
"action": "Explain this image",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"config": {
|
"config": {
|
||||||
"requireContent": false,
|
"requireContent": false,
|
||||||
"requireAttachment": true
|
"requireAttachment": true
|
||||||
@@ -148,7 +148,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Explain this code",
|
"name": "Explain this code",
|
||||||
"action": "Explain this code",
|
"action": "Explain this code",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -163,7 +163,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Translate to",
|
"name": "Translate to",
|
||||||
"action": "Translate",
|
"action": "Translate",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"params": {
|
"params": {
|
||||||
"language": {
|
"language": {
|
||||||
"default": "English",
|
"default": "English",
|
||||||
@@ -256,7 +256,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Write a twitter about this",
|
"name": "Write a twitter about this",
|
||||||
"action": "Write a twitter about this",
|
"action": "Write a twitter about this",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -271,7 +271,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Write a poem about this",
|
"name": "Write a poem about this",
|
||||||
"action": "Write a poem about this",
|
"action": "Write a poem about this",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -301,7 +301,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Write outline",
|
"name": "Write outline",
|
||||||
"action": "Write outline",
|
"action": "Write outline",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -316,7 +316,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Change tone to",
|
"name": "Change tone to",
|
||||||
"action": "Change tone",
|
"action": "Change tone",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"params": {
|
"params": {
|
||||||
"tone": {
|
"tone": {
|
||||||
"default": "professional",
|
"default": "professional",
|
||||||
@@ -337,7 +337,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Brainstorm ideas about this",
|
"name": "Brainstorm ideas about this",
|
||||||
"action": "Brainstorm ideas about this",
|
"action": "Brainstorm ideas about this",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -386,7 +386,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Improve writing for it",
|
"name": "Improve writing for it",
|
||||||
"action": "Improve writing for it",
|
"action": "Improve writing for it",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -416,7 +416,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Fix spelling for it",
|
"name": "Fix spelling for it",
|
||||||
"action": "Fix spelling for it",
|
"action": "Fix spelling for it",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -476,7 +476,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Create headings",
|
"name": "Create headings",
|
||||||
"action": "Create headings",
|
"action": "Create headings",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -521,7 +521,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Make it longer",
|
"name": "Make it longer",
|
||||||
"action": "Make it longer",
|
"action": "Make it longer",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -536,7 +536,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Make it shorter",
|
"name": "Make it shorter",
|
||||||
"action": "Make it shorter",
|
"action": "Make it shorter",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -551,7 +551,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Continue writing",
|
"name": "Continue writing",
|
||||||
"action": "Continue writing",
|
"action": "Continue writing",
|
||||||
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] },
|
"managedRoute": { "targets": ["gemini-3.7-flash"] },
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
@@ -711,8 +711,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "gemini",
|
"id": "gemini",
|
||||||
"modelId": "gemini-3.6-flash",
|
"modelId": "gemini-3.7-flash",
|
||||||
"displayName": "Gemini 3.6 Flash",
|
"displayName": "Gemini 3.7 Flash",
|
||||||
"minimumTier": "premium"
|
"minimumTier": "premium"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -81,12 +81,12 @@ mod tests {
|
|||||||
|
|
||||||
let response = llm_resolve_model_registry_variant(ModelRegistryResolveRequest {
|
let response = llm_resolve_model_registry_variant(ModelRegistryResolveRequest {
|
||||||
backend_kind: Some("gemini_api".to_string()),
|
backend_kind: Some("gemini_api".to_string()),
|
||||||
model_id: "gemini-3.6-flash".to_string(),
|
model_id: "gemini-3.7-flash".to_string(),
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(response.matched_by.as_deref(), Some("raw_model_id"));
|
assert_eq!(response.matched_by.as_deref(), Some("raw_model_id"));
|
||||||
assert_eq!(response.variant.unwrap().raw_model_id, "gemini-3.6-flash");
|
assert_eq!(response.variant.unwrap().raw_model_id, "gemini-3.7-flash");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -301,7 +301,7 @@ mod tests {
|
|||||||
attachment_kinds: None,
|
attachment_kinds: None,
|
||||||
attachment_source_kinds: None,
|
attachment_source_kinds: None,
|
||||||
has_remote_attachments: None,
|
has_remote_attachments: None,
|
||||||
model_id: Some("gemini-3.6-flash".to_string()),
|
model_id: Some("gemini-3.7-flash".to_string()),
|
||||||
output_type: Some("image".to_string()),
|
output_type: Some("image".to_string()),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -210,7 +210,7 @@ mod tests {
|
|||||||
fn should_materialize_structured_request_with_response_contract() {
|
fn should_materialize_structured_request_with_response_contract() {
|
||||||
let response = llm_build_canonical_structured_request(
|
let response = llm_build_canonical_structured_request(
|
||||||
serde_json::from_value::<CanonicalStructuredRequestContract>(json!({
|
serde_json::from_value::<CanonicalStructuredRequestContract>(json!({
|
||||||
"model": "gemini-3.6-flash",
|
"model": "gemini-3.7-flash",
|
||||||
"messages": [
|
"messages": [
|
||||||
{ "role": "user", "content": "hello" }
|
{ "role": "user", "content": "hello" }
|
||||||
],
|
],
|
||||||
@@ -226,7 +226,7 @@ mod tests {
|
|||||||
assert_eq!(
|
assert_eq!(
|
||||||
response,
|
response,
|
||||||
json!({
|
json!({
|
||||||
"model": "gemini-3.6-flash",
|
"model": "gemini-3.7-flash",
|
||||||
"messages": [
|
"messages": [
|
||||||
{
|
{
|
||||||
"role": "user",
|
"role": "user",
|
||||||
|
|||||||
@@ -597,13 +597,13 @@ mod tests {
|
|||||||
);
|
);
|
||||||
|
|
||||||
let transcript = built_in_prompt("Transcript audio structured").expect("transcript prompt");
|
let transcript = built_in_prompt("Transcript audio structured").expect("transcript prompt");
|
||||||
assert_eq!(transcript.managed_targets, ["gemini-3.5-flash-lite"]);
|
assert_eq!(transcript.managed_targets, ["gemini-3.7-flash"]);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
transcript
|
transcript
|
||||||
.managed_premium_targets
|
.managed_premium_targets
|
||||||
.as_deref()
|
.as_deref()
|
||||||
.map(|targets| targets.iter().map(String::as_str).collect::<Vec<_>>()),
|
.map(|targets| targets.iter().map(String::as_str).collect::<Vec<_>>()),
|
||||||
Some(vec!["gemini-3.6-flash"])
|
Some(vec!["gemini-3.7-flash"])
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -262,7 +262,7 @@ mod tests {
|
|||||||
CopilotManagedTier::Premium
|
CopilotManagedTier::Premium
|
||||||
)
|
)
|
||||||
.unwrap(),
|
.unwrap(),
|
||||||
["gemini-3.6-flash"]
|
["gemini-3.7-flash"]
|
||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
managed_targets(&slot("index.embedding").unwrap(), None, CopilotManagedTier::Standard).unwrap(),
|
managed_targets(&slot("index.embedding").unwrap(), None, CopilotManagedTier::Standard).unwrap(),
|
||||||
|
|||||||
@@ -326,7 +326,7 @@ mod tests {
|
|||||||
id: "vertex".to_string(),
|
id: "vertex".to_string(),
|
||||||
provider: "geminiVertex".to_string(),
|
provider: "geminiVertex".to_string(),
|
||||||
enabled: true,
|
enabled: true,
|
||||||
models: vec!["gemini-3.6-flash".to_string()],
|
models: vec!["gemini-3.7-flash".to_string()],
|
||||||
config: json!({ "project": "affine-us", "location": location }),
|
config: json!({ "project": "affine-us", "location": location }),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -294,7 +294,7 @@ test('retryTask reuses failed task and queues a new action attempt', async t =>
|
|||||||
} as never,
|
} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{
|
{
|
||||||
resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.5-flash-lite'),
|
resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.7-flash'),
|
||||||
} as never,
|
} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
@@ -303,7 +303,7 @@ test('retryTask reuses failed task and queues a new action attempt', async t =>
|
|||||||
|
|
||||||
const result = await service.retryTask('user-1', 'workspace-1', 'task-1');
|
const result = await service.retryTask('user-1', 'workspace-1', 'task-1');
|
||||||
|
|
||||||
t.is(result.status, AiJobStatus.running);
|
t.is(result?.status, AiJobStatus.running);
|
||||||
t.like(queuedJobs[0] as Record<string, unknown>, {
|
t.like(queuedJobs[0] as Record<string, unknown>, {
|
||||||
name: 'copilot.transcript.task.submit',
|
name: 'copilot.transcript.task.submit',
|
||||||
});
|
});
|
||||||
@@ -351,9 +351,7 @@ for (const status of ['ready', 'settled']) {
|
|||||||
} as never,
|
} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{
|
{
|
||||||
resolveTranscriptionModel: Sinon.stub().resolves(
|
resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.7-flash'),
|
||||||
'gemini-3.5-flash-lite'
|
|
||||||
),
|
|
||||||
} as never,
|
} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
@@ -403,7 +401,8 @@ test('transcriptTask runs native transcript recipe through action bridge when av
|
|||||||
],
|
],
|
||||||
infos: [
|
infos: [
|
||||||
{
|
{
|
||||||
url: 'data:image/png;base64,YXVkaW8=',
|
key: 'blob-1-0',
|
||||||
|
url: 'https://affine.fail/api/copilot/blob/user-1/workspace-1/blob-1-0',
|
||||||
mimeType: 'audio/opus',
|
mimeType: 'audio/opus',
|
||||||
index: 0,
|
index: 0,
|
||||||
},
|
},
|
||||||
@@ -428,7 +427,11 @@ test('transcriptTask runs native transcript recipe through action bridge when av
|
|||||||
},
|
},
|
||||||
} as never,
|
} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
{} as never,
|
{
|
||||||
|
presignGet: Sinon.stub().resolves(
|
||||||
|
'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test'
|
||||||
|
),
|
||||||
|
} as never,
|
||||||
{} as never,
|
{} as never,
|
||||||
createTranscriptPromptService() as never,
|
createTranscriptPromptService() as never,
|
||||||
createSuccessfulTranscriptBridge('run-bridge', bridgeInputs) as never
|
createSuccessfulTranscriptBridge('run-bridge', bridgeInputs) as never
|
||||||
@@ -447,6 +450,20 @@ test('transcriptTask runs native transcript recipe through action bridge when av
|
|||||||
slot: 'transcript.audio',
|
slot: 'transcript.audio',
|
||||||
builtInRouteId: 'Transcript audio structured',
|
builtInRouteId: 'Transcript audio structured',
|
||||||
});
|
});
|
||||||
|
t.deepEqual(
|
||||||
|
(
|
||||||
|
bridgeInputs[0] as {
|
||||||
|
nativeInput: { input: { infos: unknown[] } };
|
||||||
|
}
|
||||||
|
).nativeInput.input.infos,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
url: 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test',
|
||||||
|
mimeType: 'audio/opus',
|
||||||
|
index: 0,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
);
|
||||||
const messages = (
|
const messages = (
|
||||||
bridgeInputs[0] as {
|
bridgeInputs[0] as {
|
||||||
step: {
|
step: {
|
||||||
@@ -459,7 +476,10 @@ test('transcriptTask runs native transcript recipe through action bridge when av
|
|||||||
infos: [{ mimeType: 'audio/opus', index: 0 }],
|
infos: [{ mimeType: 'audio/opus', index: 0 }],
|
||||||
});
|
});
|
||||||
t.deepEqual(messages.at(-1)?.attachments, [
|
t.deepEqual(messages.at(-1)?.attachments, [
|
||||||
{ attachment: 'data:image/png;base64,YXVkaW8=', mimeType: 'audio/opus' },
|
{
|
||||||
|
attachment: 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test',
|
||||||
|
mimeType: 'audio/opus',
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
t.like(complete.firstCall.args[1], {
|
t.like(complete.firstCall.args[1], {
|
||||||
status: 'ready',
|
status: 'ready',
|
||||||
@@ -471,6 +491,7 @@ test('transcriptTask runs native transcript recipe through action bridge when av
|
|||||||
complete.firstCall.args[1].protectedResult.normalizedTranscript,
|
complete.firstCall.args[1].protectedResult.normalizedTranscript,
|
||||||
'00:00:05 A: Kickoff'
|
'00:00:05 A: Kickoff'
|
||||||
);
|
);
|
||||||
|
t.deepEqual(complete.firstCall.args[1].protectedResult.infos, payload.infos);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('transcriptTask fails task when native action bridge reports an error event', async t => {
|
test('transcriptTask fails task when native action bridge reports an error event', async t => {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import ava, { TestFn } from 'ava';
|
|||||||
import { createTestingModule, type TestingModule } from '../../__tests__/utils';
|
import { createTestingModule, type TestingModule } from '../../__tests__/utils';
|
||||||
import { Models } from '../../models';
|
import { Models } from '../../models';
|
||||||
import { BackfillPermissionProjection1765500000000 } from '../migrations/1765500000000-backfill-permission-projection';
|
import { BackfillPermissionProjection1765500000000 } from '../migrations/1765500000000-backfill-permission-projection';
|
||||||
|
import { BackfillTranscriptStorageKeys1786805802350 } from '../migrations/1786805802350-backfill-transcript-storage-keys';
|
||||||
|
|
||||||
interface Context {
|
interface Context {
|
||||||
module: TestingModule;
|
module: TestingModule;
|
||||||
@@ -81,3 +82,44 @@ test('permission backfill repairs ownerless workspaces before runtime state proj
|
|||||||
{ role: 'owner' }
|
{ role: 'owner' }
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('transcript backfill adds stable keys without removing compatibility URLs', async t => {
|
||||||
|
const payload = {
|
||||||
|
sourceAudio: { blobId: 'blob-1' },
|
||||||
|
infos: [
|
||||||
|
{
|
||||||
|
url: 'https://affine.example/api/copilot/blob/user-1/workspace-1/blob-1',
|
||||||
|
mimeType: 'audio/m4a',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
url: 'https://example.com/external.m4a',
|
||||||
|
mimeType: 'audio/m4a',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
await t.context.db.aiTranscriptTask.create({
|
||||||
|
data: {
|
||||||
|
userId: 'user-1',
|
||||||
|
workspaceId: 'workspace-1',
|
||||||
|
blobId: 'blob-1',
|
||||||
|
status: 'failed',
|
||||||
|
recipeId: 'transcript.audio',
|
||||||
|
recipeVersion: 'v1',
|
||||||
|
inputSnapshot: payload,
|
||||||
|
protectedResult: payload,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await BackfillTranscriptStorageKeys1786805802350.up(t.context.db);
|
||||||
|
await BackfillTranscriptStorageKeys1786805802350.up(t.context.db);
|
||||||
|
|
||||||
|
const task = await t.context.db.aiTranscriptTask.findFirstOrThrow({
|
||||||
|
where: { blobId: 'blob-1' },
|
||||||
|
});
|
||||||
|
const expected = {
|
||||||
|
...payload,
|
||||||
|
infos: [{ ...payload.infos[0], key: 'blob-1' }, payload.infos[1]],
|
||||||
|
};
|
||||||
|
t.deepEqual(task.inputSnapshot, expected);
|
||||||
|
t.deepEqual(task.protectedResult, expected);
|
||||||
|
});
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import { type Prisma, PrismaClient } from '@prisma/client';
|
||||||
|
|
||||||
|
const BATCH_SIZE = 100;
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return !!value && typeof value === 'object' && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function backfillTranscriptStorageKeys(
|
||||||
|
payload: Prisma.JsonValue | null,
|
||||||
|
userId: string,
|
||||||
|
workspaceId: string
|
||||||
|
): Prisma.InputJsonValue | undefined {
|
||||||
|
if (!isRecord(payload) || !Array.isArray(payload.infos)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const prefix = `/api/copilot/blob/${encodeURIComponent(userId)}/${encodeURIComponent(workspaceId)}/`;
|
||||||
|
let changed = false;
|
||||||
|
const infos = payload.infos.map(info => {
|
||||||
|
if (!isRecord(info) || info.key || typeof info.url !== 'string') {
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const url = new URL(info.url);
|
||||||
|
if (!url.pathname.startsWith(prefix)) {
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
const key = decodeURIComponent(url.pathname.slice(prefix.length));
|
||||||
|
if (!key) {
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
changed = true;
|
||||||
|
return { ...info, key };
|
||||||
|
} catch {
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return changed ? ({ ...payload, infos } as Prisma.InputJsonValue) : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class BackfillTranscriptStorageKeys1786805802350 {
|
||||||
|
static async up(db: PrismaClient) {
|
||||||
|
let cursor: string | undefined;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const tasks = await db.aiTranscriptTask.findMany({
|
||||||
|
select: {
|
||||||
|
id: true,
|
||||||
|
userId: true,
|
||||||
|
workspaceId: true,
|
||||||
|
inputSnapshot: true,
|
||||||
|
protectedResult: true,
|
||||||
|
},
|
||||||
|
orderBy: { id: 'asc' },
|
||||||
|
take: BATCH_SIZE,
|
||||||
|
...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}),
|
||||||
|
});
|
||||||
|
if (!tasks.length) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const updates = tasks.flatMap(task => {
|
||||||
|
const inputSnapshot = backfillTranscriptStorageKeys(
|
||||||
|
task.inputSnapshot,
|
||||||
|
task.userId,
|
||||||
|
task.workspaceId
|
||||||
|
);
|
||||||
|
const protectedResult = backfillTranscriptStorageKeys(
|
||||||
|
task.protectedResult,
|
||||||
|
task.userId,
|
||||||
|
task.workspaceId
|
||||||
|
);
|
||||||
|
if (!inputSnapshot && !protectedResult) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
return db.aiTranscriptTask.update({
|
||||||
|
where: { id: task.id },
|
||||||
|
data: {
|
||||||
|
...(inputSnapshot ? { inputSnapshot } : {}),
|
||||||
|
...(protectedResult ? { protectedResult } : {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
if (updates.length) {
|
||||||
|
await db.$transaction(updates);
|
||||||
|
}
|
||||||
|
cursor = tasks.at(-1)?.id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static async down(_db: PrismaClient) {}
|
||||||
|
}
|
||||||
@@ -6,3 +6,4 @@ export * from './1751966744168-correct-session-update-time';
|
|||||||
export * from './1763800000000-rebuild-manticore-mixed-script-indexes';
|
export * from './1763800000000-rebuild-manticore-mixed-script-indexes';
|
||||||
export * from './1765500000000-backfill-permission-projection';
|
export * from './1765500000000-backfill-permission-projection';
|
||||||
export * from './1765600000000-backfill-entitlement-projection';
|
export * from './1765600000000-backfill-entitlement-projection';
|
||||||
|
export * from './1786805802350-backfill-transcript-storage-keys';
|
||||||
|
|||||||
@@ -59,14 +59,31 @@ export class CopilotStorage {
|
|||||||
): Promise<StorageRuntimeGetObjectResult> {
|
): Promise<StorageRuntimeGetObjectResult> {
|
||||||
const name = `${userId}/${workspaceId}/${key}`;
|
const name = `${userId}/${workspaceId}/${key}`;
|
||||||
if (signedUrl) {
|
if (signedUrl) {
|
||||||
const presigned = await this.rt.presignGet('copilot', name);
|
const redirectUrl = await this.presignGet(userId, workspaceId, key);
|
||||||
if (presigned) {
|
if (redirectUrl) return { redirectUrl };
|
||||||
return { redirectUrl: presigned.url };
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return this.rt.getObject('copilot', name);
|
return this.rt.getObject('copilot', name);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async presignGet(userId: string, workspaceId: string, key: string) {
|
||||||
|
return (
|
||||||
|
await this.rt.presignGet('copilot', `${userId}/${workspaceId}/${key}`)
|
||||||
|
)?.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
keyFromUrl(userId: string, workspaceId: string, url: string) {
|
||||||
|
try {
|
||||||
|
const parsed = new URL(url);
|
||||||
|
const prefix = `/api/copilot/blob/${encodeURIComponent(userId)}/${encodeURIComponent(workspaceId)}/`;
|
||||||
|
if (parsed.pathname.startsWith(prefix)) {
|
||||||
|
return decodeURIComponent(parsed.pathname.slice(prefix.length));
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
@CallMetric('ai', 'blob_delete')
|
@CallMetric('ai', 'blob_delete')
|
||||||
async delete(userId: string, workspaceId: string, key: string) {
|
async delete(userId: string, workspaceId: string, key: string) {
|
||||||
await this.rt.deleteObject('copilot', `${userId}/${workspaceId}/${key}`);
|
await this.rt.deleteObject('copilot', `${userId}/${workspaceId}/${key}`);
|
||||||
|
|||||||
@@ -18,7 +18,8 @@ export const LegacyTranscriptionSchema = z.array(
|
|||||||
);
|
);
|
||||||
|
|
||||||
export const AudioBlobInfoSchema = z.object({
|
export const AudioBlobInfoSchema = z.object({
|
||||||
url: z.string(),
|
key: z.string().min(1).optional(),
|
||||||
|
url: z.string().min(1),
|
||||||
mimeType: z.string(),
|
mimeType: z.string(),
|
||||||
index: z.number().int().nullable().optional(),
|
index: z.number().int().nullable().optional(),
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -46,10 +46,6 @@ export class CopilotTranscriptionService {
|
|||||||
private readonly realtime: RealtimePublisher
|
private readonly realtime: RealtimePublisher
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
private parseTaskPayload(payload: unknown): TranscriptionPayloadV2 {
|
|
||||||
return TranscriptPayloadSchema.parse(payload);
|
|
||||||
}
|
|
||||||
|
|
||||||
private buildTaskPublicMeta(payload: TranscriptionPayloadV2) {
|
private buildTaskPublicMeta(payload: TranscriptionPayloadV2) {
|
||||||
return {
|
return {
|
||||||
sourceAudio: payload.sourceAudio,
|
sourceAudio: payload.sourceAudio,
|
||||||
@@ -68,13 +64,10 @@ export class CopilotTranscriptionService {
|
|||||||
const infos: AudioBlobInfos = [];
|
const infos: AudioBlobInfos = [];
|
||||||
for (const [idx, blob] of blobs.entries()) {
|
for (const [idx, blob] of blobs.entries()) {
|
||||||
const buffer = await readStream(blob.createReadStream());
|
const buffer = await readStream(blob.createReadStream());
|
||||||
const url = await this.storage.put(
|
const key = `${blobId}-${idx}`;
|
||||||
userId,
|
const url = await this.storage.put(userId, workspaceId, key, buffer);
|
||||||
workspaceId,
|
|
||||||
`${blobId}-${idx}`,
|
|
||||||
buffer
|
|
||||||
);
|
|
||||||
infos.push({
|
infos.push({
|
||||||
|
key,
|
||||||
url,
|
url,
|
||||||
mimeType: sniffMime(buffer, blob.mimetype) || blob.mimetype,
|
mimeType: sniffMime(buffer, blob.mimetype) || blob.mimetype,
|
||||||
index: idx,
|
index: idx,
|
||||||
@@ -103,6 +96,47 @@ export class CopilotTranscriptionService {
|
|||||||
} satisfies TranscriptionPayloadV2;
|
} satisfies TranscriptionPayloadV2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async resolveAttachmentUrl(
|
||||||
|
userId: string,
|
||||||
|
workspaceId: string,
|
||||||
|
info: AudioBlobInfos[number]
|
||||||
|
) {
|
||||||
|
if (info.url.startsWith('data:')) {
|
||||||
|
return info.url;
|
||||||
|
}
|
||||||
|
|
||||||
|
const key =
|
||||||
|
info.key ?? this.storage.keyFromUrl(userId, workspaceId, info.url);
|
||||||
|
if (!key) {
|
||||||
|
throw new Error('Transcript attachment cannot be resolved');
|
||||||
|
}
|
||||||
|
|
||||||
|
const signedUrl = await this.storage.presignGet(userId, workspaceId, key);
|
||||||
|
if (!signedUrl) {
|
||||||
|
throw new Error('Transcript attachment signing is not configured');
|
||||||
|
}
|
||||||
|
return signedUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async materializePayload(
|
||||||
|
userId: string,
|
||||||
|
workspaceId: string,
|
||||||
|
payload: TranscriptionPayloadV2
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
...payload,
|
||||||
|
infos: payload.infos
|
||||||
|
? await Promise.all(
|
||||||
|
payload.infos.map(async info => ({
|
||||||
|
url: await this.resolveAttachmentUrl(userId, workspaceId, info),
|
||||||
|
mimeType: info.mimeType,
|
||||||
|
index: info.index,
|
||||||
|
}))
|
||||||
|
)
|
||||||
|
: payload.infos,
|
||||||
|
} satisfies TranscriptionPayloadV2;
|
||||||
|
}
|
||||||
|
|
||||||
private async buildTranscriptActionMessages(payload: TranscriptionPayloadV2) {
|
private async buildTranscriptActionMessages(payload: TranscriptionPayloadV2) {
|
||||||
const prompt = await this.prompts.get(TRANSCRIPT_PROMPT_REF);
|
const prompt = await this.prompts.get(TRANSCRIPT_PROMPT_REF);
|
||||||
if (!prompt) {
|
if (!prompt) {
|
||||||
@@ -118,13 +152,12 @@ export class CopilotTranscriptionService {
|
|||||||
index: info.index ?? null,
|
index: info.index ?? null,
|
||||||
})) ?? null,
|
})) ?? null,
|
||||||
};
|
};
|
||||||
const attachments =
|
const attachments = (payload.infos ?? []).map(info => ({
|
||||||
payload.infos?.map(info => ({
|
role: 'user' as const,
|
||||||
role: 'user' as const,
|
content: `Audio attachment ${info.index ?? 0}`,
|
||||||
content: `Audio attachment ${info.index ?? 0}`,
|
attachments: [{ attachment: info.url, mimeType: info.mimeType }],
|
||||||
attachments: [{ attachment: info.url, mimeType: info.mimeType }],
|
params: { mimetype: info.mimeType },
|
||||||
params: { mimetype: info.mimeType },
|
}));
|
||||||
})) ?? [];
|
|
||||||
return [
|
return [
|
||||||
...this.prompts.finish(prompt, {
|
...this.prompts.finish(prompt, {
|
||||||
content: JSON.stringify(metadata),
|
content: JSON.stringify(metadata),
|
||||||
@@ -205,7 +238,7 @@ export class CopilotTranscriptionService {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const payload = this.parseTaskPayload(task.protectedResult);
|
const payload = TranscriptPayloadSchema.parse(task.protectedResult);
|
||||||
await this.runtime.assertRoute(
|
await this.runtime.assertRoute(
|
||||||
'transcript.audio',
|
'transcript.audio',
|
||||||
{},
|
{},
|
||||||
@@ -287,14 +320,19 @@ export class CopilotTranscriptionService {
|
|||||||
let bridgeFailed = false;
|
let bridgeFailed = false;
|
||||||
let bridgeError = 'transcript native recipe failed';
|
let bridgeError = 'transcript native recipe failed';
|
||||||
let finalResult: unknown = null;
|
let finalResult: unknown = null;
|
||||||
const messages = await this.buildTranscriptActionMessages(payload);
|
const runtimePayload = await this.materializePayload(
|
||||||
|
task.userId,
|
||||||
|
task.workspaceId,
|
||||||
|
payload
|
||||||
|
);
|
||||||
|
const messages = await this.buildTranscriptActionMessages(runtimePayload);
|
||||||
for await (const event of this.actionBridge.runStream({
|
for await (const event of this.actionBridge.runStream({
|
||||||
userId: task.userId,
|
userId: task.userId,
|
||||||
workspaceId: task.workspaceId,
|
workspaceId: task.workspaceId,
|
||||||
actionId: TRANSCRIPT_ACTION_ID,
|
actionId: TRANSCRIPT_ACTION_ID,
|
||||||
actionVersion: TRANSCRIPT_ACTION_VERSION,
|
actionVersion: TRANSCRIPT_ACTION_VERSION,
|
||||||
retryOf: retryOf ?? null,
|
retryOf: retryOf ?? null,
|
||||||
inputSnapshot: payload,
|
inputSnapshot: runtimePayload,
|
||||||
onRunCreated: async ({ runId }) => {
|
onRunCreated: async ({ runId }) => {
|
||||||
await this.models.copilotTranscriptTask.markRunning(taskId, runId);
|
await this.models.copilotTranscriptTask.markRunning(taskId, runId);
|
||||||
this.publishTaskChanged(
|
this.publishTaskChanged(
|
||||||
@@ -329,7 +367,10 @@ export class CopilotTranscriptionService {
|
|||||||
if (bridgeFailed) {
|
if (bridgeFailed) {
|
||||||
throw new Error(bridgeError);
|
throw new Error(bridgeError);
|
||||||
}
|
}
|
||||||
const parsedResult = TranscriptPayloadSchema.parse(finalResult);
|
const parsedResult = {
|
||||||
|
...TranscriptPayloadSchema.parse(finalResult),
|
||||||
|
infos: payload.infos,
|
||||||
|
} satisfies TranscriptionPayloadV2;
|
||||||
await this.models.copilotTranscriptTask.complete(taskId, {
|
await this.models.copilotTranscriptTask.complete(taskId, {
|
||||||
status: 'ready',
|
status: 'ready',
|
||||||
actionRunId,
|
actionRunId,
|
||||||
|
|||||||
Reference in New Issue
Block a user