fix(server): transcript key & bump models (#15485)

#### PR Dependency Tree


* **PR #15485** 👈

This tree was auto-generated by
[Charcoal](https://github.com/danerwilliams/charcoal)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Upgraded built-in AI actions and chat model selection to Gemini 3.7
Flash.
* Transcript attachments now retain storage keys and support secure
presigned URLs during processing.
* Completed transcript results preserve attachment metadata for reliable
access.

* **Bug Fixes**
* Improved transcript handling across uploads, retries, and native
processing.
  * Added safeguards for invalid or missing attachment URLs.

* **Maintenance**
* Added a migration to backfill storage keys for existing transcript
records without altering their URLs.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
2026-08-16 00:49:27 +08:00
committed by GitHub
parent 7b3a8afc87
commit ff1e3d9c94
14 changed files with 286 additions and 67 deletions

6
Cargo.lock generated
View File

@@ -2963,7 +2963,7 @@ dependencies = [
"libc", "libc",
"log", "log",
"rustversion", "rustversion",
"windows-link 0.2.1", "windows-link 0.1.3",
"windows-result 0.4.1", "windows-result 0.4.1",
] ]
@@ -4756,9 +4756,9 @@ dependencies = [
[[package]] [[package]]
name = "llm_adapter" name = "llm_adapter"
version = "0.2.18" version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49df35c253da1563c33f0733ef3a46da3194299807cbd7e03c48dfa12a89a72f" checksum = "c17ef06aac72e31a4fae7937998029267c4d923a0671bf1a4062c96a1ab5fa70"
dependencies = [ dependencies = [
"base64", "base64",
"jsonschema", "jsonschema",

View File

@@ -3,8 +3,8 @@
"name": "Transcript audio", "name": "Transcript audio",
"action": "Transcript audio", "action": "Transcript audio",
"managedRoute": { "managedRoute": {
"targets": ["gemini-3.5-flash-lite"], "targets": ["gemini-3.7-flash"],
"premiumTargets": ["gemini-3.6-flash"] "premiumTargets": ["gemini-3.7-flash"]
}, },
"config": { "config": {
"requireContent": false, "requireContent": false,
@@ -17,8 +17,8 @@
"name": "Transcript audio structured", "name": "Transcript audio structured",
"action": "Transcript audio structured", "action": "Transcript audio structured",
"managedRoute": { "managedRoute": {
"targets": ["gemini-3.5-flash-lite"], "targets": ["gemini-3.7-flash"],
"premiumTargets": ["gemini-3.6-flash"] "premiumTargets": ["gemini-3.7-flash"]
}, },
"config": { "config": {
"requireContent": false, "requireContent": false,
@@ -39,7 +39,7 @@
{ {
"name": "Generate a caption", "name": "Generate a caption",
"action": "Generate a caption", "action": "Generate a caption",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"config": { "config": {
"requireContent": false, "requireContent": false,
"requireAttachment": true "requireAttachment": true
@@ -129,7 +129,7 @@
{ {
"name": "Explain this image", "name": "Explain this image",
"action": "Explain this image", "action": "Explain this image",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"config": { "config": {
"requireContent": false, "requireContent": false,
"requireAttachment": true "requireAttachment": true
@@ -148,7 +148,7 @@
{ {
"name": "Explain this code", "name": "Explain this code",
"action": "Explain this code", "action": "Explain this code",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -163,7 +163,7 @@
{ {
"name": "Translate to", "name": "Translate to",
"action": "Translate", "action": "Translate",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"params": { "params": {
"language": { "language": {
"default": "English", "default": "English",
@@ -256,7 +256,7 @@
{ {
"name": "Write a twitter about this", "name": "Write a twitter about this",
"action": "Write a twitter about this", "action": "Write a twitter about this",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -271,7 +271,7 @@
{ {
"name": "Write a poem about this", "name": "Write a poem about this",
"action": "Write a poem about this", "action": "Write a poem about this",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -301,7 +301,7 @@
{ {
"name": "Write outline", "name": "Write outline",
"action": "Write outline", "action": "Write outline",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -316,7 +316,7 @@
{ {
"name": "Change tone to", "name": "Change tone to",
"action": "Change tone", "action": "Change tone",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"params": { "params": {
"tone": { "tone": {
"default": "professional", "default": "professional",
@@ -337,7 +337,7 @@
{ {
"name": "Brainstorm ideas about this", "name": "Brainstorm ideas about this",
"action": "Brainstorm ideas about this", "action": "Brainstorm ideas about this",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -386,7 +386,7 @@
{ {
"name": "Improve writing for it", "name": "Improve writing for it",
"action": "Improve writing for it", "action": "Improve writing for it",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -416,7 +416,7 @@
{ {
"name": "Fix spelling for it", "name": "Fix spelling for it",
"action": "Fix spelling for it", "action": "Fix spelling for it",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -476,7 +476,7 @@
{ {
"name": "Create headings", "name": "Create headings",
"action": "Create headings", "action": "Create headings",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -521,7 +521,7 @@
{ {
"name": "Make it longer", "name": "Make it longer",
"action": "Make it longer", "action": "Make it longer",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -536,7 +536,7 @@
{ {
"name": "Make it shorter", "name": "Make it shorter",
"action": "Make it shorter", "action": "Make it shorter",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -551,7 +551,7 @@
{ {
"name": "Continue writing", "name": "Continue writing",
"action": "Continue writing", "action": "Continue writing",
"managedRoute": { "targets": ["gemini-3.5-flash-lite"] }, "managedRoute": { "targets": ["gemini-3.7-flash"] },
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@@ -711,8 +711,8 @@
}, },
{ {
"id": "gemini", "id": "gemini",
"modelId": "gemini-3.6-flash", "modelId": "gemini-3.7-flash",
"displayName": "Gemini 3.6 Flash", "displayName": "Gemini 3.7 Flash",
"minimumTier": "premium" "minimumTier": "premium"
}, },
{ {

View File

@@ -81,12 +81,12 @@ mod tests {
let response = llm_resolve_model_registry_variant(ModelRegistryResolveRequest { let response = llm_resolve_model_registry_variant(ModelRegistryResolveRequest {
backend_kind: Some("gemini_api".to_string()), backend_kind: Some("gemini_api".to_string()),
model_id: "gemini-3.6-flash".to_string(), model_id: "gemini-3.7-flash".to_string(),
}) })
.unwrap(); .unwrap();
assert_eq!(response.matched_by.as_deref(), Some("raw_model_id")); assert_eq!(response.matched_by.as_deref(), Some("raw_model_id"));
assert_eq!(response.variant.unwrap().raw_model_id, "gemini-3.6-flash"); assert_eq!(response.variant.unwrap().raw_model_id, "gemini-3.7-flash");
} }
#[test] #[test]
@@ -301,7 +301,7 @@ mod tests {
attachment_kinds: None, attachment_kinds: None,
attachment_source_kinds: None, attachment_source_kinds: None,
has_remote_attachments: None, has_remote_attachments: None,
model_id: Some("gemini-3.6-flash".to_string()), model_id: Some("gemini-3.7-flash".to_string()),
output_type: Some("image".to_string()), output_type: Some("image".to_string()),
}, },
}) })

View File

@@ -210,7 +210,7 @@ mod tests {
fn should_materialize_structured_request_with_response_contract() { fn should_materialize_structured_request_with_response_contract() {
let response = llm_build_canonical_structured_request( let response = llm_build_canonical_structured_request(
serde_json::from_value::<CanonicalStructuredRequestContract>(json!({ serde_json::from_value::<CanonicalStructuredRequestContract>(json!({
"model": "gemini-3.6-flash", "model": "gemini-3.7-flash",
"messages": [ "messages": [
{ "role": "user", "content": "hello" } { "role": "user", "content": "hello" }
], ],
@@ -226,7 +226,7 @@ mod tests {
assert_eq!( assert_eq!(
response, response,
json!({ json!({
"model": "gemini-3.6-flash", "model": "gemini-3.7-flash",
"messages": [ "messages": [
{ {
"role": "user", "role": "user",

View File

@@ -597,13 +597,13 @@ mod tests {
); );
let transcript = built_in_prompt("Transcript audio structured").expect("transcript prompt"); let transcript = built_in_prompt("Transcript audio structured").expect("transcript prompt");
assert_eq!(transcript.managed_targets, ["gemini-3.5-flash-lite"]); assert_eq!(transcript.managed_targets, ["gemini-3.7-flash"]);
assert_eq!( assert_eq!(
transcript transcript
.managed_premium_targets .managed_premium_targets
.as_deref() .as_deref()
.map(|targets| targets.iter().map(String::as_str).collect::<Vec<_>>()), .map(|targets| targets.iter().map(String::as_str).collect::<Vec<_>>()),
Some(vec!["gemini-3.6-flash"]) Some(vec!["gemini-3.7-flash"])
); );
} }
} }

View File

@@ -262,7 +262,7 @@ mod tests {
CopilotManagedTier::Premium CopilotManagedTier::Premium
) )
.unwrap(), .unwrap(),
["gemini-3.6-flash"] ["gemini-3.7-flash"]
); );
assert_eq!( assert_eq!(
managed_targets(&slot("index.embedding").unwrap(), None, CopilotManagedTier::Standard).unwrap(), managed_targets(&slot("index.embedding").unwrap(), None, CopilotManagedTier::Standard).unwrap(),

View File

@@ -326,7 +326,7 @@ mod tests {
id: "vertex".to_string(), id: "vertex".to_string(),
provider: "geminiVertex".to_string(), provider: "geminiVertex".to_string(),
enabled: true, enabled: true,
models: vec!["gemini-3.6-flash".to_string()], models: vec!["gemini-3.7-flash".to_string()],
config: json!({ "project": "affine-us", "location": location }), config: json!({ "project": "affine-us", "location": location }),
} }
} }

View File

@@ -294,7 +294,7 @@ test('retryTask reuses failed task and queues a new action attempt', async t =>
} as never, } as never,
{} as never, {} as never,
{ {
resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.5-flash-lite'), resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.7-flash'),
} as never, } as never,
{} as never, {} as never,
{} as never, {} as never,
@@ -303,7 +303,7 @@ test('retryTask reuses failed task and queues a new action attempt', async t =>
const result = await service.retryTask('user-1', 'workspace-1', 'task-1'); const result = await service.retryTask('user-1', 'workspace-1', 'task-1');
t.is(result.status, AiJobStatus.running); t.is(result?.status, AiJobStatus.running);
t.like(queuedJobs[0] as Record<string, unknown>, { t.like(queuedJobs[0] as Record<string, unknown>, {
name: 'copilot.transcript.task.submit', name: 'copilot.transcript.task.submit',
}); });
@@ -351,9 +351,7 @@ for (const status of ['ready', 'settled']) {
} as never, } as never,
{} as never, {} as never,
{ {
resolveTranscriptionModel: Sinon.stub().resolves( resolveTranscriptionModel: Sinon.stub().resolves('gemini-3.7-flash'),
'gemini-3.5-flash-lite'
),
} as never, } as never,
{} as never, {} as never,
{} as never, {} as never,
@@ -403,7 +401,8 @@ test('transcriptTask runs native transcript recipe through action bridge when av
], ],
infos: [ infos: [
{ {
url: 'data:image/png;base64,YXVkaW8=', key: 'blob-1-0',
url: 'https://affine.fail/api/copilot/blob/user-1/workspace-1/blob-1-0',
mimeType: 'audio/opus', mimeType: 'audio/opus',
index: 0, index: 0,
}, },
@@ -428,7 +427,11 @@ test('transcriptTask runs native transcript recipe through action bridge when av
}, },
} as never, } as never,
{} as never, {} as never,
{} as never, {
presignGet: Sinon.stub().resolves(
'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test'
),
} as never,
{} as never, {} as never,
createTranscriptPromptService() as never, createTranscriptPromptService() as never,
createSuccessfulTranscriptBridge('run-bridge', bridgeInputs) as never createSuccessfulTranscriptBridge('run-bridge', bridgeInputs) as never
@@ -447,6 +450,20 @@ test('transcriptTask runs native transcript recipe through action bridge when av
slot: 'transcript.audio', slot: 'transcript.audio',
builtInRouteId: 'Transcript audio structured', builtInRouteId: 'Transcript audio structured',
}); });
t.deepEqual(
(
bridgeInputs[0] as {
nativeInput: { input: { infos: unknown[] } };
}
).nativeInput.input.infos,
[
{
url: 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test',
mimeType: 'audio/opus',
index: 0,
},
]
);
const messages = ( const messages = (
bridgeInputs[0] as { bridgeInputs[0] as {
step: { step: {
@@ -459,7 +476,10 @@ test('transcriptTask runs native transcript recipe through action bridge when av
infos: [{ mimeType: 'audio/opus', index: 0 }], infos: [{ mimeType: 'audio/opus', index: 0 }],
}); });
t.deepEqual(messages.at(-1)?.attachments, [ t.deepEqual(messages.at(-1)?.attachments, [
{ attachment: 'data:image/png;base64,YXVkaW8=', mimeType: 'audio/opus' }, {
attachment: 'https://canary.copilotcontent.affine.pro/blob-1-0?sig=test',
mimeType: 'audio/opus',
},
]); ]);
t.like(complete.firstCall.args[1], { t.like(complete.firstCall.args[1], {
status: 'ready', status: 'ready',
@@ -471,6 +491,7 @@ test('transcriptTask runs native transcript recipe through action bridge when av
complete.firstCall.args[1].protectedResult.normalizedTranscript, complete.firstCall.args[1].protectedResult.normalizedTranscript,
'00:00:05 A: Kickoff' '00:00:05 A: Kickoff'
); );
t.deepEqual(complete.firstCall.args[1].protectedResult.infos, payload.infos);
}); });
test('transcriptTask fails task when native action bridge reports an error event', async t => { test('transcriptTask fails task when native action bridge reports an error event', async t => {

View File

@@ -5,6 +5,7 @@ import ava, { TestFn } from 'ava';
import { createTestingModule, type TestingModule } from '../../__tests__/utils'; import { createTestingModule, type TestingModule } from '../../__tests__/utils';
import { Models } from '../../models'; import { Models } from '../../models';
import { BackfillPermissionProjection1765500000000 } from '../migrations/1765500000000-backfill-permission-projection'; import { BackfillPermissionProjection1765500000000 } from '../migrations/1765500000000-backfill-permission-projection';
import { BackfillTranscriptStorageKeys1786805802350 } from '../migrations/1786805802350-backfill-transcript-storage-keys';
interface Context { interface Context {
module: TestingModule; module: TestingModule;
@@ -81,3 +82,44 @@ test('permission backfill repairs ownerless workspaces before runtime state proj
{ role: 'owner' } { role: 'owner' }
); );
}); });
test('transcript backfill adds stable keys without removing compatibility URLs', async t => {
const payload = {
sourceAudio: { blobId: 'blob-1' },
infos: [
{
url: 'https://affine.example/api/copilot/blob/user-1/workspace-1/blob-1',
mimeType: 'audio/m4a',
},
{
url: 'https://example.com/external.m4a',
mimeType: 'audio/m4a',
},
],
};
await t.context.db.aiTranscriptTask.create({
data: {
userId: 'user-1',
workspaceId: 'workspace-1',
blobId: 'blob-1',
status: 'failed',
recipeId: 'transcript.audio',
recipeVersion: 'v1',
inputSnapshot: payload,
protectedResult: payload,
},
});
await BackfillTranscriptStorageKeys1786805802350.up(t.context.db);
await BackfillTranscriptStorageKeys1786805802350.up(t.context.db);
const task = await t.context.db.aiTranscriptTask.findFirstOrThrow({
where: { blobId: 'blob-1' },
});
const expected = {
...payload,
infos: [{ ...payload.infos[0], key: 'blob-1' }, payload.infos[1]],
};
t.deepEqual(task.inputSnapshot, expected);
t.deepEqual(task.protectedResult, expected);
});

View File

@@ -0,0 +1,96 @@
import { type Prisma, PrismaClient } from '@prisma/client';
const BATCH_SIZE = 100;
function isRecord(value: unknown): value is Record<string, unknown> {
return !!value && typeof value === 'object' && !Array.isArray(value);
}
function backfillTranscriptStorageKeys(
payload: Prisma.JsonValue | null,
userId: string,
workspaceId: string
): Prisma.InputJsonValue | undefined {
if (!isRecord(payload) || !Array.isArray(payload.infos)) {
return;
}
const prefix = `/api/copilot/blob/${encodeURIComponent(userId)}/${encodeURIComponent(workspaceId)}/`;
let changed = false;
const infos = payload.infos.map(info => {
if (!isRecord(info) || info.key || typeof info.url !== 'string') {
return info;
}
try {
const url = new URL(info.url);
if (!url.pathname.startsWith(prefix)) {
return info;
}
const key = decodeURIComponent(url.pathname.slice(prefix.length));
if (!key) {
return info;
}
changed = true;
return { ...info, key };
} catch {
return info;
}
});
return changed ? ({ ...payload, infos } as Prisma.InputJsonValue) : undefined;
}
export class BackfillTranscriptStorageKeys1786805802350 {
static async up(db: PrismaClient) {
let cursor: string | undefined;
while (true) {
const tasks = await db.aiTranscriptTask.findMany({
select: {
id: true,
userId: true,
workspaceId: true,
inputSnapshot: true,
protectedResult: true,
},
orderBy: { id: 'asc' },
take: BATCH_SIZE,
...(cursor ? { cursor: { id: cursor }, skip: 1 } : {}),
});
if (!tasks.length) {
return;
}
const updates = tasks.flatMap(task => {
const inputSnapshot = backfillTranscriptStorageKeys(
task.inputSnapshot,
task.userId,
task.workspaceId
);
const protectedResult = backfillTranscriptStorageKeys(
task.protectedResult,
task.userId,
task.workspaceId
);
if (!inputSnapshot && !protectedResult) {
return [];
}
return db.aiTranscriptTask.update({
where: { id: task.id },
data: {
...(inputSnapshot ? { inputSnapshot } : {}),
...(protectedResult ? { protectedResult } : {}),
},
});
});
if (updates.length) {
await db.$transaction(updates);
}
cursor = tasks.at(-1)?.id;
}
}
static async down(_db: PrismaClient) {}
}

View File

@@ -6,3 +6,4 @@ export * from './1751966744168-correct-session-update-time';
export * from './1763800000000-rebuild-manticore-mixed-script-indexes'; export * from './1763800000000-rebuild-manticore-mixed-script-indexes';
export * from './1765500000000-backfill-permission-projection'; export * from './1765500000000-backfill-permission-projection';
export * from './1765600000000-backfill-entitlement-projection'; export * from './1765600000000-backfill-entitlement-projection';
export * from './1786805802350-backfill-transcript-storage-keys';

View File

@@ -59,14 +59,31 @@ export class CopilotStorage {
): Promise<StorageRuntimeGetObjectResult> { ): Promise<StorageRuntimeGetObjectResult> {
const name = `${userId}/${workspaceId}/${key}`; const name = `${userId}/${workspaceId}/${key}`;
if (signedUrl) { if (signedUrl) {
const presigned = await this.rt.presignGet('copilot', name); const redirectUrl = await this.presignGet(userId, workspaceId, key);
if (presigned) { if (redirectUrl) return { redirectUrl };
return { redirectUrl: presigned.url };
}
} }
return this.rt.getObject('copilot', name); return this.rt.getObject('copilot', name);
} }
async presignGet(userId: string, workspaceId: string, key: string) {
return (
await this.rt.presignGet('copilot', `${userId}/${workspaceId}/${key}`)
)?.url;
}
keyFromUrl(userId: string, workspaceId: string, url: string) {
try {
const parsed = new URL(url);
const prefix = `/api/copilot/blob/${encodeURIComponent(userId)}/${encodeURIComponent(workspaceId)}/`;
if (parsed.pathname.startsWith(prefix)) {
return decodeURIComponent(parsed.pathname.slice(prefix.length));
}
} catch {
return;
}
return undefined;
}
@CallMetric('ai', 'blob_delete') @CallMetric('ai', 'blob_delete')
async delete(userId: string, workspaceId: string, key: string) { async delete(userId: string, workspaceId: string, key: string) {
await this.rt.deleteObject('copilot', `${userId}/${workspaceId}/${key}`); await this.rt.deleteObject('copilot', `${userId}/${workspaceId}/${key}`);

View File

@@ -18,7 +18,8 @@ export const LegacyTranscriptionSchema = z.array(
); );
export const AudioBlobInfoSchema = z.object({ export const AudioBlobInfoSchema = z.object({
url: z.string(), key: z.string().min(1).optional(),
url: z.string().min(1),
mimeType: z.string(), mimeType: z.string(),
index: z.number().int().nullable().optional(), index: z.number().int().nullable().optional(),
}); });

View File

@@ -46,10 +46,6 @@ export class CopilotTranscriptionService {
private readonly realtime: RealtimePublisher private readonly realtime: RealtimePublisher
) {} ) {}
private parseTaskPayload(payload: unknown): TranscriptionPayloadV2 {
return TranscriptPayloadSchema.parse(payload);
}
private buildTaskPublicMeta(payload: TranscriptionPayloadV2) { private buildTaskPublicMeta(payload: TranscriptionPayloadV2) {
return { return {
sourceAudio: payload.sourceAudio, sourceAudio: payload.sourceAudio,
@@ -68,13 +64,10 @@ export class CopilotTranscriptionService {
const infos: AudioBlobInfos = []; const infos: AudioBlobInfos = [];
for (const [idx, blob] of blobs.entries()) { for (const [idx, blob] of blobs.entries()) {
const buffer = await readStream(blob.createReadStream()); const buffer = await readStream(blob.createReadStream());
const url = await this.storage.put( const key = `${blobId}-${idx}`;
userId, const url = await this.storage.put(userId, workspaceId, key, buffer);
workspaceId,
`${blobId}-${idx}`,
buffer
);
infos.push({ infos.push({
key,
url, url,
mimeType: sniffMime(buffer, blob.mimetype) || blob.mimetype, mimeType: sniffMime(buffer, blob.mimetype) || blob.mimetype,
index: idx, index: idx,
@@ -103,6 +96,47 @@ export class CopilotTranscriptionService {
} satisfies TranscriptionPayloadV2; } satisfies TranscriptionPayloadV2;
} }
private async resolveAttachmentUrl(
userId: string,
workspaceId: string,
info: AudioBlobInfos[number]
) {
if (info.url.startsWith('data:')) {
return info.url;
}
const key =
info.key ?? this.storage.keyFromUrl(userId, workspaceId, info.url);
if (!key) {
throw new Error('Transcript attachment cannot be resolved');
}
const signedUrl = await this.storage.presignGet(userId, workspaceId, key);
if (!signedUrl) {
throw new Error('Transcript attachment signing is not configured');
}
return signedUrl;
}
private async materializePayload(
userId: string,
workspaceId: string,
payload: TranscriptionPayloadV2
) {
return {
...payload,
infos: payload.infos
? await Promise.all(
payload.infos.map(async info => ({
url: await this.resolveAttachmentUrl(userId, workspaceId, info),
mimeType: info.mimeType,
index: info.index,
}))
)
: payload.infos,
} satisfies TranscriptionPayloadV2;
}
private async buildTranscriptActionMessages(payload: TranscriptionPayloadV2) { private async buildTranscriptActionMessages(payload: TranscriptionPayloadV2) {
const prompt = await this.prompts.get(TRANSCRIPT_PROMPT_REF); const prompt = await this.prompts.get(TRANSCRIPT_PROMPT_REF);
if (!prompt) { if (!prompt) {
@@ -118,13 +152,12 @@ export class CopilotTranscriptionService {
index: info.index ?? null, index: info.index ?? null,
})) ?? null, })) ?? null,
}; };
const attachments = const attachments = (payload.infos ?? []).map(info => ({
payload.infos?.map(info => ({ role: 'user' as const,
role: 'user' as const, content: `Audio attachment ${info.index ?? 0}`,
content: `Audio attachment ${info.index ?? 0}`, attachments: [{ attachment: info.url, mimeType: info.mimeType }],
attachments: [{ attachment: info.url, mimeType: info.mimeType }], params: { mimetype: info.mimeType },
params: { mimetype: info.mimeType }, }));
})) ?? [];
return [ return [
...this.prompts.finish(prompt, { ...this.prompts.finish(prompt, {
content: JSON.stringify(metadata), content: JSON.stringify(metadata),
@@ -205,7 +238,7 @@ export class CopilotTranscriptionService {
); );
} }
const payload = this.parseTaskPayload(task.protectedResult); const payload = TranscriptPayloadSchema.parse(task.protectedResult);
await this.runtime.assertRoute( await this.runtime.assertRoute(
'transcript.audio', 'transcript.audio',
{}, {},
@@ -287,14 +320,19 @@ export class CopilotTranscriptionService {
let bridgeFailed = false; let bridgeFailed = false;
let bridgeError = 'transcript native recipe failed'; let bridgeError = 'transcript native recipe failed';
let finalResult: unknown = null; let finalResult: unknown = null;
const messages = await this.buildTranscriptActionMessages(payload); const runtimePayload = await this.materializePayload(
task.userId,
task.workspaceId,
payload
);
const messages = await this.buildTranscriptActionMessages(runtimePayload);
for await (const event of this.actionBridge.runStream({ for await (const event of this.actionBridge.runStream({
userId: task.userId, userId: task.userId,
workspaceId: task.workspaceId, workspaceId: task.workspaceId,
actionId: TRANSCRIPT_ACTION_ID, actionId: TRANSCRIPT_ACTION_ID,
actionVersion: TRANSCRIPT_ACTION_VERSION, actionVersion: TRANSCRIPT_ACTION_VERSION,
retryOf: retryOf ?? null, retryOf: retryOf ?? null,
inputSnapshot: payload, inputSnapshot: runtimePayload,
onRunCreated: async ({ runId }) => { onRunCreated: async ({ runId }) => {
await this.models.copilotTranscriptTask.markRunning(taskId, runId); await this.models.copilotTranscriptTask.markRunning(taskId, runId);
this.publishTaskChanged( this.publishTaskChanged(
@@ -329,7 +367,10 @@ export class CopilotTranscriptionService {
if (bridgeFailed) { if (bridgeFailed) {
throw new Error(bridgeError); throw new Error(bridgeError);
} }
const parsedResult = TranscriptPayloadSchema.parse(finalResult); const parsedResult = {
...TranscriptPayloadSchema.parse(finalResult),
infos: payload.infos,
} satisfies TranscriptionPayloadV2;
await this.models.copilotTranscriptTask.complete(taskId, { await this.models.copilotTranscriptTask.complete(taskId, {
status: 'ready', status: 'ready',
actionRunId, actionRunId,