diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index 2768c3726..f6d6360e5 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -69,7 +69,7 @@ "@prisma/instrumentation": "^5.22.0", "@react-email/components": "0.0.36", "@socket.io/redis-adapter": "^8.3.0", - "ai": "^4.1.51", + "ai": "^4.3.4", "bullmq": "^5.40.2", "cookie-parser": "^1.4.7", "date-fns": "^4.0.0", diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts index a47708436..047b43a7a 100644 --- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts @@ -1,6 +1,5 @@ import type { ExecutionContext, TestFn } from 'ava'; import ava from 'ava'; -import { z } from 'zod'; import { ServerFeature, ServerService } from '../core'; import { AuthService } from '../core/auth'; @@ -8,6 +7,7 @@ import { QuotaModule } from '../core/quota'; import { CopilotModule } from '../plugins/copilot'; import { prompts, PromptService } from '../plugins/copilot/prompt'; import { CopilotProviderFactory } from '../plugins/copilot/providers'; +import { TranscriptionResponseSchema } from '../plugins/copilot/transcript/types'; import { CopilotChatTextExecutor, CopilotWorkflowService, @@ -315,6 +315,7 @@ const actions = [ type: 'text' as const, }, { + name: 'Should transcribe short audio', promptName: ['Transcript audio'], messages: [ { @@ -323,23 +324,58 @@ const actions = [ attachments: [ 'https://cdn.affine.pro/copilot-test/MP9qDGuYgnY+ILoEAmHpp3h9Npuw2403EAYMEA.mp3', ], + params: { + schema: TranscriptionResponseSchema, + }, }, ], verifier: (t: ExecutionContext, result: string) => { - // cleanup json markdown wrap - const cleaned = result - .replace(/```[\w\s]+\n/g, '') - .replace(/\n```/g, '') - .trim(); t.notThrows(() => { - z.object({ - a: z.string(), - s: z.number(), - e: z.number(), - t: z.string(), - }) - .array() - .parse(JSON.parse(cleaned)); + TranscriptionResponseSchema.parse(JSON.parse(result)); + }); + }, + type: 'text' as const, + }, + { + name: 'Should transcribe middle audio', + promptName: ['Transcript audio'], + messages: [ + { + role: 'user' as const, + content: '', + attachments: [ + 'https://cdn.affine.pro/copilot-test/2ed05eo1KvZ2tWB_BAjFo67EAPZZY-w4LylUAw.m4a', + ], + params: { + schema: TranscriptionResponseSchema, + }, + }, + ], + verifier: (t: ExecutionContext, result: string) => { + t.notThrows(() => { + TranscriptionResponseSchema.parse(JSON.parse(result)); + }); + }, + type: 'text' as const, + }, + { + name: 'Should transcribe long audio', + promptName: ['Transcript audio'], + messages: [ + { + role: 'user' as const, + content: '', + attachments: [ + 'https://cdn.affine.pro/copilot-test/nC9-e7P85PPI2rU29QWwf8slBNRMy92teLIIMw.opus', + ], + params: { + schema: TranscriptionResponseSchema, + }, + }, + ], + verifier: (t: ExecutionContext, result: string) => { + t.notThrows(() => { + TranscriptionResponseSchema.parse(JSON.parse(result)); }); }, type: 'text' as const, @@ -516,7 +552,8 @@ for (const { name, promptName, messages, verifier, type } of actions) { ), ...messages, ], - prompt.model + prompt.model, + Object.assign({}, prompt.config) ); t.truthy(result, 'should return result'); verifier?.(t, result); diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index 4719caf91..6fa0622f0 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -180,7 +180,6 @@ export class OpenAIProvider options: CopilotChatOptions = {} ): Promise { await this.checkParams({ messages, model, options }); - console.log('messages', messages); try { metrics.ai.counter('chat_text_calls').add(1, { model }); @@ -215,7 +214,6 @@ export class OpenAIProvider return text.trim(); } catch (e: any) { - console.log('error', e); metrics.ai.counter('chat_text_errors').add(1, { model }); throw this.handleError(e, model, options); } diff --git a/packages/backend/server/src/plugins/copilot/providers/utils.ts b/packages/backend/server/src/plugins/copilot/providers/utils.ts index df0937808..7d12bac50 100644 --- a/packages/backend/server/src/plugins/copilot/providers/utils.ts +++ b/packages/backend/server/src/plugins/copilot/providers/utils.ts @@ -14,6 +14,12 @@ const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/; const FORMAT_INFER_MAP: Record = { pdf: 'application/pdf', mp3: 'audio/mpeg', + opus: 'audio/opus', + ogg: 'audio/ogg', + aac: 'audio/aac', + m4a: 'audio/aac', + flac: 'audio/flac', + ogv: 'video/ogg', wav: 'audio/wav', png: 'image/png', jpeg: 'image/jpeg', @@ -57,10 +63,7 @@ export async function chatToGPTMessage( if (Array.isArray(attachments)) { const contents: (TextPart | ImagePart | FilePart)[] = []; if (content.length) { - contents.push({ - type: 'text', - text: content, - }); + contents.push({ type: 'text', text: content }); } for (const url of attachments) { @@ -69,20 +72,12 @@ export async function chatToGPTMessage( typeof mimetype === 'string' ? mimetype : inferMimeType(url); if (mimeType) { if (mimeType.startsWith('image/')) { - contents.push({ - type: 'image', - image: url, - mimeType, - }); + contents.push({ type: 'image', image: url, mimeType }); } else { const data = url.startsWith('data:') ? await fetch(url).then(r => r.arrayBuffer()) : new URL(url); - contents.push({ - type: 'file' as const, - data, - mimeType, - }); + contents.push({ type: 'file' as const, data, mimeType }); } } } diff --git a/yarn.lock b/yarn.lock index 482cfbc1d..a5027666b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -929,7 +929,7 @@ __metadata: "@types/semver": "npm:^7.5.8" "@types/sinon": "npm:^17.0.3" "@types/supertest": "npm:^6.0.2" - ai: "npm:^4.1.51" + ai: "npm:^4.3.4" ava: "npm:^6.2.0" bullmq: "npm:^5.40.2" c8: "npm:^10.1.3" @@ -16349,7 +16349,7 @@ __metadata: languageName: node linkType: hard -"ai@npm:^4.1.51": +"ai@npm:^4.3.4": version: 4.3.4 resolution: "ai@npm:4.3.4" dependencies: