test(server): improve server copilot test (#11596)

packages/backend/server/package.json

@@ -69,7 +69,7 @@
     "@prisma/instrumentation": "^5.22.0",
     "@react-email/components": "0.0.36",
     "@socket.io/redis-adapter": "^8.3.0",
-    "ai": "^4.1.51",
+    "ai": "^4.3.4",
     "bullmq": "^5.40.2",
     "cookie-parser": "^1.4.7",
     "date-fns": "^4.0.0",

packages/backend/server/src/__tests__/copilot-provider.spec.ts

@@ -1,6 +1,5 @@
 import type { ExecutionContext, TestFn } from 'ava';
 import ava from 'ava';
-import { z } from 'zod';
 
 import { ServerFeature, ServerService } from '../core';
 import { AuthService } from '../core/auth';
@@ -8,6 +7,7 @@ import { QuotaModule } from '../core/quota';
 import { CopilotModule } from '../plugins/copilot';
 import { prompts, PromptService } from '../plugins/copilot/prompt';
 import { CopilotProviderFactory } from '../plugins/copilot/providers';
+import { TranscriptionResponseSchema } from '../plugins/copilot/transcript/types';
 import {
   CopilotChatTextExecutor,
   CopilotWorkflowService,
@@ -315,6 +315,7 @@ const actions = [
     type: 'text' as const,
   },
   {
+    name: 'Should transcribe short audio',
     promptName: ['Transcript audio'],
     messages: [
       {
@@ -323,23 +324,58 @@ const actions = [
         attachments: [
           'https://cdn.affine.pro/copilot-test/MP9qDGuYgnY+ILoEAmHpp3h9Npuw2403EAYMEA.mp3',
         ],
+        params: {
+          schema: TranscriptionResponseSchema,
+        },
       },
     ],
     verifier: (t: ExecutionContext<Tester>, result: string) => {
-      // cleanup json markdown wrap
-      const cleaned = result
-        .replace(/```[\w\s]+\n/g, '')
-        .replace(/\n```/g, '')
-        .trim();
       t.notThrows(() => {
-        z.object({
+        TranscriptionResponseSchema.parse(JSON.parse(result));
-          a: z.string(),
+      });
-          s: z.number(),
+    },
-          e: z.number(),
+    type: 'text' as const,
-          t: z.string(),
+  },
-        })
+  {
-          .array()
+    name: 'Should transcribe middle audio',
-          .parse(JSON.parse(cleaned));
+    promptName: ['Transcript audio'],
+    messages: [
+      {
+        role: 'user' as const,
+        content: '',
+        attachments: [
+          'https://cdn.affine.pro/copilot-test/2ed05eo1KvZ2tWB_BAjFo67EAPZZY-w4LylUAw.m4a',
+        ],
+        params: {
+          schema: TranscriptionResponseSchema,
+        },
+      },
+    ],
+    verifier: (t: ExecutionContext<Tester>, result: string) => {
+      t.notThrows(() => {
+        TranscriptionResponseSchema.parse(JSON.parse(result));
+      });
+    },
+    type: 'text' as const,
+  },
+  {
+    name: 'Should transcribe long audio',
+    promptName: ['Transcript audio'],
+    messages: [
+      {
+        role: 'user' as const,
+        content: '',
+        attachments: [
+          'https://cdn.affine.pro/copilot-test/nC9-e7P85PPI2rU29QWwf8slBNRMy92teLIIMw.opus',
+        ],
+        params: {
+          schema: TranscriptionResponseSchema,
+        },
+      },
+    ],
+    verifier: (t: ExecutionContext<Tester>, result: string) => {
+      t.notThrows(() => {
+        TranscriptionResponseSchema.parse(JSON.parse(result));
       });
     },
     type: 'text' as const,
@@ -516,7 +552,8 @@ for (const { name, promptName, messages, verifier, type } of actions) {
                 ),
                 ...messages,
               ],
-              prompt.model
+              prompt.model,
+              Object.assign({}, prompt.config)
             );
             t.truthy(result, 'should return result');
             verifier?.(t, result);

packages/backend/server/src/plugins/copilot/providers/openai.ts

@@ -180,7 +180,6 @@ export class OpenAIProvider
     options: CopilotChatOptions = {}
   ): Promise<string> {
     await this.checkParams({ messages, model, options });
-    console.log('messages', messages);
 
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model });
@@ -215,7 +214,6 @@ export class OpenAIProvider
 
       return text.trim();
     } catch (e: any) {
-      console.log('error', e);
       metrics.ai.counter('chat_text_errors').add(1, { model });
       throw this.handleError(e, model, options);
     }

packages/backend/server/src/plugins/copilot/providers/utils.ts

@@ -14,6 +14,12 @@ const SIMPLE_IMAGE_URL_REGEX = /^(https?:\/\/|data:image\/)/;
 const FORMAT_INFER_MAP: Record<string, string> = {
   pdf: 'application/pdf',
   mp3: 'audio/mpeg',
+  opus: 'audio/opus',
+  ogg: 'audio/ogg',
+  aac: 'audio/aac',
+  m4a: 'audio/aac',
+  flac: 'audio/flac',
+  ogv: 'video/ogg',
   wav: 'audio/wav',
   png: 'image/png',
   jpeg: 'image/jpeg',
@@ -57,10 +63,7 @@ export async function chatToGPTMessage(
     if (Array.isArray(attachments)) {
       const contents: (TextPart | ImagePart | FilePart)[] = [];
       if (content.length) {
-        contents.push({
+        contents.push({ type: 'text', text: content });
-          type: 'text',
-          text: content,
-        });
       }
 
       for (const url of attachments) {
@@ -69,20 +72,12 @@ export async function chatToGPTMessage(
             typeof mimetype === 'string' ? mimetype : inferMimeType(url);
           if (mimeType) {
             if (mimeType.startsWith('image/')) {
-              contents.push({
+              contents.push({ type: 'image', image: url, mimeType });
-                type: 'image',
-                image: url,
-                mimeType,
-              });
             } else {
               const data = url.startsWith('data:')
                 ? await fetch(url).then(r => r.arrayBuffer())
                 : new URL(url);
-              contents.push({
+              contents.push({ type: 'file' as const, data, mimeType });
-                type: 'file' as const,
-                data,
-                mimeType,
-              });
             }
           }
         }

yarn.lock

@@ -929,7 +929,7 @@ __metadata:
     "@types/semver": "npm:^7.5.8"
     "@types/sinon": "npm:^17.0.3"
     "@types/supertest": "npm:^6.0.2"
-    ai: "npm:^4.1.51"
+    ai: "npm:^4.3.4"
     ava: "npm:^6.2.0"
     bullmq: "npm:^5.40.2"
     c8: "npm:^10.1.3"
@@ -16349,7 +16349,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ai@npm:^4.1.51":
+"ai@npm:^4.3.4":
   version: 4.3.4
   resolution: "ai@npm:4.3.4"
   dependencies: