feat(server): add attachment fallback for ai sdk (#12639)

fix AI-161

packages/backend/server/package.json

@@ -28,11 +28,11 @@
   "dependencies": {
     "@affine/reader": "workspace:*",
     "@affine/server-native": "workspace:*",
-    "@ai-sdk/anthropic": "^1.2.10",
+    "@ai-sdk/anthropic": "^1.2.12",
     "@ai-sdk/google": "^1.2.18",
-    "@ai-sdk/google-vertex": "^2.2.22",
+    "@ai-sdk/google-vertex": "^2.2.23",
-    "@ai-sdk/openai": "^1.3.21",
+    "@ai-sdk/openai": "^1.3.22",
-    "@ai-sdk/perplexity": "^1.1.6",
+    "@ai-sdk/perplexity": "^1.1.9",
     "@apollo/server": "^4.11.3",
     "@aws-sdk/client-s3": "^3.779.0",
     "@aws-sdk/s3-request-presigner": "^3.779.0",

packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts

@@ -58,7 +58,7 @@ export abstract class AnthropicProvider<T> extends CopilotProvider<T> {
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
 
-      const [system, msgs] = await chatToGPTMessage(messages);
+      const [system, msgs] = await chatToGPTMessage(messages, true, true);
 
       const modelInstance = this.instance(model.id);
       const { text, reasoning } = await generateText({

packages/backend/server/src/plugins/copilot/providers/utils.ts

@@ -64,7 +64,10 @@ export async function inferMimeType(url: string) {
 export async function chatToGPTMessage(
   messages: PromptMessage[],
   // TODO(@darkskygit): move this logic in interface refactoring
-  withAttachment: boolean = true
+  withAttachment: boolean = true,
+  // NOTE: some providers in vercel ai sdk are not able to handle url attachments yet
+  //       so we need to use base64 encoded attachments instead
+  useBase64Attachment: boolean = false
 ): Promise<[string | undefined, ChatMessage[], ZodType?]> {
   const system = messages[0]?.role === 'system' ? messages.shift() : undefined;
   const schema =
@@ -101,9 +104,10 @@ export async function chatToGPTMessage(
             if (mimeType.startsWith('image/')) {
               contents.push({ type: 'image', image: attachment, mimeType });
             } else {
-              const data = attachment.startsWith('data:')
+              const data =
-                ? await fetch(attachment).then(r => r.arrayBuffer())
+                attachment.startsWith('data:') || useBase64Attachment
-                : new URL(attachment);
+                  ? await fetch(attachment).then(r => r.arrayBuffer())
+                  : new URL(attachment);
               contents.push({ type: 'file' as const, data, mimeType });
             }
           }

yarn.lock

@@ -908,11 +908,11 @@ __metadata:
     "@affine/graphql": "workspace:*"
     "@affine/reader": "workspace:*"
     "@affine/server-native": "workspace:*"
-    "@ai-sdk/anthropic": "npm:^1.2.10"
+    "@ai-sdk/anthropic": "npm:^1.2.12"
     "@ai-sdk/google": "npm:^1.2.18"
-    "@ai-sdk/google-vertex": "npm:^2.2.22"
+    "@ai-sdk/google-vertex": "npm:^2.2.23"
-    "@ai-sdk/openai": "npm:^1.3.21"
+    "@ai-sdk/openai": "npm:^1.3.22"
-    "@ai-sdk/perplexity": "npm:^1.1.6"
+    "@ai-sdk/perplexity": "npm:^1.1.9"
     "@apollo/server": "npm:^4.11.3"
     "@aws-sdk/client-s3": "npm:^3.779.0"
     "@aws-sdk/s3-request-presigner": "npm:^3.779.0"
@@ -1075,30 +1075,30 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@ai-sdk/anthropic@npm:1.2.11, @ai-sdk/anthropic@npm:^1.2.10":
+"@ai-sdk/anthropic@npm:1.2.12, @ai-sdk/anthropic@npm:^1.2.12":
-  version: 1.2.11
+  version: 1.2.12
-  resolution: "@ai-sdk/anthropic@npm:1.2.11"
+  resolution: "@ai-sdk/anthropic@npm:1.2.12"
   dependencies:
     "@ai-sdk/provider": "npm:1.1.3"
     "@ai-sdk/provider-utils": "npm:2.2.8"
   peerDependencies:
     zod: ^3.0.0
-  checksum: 10/89f48e23f406bef946d2b9153e67eb8417ba01693e291566693f4d49ca4eb9e41e7e49ec83cbb738c7ac2eaf8c3aa572327a0ab6ae7f8f8bf50f08d3be0c2471
+  checksum: 10/ee09f00328c88954fba8d00795f6c379091f95b7717bb90292acf32f41b072b2c89b178c1c18dfc23fcec7532e04fcd13233b9a813ce4881e0a16cae20a878c8
   languageName: node
   linkType: hard
 
-"@ai-sdk/google-vertex@npm:^2.2.22":
+"@ai-sdk/google-vertex@npm:^2.2.23":
-  version: 2.2.22
+  version: 2.2.23
-  resolution: "@ai-sdk/google-vertex@npm:2.2.22"
+  resolution: "@ai-sdk/google-vertex@npm:2.2.23"
   dependencies:
-    "@ai-sdk/anthropic": "npm:1.2.11"
+    "@ai-sdk/anthropic": "npm:1.2.12"
     "@ai-sdk/google": "npm:1.2.18"
     "@ai-sdk/provider": "npm:1.1.3"
     "@ai-sdk/provider-utils": "npm:2.2.8"
     google-auth-library: "npm:^9.15.0"
   peerDependencies:
     zod: ^3.0.0
-  checksum: 10/852928d35797cc14802d8c4a7dba2794197f019507a24124e7f87e456ba19f2b4e50438af626dd30e3a2a0f9f10819d6534a31e23c08f7f3d148d56b68167d0a
+  checksum: 10/19b7120879662e3597e95b84d30a587ada24d5fbd4306e4551f88605ae4e7c9d3f52bc60b6ea3bfbfc487a58c0a0ad35d1e058f7408140ea84e2ff86e6a1d052
   languageName: node
   linkType: hard
 
@@ -1114,7 +1114,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@ai-sdk/openai@npm:^1.3.21":
+"@ai-sdk/openai@npm:^1.3.22":
   version: 1.3.22
   resolution: "@ai-sdk/openai@npm:1.3.22"
   dependencies:
@@ -1126,7 +1126,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@ai-sdk/perplexity@npm:^1.1.6":
+"@ai-sdk/perplexity@npm:^1.1.9":
   version: 1.1.9
   resolution: "@ai-sdk/perplexity@npm:1.1.9"
   dependencies: