From 063072457cc4df843c9c011c3c7e7b6ff5d5c4bc Mon Sep 17 00:00:00 2001
From: darkskygit <darksky.graphite@toeverything.info>
Date: Wed, 4 Jun 2025 08:51:02 +0000
Subject: [PATCH] fix(server): chat with image (#12699)

<!-- This is an auto-generated comment: release notes by coderabbit.ai -->
## Summary by CodeRabbit

- **Refactor**
  - Improved the handling of attachments in chat messages for more efficient processing of images and files without impacting user experience.
- **Chores**
  - Added internal logging to enhance monitoring of AI model interactions.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
---
 .../plugins/copilot/providers/anthropic/anthropic.ts   |  4 ++--
 .../server/src/plugins/copilot/providers/utils.ts      | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts b/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts
index 23d681bb4..1fd22d6aa 100644
--- a/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts
+++ b/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts
@@ -57,7 +57,6 @@ export abstract class AnthropicProvider<T> extends CopilotProvider<T> {
 
     try {
       metrics.ai.counter('chat_text_calls').add(1, { model: model.id });
-
       const [system, msgs] = await chatToGPTMessage(messages, true, true);
 
       const modelInstance = this.instance(model.id);
@@ -94,7 +93,8 @@ export abstract class AnthropicProvider<T> extends CopilotProvider<T> {
 
     try {
       metrics.ai.counter('chat_text_stream_calls').add(1, { model: model.id });
-      const [system, msgs] = await chatToGPTMessage(messages);
+      const [system, msgs] = await chatToGPTMessage(messages, true, true);
+
       const { fullStream } = streamText({
         model: this.instance(model.id),
         system,
diff --git a/packages/backend/server/src/plugins/copilot/providers/utils.ts b/packages/backend/server/src/plugins/copilot/providers/utils.ts
index ddee82889..f98654c83 100644
--- a/packages/backend/server/src/plugins/copilot/providers/utils.ts
+++ b/packages/backend/server/src/plugins/copilot/providers/utils.ts
@@ -101,13 +101,13 @@ export async function chatToGPTMessage(
             ({ attachment, mimeType } = attachment);
           }
           if (SIMPLE_IMAGE_URL_REGEX.test(attachment)) {
+            const data =
+              attachment.startsWith('data:') || useBase64Attachment
+                ? await fetch(attachment).then(r => r.arrayBuffer())
+                : new URL(attachment);
             if (mimeType.startsWith('image/')) {
-              contents.push({ type: 'image', image: attachment, mimeType });
+              contents.push({ type: 'image', image: data, mimeType });
             } else {
-              const data =
-                attachment.startsWith('data:') || useBase64Attachment
-                  ? await fetch(attachment).then(r => r.arrayBuffer())
-                  : new URL(attachment);
               contents.push({ type: 'file' as const, data, mimeType });
             }
           }