diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index ffadd5e9a..097836224 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -669,7 +669,7 @@ }, "scenarios": { "type": "object", - "description": "Use custom models in scenarios and override default settings.\n@default {\"override_enabled\":false,\"scenarios\":{\"audio_transcribing\":\"gemini-2.5-flash\",\"chat\":\"claude-sonnet-4@20250514\",\"embedding\":\"gemini-embedding-001\",\"image\":\"gpt-image-1\",\"rerank\":\"gpt-4.1\",\"coding\":\"claude-sonnet-4@20250514\",\"complex_text_generation\":\"gpt-4o-2024-08-06\",\"quick_decision_making\":\"gpt-4.1-mini\",\"quick_text_generation\":\"gemini-2.5-flash\",\"polish_and_summarize\":\"gemini-2.5-flash\"}}", + "description": "Use custom models in scenarios and override default settings.\n@default {\"override_enabled\":false,\"scenarios\":{\"audio_transcribing\":\"gemini-2.5-flash\",\"chat\":\"claude-sonnet-4@20250514\",\"embedding\":\"gemini-embedding-001\",\"image\":\"gpt-image-1\",\"rerank\":\"gpt-4.1\",\"coding\":\"claude-sonnet-4@20250514\",\"complex_text_generation\":\"gpt-4o-2024-08-06\",\"quick_decision_making\":\"gpt-5-mini\",\"quick_text_generation\":\"gemini-2.5-flash\",\"polish_and_summarize\":\"gemini-2.5-flash\"}}", "default": { "override_enabled": false, "scenarios": { @@ -680,7 +680,7 @@ "rerank": "gpt-4.1", "coding": "claude-sonnet-4@20250514", "complex_text_generation": "gpt-4o-2024-08-06", - "quick_decision_making": "gpt-4.1-mini", + "quick_decision_making": "gpt-5-mini", "quick_text_generation": "gemini-2.5-flash", "polish_and_summarize": "gemini-2.5-flash" } diff --git a/Cargo.lock b/Cargo.lock index eb2296214..f99bbc88e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,7 +93,7 @@ dependencies = [ "symphonia", "thiserror 2.0.12", "uuid", - "windows 0.61.1", + "windows 0.61.3", "windows-core 0.61.2", ] @@ -1691,7 +1691,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows 0.61.1", + "windows 0.61.3", ] [[package]] @@ -2284,7 +2284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -4732,9 +4732,9 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.25.5" +version = "0.25.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac5fff5c47490dfdf473b5228039bfacad9d765d9b6939d26bf7cc064c1c7822" +checksum = "6d7b8994f367f16e6fa14b5aebbcb350de5d7cbea82dc5b00ae997dd71680dd2" dependencies = [ "cc", "regex", @@ -4842,9 +4842,9 @@ dependencies = [ [[package]] name = "tree-sitter-scala" -version = "0.23.4" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efde5e68b4736e9eac17bfa296c6f104a26bffab363b365eb898c40a63c15d2f" +checksum = "7516aeb3d1f40ede8e3045b163e86993b3434514dd06c34c0b75e782d9a0b251" dependencies = [ "cc", "tree-sitter-language", @@ -5334,7 +5334,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -5365,9 +5365,9 @@ dependencies = [ [[package]] name = "windows" -version = "0.61.1" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", "windows-core 0.61.2", @@ -5477,9 +5477,9 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] name = "windows-numerics" diff --git a/Cargo.toml b/Cargo.toml index 424c03e22..c75b0795d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -93,7 +93,7 @@ tree-sitter-javascript = { version = "0.23" } tree-sitter-kotlin-ng = { version = "1.1" } tree-sitter-python = { version = "0.23" } tree-sitter-rust = { version = "0.24" } -tree-sitter-scala = { version = "0.23" } +tree-sitter-scala = { version = "0.24" } tree-sitter-typescript = { version = "0.23" } uniffi = "0.29" url = { version = "2.5" } diff --git a/packages/backend/native/src/tiktoken.rs b/packages/backend/native/src/tiktoken.rs index 919e95294..112ad83db 100644 --- a/packages/backend/native/src/tiktoken.rs +++ b/packages/backend/native/src/tiktoken.rs @@ -1,5 +1,7 @@ use std::collections::HashSet; +use tiktoken_rs::{get_bpe_from_tokenizer, tokenizer::Tokenizer as TiktokenTokenizer}; + #[napi] pub struct Tokenizer { inner: tiktoken_rs::CoreBPE, @@ -7,6 +9,10 @@ pub struct Tokenizer { #[napi] pub fn from_model_name(model_name: String) -> Option { + if model_name.starts_with("gpt-5") { + let bpe = get_bpe_from_tokenizer(TiktokenTokenizer::O200kBase).ok()?; + return Some(Tokenizer { inner: bpe }); + } let bpe = tiktoken_rs::get_bpe_from_model(&model_name).ok()?; Some(Tokenizer { inner: bpe }) } @@ -31,7 +37,7 @@ mod tests { #[test] fn test_tokenizer() { - let tokenizer = from_model_name("gpt-4.1".to_string()).unwrap(); + let tokenizer = from_model_name("gpt-5".to_string()).unwrap(); let content = "Hello, world!"; let count = tokenizer.count(content.to_string(), None); assert!(count > 0); diff --git a/packages/backend/server/package.json b/packages/backend/server/package.json index ee0d454ff..3bfa14124 100644 --- a/packages/backend/server/package.json +++ b/packages/backend/server/package.json @@ -28,12 +28,12 @@ "dependencies": { "@affine/reader": "workspace:*", "@affine/server-native": "workspace:*", - "@ai-sdk/anthropic": "^1.2.12", - "@ai-sdk/google": "^1.2.18", - "@ai-sdk/google-vertex": "^2.2.23", - "@ai-sdk/openai": "^1.3.22", - "@ai-sdk/openai-compatible": "^0.2.14", - "@ai-sdk/perplexity": "^1.1.9", + "@ai-sdk/anthropic": "^2.0.1", + "@ai-sdk/google": "^2.0.4", + "@ai-sdk/google-vertex": "^3.0.5", + "@ai-sdk/openai": "^2.0.10", + "@ai-sdk/openai-compatible": "^1.0.5", + "@ai-sdk/perplexity": "^2.0.1", "@apollo/server": "^4.11.3", "@aws-sdk/client-s3": "^3.779.0", "@aws-sdk/s3-request-presigner": "^3.779.0", @@ -75,7 +75,7 @@ "@prisma/instrumentation": "^6.7.0", "@react-email/components": "0.0.38", "@socket.io/redis-adapter": "^8.3.0", - "ai": "^4.3.4", + "ai": "^5.0.10", "bullmq": "^5.40.2", "cookie-parser": "^1.4.7", "cross-env": "^7.0.3", diff --git a/packages/backend/server/src/__tests__/copilot-provider.spec.ts b/packages/backend/server/src/__tests__/copilot-provider.spec.ts index 3fb35df7f..80874c8b4 100644 --- a/packages/backend/server/src/__tests__/copilot-provider.spec.ts +++ b/packages/backend/server/src/__tests__/copilot-provider.spec.ts @@ -118,11 +118,11 @@ test.serial.before(async t => { enabled: true, scenarios: { image: 'flux-1/schnell', - rerank: 'gpt-4.1-mini', - complex_text_generation: 'gpt-4.1-mini', - coding: 'gpt-4.1-mini', - quick_decision_making: 'gpt-4.1-mini', - quick_text_generation: 'gpt-4.1-mini', + rerank: 'gpt-5-mini', + complex_text_generation: 'gpt-5-mini', + coding: 'gpt-5-mini', + quick_decision_making: 'gpt-5-mini', + quick_text_generation: 'gpt-5-mini', polish_and_summarize: 'gemini-2.5-flash', }, }, diff --git a/packages/backend/server/src/__tests__/copilot.spec.ts b/packages/backend/server/src/__tests__/copilot.spec.ts index d9fe4a9ca..978cee77f 100644 --- a/packages/backend/server/src/__tests__/copilot.spec.ts +++ b/packages/backend/server/src/__tests__/copilot.spec.ts @@ -5,6 +5,7 @@ import { ProjectRoot } from '@affine-tools/utils/path'; import { PrismaClient } from '@prisma/client'; import type { TestFn } from 'ava'; import ava from 'ava'; +import { nanoid } from 'nanoid'; import Sinon from 'sinon'; import { EventBus, JobQueue } from '../base'; @@ -1340,16 +1341,16 @@ test('TextStreamParser should format different types of chunks correctly', t => textDelta: { chunk: { type: 'text-delta' as const, - textDelta: 'Hello world', - } as any, + text: 'Hello world', + }, expected: 'Hello world', description: 'should format text-delta correctly', }, reasoning: { chunk: { - type: 'reasoning' as const, - textDelta: 'I need to think about this', - } as any, + type: 'reasoning-delta' as const, + text: 'I need to think about this', + }, expected: '\n> [!]\n> I need to think about this', description: 'should format reasoning as callout', }, @@ -1358,8 +1359,8 @@ test('TextStreamParser should format different types of chunks correctly', t => type: 'tool-call' as const, toolName: 'web_search_exa' as const, toolCallId: 'test-id-1', - args: { query: 'test query', mode: 'AUTO' as const }, - } as any, + input: { query: 'test query', mode: 'AUTO' as const }, + }, expected: '\n> [!]\n> \n> Searching the web "test query"\n> ', description: 'should format web search tool call correctly', }, @@ -1368,8 +1369,8 @@ test('TextStreamParser should format different types of chunks correctly', t => type: 'tool-call' as const, toolName: 'web_crawl_exa' as const, toolCallId: 'test-id-2', - args: { url: 'https://example.com' }, - } as any, + input: { url: 'https://example.com' }, + }, expected: '\n> [!]\n> \n> Crawling the web "https://example.com"\n> ', description: 'should format web crawl tool call correctly', }, @@ -1378,8 +1379,8 @@ test('TextStreamParser should format different types of chunks correctly', t => type: 'tool-result' as const, toolName: 'web_search_exa' as const, toolCallId: 'test-id-1', - args: { query: 'test query', mode: 'AUTO' as const }, - result: [ + input: { query: 'test query', mode: 'AUTO' as const }, + output: [ { title: 'Test Title', url: 'https://test.com', @@ -1406,7 +1407,7 @@ test('TextStreamParser should format different types of chunks correctly', t => chunk: { type: 'error' as const, error: { type: 'testError', message: 'Test error message' }, - } as any, + }, errorMessage: 'Test error message', description: 'should throw error for error chunks', }, @@ -1436,78 +1437,85 @@ test('TextStreamParser should process a sequence of message chunks', t => { chunks: [ // Reasoning chunks { - type: 'reasoning' as const, - textDelta: 'The user is asking about', - } as any, + id: nanoid(), + type: 'reasoning-delta' as const, + text: 'The user is asking about', + }, { - type: 'reasoning' as const, - textDelta: ' recent advances in quantum computing', - } as any, + id: nanoid(), + type: 'reasoning-delta' as const, + text: ' recent advances in quantum computing', + }, { - type: 'reasoning' as const, - textDelta: ' and how it might impact', - } as any, + id: nanoid(), + type: 'reasoning-delta' as const, + text: ' and how it might impact', + }, { - type: 'reasoning' as const, - textDelta: ' cryptography and data security.', - } as any, + id: nanoid(), + type: 'reasoning-delta' as const, + text: ' cryptography and data security.', + }, { - type: 'reasoning' as const, - textDelta: - ' I should provide information on quantum supremacy achievements', - } as any, + id: nanoid(), + type: 'reasoning-delta' as const, + text: ' I should provide information on quantum supremacy achievements', + }, // Text delta { + id: nanoid(), type: 'text-delta' as const, - textDelta: - 'Let me search for the latest breakthroughs in quantum computing and their ', - } as any, + text: 'Let me search for the latest breakthroughs in quantum computing and their ', + }, // Tool call { type: 'tool-call' as const, toolCallId: 'toolu_01ABCxyz123456789', toolName: 'web_search_exa' as const, - args: { + input: { query: 'latest quantum computing breakthroughs cryptography impact', }, - } as any, + }, // Tool result { type: 'tool-result' as const, toolCallId: 'toolu_01ABCxyz123456789', toolName: 'web_search_exa' as const, - args: { + input: { query: 'latest quantum computing breakthroughs cryptography impact', }, - result: [ + output: [ { title: 'IBM Unveils 1000-Qubit Quantum Processor', url: 'https://example.com/tech/quantum-computing-milestone', }, ], - } as any, + }, // More text deltas { + id: nanoid(), type: 'text-delta' as const, - textDelta: 'implications for security.', - } as any, + text: 'implications for security.', + }, { + id: nanoid(), type: 'text-delta' as const, - textDelta: '\n\nQuantum computing has made ', - } as any, + text: '\n\nQuantum computing has made ', + }, { + id: nanoid(), type: 'text-delta' as const, - textDelta: 'remarkable progress in the past year. ', - } as any, + text: 'remarkable progress in the past year. ', + }, { + id: nanoid(), type: 'text-delta' as const, - textDelta: - 'The development of more stable qubits has accelerated research significantly.', - } as any, + text: 'The development of more stable qubits has accelerated research significantly.', + }, ], expected: '\n> [!]\n> The user is asking about recent advances in quantum computing and how it might impact cryptography and data security. I should provide information on quantum supremacy achievements\n\nLet me search for the latest breakthroughs in quantum computing and their \n> [!]\n> \n> Searching the web "latest quantum computing breakthroughs cryptography impact"\n> \n> \n> \n> [IBM Unveils 1000-Qubit Quantum Processor](https://example.com/tech/quantum-computing-milestone)\n> \n> \n> \n\nimplications for security.\n\nQuantum computing has made remarkable progress in the past year. The development of more stable qubits has accelerated research significantly.', diff --git a/packages/backend/server/src/__tests__/mocks/copilot.mock.ts b/packages/backend/server/src/__tests__/mocks/copilot.mock.ts index 4b88ea2b2..a092329df 100644 --- a/packages/backend/server/src/__tests__/mocks/copilot.mock.ts +++ b/packages/backend/server/src/__tests__/mocks/copilot.mock.ts @@ -57,15 +57,6 @@ export class MockCopilotProvider extends OpenAIProvider { }, ], }, - { - id: 'gpt-4.1', - capabilities: [ - { - input: [ModelInputType.Text, ModelInputType.Image], - output: [ModelOutputType.Text, ModelOutputType.Object], - }, - ], - }, { id: 'gpt-4.1-2025-04-14', capabilities: [ @@ -76,7 +67,25 @@ export class MockCopilotProvider extends OpenAIProvider { ], }, { - id: 'gpt-4.1-mini', + id: 'gpt-5', + capabilities: [ + { + input: [ModelInputType.Text, ModelInputType.Image], + output: [ModelOutputType.Text, ModelOutputType.Object], + }, + ], + }, + { + id: 'gpt-5-2025-08-07', + capabilities: [ + { + input: [ModelInputType.Text, ModelInputType.Image], + output: [ModelOutputType.Text, ModelOutputType.Object], + }, + ], + }, + { + id: 'gpt-5-mini', capabilities: [ { input: [ModelInputType.Text, ModelInputType.Image], diff --git a/packages/backend/server/src/__tests__/models/copilot-context.spec.ts b/packages/backend/server/src/__tests__/models/copilot-context.spec.ts index 481daaf7d..1cea7d200 100644 --- a/packages/backend/server/src/__tests__/models/copilot-context.spec.ts +++ b/packages/backend/server/src/__tests__/models/copilot-context.spec.ts @@ -48,7 +48,7 @@ let docId = 'doc1'; test.beforeEach(async t => { await t.context.module.initTestingDB(); - await t.context.copilotSession.createPrompt('prompt-name', 'gpt-4.1'); + await t.context.copilotSession.createPrompt('prompt-name', 'gpt-5-mini'); user = await t.context.user.create({ email: 'test@affine.pro', }); diff --git a/packages/backend/server/src/__tests__/models/copilot-session.spec.ts b/packages/backend/server/src/__tests__/models/copilot-session.spec.ts index f85165731..06d44e1f0 100644 --- a/packages/backend/server/src/__tests__/models/copilot-session.spec.ts +++ b/packages/backend/server/src/__tests__/models/copilot-session.spec.ts @@ -58,9 +58,9 @@ const createTestPrompts = async ( copilotSession: CopilotSessionModel, db: PrismaClient ) => { - await copilotSession.createPrompt(TEST_PROMPTS.NORMAL, 'gpt-4.1'); + await copilotSession.createPrompt(TEST_PROMPTS.NORMAL, 'gpt-5-mini'); await db.aiPrompt.create({ - data: { name: TEST_PROMPTS.ACTION, model: 'gpt-4.1', action: 'edit' }, + data: { name: TEST_PROMPTS.ACTION, model: 'gpt-5-mini', action: 'edit' }, }); }; @@ -116,7 +116,7 @@ const addMessagesToSession = async ( await copilotSession.updateMessages({ sessionId, userId: user.id, - prompt: { model: 'gpt-4.1' }, + prompt: { model: 'gpt-5-mini' }, messages: [ { role: 'user', @@ -807,7 +807,7 @@ test('should handle fork and session attachment operations', async t => { pinned: forkConfig.pinned, title: null, parentSessionId, - prompt: { name: TEST_PROMPTS.NORMAL, action: null, model: 'gpt-4.1' }, + prompt: { name: TEST_PROMPTS.NORMAL, action: null, model: 'gpt-5-mini' }, messages: [ { role: 'user', diff --git a/packages/backend/server/src/plugins/copilot/config.ts b/packages/backend/server/src/plugins/copilot/config.ts index bd43b2093..813e74601 100644 --- a/packages/backend/server/src/plugins/copilot/config.ts +++ b/packages/backend/server/src/plugins/copilot/config.ts @@ -57,7 +57,7 @@ defineModuleConfig('copilot', { rerank: 'gpt-4.1', coding: 'claude-sonnet-4@20250514', complex_text_generation: 'gpt-4o-2024-08-06', - quick_decision_making: 'gpt-4.1-mini', + quick_decision_making: 'gpt-5-mini', quick_text_generation: 'gemini-2.5-flash', polish_and_summarize: 'gemini-2.5-flash', }, diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index fce226c53..1b53ea983 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -107,7 +107,7 @@ const workflows: Prompt[] = [ { name: 'workflow:presentation:step1', action: 'workflow:presentation:step1', - model: 'gpt-4.1-mini', + model: 'gpt-5-mini', config: { temperature: 0.7 }, messages: [ { @@ -170,7 +170,7 @@ const workflows: Prompt[] = [ { name: 'workflow:brainstorm:step1', action: 'workflow:brainstorm:step1', - model: 'gpt-4.1-mini', + model: 'gpt-5-mini', config: { temperature: 0.7 }, messages: [ { @@ -221,7 +221,7 @@ const workflows: Prompt[] = [ { name: 'workflow:image-sketch:step2', action: 'workflow:image-sketch:step2', - model: 'gpt-4.1-mini', + model: 'gpt-5-mini', messages: [ { role: 'system', @@ -262,7 +262,7 @@ const workflows: Prompt[] = [ { name: 'workflow:image-clay:step2', action: 'workflow:image-clay:step2', - model: 'gpt-4.1-mini', + model: 'gpt-5-mini', messages: [ { role: 'system', @@ -303,7 +303,7 @@ const workflows: Prompt[] = [ { name: 'workflow:image-anime:step2', action: 'workflow:image-anime:step2', - model: 'gpt-4.1-mini', + model: 'gpt-5-mini', messages: [ { role: 'system', @@ -344,7 +344,7 @@ const workflows: Prompt[] = [ { name: 'workflow:image-pixel:step2', action: 'workflow:image-pixel:step2', - model: 'gpt-4.1-mini', + model: 'gpt-5-mini', messages: [ { role: 'system', @@ -432,7 +432,7 @@ Convert a multi-speaker audio recording into a structured JSON format by transcr { name: 'Generate a caption', action: 'Generate a caption', - model: 'gpt-4.1-mini', + model: 'gpt-5-mini', messages: [ { role: 'user', @@ -1931,6 +1931,7 @@ const CHAT_PROMPT: Omit = { model: 'claude-sonnet-4@20250514', optionalModels: [ 'gpt-4.1', + 'gpt-5', 'o3', 'o4-mini', 'gemini-2.5-flash', diff --git a/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts b/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts index e2912a7cf..2bd16a47e 100644 --- a/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts +++ b/packages/backend/server/src/plugins/copilot/providers/anthropic/anthropic.ts @@ -3,7 +3,7 @@ import { type AnthropicProviderOptions, } from '@ai-sdk/anthropic'; import { type GoogleVertexAnthropicProvider } from '@ai-sdk/google-vertex/anthropic'; -import { AISDKError, generateText, streamText } from 'ai'; +import { AISDKError, generateText, stepCountIs, streamText } from 'ai'; import { CopilotProviderSideError, @@ -75,8 +75,7 @@ export abstract class AnthropicProvider extends CopilotProvider { anthropic: this.getAnthropicOptions(options, model.id), }, tools: await this.getTools(options, model.id), - maxSteps: this.MAX_STEPS, - experimental_continueSteps: true, + stopWhen: stepCountIs(this.MAX_STEPS), }); if (!text) throw new Error('Failed to generate text'); @@ -169,8 +168,7 @@ export abstract class AnthropicProvider extends CopilotProvider { anthropic: this.getAnthropicOptions(options, model.id), }, tools: await this.getTools(options, model.id), - maxSteps: this.MAX_STEPS, - experimental_continueSteps: true, + stopWhen: stepCountIs(this.MAX_STEPS), }); return fullStream; } diff --git a/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts b/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts index 075499e79..f2d200492 100644 --- a/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts +++ b/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts @@ -38,8 +38,6 @@ import { export const DEFAULT_DIMENSIONS = 256; export abstract class GeminiProvider extends CopilotProvider { - private readonly MAX_STEPS = 20; - protected abstract instance: | GoogleGenerativeAIProvider | GoogleVertexProvider; @@ -87,8 +85,6 @@ export abstract class GeminiProvider extends CopilotProvider { google: this.getGeminiOptions(options, model.id), }, tools: await this.getTools(options, model.id), - maxSteps: this.MAX_STEPS, - experimental_continueSteps: true, }); if (!text) throw new Error('Failed to generate text'); @@ -116,9 +112,7 @@ export abstract class GeminiProvider extends CopilotProvider { throw new CopilotPromptInvalid('Schema is required'); } - const modelInstance = this.instance(model.id, { - structuredOutputs: true, - }); + const modelInstance = this.instance(model.id); const { object } = await generateObject({ model: modelInstance, system, @@ -238,14 +232,21 @@ export abstract class GeminiProvider extends CopilotProvider { .counter('generate_embedding_calls') .add(1, { model: model.id }); - const modelInstance = this.instance.textEmbeddingModel(model.id, { - outputDimensionality: options.dimensions || DEFAULT_DIMENSIONS, - taskType: 'RETRIEVAL_DOCUMENT', - }); + const modelInstance = this.instance.textEmbeddingModel(model.id); const embeddings = await Promise.allSettled( messages.map(m => - embedMany({ model: modelInstance, values: [m], maxRetries: 3 }) + embedMany({ + model: modelInstance, + values: [m], + maxRetries: 3, + providerOptions: { + google: { + outputDimensionality: options.dimensions || DEFAULT_DIMENSIONS, + taskType: 'RETRIEVAL_DOCUMENT', + }, + }, + }) ) ); @@ -275,8 +276,6 @@ export abstract class GeminiProvider extends CopilotProvider { google: this.getGeminiOptions(options, model.id), }, tools: await this.getTools(options, model.id), - maxSteps: this.MAX_STEPS, - experimental_continueSteps: true, }); return fullStream; } diff --git a/packages/backend/server/src/plugins/copilot/providers/openai.ts b/packages/backend/server/src/plugins/copilot/providers/openai.ts index 90c67b740..48991cac2 100644 --- a/packages/backend/server/src/plugins/copilot/providers/openai.ts +++ b/packages/backend/server/src/plugins/copilot/providers/openai.ts @@ -10,6 +10,7 @@ import { experimental_generateImage as generateImage, generateObject, generateText, + stepCountIs, streamText, Tool, } from 'ai'; @@ -65,6 +66,18 @@ const ImageResponseSchema = z.union([ }), }), ]); +const LogProbsSchema = z.array( + z.object({ + token: z.string(), + logprob: z.number(), + top_logprobs: z.array( + z.object({ + token: z.string(), + logprob: z.number(), + }) + ), + }) +); export class OpenAIProvider extends CopilotProvider { readonly type = CopilotProviderType.OpenAI; @@ -162,6 +175,58 @@ export class OpenAIProvider extends CopilotProvider { }, ], }, + { + id: 'gpt-5', + capabilities: [ + { + input: [ModelInputType.Text, ModelInputType.Image], + output: [ + ModelOutputType.Text, + ModelOutputType.Object, + ModelOutputType.Structured, + ], + }, + ], + }, + { + id: 'gpt-5-2025-08-07', + capabilities: [ + { + input: [ModelInputType.Text, ModelInputType.Image], + output: [ + ModelOutputType.Text, + ModelOutputType.Object, + ModelOutputType.Structured, + ], + }, + ], + }, + { + id: 'gpt-5-mini', + capabilities: [ + { + input: [ModelInputType.Text, ModelInputType.Image], + output: [ + ModelOutputType.Text, + ModelOutputType.Object, + ModelOutputType.Structured, + ], + }, + ], + }, + { + id: 'gpt-5-nano', + capabilities: [ + { + input: [ModelInputType.Text, ModelInputType.Image], + output: [ + ModelOutputType.Text, + ModelOutputType.Object, + ModelOutputType.Structured, + ], + }, + ], + }, { id: 'o1', capabilities: [ @@ -299,7 +364,7 @@ export class OpenAIProvider extends CopilotProvider { model: string ): [string, Tool?] | undefined { if (toolName === 'webSearch' && !this.isReasoningModel(model)) { - return ['web_search_preview', openai.tools.webSearchPreview()]; + return ['web_search_preview', openai.tools.webSearchPreview({})]; } else if (toolName === 'docEdit') { return ['doc_edit', undefined]; } @@ -330,12 +395,12 @@ export class OpenAIProvider extends CopilotProvider { system, messages: msgs, temperature: options.temperature ?? 0, - maxTokens: options.maxTokens ?? 4096, + maxOutputTokens: options.maxTokens ?? 4096, providerOptions: { openai: this.getOpenAIOptions(options, model.id), }, tools: await this.getTools(options, model.id), - maxSteps: this.MAX_STEPS, + stopWhen: stepCountIs(this.MAX_STEPS), abortSignal: options.signal, }); @@ -451,7 +516,7 @@ export class OpenAIProvider extends CopilotProvider { system, messages: msgs, temperature: options.temperature ?? 0, - maxTokens: options.maxTokens ?? 4096, + maxOutputTokens: options.maxTokens ?? 4096, maxRetries: options.maxRetries ?? 3, schema, providerOptions: { @@ -476,36 +541,37 @@ export class OpenAIProvider extends CopilotProvider { await this.checkParams({ messages: [], cond: fullCond, options }); const model = this.selectModel(fullCond); // get the log probability of "yes"/"no" - const instance = this.#instance(model.id, { logprobs: 16 }); + const instance = this.#instance.chat(model.id); const scores = await Promise.all( chunkMessages.map(async messages => { const [system, msgs] = await chatToGPTMessage(messages); - const { logprobs } = await generateText({ + const result = await generateText({ model: instance, system, messages: msgs, temperature: 0, - maxTokens: 16, + maxOutputTokens: 16, providerOptions: { openai: { ...this.getOpenAIOptions(options, model.id), + logprobs: 16, }, }, abortSignal: options.signal, }); - const topMap: Record = ( - logprobs?.[0]?.topLogprobs ?? [] - ).reduce>( + const topMap: Record = LogProbsSchema.parse( + result.providerMetadata?.openai?.logprobs + )[0].top_logprobs.reduce>( (acc, { token, logprob }) => ({ ...acc, [token]: logprob }), {} ); const findLogProb = (token: string): number => { // OpenAI often includes a leading space, so try matching '.yes', '_yes', ' yes' and 'yes' - return [`.${token}`, `_${token}`, ` ${token}`, token] + return [...'_:. "-\t,(=_“'.split('').map(c => c + token), token] .flatMap(v => [v, v.toLowerCase(), v.toUpperCase()]) .reduce( (best, key) => @@ -544,12 +610,12 @@ export class OpenAIProvider extends CopilotProvider { frequencyPenalty: options.frequencyPenalty ?? 0, presencePenalty: options.presencePenalty ?? 0, temperature: options.temperature ?? 0, - maxTokens: options.maxTokens ?? 4096, + maxOutputTokens: options.maxTokens ?? 4096, providerOptions: { openai: this.getOpenAIOptions(options, model.id), }, tools: await this.getTools(options, model.id), - maxSteps: this.MAX_STEPS, + stopWhen: stepCountIs(this.MAX_STEPS), abortSignal: options.signal, }); return fullStream; @@ -676,14 +742,16 @@ export class OpenAIProvider extends CopilotProvider { .counter('generate_embedding_calls') .add(1, { model: model.id }); - const modelInstance = this.#instance.embedding(model.id, { - dimensions: options.dimensions || DEFAULT_DIMENSIONS, - user: options.user, - }); + const modelInstance = this.#instance.embedding(model.id); const { embeddings } = await embedMany({ model: modelInstance, values: messages, + providerOptions: { + openai: { + dimensions: options.dimensions || DEFAULT_DIMENSIONS, + }, + }, }); return embeddings.filter(v => v && Array.isArray(v)); diff --git a/packages/backend/server/src/plugins/copilot/providers/perplexity.ts b/packages/backend/server/src/plugins/copilot/providers/perplexity.ts index 706f948ae..b49f7ece1 100644 --- a/packages/backend/server/src/plugins/copilot/providers/perplexity.ts +++ b/packages/backend/server/src/plugins/copilot/providers/perplexity.ts @@ -125,12 +125,12 @@ export class PerplexityProvider extends CopilotProvider { system, messages: msgs, temperature: options.temperature ?? 0, - maxTokens: options.maxTokens ?? 4096, + maxOutputTokens: options.maxTokens ?? 4096, abortSignal: options.signal, }); const parser = new CitationParser(); - for (const source of sources) { + for (const source of sources.filter(s => s.sourceType === 'url')) { parser.push(source.url); } @@ -165,7 +165,7 @@ export class PerplexityProvider extends CopilotProvider { system, messages: msgs, temperature: options.temperature ?? 0, - maxTokens: options.maxTokens ?? 4096, + maxOutputTokens: options.maxTokens ?? 4096, abortSignal: options.signal, }); @@ -173,19 +173,18 @@ export class PerplexityProvider extends CopilotProvider { for await (const chunk of stream.fullStream) { switch (chunk.type) { case 'source': { - parser.push(chunk.source.url); + if (chunk.sourceType === 'url') { + parser.push(chunk.url); + } break; } case 'text-delta': { - const text = chunk.textDelta.replaceAll( - /<\/?think>\n?/g, - '\n---\n' - ); + const text = chunk.text.replaceAll(/<\/?think>\n?/g, '\n---\n'); const result = parser.parse(text); yield result; break; } - case 'step-finish': { + case 'finish-step': { const result = parser.end(); yield result; break; diff --git a/packages/backend/server/src/plugins/copilot/providers/utils.ts b/packages/backend/server/src/plugins/copilot/providers/utils.ts index f39509ab7..ae350eb52 100644 --- a/packages/backend/server/src/plugins/copilot/providers/utils.ts +++ b/packages/backend/server/src/plugins/copilot/providers/utils.ts @@ -94,24 +94,24 @@ export async function chatToGPTMessage( if (withAttachment) { for (let attachment of attachments) { - let mimeType: string; + let mediaType: string; if (typeof attachment === 'string') { - mimeType = + mediaType = typeof mimetype === 'string' ? mimetype : await inferMimeType(attachment); } else { - ({ attachment, mimeType } = attachment); + ({ attachment, mimeType: mediaType } = attachment); } if (SIMPLE_IMAGE_URL_REGEX.test(attachment)) { const data = attachment.startsWith('data:') || useBase64Attachment ? await fetch(attachment).then(r => r.arrayBuffer()) : new URL(attachment); - if (mimeType.startsWith('image/')) { - contents.push({ type: 'image', image: data, mimeType }); + if (mediaType.startsWith('image/')) { + contents.push({ type: 'image', image: data, mediaType }); } else { - contents.push({ type: 'file' as const, data, mimeType }); + contents.push({ type: 'file' as const, data, mediaType }); } } } @@ -417,12 +417,12 @@ export class TextStreamParser { if (!this.prefix) { this.resetPrefix(); } - result = chunk.textDelta; + result = chunk.text; result = this.addNewline(chunk.type, result); break; } - case 'reasoning': { - result = chunk.textDelta; + case 'reasoning-delta': { + result = chunk.text; result = this.addPrefix(result); result = this.markAsCallout(result); break; @@ -438,28 +438,28 @@ export class TextStreamParser { break; } case 'web_search_exa': { - result += `\nSearching the web "${chunk.args.query}"\n`; + result += `\nSearching the web "${chunk.input.query}"\n`; break; } case 'web_crawl_exa': { - result += `\nCrawling the web "${chunk.args.url}"\n`; + result += `\nCrawling the web "${chunk.input.url}"\n`; break; } case 'doc_keyword_search': { - result += `\nSearching the keyword "${chunk.args.query}"\n`; + result += `\nSearching the keyword "${chunk.input.query}"\n`; break; } case 'doc_read': { - result += `\nReading the doc "${chunk.args.doc_id}"\n`; + result += `\nReading the doc "${chunk.input.doc_id}"\n`; break; } case 'doc_compose': { - result += `\nWriting document "${chunk.args.title}"\n`; + result += `\nWriting document "${chunk.input.title}"\n`; break; } case 'doc_edit': { this.docEditFootnotes.push({ - intent: chunk.args.instructions, + intent: chunk.input.instructions, result: '', }); break; @@ -475,12 +475,12 @@ export class TextStreamParser { result = this.addPrefix(result); switch (chunk.toolName) { case 'doc_edit': { - if ( - chunk.result && - typeof chunk.result === 'object' && - Array.isArray(chunk.result.result) - ) { - result += chunk.result.result + const array = + chunk.output && typeof chunk.output === 'object' + ? chunk.output.result + : undefined; + if (Array.isArray(array)) { + result += array .map(item => { return `\n${item.changedContent}\n`; }) @@ -493,37 +493,37 @@ export class TextStreamParser { break; } case 'doc_semantic_search': { - if (Array.isArray(chunk.result)) { - result += `\nFound ${chunk.result.length} document${chunk.result.length !== 1 ? 's' : ''} related to “${chunk.args.query}”.\n`; - } else if (typeof chunk.result === 'string') { - result += `\n${chunk.result}\n`; + const output = chunk.output; + if (Array.isArray(output)) { + result += `\nFound ${output.length} document${output.length !== 1 ? 's' : ''} related to “${chunk.input.query}”.\n`; + } else if (typeof output === 'string') { + result += `\n${output}\n`; } else { this.logger.warn( - `Unexpected result type for doc_semantic_search: ${chunk.result?.message || 'Unknown error'}` + `Unexpected result type for doc_semantic_search: ${output?.message || 'Unknown error'}` ); } break; } case 'doc_keyword_search': { - if (Array.isArray(chunk.result)) { - result += `\nFound ${chunk.result.length} document${chunk.result.length !== 1 ? 's' : ''} related to “${chunk.args.query}”.\n`; - result += `\n${this.getKeywordSearchLinks(chunk.result)}\n`; + const output = chunk.output; + if (Array.isArray(output)) { + result += `\nFound ${output.length} document${output.length !== 1 ? 's' : ''} related to “${chunk.input.query}”.\n`; + result += `\n${this.getKeywordSearchLinks(output)}\n`; } break; } case 'doc_compose': { - if ( - chunk.result && - typeof chunk.result === 'object' && - 'title' in chunk.result - ) { - result += `\nDocument "${chunk.result.title}" created successfully with ${chunk.result.wordCount} words.\n`; + const output = chunk.output; + if (output && typeof output === 'object' && 'title' in output) { + result += `\nDocument "${output.title}" created successfully with ${output.wordCount} words.\n`; } break; } case 'web_search_exa': { - if (Array.isArray(chunk.result)) { - result += `\n${this.getWebSearchLinks(chunk.result)}\n`; + const output = chunk.output; + if (Array.isArray(output)) { + result += `\n${this.getWebSearchLinks(output)}\n`; } break; } @@ -598,11 +598,18 @@ export class TextStreamParser { export class StreamObjectParser { public parse(chunk: TextStreamPart) { switch (chunk.type) { - case 'reasoning': - case 'text-delta': + case 'reasoning-delta': { + return { type: 'reasoning' as const, textDelta: chunk.text }; + } + case 'text-delta': { + const { type, text: textDelta } = chunk; + return { type, textDelta }; + } case 'tool-call': case 'tool-result': { - return chunk; + const { type, toolCallId, toolName, input: args } = chunk; + const result = 'output' in chunk ? chunk.output : undefined; + return { type, toolCallId, toolName, args, result } as StreamObject; } case 'error': { throw toError(chunk.error); diff --git a/packages/backend/server/src/plugins/copilot/tools/blob-read.ts b/packages/backend/server/src/plugins/copilot/tools/blob-read.ts index fe512def9..bda5a3140 100644 --- a/packages/backend/server/src/plugins/copilot/tools/blob-read.ts +++ b/packages/backend/server/src/plugins/copilot/tools/blob-read.ts @@ -52,7 +52,7 @@ export const createBlobReadTool = ( return tool({ description: 'Return the content and basic metadata of a single attachment identified by blobId; more inclined to use search tools rather than this tool.', - parameters: z.object({ + inputSchema: z.object({ blob_id: z.string().describe('The target blob in context to read'), chunk: z .number() diff --git a/packages/backend/server/src/plugins/copilot/tools/code-artifact.ts b/packages/backend/server/src/plugins/copilot/tools/code-artifact.ts index 7b0a86894..9058448fb 100644 --- a/packages/backend/server/src/plugins/copilot/tools/code-artifact.ts +++ b/packages/backend/server/src/plugins/copilot/tools/code-artifact.ts @@ -19,7 +19,7 @@ export const createCodeArtifactTool = ( return tool({ description: 'Generate a single-file HTML snippet (with inline