diff --git a/.docker/selfhost/schema.json b/.docker/selfhost/schema.json index cfd610f36..e9bb9f3a7 100644 --- a/.docker/selfhost/schema.json +++ b/.docker/selfhost/schema.json @@ -1467,6 +1467,237 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "boolean", "default": false + }, + "providers.profiles": { + "type": "array", + "description": "The profile list for copilot providers.\n@default []", + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "CopilotManagedCapabilityToken": { + "enum": [ + "chat", + "tools", + "vision", + "structured", + "embedding", + "rerank", + "image" + ], + "type": "string" + }, + "CopilotManagedModelConfigFile": { + "anyOf": [ + { + "type": "string" + }, + { + "properties": { + "capabilities": { + "default": [], + "items": { + "$ref": "#/definitions/CopilotManagedCapabilityToken" + }, + "type": "array" + }, + "enabled": { + "default": false, + "type": "boolean" + }, + "id": { + "type": "string" + } + }, + "required": [ + "id" + ], + "type": "object" + } + ] + }, + "CopilotManagedProfileConfigFile": { + "properties": { + "apiKey": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "baseUrl": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "config": { + "additionalProperties": true, + "default": {}, + "description": "Legacy nested form. New configs use the flat baseUrl/apiKey/dialect fields.", + "type": "object" + }, + "dialect": { + "default": null, + "type": [ + "string", + "null" + ] + }, + "displayName": { + "type": [ + "string", + "null" + ] + }, + "enabled": { + "default": true, + "type": "boolean" + }, + "id": { + "type": "string" + }, + "middleware": { + "anyOf": [ + { + "$ref": "#/definitions/CopilotProviderMiddlewareConfigFile" + }, + { + "type": "null" + } + ] + }, + "models": { + "items": { + "$ref": "#/definitions/CopilotManagedModelConfigFile" + }, + "type": [ + "array", + "null" + ] + }, + "priority": { + "format": "double", + "type": [ + "number", + "null" + ] + }, + "type": { + "$ref": "#/definitions/CopilotManagedProvider" + } + }, + "required": [ + "id", + "type" + ], + "type": "object" + }, + "CopilotManagedProvider": { + "enum": [ + "anthropic", + "anthropicVertex", + "cloudflareWorkersAi", + "fal", + "gemini", + "geminiVertex", + "openai", + "openaiCompatible" + ], + "type": "string" + }, + "CopilotNodeMiddlewareConfigFile": { + "properties": { + "text": { + "items": { + "$ref": "#/definitions/CopilotNodeTextMiddleware" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, + "CopilotNodeTextMiddleware": { + "enum": [ + "citation_footnote", + "callout", + "thinking_format" + ], + "type": "string" + }, + "CopilotProviderMiddlewareConfigFile": { + "properties": { + "node": { + "anyOf": [ + { + "$ref": "#/definitions/CopilotNodeMiddlewareConfigFile" + }, + { + "type": "null" + } + ] + }, + "rust": { + "anyOf": [ + { + "$ref": "#/definitions/CopilotRustMiddlewareConfigFile" + }, + { + "type": "null" + } + ] + } + }, + "type": "object" + }, + "CopilotRustMiddlewareConfigFile": { + "properties": { + "request": { + "items": { + "$ref": "#/definitions/CopilotRustRequestMiddleware" + }, + "type": [ + "array", + "null" + ] + }, + "stream": { + "items": { + "$ref": "#/definitions/CopilotRustStreamMiddleware" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, + "CopilotRustRequestMiddleware": { + "enum": [ + "normalize_messages", + "clamp_max_tokens", + "tool_schema_rewrite", + "openai_request_compat", + "omit_tool_choice" + ], + "type": "string" + }, + "CopilotRustStreamMiddleware": { + "enum": [ + "stream_event_normalize", + "citation_indexing" + ], + "type": "string" + } + }, + "items": { + "$ref": "#/definitions/CopilotManagedProfileConfigFile" + }, + "title": "Array_of_CopilotManagedProfileConfigFile", + "default": [] } } }, diff --git a/packages/backend/native/index.d.ts b/packages/backend/native/index.d.ts index 8445e9117..725672bc3 100644 --- a/packages/backend/native/index.d.ts +++ b/packages/backend/native/index.d.ts @@ -87,6 +87,7 @@ export declare class BackendRuntime { rotateByokCredential(input: RotateByokCredentialInput): Promise probeByokProfile(input: ProbeByokProfileInput): Promise probeByokDraft(input: ProbeByokDraftInput): Promise + probeManagedCopilotProfile(profileId: string, checks: Array): Promise deleteByokProfile(workspaceId: string, profileId: string): Promise reorderByokProfiles(input: ReorderByokProfilesInput): Promise> createByokLocalLease(input: CreateByokLocalLeaseInput): Promise diff --git a/packages/backend/native/src/runtime/backend_runtime/byok/mod.rs b/packages/backend/native/src/runtime/backend_runtime/byok/mod.rs index 14f359879..dd5b74f2c 100644 --- a/packages/backend/native/src/runtime/backend_runtime/byok/mod.rs +++ b/packages/backend/native/src/runtime/backend_runtime/byok/mod.rs @@ -3,6 +3,7 @@ mod probe; mod profile; pub(super) use local::{LocalLeasePayload, create as create_local_lease}; +pub(in crate::runtime::backend_runtime) use probe::execute_probe_with_policy; pub(super) use profile::{create, delete, list, probe_draft, probe_profile, reorder, replace, rotate}; use profile::{envelope_key, require_text}; diff --git a/packages/backend/native/src/runtime/backend_runtime/byok/probe.rs b/packages/backend/native/src/runtime/backend_runtime/byok/probe.rs index e9ac38616..4eea4eafd 100644 --- a/packages/backend/native/src/runtime/backend_runtime/byok/probe.rs +++ b/packages/backend/native/src/runtime/backend_runtime/byok/probe.rs @@ -28,6 +28,26 @@ pub(super) async fn execute_probe( credential: SensitiveCredential, policy: &ByokPolicy, checks: Vec, +) -> RuntimeResult { + execute_probe_with_policy( + provider, + definition, + credential, + policy.egress_policy(&definition.endpoint), + checks, + ) + .await +} + +/// Probe with an explicit egress policy. Managed (admin-configured) profiles +/// pass `AllowPrivate` for self-hosted endpoints; workspace BYOK always +/// derives the policy from [`ByokPolicy`]. +pub(in crate::runtime::backend_runtime) async fn execute_probe_with_policy( + provider: &str, + definition: &ByokProfileDefinition, + credential: SensitiveCredential, + egress_policy: EgressPolicy, + checks: Vec, ) -> RuntimeResult { let tested_at_ms = chrono::Utc::now().timestamp_millis(); let mut requested = Vec::new(); @@ -71,7 +91,6 @@ pub(super) async fn execute_probe( let credential = String::from_utf8(credential.expose().to_vec()) .map_err(|_| RuntimeError::invalid_state("credential_unavailable"))?; let operation_for_task = operation.clone(); - let egress_policy = policy.egress_policy(&endpoint); tokio::task::spawn_blocking(move || { dispatch_check( &provider, diff --git a/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs b/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs index b63a115e1..ed58b21a6 100644 --- a/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs +++ b/packages/backend/native/src/runtime/backend_runtime/copilot/context.rs @@ -194,7 +194,7 @@ fn load_managed_profiles( .providers .profiles .iter() - .filter(|profile| profile.enabled && profile.models.iter().any(|model| model == model_id)) + .filter(|profile| profile.enabled && profile.models.iter().any(|model| model.id == *model_id)) .collect::>(); let Some(profile) = matches.first() else { return Ok(None); @@ -204,15 +204,23 @@ fn load_managed_profiles( "built-in managed route model matches multiple profiles", )); } - let capabilities = provider_default_capability_upper_bound(&profile.provider, model_id) - .ok_or_else(|| RuntimeError::invalid_state("built-in managed route model is incompatible with its profile"))?; + let declared = matches + .first() + .and_then(|profile| profile.models.iter().find(|model| model.id == *model_id)) + .and_then(|model| model.capabilities.clone()); + let capabilities = match declared { + Some(capabilities) => capabilities, + None => provider_default_capability_upper_bound(&profile.provider, model_id).ok_or_else(|| { + RuntimeError::invalid_state("built-in managed route model is incompatible with its profile") + })?, + }; let endpoint = managed_endpoint(profile)?; Ok(Some(AuthorizedProviderProfile { profile_id: profile.id.clone(), source: ProfileSource::Managed, provider: profile.provider.clone(), endpoint, - openai_dialect: (profile.provider == "openai").then_some(OpenAiDialect::Responses), + openai_dialect: openai_dialect_for(profile), egress_policy: llm_adapter::target::EgressPolicy::PublicOnly, models: vec![crate::llm::byok::ByokModelDeclaration { model_id: model_id.clone(), @@ -229,6 +237,19 @@ fn load_managed_profiles( .collect() } +fn openai_dialect_for(profile: &CopilotManagedProfileConfig) -> Option { + match profile.provider.as_str() { + "openai" => Some(OpenAiDialect::Responses), + "openaiCompatible" => Some( + match profile.config.get("dialect").and_then(serde_json::Value::as_str) { + Some("responses") => OpenAiDialect::Responses, + _ => OpenAiDialect::ChatCompletions, + }, + ), + _ => None, + } +} + fn managed_endpoint(profile: &CopilotManagedProfileConfig) -> RuntimeResult { if let Some(base_url) = profile.config.get("baseURL").and_then(serde_json::Value::as_str) { return llm_adapter::target::canonicalize_endpoint(base_url) @@ -315,14 +336,18 @@ pub(super) fn required_config_text<'a>( mod tests { use serde_json::json; - use super::{BackendEndpoint, CopilotManagedProfileConfig, managed_endpoint}; + use super::{BackendEndpoint, CopilotManagedProfileConfig, managed_endpoint, openai_dialect_for}; + use crate::runtime::config::CopilotManagedModel; fn vertex_profile(location: &str) -> CopilotManagedProfileConfig { CopilotManagedProfileConfig { id: "vertex".to_string(), provider: "geminiVertex".to_string(), enabled: true, - models: vec!["gemini-3.7-flash".to_string()], + models: vec![CopilotManagedModel { + id: "gemini-3.7-flash".to_string(), + capabilities: None, + }], config: json!({ "project": "affine-us", "location": location }), } } @@ -347,4 +372,32 @@ mod tests { ) ); } + + #[test] + fn openai_compatible_dialect_defaults_to_chat_completions() { + let profile = CopilotManagedProfileConfig { + id: "vllm".to_string(), + provider: "openaiCompatible".to_string(), + enabled: true, + models: vec![CopilotManagedModel { + id: "qwen3-32b".to_string(), + capabilities: None, + }], + config: json!({ "baseURL": "http://127.0.0.1:8000/v1", "apiKey": "x" }), + }; + assert!(managed_endpoint(&profile).is_ok()); + assert_eq!( + openai_dialect_for(&profile), + Some(llm_adapter::target::OpenAiDialect::ChatCompletions) + ); + + let profile = CopilotManagedProfileConfig { + config: json!({ "dialect": "responses" }), + ..profile + }; + assert_eq!( + openai_dialect_for(&profile), + Some(llm_adapter::target::OpenAiDialect::Responses) + ); + } } diff --git a/packages/backend/native/src/runtime/backend_runtime/copilot/dispatch.rs b/packages/backend/native/src/runtime/backend_runtime/copilot/dispatch.rs index 0436758b8..86e3312e5 100644 --- a/packages/backend/native/src/runtime/backend_runtime/copilot/dispatch.rs +++ b/packages/backend/native/src/runtime/backend_runtime/copilot/dispatch.rs @@ -344,6 +344,7 @@ pub(super) async fn create_vertex_token_provider( pub(in crate::runtime::backend_runtime) fn provider(value: &str) -> RuntimeResult { match value { "openai" => Ok(BackendProvider::OpenAi), + "openaiCompatible" => Ok(BackendProvider::OpenAi), "anthropic" => Ok(BackendProvider::Anthropic), "anthropicVertex" => Ok(BackendProvider::AnthropicVertex), "gemini" => Ok(BackendProvider::Gemini), diff --git a/packages/backend/native/src/runtime/backend_runtime/copilot/managed_probe.rs b/packages/backend/native/src/runtime/backend_runtime/copilot/managed_probe.rs new file mode 100644 index 000000000..61ca52e41 --- /dev/null +++ b/packages/backend/native/src/runtime/backend_runtime/copilot/managed_probe.rs @@ -0,0 +1,93 @@ +use std::collections::HashSet; + +use llm_adapter::target::EgressPolicy; + +use crate::{ + llm::{ + ByokProbeCheckInput, ByokProbeResultOutput, + byok::{ByokEndpoint, SensitiveCredential}, + }, + runtime::{BackendRuntimeConfig, CopilotManagedProfileConfig, RuntimeError, RuntimeResult}, +}; + +/// Build a BYOK-equivalent definition from a managed profile so the shared +/// probe engine can compile targets and dispatch real requests. Managed +/// profiles are admin-controlled, so private endpoints (self-hosted vLLM, +/// Ollama, LiteLLM) are allowed and no DNS admission check applies. +pub(super) fn managed_definition( + managed: &CopilotManagedProfileConfig, +) -> RuntimeResult<(crate::llm::byok::ByokProfileDefinition, EgressPolicy)> { + use crate::llm::byok::{ByokModelDeclaration, ByokProfileDefinition}; + + let endpoint = if let Some(base_url) = managed.config.get("baseURL").and_then(serde_json::Value::as_str) { + ByokEndpoint::OpenAiCompatible { + url: llm_adapter::target::canonicalize_endpoint(base_url) + .map_err(|error| RuntimeError::invalid_state(error.to_string()))?, + dialect: openai_dialect(managed).unwrap_or(llm_adapter::target::OpenAiDialect::ChatCompletions), + } + } else { + ByokEndpoint::ProviderDefault + }; + let mut ids = HashSet::new(); + let models = managed + .models + .iter() + .map(|model| { + if !ids.insert(model.id.clone()) { + return Err(RuntimeError::invalid_state( + "managed copilot profile models must be unique", + )); + } + Ok(ByokModelDeclaration { + model_id: model.id.clone(), + enabled: true, + capabilities: model.capabilities.clone().unwrap_or_default(), + }) + }) + .collect::>>()?; + let egress_policy = if matches!(endpoint, ByokEndpoint::OpenAiCompatible { .. }) { + EgressPolicy::AllowPrivate + } else { + EgressPolicy::PublicOnly + }; + Ok((ByokProfileDefinition { endpoint, models }, egress_policy)) +} + +fn openai_dialect(managed: &CopilotManagedProfileConfig) -> Option { + match managed.provider.as_str() { + "openai" => Some(llm_adapter::target::OpenAiDialect::Responses), + "openaiCompatible" => Some( + match managed.config.get("dialect").and_then(serde_json::Value::as_str) { + Some("responses") => llm_adapter::target::OpenAiDialect::Responses, + _ => llm_adapter::target::OpenAiDialect::ChatCompletions, + }, + ), + _ => None, + } +} + +/// Probe a managed profile from the active runtime config without touching +/// per-workspace BYOK storage. Used by the admin console "Test" action. +pub(in crate::runtime::backend_runtime) async fn probe_managed( + config: &BackendRuntimeConfig, + profile_id: &str, + checks: Vec, +) -> RuntimeResult { + let managed = super::context::managed_profile(&config.copilot, profile_id)?; + let (definition, egress_policy) = managed_definition(managed)?; + // Vertex credentials need a token provider; static providers read config. + if matches!(managed.provider.as_str(), "geminiVertex" | "anthropicVertex") { + return Err(RuntimeError::invalid_input( + "probe for vertex managed profiles is not supported; use workspace BYOK", + )); + } + let credential = super::dispatch::managed_credential(managed, None).await?; + crate::runtime::backend_runtime::byok::execute_probe_with_policy( + &managed.provider, + &definition, + SensitiveCredential::new(credential.into_bytes()), + egress_policy, + checks, + ) + .await +} diff --git a/packages/backend/native/src/runtime/backend_runtime/copilot/mod.rs b/packages/backend/native/src/runtime/backend_runtime/copilot/mod.rs index 31ed39f56..f2b96162a 100644 --- a/packages/backend/native/src/runtime/backend_runtime/copilot/mod.rs +++ b/packages/backend/native/src/runtime/backend_runtime/copilot/mod.rs @@ -1,5 +1,6 @@ mod context; mod dispatch; +pub(in crate::runtime::backend_runtime) mod managed_probe; mod stream; use std::{ diff --git a/packages/backend/native/src/runtime/backend_runtime/mod.rs b/packages/backend/native/src/runtime/backend_runtime/mod.rs index 02c7ba372..127721fa8 100644 --- a/packages/backend/native/src/runtime/backend_runtime/mod.rs +++ b/packages/backend/native/src/runtime/backend_runtime/mod.rs @@ -45,9 +45,9 @@ pub(super) use super::{ napi_error, to_napi_error, webpki_tls_config, }; use crate::llm::{ - ByokLocalLeaseOutput, ByokPolicyOutput, ByokProbeResultOutput, ByokProfileOutput, CreateByokLocalLeaseInput, - CreateByokProfileInput, ProbeByokDraftInput, ProbeByokProfileInput, ReorderByokProfilesInput, - ReplaceByokProfileInput, RotateByokCredentialInput, + ByokLocalLeaseOutput, ByokPolicyOutput, ByokProbeCheckInput, ByokProbeResultOutput, ByokProfileOutput, + CreateByokLocalLeaseInput, CreateByokProfileInput, ProbeByokDraftInput, ProbeByokProfileInput, + ReorderByokProfilesInput, ReplaceByokProfileInput, RotateByokCredentialInput, }; pub(super) fn token_hash(token: &str) -> String { @@ -710,6 +710,18 @@ impl BackendRuntime { .map_err(to_napi_error) } + #[napi] + pub async fn probe_managed_copilot_profile( + &self, + profile_id: String, + checks: Vec, + ) -> Result { + let config = self.config()?; + copilot::managed_probe::probe_managed(&config, &profile_id, checks) + .await + .map_err(to_napi_error) + } + #[napi] pub async fn delete_byok_profile(&self, workspace_id: String, profile_id: String) -> Result { let deleted = byok::delete(&self.pool().await?, &workspace_id, &profile_id) diff --git a/packages/backend/native/src/runtime/config.rs b/packages/backend/native/src/runtime/config.rs index b4779d8c3..58e94a3e2 100644 --- a/packages/backend/native/src/runtime/config.rs +++ b/packages/backend/native/src/runtime/config.rs @@ -5,7 +5,7 @@ use std::{ sync::Arc, }; -use llm_adapter::capability::provider_default_capability_upper_bound; +use llm_adapter::capability::{DeclaredModelCapability, ModelFeature, provider_default_capability_upper_bound}; use serde::Deserialize; use serde_json::Map; use sqlx::{PgPool, Row}; @@ -99,9 +99,7 @@ impl ConfigSource { self.exact() || self.override_path.as_deref() == Some(path) } } - -#[derive(Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase", default)] +#[derive(Clone, Default)] pub(crate) struct CopilotRuntimeConfig { pub(crate) enabled: bool, pub(crate) byok: CopilotByokRuntimeConfig, @@ -135,25 +133,26 @@ fn default_allowed_providers() -> Vec { SUPPORTED_BYOK_PROVIDERS.into_iter().map(str::to_string).collect() } -#[derive(Clone, Default, Deserialize)] -#[serde(rename_all = "camelCase", default)] +#[derive(Clone, Default)] pub(crate) struct CopilotProvidersRuntimeConfig { pub(crate) profiles: Vec, } -#[derive(Clone, Deserialize)] -#[serde(rename_all = "camelCase")] +#[derive(Clone)] pub(crate) struct CopilotManagedProfileConfig { pub(crate) id: String, - #[serde(rename = "type")] pub(crate) provider: String, - #[serde(default = "enabled_by_default")] pub(crate) enabled: bool, - #[serde(default)] - pub(crate) models: Vec, + pub(crate) models: Vec, pub(crate) config: serde_json::Value, } +#[derive(Clone)] +pub(crate) struct CopilotManagedModel { + pub(crate) id: String, + pub(crate) capabilities: Option>, +} + fn enabled_by_default() -> bool { true } @@ -182,11 +181,44 @@ pub(crate) struct CopilotManagedProfileConfigFile { priority: Option, #[serde(default = "enabled_by_default")] enabled: bool, - models: Option>, + #[serde(default)] + base_url: Option, + #[serde(default)] + api_key: Option, + #[serde(default)] + dialect: Option, + models: Option>, middleware: Option, + /// Legacy nested form. New configs use the flat baseUrl/apiKey/dialect fields. + #[serde(default)] config: Map, } +#[derive(Clone, Deserialize, serde::Serialize, schemars::JsonSchema)] +#[serde(untagged)] +pub(crate) enum CopilotManagedModelConfigFile { + Id(String), + Declared { + id: String, + #[serde(default)] + enabled: bool, + #[serde(default)] + capabilities: Vec, + }, +} + +#[derive(Clone, Copy, PartialEq, Deserialize, serde::Serialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub(crate) enum CopilotManagedCapabilityToken { + Chat, + Tools, + Vision, + Structured, + Embedding, + Rerank, + Image, +} + #[derive(Clone, Copy, Deserialize, serde::Serialize, schemars::JsonSchema)] enum CopilotManagedProvider { #[serde(rename = "anthropic")] @@ -203,6 +235,8 @@ enum CopilotManagedProvider { GeminiVertex, #[serde(rename = "openai")] OpenAi, + #[serde(rename = "openaiCompatible")] + OpenAiCompatible, } impl CopilotManagedProvider { @@ -215,12 +249,14 @@ impl CopilotManagedProvider { Self::Gemini => "gemini", Self::GeminiVertex => "geminiVertex", Self::OpenAi => "openai", + Self::OpenAiCompatible => "openaiCompatible", } } fn legacy_models(self) -> Vec { let models: &[&str] = match self { Self::OpenAi => &["gpt-5.6-luna", "gpt-5.6-terra", "gpt-image-1", "gpt-4o-mini"], + Self::OpenAiCompatible => &[], Self::CloudflareWorkersAi => &["@cf/baai/bge-reranker-base"], Self::Fal => &["lora/image-to-image", "workflowutils/teed"], Self::Gemini => &["gemini-3.7-flash", "gemini-embedding-001"], @@ -273,6 +309,65 @@ enum CopilotNodeTextMiddleware { ThinkingFormat, } +impl TryFrom for CopilotManagedProfileConfig { + type Error = RuntimeError; + + fn try_from(value: CopilotManagedProfileConfigFile) -> Result { + if value.id.is_empty() + || !value + .id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + return Err(RuntimeError::invalid_state( + "managed copilot profile id must contain only letters, numbers, hyphens, and underscores", + )); + } + let models = value.models.unwrap_or_else(|| { + value + .provider + .legacy_models() + .into_iter() + .map(|id| CopilotManagedModelConfigFile::Id(id)) + .collect() + }); + + let models = models + .into_iter() + .map(|model| match model { + CopilotManagedModelConfigFile::Id(id) => Ok(CopilotManagedModel { id, capabilities: None }), + CopilotManagedModelConfigFile::Declared { + id, + enabled: _, + capabilities, + } => { + let capabilities = (!capabilities.is_empty()) + .then(|| capabilities.iter().map(managed_capability).collect()) + .transpose()?; + Ok(CopilotManagedModel { id, capabilities }) + } + }) + .collect::>>()?; + let mut config = value.config; + if let Some(base_url) = value.base_url { + config.insert("baseURL".to_string(), serde_json::Value::String(base_url)); + } + if let Some(api_key) = value.api_key { + config.insert("apiKey".to_string(), serde_json::Value::String(api_key)); + } + if let Some(dialect) = value.dialect { + config.insert("dialect".to_string(), serde_json::Value::String(dialect)); + } + Ok(Self { + id: value.id, + provider: value.provider.as_str().to_string(), + enabled: value.enabled, + models, + config: serde_json::Value::Object(config), + }) + } +} + impl TryFrom for CopilotRuntimeConfig { type Error = RuntimeError; @@ -292,29 +387,62 @@ impl TryFrom for CopilotRuntimeConfig { } } -impl TryFrom for CopilotManagedProfileConfig { - type Error = RuntimeError; - - fn try_from(value: CopilotManagedProfileConfigFile) -> Result { - if value.id.is_empty() - || !value - .id - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) - { - return Err(RuntimeError::invalid_state( - "managed copilot profile id must contain only letters, numbers, hyphens, and underscores", - )); - } - let models = value.models.unwrap_or_else(|| value.provider.legacy_models()); - Ok(Self { - id: value.id, - provider: value.provider.as_str().to_string(), - enabled: value.enabled, - models, - config: serde_json::Value::Object(value.config), - }) - } +fn managed_capability(token: &CopilotManagedCapabilityToken) -> RuntimeResult { + use llm_adapter::capability::{AttachmentKind, AttachmentSource, ModelInput, ModelOutput}; + let capability = match token { + CopilotManagedCapabilityToken::Chat => DeclaredModelCapability { + input: vec![ModelInput::Text], + output: vec![ModelOutput::Text], + features: vec![], + attachment_kinds: vec![], + attachment_sources: vec![], + }, + CopilotManagedCapabilityToken::Tools => DeclaredModelCapability { + input: vec![ModelInput::Text], + output: vec![ModelOutput::Text], + features: vec![ModelFeature::ToolCalling], + attachment_kinds: vec![], + attachment_sources: vec![], + }, + CopilotManagedCapabilityToken::Vision => DeclaredModelCapability { + input: vec![ModelInput::Text, ModelInput::Image], + output: vec![ModelOutput::Text], + features: vec![], + attachment_kinds: vec![AttachmentKind::Image], + attachment_sources: vec![AttachmentSource::Url, AttachmentSource::Data, AttachmentSource::Bytes], + }, + CopilotManagedCapabilityToken::Structured => DeclaredModelCapability { + input: vec![ModelInput::Text], + output: vec![ModelOutput::Structured], + features: vec![], + attachment_kinds: vec![], + attachment_sources: vec![], + }, + CopilotManagedCapabilityToken::Embedding => DeclaredModelCapability { + input: vec![ModelInput::Text], + output: vec![ModelOutput::Embedding], + features: vec![], + attachment_kinds: vec![], + attachment_sources: vec![], + }, + CopilotManagedCapabilityToken::Rerank => DeclaredModelCapability { + input: vec![ModelInput::Text], + output: vec![ModelOutput::Rerank], + features: vec![], + attachment_kinds: vec![], + attachment_sources: vec![], + }, + CopilotManagedCapabilityToken::Image => DeclaredModelCapability { + input: vec![ModelInput::Text], + output: vec![ModelOutput::Image], + features: vec![], + attachment_kinds: vec![], + attachment_sources: vec![], + }, + }; + llm_adapter::capability::validate_declared_capability(&capability) + .map(|_| capability) + .map_err(|error| RuntimeError::invalid_state(format!("managed copilot profile capability invalid: {error}"))) } #[derive(Clone, Debug)] @@ -459,12 +587,17 @@ pub(super) fn validate_copilot_config(config: &CopilotRuntimeConfig) -> RuntimeR } let mut models = std::collections::HashSet::new(); for model in &profile.models { - if model.trim().is_empty() || !models.insert(model.as_str()) { + if model.id.trim().is_empty() || !models.insert(model.id.as_str()) { return Err(RuntimeError::invalid_state( "managed copilot profile models must be non-empty and unique", )); } - provider_default_capability_upper_bound(&profile.provider, model) + // openaiCompatible endpoints serve arbitrary model ids; declared or + // probed capabilities replace the built-in catalog lookup. + if profile.provider == "openaiCompatible" { + continue; + } + provider_default_capability_upper_bound(&profile.provider, &model.id) .ok_or_else(|| RuntimeError::invalid_state("managed copilot profile model is unsupported"))?; } } @@ -854,7 +987,14 @@ mod tests { .unwrap(); let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap(); validate_copilot_config(&copilot).unwrap(); - assert_eq!(copilot.providers.profiles[0].models, expected_models); + assert_eq!( + copilot.providers.profiles[0] + .models + .iter() + .map(|m| m.id.as_str()) + .collect::>(), + expected_models + ); } let app_config = app_config_from_flat_overrides([( @@ -1025,4 +1165,60 @@ mod tests { Some("workspace_invitation") ); } + + #[test] + fn openai_compatible_profile_accepts_arbitrary_models_and_flat_fields() { + let app_config = app_config_from_module_json(serde_json::json!({ + "copilot": { + "enabled": true, + "providers": { + "profiles": [{ + "id": "my-vllm", + "type": "openaiCompatible", + "baseUrl": "http://127.0.0.1:8000/v1", + "apiKey": "sk-test", + "models": [ + "qwen3-32b", + { "id": "bge-m3", "capabilities": ["embedding"] } + ] + }] + } + } + })) + .unwrap(); + let copilot: CopilotRuntimeConfig = app_config.copilot.unwrap().try_into().unwrap(); + validate_copilot_config(&copilot).unwrap(); + + let profile = &copilot.providers.profiles[0]; + assert_eq!(profile.provider, "openaiCompatible"); + assert_eq!(profile.models.len(), 2); + assert_eq!(profile.models[0].id, "qwen3-32b"); + assert!(profile.models[0].capabilities.is_none()); + assert_eq!( + profile.config.get("baseURL").and_then(serde_json::Value::as_str), + Some("http://127.0.0.1:8000/v1") + ); + assert_eq!( + profile.config.get("apiKey").and_then(serde_json::Value::as_str), + Some("sk-test") + ); + + let declared = profile.models[1].capabilities.as_ref().unwrap(); + use llm_adapter::capability::ModelOutput; + assert!( + declared + .iter() + .any(|capability| capability.output.contains(&ModelOutput::Embedding)) + ); + + // unknown provider types are rejected at deserialization, not validation + let app_config = app_config_from_flat_overrides([( + "copilot.providers.profiles", + serde_json::json!([{ "id": "x", "type": "totally-unknown-provider", "models": ["any"] }]), + )]); + assert!( + app_config.is_err(), + "unknown provider type must be rejected at deserialization" + ); + } } diff --git a/packages/backend/native/src/runtime/config_descriptor.rs b/packages/backend/native/src/runtime/config_descriptor.rs index c274ff5b4..8b72b5068 100644 --- a/packages/backend/native/src/runtime/config_descriptor.rs +++ b/packages/backend/native/src/runtime/config_descriptor.rs @@ -69,7 +69,7 @@ fn descriptors() -> Vec { description: "The profile list for copilot providers.".to_string(), default_value: json!(defaults.providers.profiles), schema: schema_for::>(), - internal: true, + internal: false, }, ] } @@ -161,7 +161,7 @@ mod tests { ] ); assert_eq!(descriptors[0].default_value, json!(true)); - assert!(descriptors[4].internal); + assert!(!descriptors[4].internal); assert!( validate_app_config_value( "copilot".to_string(), @@ -218,4 +218,35 @@ mod tests { ); } } + + #[test] + fn openai_compatible_profiles_validate_with_flat_fields_and_arbitrary_models() { + assert!( + validate_app_config_value( + "copilot".to_string(), + "providers.profiles".to_string(), + json!([{ + "id": "my-vllm", + "type": "openaiCompatible", + "enabled": true, + "baseUrl": "http://127.0.0.1:8000/v1", + "apiKey": "sk-test", + "dialect": "chat_completions", + "models": ["qwen3-32b", { "id": "bge-m3", "capabilities": ["embedding"] }] + }]), + ) + .unwrap() + .is_empty() + ); + // unknown model ids on catalog providers still fail + assert!( + !validate_app_config_value( + "copilot".to_string(), + "providers.profiles".to_string(), + json!([{ "id": "x", "type": "openai", "models": ["not-in-catalog"] }]), + ) + .unwrap() + .is_empty() + ); + } } diff --git a/packages/backend/server/src/core/backend-runtime/provider.ts b/packages/backend/server/src/core/backend-runtime/provider.ts index 995422447..98bd5c3ed 100644 --- a/packages/backend/server/src/core/backend-runtime/provider.ts +++ b/packages/backend/server/src/core/backend-runtime/provider.ts @@ -629,6 +629,15 @@ export class BackendRuntimeProvider ); } + async probeManagedProfile( + profileId: string, + checks: Array<{ modelId: string; operation: string }> + ): Promise { + return await this.measured('probeManagedProfile', runtime => + runtime.probeManagedCopilotProfile(profileId, checks) + ); + } + async deleteByokProfile(workspaceId: string, profileId: string) { return await this.measured('deleteByokProfile', runtime => runtime.deleteByokProfile(workspaceId, profileId) diff --git a/packages/backend/server/src/plugins/copilot/config.ts b/packages/backend/server/src/plugins/copilot/config.ts index b5a99cb5a..e56ff7a4b 100644 --- a/packages/backend/server/src/plugins/copilot/config.ts +++ b/packages/backend/server/src/plugins/copilot/config.ts @@ -35,13 +35,17 @@ type CopilotProviderProfileCommon = { displayName?: string; priority?: number; enabled?: boolean; - models?: string[]; + baseUrl?: string; + apiKey?: string; + dialect?: 'responses' | 'chat_completions'; + models?: Array; middleware?: ProviderMiddlewareConfig; }; export type CopilotProviderProfile = CopilotProviderProfileCommon & { type: CopilotProviderType; - config: ProviderSpecificConfig; + /** Legacy nested form. New configs use the flat baseUrl/apiKey/dialect fields. */ + config?: ProviderSpecificConfig; }; declare global { diff --git a/packages/backend/server/src/plugins/copilot/managed-profile.resolver.ts b/packages/backend/server/src/plugins/copilot/managed-profile.resolver.ts new file mode 100644 index 000000000..2c036feff --- /dev/null +++ b/packages/backend/server/src/plugins/copilot/managed-profile.resolver.ts @@ -0,0 +1,118 @@ +import { + Args, + Field, + ID, + InputType, + Mutation, + ObjectType, + Resolver, +} from '@nestjs/graphql'; + +import { Throttle } from '../../base'; +import { BackendRuntimeProvider } from '../../core/backend-runtime'; +import { Admin } from '../../core/common'; +import { ByokProbeOperation, ByokProbeStatusKind } from './byok/types'; + +@InputType() +class ManagedProfileProbeCheckInput { + @Field(() => String) + modelId!: string; + + @Field(() => ByokProbeOperation) + operation!: ByokProbeOperation; +} + +@ObjectType() +class ManagedProfileProbeStatusType { + @Field(() => ByokProbeStatusKind) + kind!: ByokProbeStatusKind; + + @Field(() => Date, { nullable: true }) + testedAt!: Date | null; + + @Field(() => String, { nullable: true }) + errorKind!: string | null; +} + +@ObjectType() +class ManagedProfileModelProbeCheckType { + @Field(() => ByokProbeOperation) + operation!: ByokProbeOperation; + + @Field(() => ManagedProfileProbeStatusType) + status!: ManagedProfileProbeStatusType; +} + +@ObjectType() +class ManagedProfileModelProbeType { + @Field(() => String) + modelId!: string; + + @Field(() => [ManagedProfileModelProbeCheckType]) + checks!: ManagedProfileModelProbeCheckType[]; +} + +@ObjectType() +class ManagedProfileProbeResultType { + @Field(() => String) + definitionFingerprint!: string; + + @Field(() => ManagedProfileProbeStatusType) + connection!: ManagedProfileProbeStatusType; + + @Field(() => [ManagedProfileModelProbeType]) + models!: ManagedProfileModelProbeType[]; +} + +function projectStatus(probe: { + kind: string; + testedAtMs?: number; + errorKind?: string; +}) { + return { + kind: probe.kind, + testedAt: probe.testedAtMs ? new Date(probe.testedAtMs) : null, + errorKind: probe.errorKind ?? null, + }; +} + +/** + * Admin-only surface for testing server-managed copilot provider profiles + * (copilot.providers.profiles) configured through the admin console. + */ +@Admin() +@Resolver(() => ManagedProfileProbeResultType) +export class ManagedCopilotProfileResolver { + constructor(private readonly runtime: BackendRuntimeProvider) {} + + @Mutation(() => ManagedProfileProbeResultType, { + description: + 'Test a server-managed copilot provider profile by dispatching real probe requests.', + }) + @Throttle('strict') + async probeManagedCopilotProfile( + @Args('profileId', { type: () => ID }) profileId: string, + @Args('checks', { type: () => [ManagedProfileProbeCheckInput] }) + checks: ManagedProfileProbeCheckInput[] + ) { + const result = await this.runtime.probeManagedProfile( + profileId, + checks.map(check => ({ + modelId: check.modelId, + operation: check.operation, + })) + ); + return { + definitionFingerprint: result.definitionFingerprint, + stale: result.stale, + connection: projectStatus(result.connection), + models: result.models.map(model => ({ + modelId: model.modelId, + checks: model.checks.map(check => ({ + operation: check.operation, + status: projectStatus(check.status), + })), + })), + }; + } +} diff --git a/packages/backend/server/src/plugins/copilot/module-providers.ts b/packages/backend/server/src/plugins/copilot/module-providers.ts index ce51f05c0..a23988697 100644 --- a/packages/backend/server/src/plugins/copilot/module-providers.ts +++ b/packages/backend/server/src/plugins/copilot/module-providers.ts @@ -16,6 +16,7 @@ import { NativeEmbeddingService, } from './embedding'; import { CopilotEmbeddingRealtimeProvider } from './embedding/realtime'; +import { ManagedCopilotProfileResolver } from './managed-profile.resolver'; import { WorkspaceMcpProvider } from './mcp/provider'; import { PromptService } from './prompt'; import { CopilotResolver, UserCopilotResolver } from './resolver'; @@ -112,6 +113,7 @@ export const COPILOT_RESOLVER_PROVIDERS = [ CopilotResolver, UserCopilotResolver, WorkspaceByokResolver, + ManagedCopilotProfileResolver, ]; export const COPILOT_JOB_PROVIDERS = [CopilotCronJobs]; diff --git a/packages/backend/server/src/plugins/copilot/providers/types.ts b/packages/backend/server/src/plugins/copilot/providers/types.ts index c63713642..71d7af00b 100644 --- a/packages/backend/server/src/plugins/copilot/providers/types.ts +++ b/packages/backend/server/src/plugins/copilot/providers/types.ts @@ -30,6 +30,7 @@ export enum CopilotProviderType { Gemini = 'gemini', GeminiVertex = 'geminiVertex', OpenAI = 'openai', + OpenAICompatible = 'openaiCompatible', } export const CopilotProviderSchema = z.object({ diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index f521ee04c..583f82ca0 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -1448,6 +1448,33 @@ input ManageUserInput { name: String } +type ManagedProfileModelProbeCheckType { + operation: ByokProbeOperation! + status: ManagedProfileProbeStatusType! +} + +type ManagedProfileModelProbeType { + checks: [ManagedProfileModelProbeCheckType!]! + modelId: String! +} + +input ManagedProfileProbeCheckInput { + modelId: String! + operation: ByokProbeOperation! +} + +type ManagedProfileProbeResultType { + connection: ManagedProfileProbeStatusType! + definitionFingerprint: String! + models: [ManagedProfileModelProbeType!]! +} + +type ManagedProfileProbeStatusType { + errorKind: String + kind: ByokProbeStatusKind! + testedAt: DateTime +} + enum McpAccessMode { READ_ONLY READ_WRITE @@ -1631,6 +1658,11 @@ type Mutation { """mention user in a doc""" mentionUser(input: MentionInput!): ID! previewLicense(license: Upload!): AdminLicensePreview! + + """ + Test a server-managed copilot provider profile by dispatching real probe requests. + """ + probeManagedCopilotProfile(checks: [ManagedProfileProbeCheckInput!]!, profileId: ID!): ManagedProfileProbeResultType! probeWorkspaceByokDraft(input: ProbeWorkspaceByokDraftInput!): WorkspaceByokProbeResultType! probeWorkspaceByokProfile(input: ProbeWorkspaceByokProfileInput!): WorkspaceByokProbeResultType! publishDoc(docId: String!, mode: PublicDocMode = Page, workspaceId: String!): DocType! diff --git a/packages/common/graphql/src/graphql/admin/managed-profile.gql b/packages/common/graphql/src/graphql/admin/managed-profile.gql new file mode 100644 index 000000000..19b6e21a3 --- /dev/null +++ b/packages/common/graphql/src/graphql/admin/managed-profile.gql @@ -0,0 +1,20 @@ +mutation probeManagedCopilotProfile($profileId: ID!, $checks: [ManagedProfileProbeCheckInput!]!) { + probeManagedCopilotProfile(profileId: $profileId, checks: $checks) { + connection { + kind + testedAt + errorKind + } + models { + modelId + checks { + operation + status { + kind + testedAt + errorKind + } + } + } + } +} diff --git a/packages/common/graphql/src/graphql/index.ts b/packages/common/graphql/src/graphql/index.ts index 3632fa0cb..c6c0306cc 100644 --- a/packages/common/graphql/src/graphql/index.ts +++ b/packages/common/graphql/src/graphql/index.ts @@ -512,6 +512,31 @@ export const listUsersQuery = { }`, }; +export const probeManagedCopilotProfileMutation = { + id: 'probeManagedCopilotProfileMutation' as const, + op: 'probeManagedCopilotProfile', + query: `mutation probeManagedCopilotProfile($profileId: ID!, $checks: [ManagedProfileProbeCheckInput!]!) { + probeManagedCopilotProfile(profileId: $profileId, checks: $checks) { + connection { + kind + testedAt + errorKind + } + models { + modelId + checks { + operation + status { + kind + testedAt + errorKind + } + } + } + } +}`, +}; + export const rotateAuthSigningKeyMutation = { id: 'rotateAuthSigningKeyMutation' as const, op: 'rotateAuthSigningKey', diff --git a/packages/common/graphql/src/schema.ts b/packages/common/graphql/src/schema.ts index f41146d59..bdd9ec921 100644 --- a/packages/common/graphql/src/schema.ts +++ b/packages/common/graphql/src/schema.ts @@ -1641,6 +1641,36 @@ export interface ManageUserInput { name?: InputMaybe; } +export interface ManagedProfileModelProbeCheckType { + __typename?: 'ManagedProfileModelProbeCheckType'; + operation: ByokProbeOperation; + status: ManagedProfileProbeStatusType; +} + +export interface ManagedProfileModelProbeType { + __typename?: 'ManagedProfileModelProbeType'; + checks: Array; + modelId: Scalars['String']['output']; +} + +export interface ManagedProfileProbeCheckInput { + modelId: Scalars['String']['input']; + operation: ByokProbeOperation; +} + +export interface ManagedProfileProbeResultType { + __typename?: 'ManagedProfileProbeResultType'; + connection: ManagedProfileProbeStatusType; + models: Array; +} + +export interface ManagedProfileProbeStatusType { + __typename?: 'ManagedProfileProbeStatusType'; + errorKind: Maybe; + kind: ByokProbeStatusKind; + testedAt: Maybe; +} + export enum McpAccessMode { READ_ONLY = 'READ_ONLY', READ_WRITE = 'READ_WRITE', @@ -1813,6 +1843,8 @@ export interface Mutation { /** mention user in a doc */ mentionUser: Scalars['ID']['output']; previewLicense: AdminLicensePreview; + /** Test a server-managed copilot provider profile by dispatching real probe requests. */ + probeManagedCopilotProfile: ManagedProfileProbeResultType; probeWorkspaceByokDraft: WorkspaceByokProbeResultType; probeWorkspaceByokProfile: WorkspaceByokProbeResultType; publishDoc: DocType; @@ -2115,6 +2147,11 @@ export interface MutationPreviewLicenseArgs { license: Scalars['Upload']['input']; } +export interface MutationProbeManagedCopilotProfileArgs { + checks: Array; + profileId: Scalars['ID']['input']; +} + export interface MutationProbeWorkspaceByokDraftArgs { input: ProbeWorkspaceByokDraftInput; } @@ -4278,6 +4315,38 @@ export type ListUsersQuery = { }>; }; +export type ProbeManagedCopilotProfileMutationVariables = Exact<{ + profileId: Scalars['ID']['input']; + checks: Array | ManagedProfileProbeCheckInput; +}>; + +export type ProbeManagedCopilotProfileMutation = { + __typename?: 'Mutation'; + probeManagedCopilotProfile: { + __typename?: 'ManagedProfileProbeResultType'; + connection: { + __typename?: 'ManagedProfileProbeStatusType'; + kind: ByokProbeStatusKind; + testedAt: string | null; + errorKind: string | null; + }; + models: Array<{ + __typename?: 'ManagedProfileModelProbeType'; + modelId: string; + checks: Array<{ + __typename?: 'ManagedProfileModelProbeCheckType'; + operation: ByokProbeOperation; + status: { + __typename?: 'ManagedProfileProbeStatusType'; + kind: ByokProbeStatusKind; + testedAt: string | null; + errorKind: string | null; + }; + }>; + }>; + }; +}; + export type RotateAuthSigningKeyMutationVariables = Exact<{ expectedActiveKeyId: Scalars['String']['input']; }>; @@ -8125,6 +8194,11 @@ export type Mutations = variables: ImportUsersMutationVariables; response: ImportUsersMutation; } + | { + name: 'probeManagedCopilotProfileMutation'; + variables: ProbeManagedCopilotProfileMutationVariables; + response: ProbeManagedCopilotProfileMutation; + } | { name: 'rotateAuthSigningKeyMutation'; variables: RotateAuthSigningKeyMutationVariables; diff --git a/packages/frontend/admin/src/config.json b/packages/frontend/admin/src/config.json index 89b0877e1..57eaafc7f 100644 --- a/packages/frontend/admin/src/config.json +++ b/packages/frontend/admin/src/config.json @@ -389,6 +389,10 @@ "byok.allowPrivateEndpoint": { "type": "Boolean", "desc": "Whether workspace BYOK custom endpoints may resolve to private network targets. Enabling this allows workspace owners and admins to send provider probe requests to the private network." + }, + "providers.profiles": { + "type": "Array", + "desc": "The profile list for copilot providers." } }, "indexer": { diff --git a/packages/frontend/admin/src/modules/settings/config.ts b/packages/frontend/admin/src/modules/settings/config.ts index 15ce4a3a4..7cf968343 100644 --- a/packages/frontend/admin/src/modules/settings/config.ts +++ b/packages/frontend/admin/src/modules/settings/config.ts @@ -4,6 +4,7 @@ import type { ComponentType } from 'react'; import CONFIG_DESCRIPTORS from '../../config.json'; import type { ConfigInputProps } from './config-input-row'; import { AuthSigningKeys } from './operations/auth-signing-keys'; +import { ProbeManagedProfiles } from './operations/probe-managed-profiles'; import { SendTestEmail } from './operations/send-test-email'; export type ConfigType = 'String' | 'Number' | 'Boolean' | 'JSON' | 'Enum'; @@ -163,6 +164,18 @@ export const KNOWN_CONFIG_GROUPS = [ }, ], } as ConfigGroup<'copilot'>, + { + name: 'AI Providers', + module: 'copilot', + fields: [ + { + key: 'providers.profiles', + desc: 'Server-managed AI provider profiles. Edit as JSON, then use Test to verify connectivity. Example: [{"id":"local-vllm","provider":"openaiCompatible","name":"Local vLLM","enabled":true,"config":{"baseURL":"http://localhost:8000/v1","apiKey":"none","dialect":"chat_completions"},"models":["qwen2.5-7b-instruct"]}].', + type: 'JSON', + }, + ], + operations: [ProbeManagedProfiles], + } as ConfigGroup<'copilot'>, { name: 'Indexer', module: 'indexer', diff --git a/packages/frontend/admin/src/modules/settings/operations/probe-managed-profiles.spec.tsx b/packages/frontend/admin/src/modules/settings/operations/probe-managed-profiles.spec.tsx new file mode 100644 index 000000000..f556d346a --- /dev/null +++ b/packages/frontend/admin/src/modules/settings/operations/probe-managed-profiles.spec.tsx @@ -0,0 +1,144 @@ +/** + * @vitest-environment happy-dom + */ +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +const triggerMock = vi.fn(); + +vi.mock('@affine/admin/use-mutation', () => ({ + useMutation: () => ({ trigger: triggerMock, isMutating: false }), +})); + +vi.mock('@affine/component', () => ({ + notify: { success: vi.fn(), error: vi.fn() }, +})); + +import { ProbeManagedProfiles } from './probe-managed-profiles'; + +describe('ProbeManagedProfiles', () => { + beforeEach(() => { + triggerMock.mockReset(); + }); + + afterEach(() => { + cleanup(); + }); + + test('renders nothing when no managed profiles configured', () => { + const { container } = render( + + ); + expect(container.textContent).toBe(''); + }); + + test('probes each enabled profile model with declared capabilities', async () => { + triggerMock.mockResolvedValue({ + probeManagedCopilotProfile: { + connection: { kind: 'verified', errorKind: null }, + models: [ + { + modelId: 'qwen', + checks: [ + { + operation: 'chat', + status: { kind: 'verified', errorKind: null }, + }, + ], + }, + ], + }, + }); + + render( + + ); + + const button = screen.getByRole('button', { name: 'Test Local vLLM' }); + expect( + screen.queryByRole('button', { name: /disabled-profile/ }) + ).toBeNull(); + + fireEvent.click(button); + + await waitFor(() => { + expect(triggerMock).toHaveBeenCalled(); + }); + expect(triggerMock).toHaveBeenCalledWith({ + profileId: 'local-vllm', + checks: [ + { modelId: 'qwen', operation: 'chat' }, + { modelId: 'qwen', operation: 'tool_calling' }, + ], + }); + await waitFor(() => { + expect(screen.getByText(/Connection:/)).toBeDefined(); + expect(screen.getByText(/qwen: chat verified/)).toBeDefined(); + }); + }); + + test('defaults to chat check when capabilities are undeclared', async () => { + triggerMock.mockResolvedValue({ + probeManagedCopilotProfile: { + connection: { kind: 'failed', errorKind: 'http_401' }, + models: [ + { + modelId: 'm1', + checks: [ + { + operation: 'chat', + status: { kind: 'failed', errorKind: 'http_401' }, + }, + ], + }, + ], + }, + }); + + render( + + ); + + fireEvent.click(screen.getByRole('button', { name: 'Test p1' })); + + await waitFor(() => { + expect(triggerMock).toHaveBeenCalledWith({ + profileId: 'p1', + checks: [{ modelId: 'm1', operation: 'chat' }], + }); + }); + await waitFor(() => { + expect(screen.getByText(/failed \(http_401\)/)).toBeDefined(); + }); + }); +}); diff --git a/packages/frontend/admin/src/modules/settings/operations/probe-managed-profiles.tsx b/packages/frontend/admin/src/modules/settings/operations/probe-managed-profiles.tsx new file mode 100644 index 000000000..5ab194963 --- /dev/null +++ b/packages/frontend/admin/src/modules/settings/operations/probe-managed-profiles.tsx @@ -0,0 +1,171 @@ +import { Button } from '@affine/admin/components/ui/button'; +import { useMutation } from '@affine/admin/use-mutation'; +import { notify } from '@affine/component'; +import { + type ByokProbeOperation, + type ByokProbeStatusKind, + probeManagedCopilotProfileMutation, +} from '@affine/graphql'; +import { useCallback, useMemo, useState } from 'react'; + +import type { AppConfig } from '../config'; + +type ProfileLike = { + id?: string; + name?: string; + enabled?: boolean; + models?: Array; +}; + +const CAPABILITY_OPERATION: Record = { + chat: 'chat', + structured: 'structured', + tools: 'tool_calling', + vision: 'vision', + embedding: 'embedding', + rerank: 'rerank', + image: 'image', +}; + +function checksForModel( + model: NonNullable[number] +): Array<{ + modelId: string; + operation: ByokProbeOperation; +}> { + const modelId = typeof model === 'string' ? model : model.id; + const capabilities = + typeof model === 'string' ? [] : (model.capabilities ?? []); + if (capabilities.length === 0) { + // Undeclared capabilities: probe basic chat as the default smoke test. + return [{ modelId, operation: 'chat' }]; + } + return capabilities + .map( + (capability: string): ByokProbeOperation | undefined => + CAPABILITY_OPERATION[capability] + ) + .filter((operation): operation is ByokProbeOperation => Boolean(operation)) + .map(operation => ({ modelId, operation })); +} + +const STATUS_LABEL: Record = { + verified: 'verified', + failed: 'failed', + not_tested: 'not tested', +}; + +export function ProbeManagedProfiles({ appConfig }: { appConfig: AppConfig }) { + const { trigger, isMutating } = useMutation({ + mutation: probeManagedCopilotProfileMutation, + }); + const [results, setResults] = useState<{ + profileId: string; + connection: { kind: string; errorKind?: string | null }; + models: Array<{ + modelId: string; + checks: Array<{ + operation: string; + status: { kind: string; errorKind?: string | null }; + }>; + }>; + } | null>(null); + const [error, setError] = useState(null); + + const profiles = useMemo(() => { + const raw = appConfig?.copilot?.['providers.profiles']; + return Array.isArray(raw) ? (raw as ProfileLike[]) : []; + }, [appConfig]); + + const onTest = useCallback( + async (profile: ProfileLike) => { + if (!profile.id) { + return; + } + setError(null); + setResults(null); + const checks = (profile.models ?? []).flatMap(checksForModel); + try { + const result = await trigger({ + profileId: profile.id, + checks: checks.length > 0 ? checks : [], + }); + setResults({ + profileId: profile.id, + ...result.probeManagedCopilotProfile, + }); + notify.success({ + title: `Tested ${profile.name || profile.id}`, + message: `Connection ${result.probeManagedCopilotProfile.connection.kind}.`, + }); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } + }, + [trigger] + ); + + if (profiles.length === 0) { + return null; + } + + return ( +
+
+ Test provider connectivity +
+
+ {profiles.map(profile => + profile.id && profile.enabled !== false ? ( +
+ +
+ ) : null + )} +
+ {error &&
{error}
} + {results && ( +
+
+ Connection:{' '} + + {results.connection.kind} + + {results.connection.errorKind + ? ` (${results.connection.errorKind})` + : ''} +
+ {results.models.map(model => ( +
+ {model.modelId}:{' '} + {model.checks + .map( + check => + `${check.operation} ${STATUS_LABEL[check.status.kind] ?? check.status.kind}${ + check.status.errorKind + ? ` (${check.status.errorKind})` + : '' + }` + ) + .join(', ')} +
+ ))} +
+ )} +
+ ); +}