feat(copilot): managed profile testing and openaiCompatible provider support
- add native openaiCompatible provider type with flat baseUrl/apiKey/dialect config fields and declared per-model capabilities - add admin-only probeManagedCopilotProfile GraphQL mutation that dispatches real probe requests against server-managed copilot profiles, reusing the BYOK probe engine with an AllowPrivate egress policy for self-hosted endpoints (Vertex profiles rejected; use workspace BYOK instead) - expose providers.profiles app config descriptor to the admin console - add 'AI Providers' settings group with JSON editor and per-profile Test action that probes declared capabilities and reports verified/failed status per model and operation
This commit is contained in:
@@ -389,6 +389,10 @@
|
||||
"byok.allowPrivateEndpoint": {
|
||||
"type": "Boolean",
|
||||
"desc": "Whether workspace BYOK custom endpoints may resolve to private network targets. Enabling this allows workspace owners and admins to send provider probe requests to the private network."
|
||||
},
|
||||
"providers.profiles": {
|
||||
"type": "Array",
|
||||
"desc": "The profile list for copilot providers."
|
||||
}
|
||||
},
|
||||
"indexer": {
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { ComponentType } from 'react';
|
||||
import CONFIG_DESCRIPTORS from '../../config.json';
|
||||
import type { ConfigInputProps } from './config-input-row';
|
||||
import { AuthSigningKeys } from './operations/auth-signing-keys';
|
||||
import { ProbeManagedProfiles } from './operations/probe-managed-profiles';
|
||||
import { SendTestEmail } from './operations/send-test-email';
|
||||
export type ConfigType = 'String' | 'Number' | 'Boolean' | 'JSON' | 'Enum';
|
||||
|
||||
@@ -163,6 +164,18 @@ export const KNOWN_CONFIG_GROUPS = [
|
||||
},
|
||||
],
|
||||
} as ConfigGroup<'copilot'>,
|
||||
{
|
||||
name: 'AI Providers',
|
||||
module: 'copilot',
|
||||
fields: [
|
||||
{
|
||||
key: 'providers.profiles',
|
||||
desc: 'Server-managed AI provider profiles. Edit as JSON, then use Test to verify connectivity. Example: [{"id":"local-vllm","provider":"openaiCompatible","name":"Local vLLM","enabled":true,"config":{"baseURL":"http://localhost:8000/v1","apiKey":"none","dialect":"chat_completions"},"models":["qwen2.5-7b-instruct"]}].',
|
||||
type: 'JSON',
|
||||
},
|
||||
],
|
||||
operations: [ProbeManagedProfiles],
|
||||
} as ConfigGroup<'copilot'>,
|
||||
{
|
||||
name: 'Indexer',
|
||||
module: 'indexer',
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* @vitest-environment happy-dom
|
||||
*/
|
||||
import {
|
||||
cleanup,
|
||||
fireEvent,
|
||||
render,
|
||||
screen,
|
||||
waitFor,
|
||||
} from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
|
||||
|
||||
const triggerMock = vi.fn();
|
||||
|
||||
vi.mock('@affine/admin/use-mutation', () => ({
|
||||
useMutation: () => ({ trigger: triggerMock, isMutating: false }),
|
||||
}));
|
||||
|
||||
vi.mock('@affine/component', () => ({
|
||||
notify: { success: vi.fn(), error: vi.fn() },
|
||||
}));
|
||||
|
||||
import { ProbeManagedProfiles } from './probe-managed-profiles';
|
||||
|
||||
describe('ProbeManagedProfiles', () => {
|
||||
beforeEach(() => {
|
||||
triggerMock.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
test('renders nothing when no managed profiles configured', () => {
|
||||
const { container } = render(
|
||||
<ProbeManagedProfiles appConfig={{ copilot: {} }} />
|
||||
);
|
||||
expect(container.textContent).toBe('');
|
||||
});
|
||||
|
||||
test('probes each enabled profile model with declared capabilities', async () => {
|
||||
triggerMock.mockResolvedValue({
|
||||
probeManagedCopilotProfile: {
|
||||
connection: { kind: 'verified', errorKind: null },
|
||||
models: [
|
||||
{
|
||||
modelId: 'qwen',
|
||||
checks: [
|
||||
{
|
||||
operation: 'chat',
|
||||
status: { kind: 'verified', errorKind: null },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
render(
|
||||
<ProbeManagedProfiles
|
||||
appConfig={{
|
||||
copilot: {
|
||||
'providers.profiles': [
|
||||
{
|
||||
id: 'local-vllm',
|
||||
name: 'Local vLLM',
|
||||
enabled: true,
|
||||
models: [{ id: 'qwen', capabilities: ['chat', 'tools'] }],
|
||||
},
|
||||
{
|
||||
id: 'disabled-profile',
|
||||
enabled: false,
|
||||
models: ['m'],
|
||||
},
|
||||
],
|
||||
},
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
const button = screen.getByRole('button', { name: 'Test Local vLLM' });
|
||||
expect(
|
||||
screen.queryByRole('button', { name: /disabled-profile/ })
|
||||
).toBeNull();
|
||||
|
||||
fireEvent.click(button);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(triggerMock).toHaveBeenCalled();
|
||||
});
|
||||
expect(triggerMock).toHaveBeenCalledWith({
|
||||
profileId: 'local-vllm',
|
||||
checks: [
|
||||
{ modelId: 'qwen', operation: 'chat' },
|
||||
{ modelId: 'qwen', operation: 'tool_calling' },
|
||||
],
|
||||
});
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Connection:/)).toBeDefined();
|
||||
expect(screen.getByText(/qwen: chat verified/)).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
test('defaults to chat check when capabilities are undeclared', async () => {
|
||||
triggerMock.mockResolvedValue({
|
||||
probeManagedCopilotProfile: {
|
||||
connection: { kind: 'failed', errorKind: 'http_401' },
|
||||
models: [
|
||||
{
|
||||
modelId: 'm1',
|
||||
checks: [
|
||||
{
|
||||
operation: 'chat',
|
||||
status: { kind: 'failed', errorKind: 'http_401' },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
render(
|
||||
<ProbeManagedProfiles
|
||||
appConfig={{
|
||||
copilot: {
|
||||
'providers.profiles': [{ id: 'p1', models: ['m1'] }],
|
||||
},
|
||||
}}
|
||||
/>
|
||||
);
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Test p1' }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(triggerMock).toHaveBeenCalledWith({
|
||||
profileId: 'p1',
|
||||
checks: [{ modelId: 'm1', operation: 'chat' }],
|
||||
});
|
||||
});
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/failed \(http_401\)/)).toBeDefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,171 @@
|
||||
import { Button } from '@affine/admin/components/ui/button';
|
||||
import { useMutation } from '@affine/admin/use-mutation';
|
||||
import { notify } from '@affine/component';
|
||||
import {
|
||||
type ByokProbeOperation,
|
||||
type ByokProbeStatusKind,
|
||||
probeManagedCopilotProfileMutation,
|
||||
} from '@affine/graphql';
|
||||
import { useCallback, useMemo, useState } from 'react';
|
||||
|
||||
import type { AppConfig } from '../config';
|
||||
|
||||
type ProfileLike = {
|
||||
id?: string;
|
||||
name?: string;
|
||||
enabled?: boolean;
|
||||
models?: Array<string | { id: string; capabilities?: string[] }>;
|
||||
};
|
||||
|
||||
const CAPABILITY_OPERATION: Record<string, ByokProbeOperation> = {
|
||||
chat: 'chat',
|
||||
structured: 'structured',
|
||||
tools: 'tool_calling',
|
||||
vision: 'vision',
|
||||
embedding: 'embedding',
|
||||
rerank: 'rerank',
|
||||
image: 'image',
|
||||
};
|
||||
|
||||
function checksForModel(
|
||||
model: NonNullable<ProfileLike['models']>[number]
|
||||
): Array<{
|
||||
modelId: string;
|
||||
operation: ByokProbeOperation;
|
||||
}> {
|
||||
const modelId = typeof model === 'string' ? model : model.id;
|
||||
const capabilities =
|
||||
typeof model === 'string' ? [] : (model.capabilities ?? []);
|
||||
if (capabilities.length === 0) {
|
||||
// Undeclared capabilities: probe basic chat as the default smoke test.
|
||||
return [{ modelId, operation: 'chat' }];
|
||||
}
|
||||
return capabilities
|
||||
.map(
|
||||
(capability: string): ByokProbeOperation | undefined =>
|
||||
CAPABILITY_OPERATION[capability]
|
||||
)
|
||||
.filter((operation): operation is ByokProbeOperation => Boolean(operation))
|
||||
.map(operation => ({ modelId, operation }));
|
||||
}
|
||||
|
||||
const STATUS_LABEL: Record<ByokProbeStatusKind, string> = {
|
||||
verified: 'verified',
|
||||
failed: 'failed',
|
||||
not_tested: 'not tested',
|
||||
};
|
||||
|
||||
export function ProbeManagedProfiles({ appConfig }: { appConfig: AppConfig }) {
|
||||
const { trigger, isMutating } = useMutation({
|
||||
mutation: probeManagedCopilotProfileMutation,
|
||||
});
|
||||
const [results, setResults] = useState<{
|
||||
profileId: string;
|
||||
connection: { kind: string; errorKind?: string | null };
|
||||
models: Array<{
|
||||
modelId: string;
|
||||
checks: Array<{
|
||||
operation: string;
|
||||
status: { kind: string; errorKind?: string | null };
|
||||
}>;
|
||||
}>;
|
||||
} | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const profiles = useMemo(() => {
|
||||
const raw = appConfig?.copilot?.['providers.profiles'];
|
||||
return Array.isArray(raw) ? (raw as ProfileLike[]) : [];
|
||||
}, [appConfig]);
|
||||
|
||||
const onTest = useCallback(
|
||||
async (profile: ProfileLike) => {
|
||||
if (!profile.id) {
|
||||
return;
|
||||
}
|
||||
setError(null);
|
||||
setResults(null);
|
||||
const checks = (profile.models ?? []).flatMap(checksForModel);
|
||||
try {
|
||||
const result = await trigger({
|
||||
profileId: profile.id,
|
||||
checks: checks.length > 0 ? checks : [],
|
||||
});
|
||||
setResults({
|
||||
profileId: profile.id,
|
||||
...result.probeManagedCopilotProfile,
|
||||
});
|
||||
notify.success({
|
||||
title: `Tested ${profile.name || profile.id}`,
|
||||
message: `Connection ${result.probeManagedCopilotProfile.connection.kind}.`,
|
||||
});
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
},
|
||||
[trigger]
|
||||
);
|
||||
|
||||
if (profiles.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="text-sm font-semibold leading-6 text-foreground">
|
||||
Test provider connectivity
|
||||
</div>
|
||||
<div className="flex flex-col gap-2">
|
||||
{profiles.map(profile =>
|
||||
profile.id && profile.enabled !== false ? (
|
||||
<div key={profile.id} className="flex items-center gap-3">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
className="h-8"
|
||||
disabled={isMutating}
|
||||
onClick={() => void onTest(profile)}
|
||||
>
|
||||
Test {profile.name || profile.id}
|
||||
</Button>
|
||||
</div>
|
||||
) : null
|
||||
)}
|
||||
</div>
|
||||
{error && <div className="text-sm text-destructive">{error}</div>}
|
||||
{results && (
|
||||
<div className="rounded-md border border-border/60 p-3 text-sm">
|
||||
<div>
|
||||
Connection:{' '}
|
||||
<span
|
||||
className={
|
||||
results.connection.kind === 'verified'
|
||||
? 'text-green-600'
|
||||
: 'text-destructive'
|
||||
}
|
||||
>
|
||||
{results.connection.kind}
|
||||
</span>
|
||||
{results.connection.errorKind
|
||||
? ` (${results.connection.errorKind})`
|
||||
: ''}
|
||||
</div>
|
||||
{results.models.map(model => (
|
||||
<div key={model.modelId} className="mt-1 pl-3">
|
||||
{model.modelId}:{' '}
|
||||
{model.checks
|
||||
.map(
|
||||
check =>
|
||||
`${check.operation} ${STATUS_LABEL[check.status.kind] ?? check.status.kind}${
|
||||
check.status.errorKind
|
||||
? ` (${check.status.errorKind})`
|
||||
: ''
|
||||
}`
|
||||
)
|
||||
.join(', ')}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user