From bdf138925896b5dc46d15645b90af55860756f10 Mon Sep 17 00:00:00 2001 From: DarkSky <25152247+darkskygit@users.noreply.github.com> Date: Thu, 17 Jul 2025 17:20:14 +0800 Subject: [PATCH] feat(server): improve transcript (#13253) fix AF-2758 fix AF-2759 --- .../src/plugins/copilot/prompt/prompts.ts | 3 +- .../copilot/providers/gemini/gemini.ts | 9 +++ .../plugins/copilot/transcript/resolver.ts | 19 ++---- .../src/plugins/copilot/transcript/service.ts | 60 +++++++++++++++---- .../src/plugins/copilot/transcript/types.ts | 1 + 5 files changed, 66 insertions(+), 26 deletions(-) diff --git a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts index c17ccd136..e419c74a5 100644 --- a/packages/backend/server/src/plugins/copilot/prompt/prompts.ts +++ b/packages/backend/server/src/plugins/copilot/prompt/prompts.ts @@ -303,7 +303,7 @@ const textActions: Prompt[] = [ { name: 'Transcript audio', action: 'Transcript audio', - model: 'gemini-2.5-pro', + model: 'gemini-2.5-flash', optionalModels: ['gemini-2.5-flash', 'gemini-2.5-pro'], messages: [ { @@ -334,6 +334,7 @@ Convert a multi-speaker audio recording into a structured JSON format by transcr config: { requireContent: false, requireAttachment: true, + maxRetries: 1, }, }, { diff --git a/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts b/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts index 3e999aa7d..936b07f21 100644 --- a/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts +++ b/packages/backend/server/src/plugins/copilot/providers/gemini/gemini.ts @@ -129,7 +129,16 @@ export abstract class GeminiProvider extends CopilotProvider { system, messages: msgs, schema, + providerOptions: { + google: { + thinkingConfig: { + thinkingBudget: -1, + includeThoughts: false, + }, + }, + }, abortSignal: options.signal, + maxRetries: options.maxRetries || 3, experimental_repairText: async ({ text, error }) => { if (error instanceof JSONParseError) { // strange fixed response, temporarily replace it diff --git a/packages/backend/server/src/plugins/copilot/transcript/resolver.ts b/packages/backend/server/src/plugins/copilot/transcript/resolver.ts index 24cbd5268..8f8dbb896 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/resolver.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/resolver.ts @@ -15,7 +15,6 @@ import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs'; import { CopilotTranscriptionAudioNotProvided, - CopilotTranscriptionJobNotFound, type FileUpload, } from '../../../base'; import { CurrentUser } from '../../../core/auth'; @@ -74,7 +73,7 @@ const FinishedStatus: Set = new Set([ export class CopilotTranscriptionResolver { constructor( private readonly ac: AccessController, - private readonly service: CopilotTranscriptionService + private readonly transcript: CopilotTranscriptionService ) {} private handleJobResult( @@ -122,7 +121,7 @@ export class CopilotTranscriptionResolver { throw new CopilotTranscriptionAudioNotProvided(); } - const jobResult = await this.service.submitTranscriptionJob( + const jobResult = await this.transcript.submitJob( user.id, workspaceId, blobId, @@ -144,19 +143,11 @@ export class CopilotTranscriptionResolver { .allowLocal() .assert('Workspace.Copilot'); - const job = await this.service.queryTranscriptionJob( + const jobResult = await this.transcript.retryJob( user.id, workspaceId, jobId ); - if (!job || !job.infos) { - throw new CopilotTranscriptionJobNotFound(); - } - - const jobResult = await this.service.executeTranscriptionJob( - job.id, - job.infos - ); return this.handleJobResult(jobResult); } @@ -166,7 +157,7 @@ export class CopilotTranscriptionResolver { @CurrentUser() user: CurrentUser, @Args('jobId') jobId: string ): Promise { - const job = await this.service.claimTranscriptionJob(user.id, jobId); + const job = await this.transcript.claimJob(user.id, jobId); return this.handleJobResult(job); } @@ -190,7 +181,7 @@ export class CopilotTranscriptionResolver { .allowLocal() .assert('Workspace.Copilot'); - const job = await this.service.queryTranscriptionJob( + const job = await this.transcript.queryJob( user.id, copilot.workspaceId, jobId, diff --git a/packages/backend/server/src/plugins/copilot/transcript/service.ts b/packages/backend/server/src/plugins/copilot/transcript/service.ts index d4411be9b..4dc0ec37e 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/service.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/service.ts @@ -49,7 +49,17 @@ export class CopilotTranscriptionService { private readonly providerFactory: CopilotProviderFactory ) {} - async submitTranscriptionJob( + private async getModel(userId: string) { + const prompt = await this.prompt.get('Transcript audio'); + const hasAccess = await this.models.userFeature.has( + userId, + 'unlimited_copilot' + ); + // choose the pro model if user has copilot plan + return prompt?.optionalModels[hasAccess ? 1 : 0]; + } + + async submitJob( userId: string, workspaceId: string, blobId: string, @@ -78,12 +88,26 @@ export class CopilotTranscriptionService { infos.push({ url, mimeType: blob.mimetype }); } - return await this.executeTranscriptionJob(jobId, infos); + const model = await this.getModel(userId); + return await this.executeJob(jobId, infos, model); } - async executeTranscriptionJob( + async retryJob(userId: string, workspaceId: string, jobId: string) { + const job = await this.queryJob(userId, workspaceId, jobId); + if (!job || !job.infos) { + throw new CopilotTranscriptionJobNotFound(); + } + + const model = await this.getModel(userId); + const jobResult = await this.executeJob(job.id, job.infos, model); + + return jobResult; + } + + async executeJob( jobId: string, - infos: AudioBlobInfos + infos: AudioBlobInfos, + modelId?: string ): Promise { const status = AiJobStatus.running; const success = await this.models.copilotJob.update(jobId, { @@ -98,12 +122,13 @@ export class CopilotTranscriptionService { await this.job.add('copilot.transcript.submit', { jobId, infos, + modelId, }); return { id: jobId, status }; } - async claimTranscriptionJob( + async claimJob( userId: string, jobId: string ): Promise { @@ -118,7 +143,7 @@ export class CopilotTranscriptionService { return null; } - async queryTranscriptionJob( + async queryJob( userId: string, workspaceId: string, jobId?: string, @@ -181,14 +206,20 @@ export class CopilotTranscriptionService { promptName: string, message: Partial, schema?: ZodType, - prefer?: CopilotProviderType + prefer?: CopilotProviderType, + modelId?: string ): Promise { const prompt = await this.prompt.get(promptName); if (!prompt) { throw new CopilotPromptNotFound({ name: promptName }); } - const cond = { modelId: prompt.model }; + const cond = { + modelId: + modelId && prompt.optionalModels.includes(modelId) + ? modelId + : prompt.model, + }; const msg = { role: 'user' as const, content: '', ...message }; const config = Object.assign({}, prompt.config); if (schema) { @@ -231,13 +262,19 @@ export class CopilotTranscriptionService { return `${hoursStr}:${minutesStr}:${secondsStr}`; } - private async callTranscript(url: string, mimeType: string, offset: number) { + private async callTranscript( + url: string, + mimeType: string, + offset: number, + modelId?: string + ) { // NOTE: Vertex provider not support transcription yet, we always use Gemini here const result = await this.chatWithPrompt( 'Transcript audio', { attachments: [url], params: { mimetype: mimeType } }, TranscriptionResponseSchema, - CopilotProviderType.Gemini + CopilotProviderType.Gemini, + modelId ); const transcription = TranscriptionResponseSchema.parse( @@ -256,6 +293,7 @@ export class CopilotTranscriptionService { async transcriptAudio({ jobId, infos, + modelId, // @deprecated url, mimeType, @@ -264,7 +302,7 @@ export class CopilotTranscriptionService { const blobInfos = this.mergeInfos(infos, url, mimeType); const transcriptions = await Promise.all( Array.from(blobInfos.entries()).map(([idx, { url, mimeType }]) => - this.callTranscript(url, mimeType, idx * 10 * 60) + this.callTranscript(url, mimeType, idx * 10 * 60, modelId) ) ); diff --git a/packages/backend/server/src/plugins/copilot/transcript/types.ts b/packages/backend/server/src/plugins/copilot/transcript/types.ts index 77e2a7ca5..b8ef63f82 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/types.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/types.ts @@ -56,6 +56,7 @@ declare global { 'copilot.transcript.submit': { jobId: string; infos?: AudioBlobInfos; + modelId?: string; /// @deprecated use `infos` instead url?: string; /// @deprecated use `infos` instead