From 501b5f6a9727e7ccdfad932bbe70aec8f214ed28 Mon Sep 17 00:00:00 2001 From: darkskygit Date: Wed, 2 Apr 2025 12:02:07 +0000 Subject: [PATCH] feat(server): retry transcript job (#11414) --- packages/backend/server/src/base/error/def.ts | 4 ++ .../server/src/base/error/errors.gen.ts | 7 ++++ .../plugins/copilot/transcript/resolver.ts | 39 +++++++++++++++++-- .../src/plugins/copilot/transcript/service.ts | 33 ++++++++++++---- .../src/plugins/copilot/transcript/types.ts | 2 + packages/backend/server/src/schema.gql | 2 + .../copilot-jobs-transcription-retry.gql | 6 +++ packages/common/graphql/src/graphql/index.ts | 11 ++++++ packages/common/graphql/src/schema.ts | 26 +++++++++++++ packages/frontend/i18n/src/i18n.gen.ts | 4 ++ packages/frontend/i18n/src/resources/en.json | 1 + 11 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 packages/common/graphql/src/graphql/copilot-jobs-transcription-retry.gql diff --git a/packages/backend/server/src/base/error/def.ts b/packages/backend/server/src/base/error/def.ts index e7bea238e..aeaa40267 100644 --- a/packages/backend/server/src/base/error/def.ts +++ b/packages/backend/server/src/base/error/def.ts @@ -705,6 +705,10 @@ export const USER_FRIENDLY_ERRORS = { type: 'bad_request', message: () => 'Transcription job already exists', }, + copilot_transcription_job_not_found: { + type: 'bad_request', + message: () => `Transcription job not found.`, + }, // Quota & Limit errors blob_quota_exceeded: { diff --git a/packages/backend/server/src/base/error/errors.gen.ts b/packages/backend/server/src/base/error/errors.gen.ts index eb8aa4966..419982609 100644 --- a/packages/backend/server/src/base/error/errors.gen.ts +++ b/packages/backend/server/src/base/error/errors.gen.ts @@ -765,6 +765,12 @@ export class CopilotTranscriptionJobExists extends UserFriendlyError { } } +export class CopilotTranscriptionJobNotFound extends UserFriendlyError { + constructor(message?: string) { + super('bad_request', 'copilot_transcription_job_not_found', message); + } +} + export class BlobQuotaExceeded extends UserFriendlyError { constructor(message?: string) { super('quota_exceeded', 'blob_quota_exceeded', message); @@ -1020,6 +1026,7 @@ export enum ErrorNames { COPILOT_FAILED_TO_MATCH_CONTEXT, COPILOT_EMBEDDING_UNAVAILABLE, COPILOT_TRANSCRIPTION_JOB_EXISTS, + COPILOT_TRANSCRIPTION_JOB_NOT_FOUND, BLOB_QUOTA_EXCEEDED, STORAGE_QUOTA_EXCEEDED, MEMBER_QUOTA_EXCEEDED, diff --git a/packages/backend/server/src/plugins/copilot/transcript/resolver.ts b/packages/backend/server/src/plugins/copilot/transcript/resolver.ts index 23b732dbd..2ea0bd6e8 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/resolver.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/resolver.ts @@ -13,7 +13,10 @@ import { import { AiJobStatus } from '@prisma/client'; import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs'; -import type { FileUpload } from '../../../base'; +import { + CopilotTranscriptionJobNotFound, + type FileUpload, +} from '../../../base'; import { CurrentUser } from '../../../core/auth'; import { AccessController } from '../../../core/permission'; import { CopilotType } from '../resolver'; @@ -106,14 +109,44 @@ export class CopilotTranscriptionResolver { .allowLocal() .assert('Workspace.Copilot'); - const job = await this.service.submitTranscriptionJob( + const jobResult = await this.service.submitTranscriptionJob( user.id, workspaceId, blobId, blob ); - return this.handleJobResult(job); + return this.handleJobResult(jobResult); + } + + @Mutation(() => TranscriptionResultType, { nullable: true }) + async retryAudioTranscription( + @CurrentUser() user: CurrentUser, + @Args('workspaceId') workspaceId: string, + @Args('jobId') jobId: string + ): Promise { + await this.ac + .user(user.id) + .workspace(workspaceId) + .allowLocal() + .assert('Workspace.Copilot'); + + const job = await this.service.queryTranscriptionJob( + user.id, + workspaceId, + jobId + ); + if (!job || !job.url || !job.mimeType) { + throw new CopilotTranscriptionJobNotFound(); + } + + const jobResult = await this.service.executeTranscriptionJob( + job.id, + job.url, + job.mimeType + ); + + return this.handleJobResult(jobResult); } @Mutation(() => TranscriptionResultType, { nullable: true }) diff --git a/packages/backend/server/src/plugins/copilot/transcript/service.ts b/packages/backend/server/src/plugins/copilot/transcript/service.ts index c18baae46..3c2b03fcc 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/service.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/service.ts @@ -5,6 +5,7 @@ import { ZodType } from 'zod'; import { CopilotPromptNotFound, CopilotTranscriptionJobExists, + CopilotTranscriptionJobNotFound, EventBus, type FileUpload, JobQueue, @@ -31,6 +32,8 @@ import { readStream } from './utils'; export type TranscriptionJob = { id: string; status: AiJobStatus; + url?: string; + mimeType?: string; transcription?: TranscriptionPayload; }; @@ -55,7 +58,7 @@ export class CopilotTranscriptionService { throw new CopilotTranscriptionJobExists(); } - const { id: jobId, status } = await this.models.copilotJob.create({ + const { id: jobId } = await this.models.copilotJob.create({ workspaceId, blobId, createdBy: userId, @@ -65,14 +68,28 @@ export class CopilotTranscriptionService { const buffer = await readStream(blob.createReadStream()); const url = await this.storage.put(userId, workspaceId, blobId, buffer); - await this.models.copilotJob.update(jobId, { - status: AiJobStatus.running, + return await this.executeTranscriptionJob(jobId, url, blob.mimetype); + } + + async executeTranscriptionJob( + jobId: string, + url: string, + mimeType: string + ): Promise { + const status = AiJobStatus.running; + const success = await this.models.copilotJob.update(jobId, { + status, + payload: { url, mimeType }, }); + if (!success) { + throw new CopilotTranscriptionJobNotFound(); + } + await this.job.add('copilot.transcript.submit', { jobId, url, - mimeType: blob.mimetype, + mimeType, }); return { id: jobId, status }; @@ -113,9 +130,11 @@ export class CopilotTranscriptionService { const ret: TranscriptionJob = { id: job.id, status: job.status }; - if (job.status === AiJobStatus.claimed) { - const payload = TranscriptPayloadSchema.safeParse(job.payload); - if (payload.success) { + const payload = TranscriptPayloadSchema.safeParse(job.payload); + if (payload.success) { + ret.url = payload.data.url || undefined; + ret.mimeType = payload.data.mimeType || undefined; + if (job.status === AiJobStatus.claimed) { ret.transcription = payload.data; } } diff --git a/packages/backend/server/src/plugins/copilot/transcript/types.ts b/packages/backend/server/src/plugins/copilot/transcript/types.ts index c9c3d1323..97ef51bea 100644 --- a/packages/backend/server/src/plugins/copilot/transcript/types.ts +++ b/packages/backend/server/src/plugins/copilot/transcript/types.ts @@ -21,6 +21,8 @@ const TranscriptionItemSchema = z.object({ export const TranscriptionSchema = z.array(TranscriptionItemSchema); export const TranscriptPayloadSchema = z.object({ + url: z.string().nullable().optional(), + mimeType: z.string().nullable().optional(), title: z.string().nullable().optional(), summary: z.string().nullable().optional(), transcription: TranscriptionSchema.nullable().optional(), diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gql index 07e1d10bb..cddbdf248 100644 --- a/packages/backend/server/src/schema.gql +++ b/packages/backend/server/src/schema.gql @@ -436,6 +436,7 @@ enum ErrorNames { COPILOT_SESSION_DELETED COPILOT_SESSION_NOT_FOUND COPILOT_TRANSCRIPTION_JOB_EXISTS + COPILOT_TRANSCRIPTION_JOB_NOT_FOUND CUSTOMER_PORTAL_CREATE_FAILED DOC_ACTION_DENIED DOC_DEFAULT_ROLE_CAN_NOT_BE_OWNER @@ -990,6 +991,7 @@ type Mutation { removeContextFile(options: RemoveContextFileInput!): Boolean! removeWorkspaceFeature(feature: FeatureType!, workspaceId: String!): Boolean! resumeSubscription(idempotencyKey: String @deprecated(reason: "use header `Idempotency-Key`"), plan: SubscriptionPlan = Pro, workspaceId: String): SubscriptionType! + retryAudioTranscription(jobId: String!, workspaceId: String!): TranscriptionResultType revoke(userId: String!, workspaceId: String!): Boolean! revokeDocUserRoles(input: RevokeDocUserRoleInput!): Boolean! revokeInviteLink(workspaceId: String!): Boolean! diff --git a/packages/common/graphql/src/graphql/copilot-jobs-transcription-retry.gql b/packages/common/graphql/src/graphql/copilot-jobs-transcription-retry.gql new file mode 100644 index 000000000..54a8bf18f --- /dev/null +++ b/packages/common/graphql/src/graphql/copilot-jobs-transcription-retry.gql @@ -0,0 +1,6 @@ +mutation retryAudioTranscription($workspaceId: String!, $jobId: String!) { + retryAudioTranscription(workspaceId: $workspaceId, jobId: $jobId) { + id + status + } +} diff --git a/packages/common/graphql/src/graphql/index.ts b/packages/common/graphql/src/graphql/index.ts index e1c314008..bb76038d3 100644 --- a/packages/common/graphql/src/graphql/index.ts +++ b/packages/common/graphql/src/graphql/index.ts @@ -656,6 +656,17 @@ export const getAudioTranscriptionQuery = { }`, }; +export const retryAudioTranscriptionMutation = { + id: 'retryAudioTranscriptionMutation' as const, + op: 'retryAudioTranscription', + query: `mutation retryAudioTranscription($workspaceId: String!, $jobId: String!) { + retryAudioTranscription(workspaceId: $workspaceId, jobId: $jobId) { + id + status + } +}`, +}; + export const createCopilotMessageMutation = { id: 'createCopilotMessageMutation' as const, op: 'createCopilotMessage', diff --git a/packages/common/graphql/src/schema.ts b/packages/common/graphql/src/schema.ts index c153674fc..ead0af69d 100644 --- a/packages/common/graphql/src/schema.ts +++ b/packages/common/graphql/src/schema.ts @@ -581,6 +581,7 @@ export enum ErrorNames { COPILOT_SESSION_DELETED = 'COPILOT_SESSION_DELETED', COPILOT_SESSION_NOT_FOUND = 'COPILOT_SESSION_NOT_FOUND', COPILOT_TRANSCRIPTION_JOB_EXISTS = 'COPILOT_TRANSCRIPTION_JOB_EXISTS', + COPILOT_TRANSCRIPTION_JOB_NOT_FOUND = 'COPILOT_TRANSCRIPTION_JOB_NOT_FOUND', CUSTOMER_PORTAL_CREATE_FAILED = 'CUSTOMER_PORTAL_CREATE_FAILED', DOC_ACTION_DENIED = 'DOC_ACTION_DENIED', DOC_DEFAULT_ROLE_CAN_NOT_BE_OWNER = 'DOC_DEFAULT_ROLE_CAN_NOT_BE_OWNER', @@ -1102,6 +1103,7 @@ export interface Mutation { removeContextFile: Scalars['Boolean']['output']; removeWorkspaceFeature: Scalars['Boolean']['output']; resumeSubscription: SubscriptionType; + retryAudioTranscription: Maybe; revoke: Scalars['Boolean']['output']; revokeDocUserRoles: Scalars['Boolean']['output']; revokeInviteLink: Scalars['Boolean']['output']; @@ -1367,6 +1369,11 @@ export interface MutationResumeSubscriptionArgs { workspaceId?: InputMaybe; } +export interface MutationRetryAudioTranscriptionArgs { + jobId: Scalars['String']['input']; + workspaceId: Scalars['String']['input']; +} + export interface MutationRevokeArgs { userId: Scalars['String']['input']; workspaceId: Scalars['String']['input']; @@ -3048,6 +3055,20 @@ export type GetAudioTranscriptionQuery = { } | null; }; +export type RetryAudioTranscriptionMutationVariables = Exact<{ + workspaceId: Scalars['String']['input']; + jobId: Scalars['String']['input']; +}>; + +export type RetryAudioTranscriptionMutation = { + __typename?: 'Mutation'; + retryAudioTranscription: { + __typename?: 'TranscriptionResultType'; + id: string; + status: AiJobStatus; + } | null; +}; + export type CreateCopilotMessageMutationVariables = Exact<{ options: CreateChatMessageInput; }>; @@ -4706,6 +4727,11 @@ export type Mutations = variables: ClaimAudioTranscriptionMutationVariables; response: ClaimAudioTranscriptionMutation; } + | { + name: 'retryAudioTranscriptionMutation'; + variables: RetryAudioTranscriptionMutationVariables; + response: RetryAudioTranscriptionMutation; + } | { name: 'createCopilotMessageMutation'; variables: CreateCopilotMessageMutationVariables; diff --git a/packages/frontend/i18n/src/i18n.gen.ts b/packages/frontend/i18n/src/i18n.gen.ts index 8749010cc..0d757dff3 100644 --- a/packages/frontend/i18n/src/i18n.gen.ts +++ b/packages/frontend/i18n/src/i18n.gen.ts @@ -8087,6 +8087,10 @@ export function useAFFiNEI18N(): { * `Transcription job already exists` */ ["error.COPILOT_TRANSCRIPTION_JOB_EXISTS"](): string; + /** + * `Transcription job not found.` + */ + ["error.COPILOT_TRANSCRIPTION_JOB_NOT_FOUND"](): string; /** * `You have exceeded your blob size quota.` */ diff --git a/packages/frontend/i18n/src/resources/en.json b/packages/frontend/i18n/src/resources/en.json index 983cda218..0c51356f7 100644 --- a/packages/frontend/i18n/src/resources/en.json +++ b/packages/frontend/i18n/src/resources/en.json @@ -1999,6 +1999,7 @@ "error.COPILOT_FAILED_TO_MATCH_CONTEXT": "Failed to match context {{contextId}} with \"%7B%7Bcontent%7D%7D\": {{message}}", "error.COPILOT_EMBEDDING_UNAVAILABLE": "Embedding feature not available, you may need to install pgvector extension to your database", "error.COPILOT_TRANSCRIPTION_JOB_EXISTS": "Transcription job already exists", + "error.COPILOT_TRANSCRIPTION_JOB_NOT_FOUND": "Transcription job not found.", "error.BLOB_QUOTA_EXCEEDED": "You have exceeded your blob size quota.", "error.STORAGE_QUOTA_EXCEEDED": "You have exceeded your storage quota.", "error.MEMBER_QUOTA_EXCEEDED": "You have exceeded your workspace member quota.",