feat(server): refactor record schema (#14729)
#### PR Dependency Tree * **PR #14729** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Transcriptions now produce structured meeting summaries (strict JSON) and a normalized, speaker‑tagged, non‑overlapping transcript with legacy projection support. * **API** * Submission accepts richer transcription input; results return source‑audio metadata, slice manifest, quality indicators, normalized segments/transcript, and structured summary JSON. * **Frontend** * Recording flow stores transcription metadata and uploads preprocessed audio slices with slice/quality info; UI-side result normalization applied. * **Tests** * Expanded unit, contract, and e2e coverage for normalization, payload parsing, persistence/retry, and end‑to‑end transcription flows. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
DarkSky
@@ -0,0 +1,586 @@
|
||||
import { AiJobStatus } from '@prisma/client';
|
||||
import test from 'ava';
|
||||
import Sinon from 'sinon';
|
||||
|
||||
import {
|
||||
buildLegacyProjection,
|
||||
buildNormalizedTranscript,
|
||||
normalizeTranscriptSegments,
|
||||
} from '../../plugins/copilot/transcript/projection';
|
||||
import { TranscriptPayloadSchema } from '../../plugins/copilot/transcript/schema';
|
||||
import { CopilotTranscriptionService } from '../../plugins/copilot/transcript/service';
|
||||
|
||||
test('normalizeTranscriptSegments trims, sorts and clips overlaps', t => {
|
||||
const normalized = normalizeTranscriptSegments([
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 1,
|
||||
speaker: ' B ',
|
||||
startSec: 12,
|
||||
endSec: 16,
|
||||
text: ' second ',
|
||||
},
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 0,
|
||||
speaker: 'A',
|
||||
startSec: 10,
|
||||
endSec: 13,
|
||||
text: ' first ',
|
||||
},
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 1,
|
||||
speaker: 'B',
|
||||
startSec: 12,
|
||||
endSec: 16,
|
||||
text: 'second',
|
||||
},
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 2,
|
||||
speaker: '',
|
||||
startSec: 16,
|
||||
endSec: 18,
|
||||
text: ' ',
|
||||
},
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 2,
|
||||
speaker: 'C',
|
||||
startSec: 15,
|
||||
endSec: 20,
|
||||
text: 'third',
|
||||
},
|
||||
]);
|
||||
|
||||
t.deepEqual(normalized, [
|
||||
{
|
||||
speaker: 'A',
|
||||
startSec: 10,
|
||||
endSec: 13,
|
||||
start: '00:00:10',
|
||||
end: '00:00:13',
|
||||
text: 'first',
|
||||
},
|
||||
{
|
||||
speaker: 'B',
|
||||
startSec: 13,
|
||||
endSec: 16,
|
||||
start: '00:00:13',
|
||||
end: '00:00:16',
|
||||
text: 'second',
|
||||
},
|
||||
{
|
||||
speaker: 'C',
|
||||
startSec: 16,
|
||||
endSec: 20,
|
||||
start: '00:00:16',
|
||||
end: '00:00:20',
|
||||
text: 'third',
|
||||
},
|
||||
]);
|
||||
|
||||
t.is(
|
||||
buildNormalizedTranscript(normalized),
|
||||
['00:00:10 A: first', '00:00:13 B: second', '00:00:16 C: third'].join('\n')
|
||||
);
|
||||
});
|
||||
|
||||
test('buildLegacyProjection backfills summary, actions and transcription', t => {
|
||||
const legacy = buildLegacyProjection({
|
||||
normalizedSegments: [
|
||||
{
|
||||
speaker: 'A',
|
||||
startSec: 10,
|
||||
endSec: 12,
|
||||
start: '00:00:10',
|
||||
end: '00:00:12',
|
||||
text: 'Kickoff',
|
||||
},
|
||||
],
|
||||
summaryJson: {
|
||||
title: 'Weekly Sync',
|
||||
durationMinutes: 30,
|
||||
attendees: ['A', 'B'],
|
||||
keyPoints: ['Reviewed launch status'],
|
||||
actionItems: [
|
||||
{
|
||||
description: 'Send recap',
|
||||
owner: 'A',
|
||||
deadline: 'Friday',
|
||||
},
|
||||
],
|
||||
decisions: ['Ship on Monday'],
|
||||
openQuestions: ['Need final QA sign-off'],
|
||||
blockers: ['Missing analytics dashboard'],
|
||||
},
|
||||
});
|
||||
|
||||
t.is(legacy.title, 'Weekly Sync');
|
||||
t.true(legacy.summary?.includes('Reviewed launch status') ?? false);
|
||||
t.true(legacy.summary?.includes('## Decisions') ?? false);
|
||||
t.is(legacy.actions, '- [ ] Send recap (A · Friday)');
|
||||
t.deepEqual(legacy.transcription, [
|
||||
{
|
||||
speaker: 'A',
|
||||
start: '00:00:10',
|
||||
end: '00:00:12',
|
||||
transcription: 'Kickoff',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('TranscriptPayloadSchema keeps legacy payload readable as v2', t => {
|
||||
const parsed = TranscriptPayloadSchema.parse({
|
||||
url: 'https://example.com/audio.opus',
|
||||
mimeType: 'audio/opus',
|
||||
title: 'Legacy title',
|
||||
summary: '- summary',
|
||||
actions: '- [ ] task',
|
||||
transcription: [
|
||||
{
|
||||
speaker: 'A',
|
||||
start: '00:00:01',
|
||||
end: '00:00:03',
|
||||
transcription: 'legacy line',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
t.deepEqual(parsed.infos, [
|
||||
{
|
||||
url: 'https://example.com/audio.opus',
|
||||
mimeType: 'audio/opus',
|
||||
index: 0,
|
||||
},
|
||||
]);
|
||||
t.deepEqual(parsed.legacy, {
|
||||
title: 'Legacy title',
|
||||
summary: '- summary',
|
||||
actions: '- [ ] task',
|
||||
transcription: [
|
||||
{
|
||||
speaker: 'A',
|
||||
start: '00:00:01',
|
||||
end: '00:00:03',
|
||||
transcription: 'legacy line',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test('TranscriptPayloadSchema rejects empty legacy payloads', t => {
|
||||
const emptyError = t.throws(() => TranscriptPayloadSchema.parse({}));
|
||||
t.truthy(emptyError);
|
||||
|
||||
const unknownOnlyError = t.throws(() =>
|
||||
TranscriptPayloadSchema.parse({ foo: 'bar' })
|
||||
);
|
||||
t.truthy(unknownOnlyError);
|
||||
});
|
||||
|
||||
test('transcriptAudio persists transcript payload before summary failure', async t => {
|
||||
const event = { emit: Sinon.spy() };
|
||||
const persistedPayloads: any[] = [];
|
||||
let currentPayload: any = {};
|
||||
const service = new CopilotTranscriptionService(
|
||||
event as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never
|
||||
);
|
||||
|
||||
Sinon.stub(service as any, 'callTranscript').resolves([
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 0,
|
||||
speaker: 'A',
|
||||
startSec: 5,
|
||||
endSec: 9,
|
||||
text: 'Kickoff',
|
||||
},
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 0,
|
||||
speaker: 'B',
|
||||
startSec: 10,
|
||||
endSec: 14,
|
||||
text: 'Status update',
|
||||
},
|
||||
]);
|
||||
Sinon.stub(service as any, 'summarizeMeeting').rejects(
|
||||
new Error('summary provider unavailable')
|
||||
);
|
||||
Sinon.stub(service as any, 'updatePayload').callsFake(
|
||||
async (...args: any[]) => {
|
||||
const updater = args[1] as (payload: any) => any;
|
||||
currentPayload = await updater(currentPayload);
|
||||
persistedPayloads.push(currentPayload);
|
||||
return currentPayload;
|
||||
}
|
||||
);
|
||||
|
||||
await t.throwsAsync(() =>
|
||||
service.transcriptAudio({
|
||||
jobId: 'job-1',
|
||||
modelId: 'model-1',
|
||||
payload: {
|
||||
infos: [
|
||||
{
|
||||
url: 'https://example.com/audio-0.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
sliceManifest: [
|
||||
{
|
||||
index: 0,
|
||||
fileName: 'audio-0.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
startSec: 0,
|
||||
durationSec: 30,
|
||||
},
|
||||
],
|
||||
},
|
||||
} as Jobs['copilot.transcript.submit'])
|
||||
);
|
||||
|
||||
t.is(persistedPayloads.length, 2);
|
||||
t.deepEqual(
|
||||
persistedPayloads[0].rawSegments?.map((segment: any) => segment.text),
|
||||
['Kickoff', 'Status update']
|
||||
);
|
||||
t.deepEqual(
|
||||
persistedPayloads[0].normalizedSegments?.map(
|
||||
(segment: any) => segment.start
|
||||
),
|
||||
['00:00:05', '00:00:10']
|
||||
);
|
||||
t.is(
|
||||
persistedPayloads[0].normalizedTranscript,
|
||||
['00:00:05 A: Kickoff', '00:00:10 B: Status update'].join('\n')
|
||||
);
|
||||
t.is(persistedPayloads[0].summaryJson, null);
|
||||
t.deepEqual(persistedPayloads[1].retryMeta, { skipAsrOnRetry: true });
|
||||
Sinon.assert.calledWith(event.emit, 'workspace.file.transcript.failed', {
|
||||
jobId: 'job-1',
|
||||
});
|
||||
});
|
||||
|
||||
test('transcriptAudio reuses persisted transcript once after summary failure', async t => {
|
||||
const event = { emit: Sinon.spy() };
|
||||
let currentPayload: any = {
|
||||
infos: [
|
||||
{
|
||||
url: 'https://example.com/audio-0.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
rawSegments: [
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 0,
|
||||
speaker: 'A',
|
||||
startSec: 5,
|
||||
endSec: 9,
|
||||
text: 'Kickoff',
|
||||
},
|
||||
],
|
||||
normalizedSegments: [
|
||||
{
|
||||
speaker: 'A',
|
||||
startSec: 5,
|
||||
endSec: 9,
|
||||
start: '00:00:05',
|
||||
end: '00:00:09',
|
||||
text: 'Kickoff',
|
||||
},
|
||||
],
|
||||
normalizedTranscript: '00:00:05 A: Kickoff',
|
||||
summaryJson: null,
|
||||
retryMeta: { skipAsrOnRetry: true },
|
||||
};
|
||||
const service = new CopilotTranscriptionService(
|
||||
event as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never
|
||||
);
|
||||
|
||||
const callTranscript = Sinon.stub(service as any, 'callTranscript');
|
||||
Sinon.stub(service as any, 'summarizeMeeting').resolves({
|
||||
title: 'Weekly Sync',
|
||||
durationMinutes: 12,
|
||||
attendees: ['A'],
|
||||
keyPoints: ['Kickoff'],
|
||||
actionItems: [],
|
||||
decisions: [],
|
||||
openQuestions: [],
|
||||
blockers: [],
|
||||
});
|
||||
Sinon.stub(service as any, 'updatePayload').callsFake(
|
||||
async (...args: any[]) => {
|
||||
const updater = args[1] as (payload: any) => any;
|
||||
currentPayload = await updater(currentPayload);
|
||||
return currentPayload;
|
||||
}
|
||||
);
|
||||
|
||||
await service.transcriptAudio({
|
||||
jobId: 'job-2',
|
||||
modelId: 'model-1',
|
||||
payload: currentPayload,
|
||||
} as Jobs['copilot.transcript.submit']);
|
||||
|
||||
Sinon.assert.notCalled(callTranscript);
|
||||
t.is(currentPayload.summaryJson?.title, 'Weekly Sync');
|
||||
t.is(currentPayload.retryMeta, undefined);
|
||||
Sinon.assert.calledWith(event.emit, 'workspace.file.transcript.finished', {
|
||||
jobId: 'job-2',
|
||||
});
|
||||
});
|
||||
|
||||
test('transcriptAudio clears reuse flag after repeated summary failure', async t => {
|
||||
const event = { emit: Sinon.spy() };
|
||||
let currentPayload: any = {
|
||||
infos: [
|
||||
{
|
||||
url: 'https://example.com/audio-0.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
rawSegments: [
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 0,
|
||||
speaker: 'A',
|
||||
startSec: 5,
|
||||
endSec: 9,
|
||||
text: 'Kickoff',
|
||||
},
|
||||
],
|
||||
normalizedSegments: [
|
||||
{
|
||||
speaker: 'A',
|
||||
startSec: 5,
|
||||
endSec: 9,
|
||||
start: '00:00:05',
|
||||
end: '00:00:09',
|
||||
text: 'Kickoff',
|
||||
},
|
||||
],
|
||||
normalizedTranscript: '00:00:05 A: Kickoff',
|
||||
summaryJson: null,
|
||||
retryMeta: { skipAsrOnRetry: true },
|
||||
};
|
||||
const service = new CopilotTranscriptionService(
|
||||
event as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never
|
||||
);
|
||||
|
||||
const callTranscript = Sinon.stub(service as any, 'callTranscript');
|
||||
Sinon.stub(service as any, 'summarizeMeeting').rejects(
|
||||
new Error('summary still unavailable')
|
||||
);
|
||||
Sinon.stub(service as any, 'updatePayload').callsFake(
|
||||
async (...args: any[]) => {
|
||||
const updater = args[1] as (payload: any) => any;
|
||||
currentPayload = await updater(currentPayload);
|
||||
return currentPayload;
|
||||
}
|
||||
);
|
||||
|
||||
await t.throwsAsync(() =>
|
||||
service.transcriptAudio({
|
||||
jobId: 'job-3',
|
||||
modelId: 'model-1',
|
||||
payload: currentPayload,
|
||||
} as Jobs['copilot.transcript.submit'])
|
||||
);
|
||||
|
||||
Sinon.assert.notCalled(callTranscript);
|
||||
t.is(currentPayload.retryMeta, undefined);
|
||||
t.is(currentPayload.normalizedTranscript, '00:00:05 A: Kickoff');
|
||||
Sinon.assert.calledWith(event.emit, 'workspace.file.transcript.failed', {
|
||||
jobId: 'job-3',
|
||||
});
|
||||
});
|
||||
|
||||
test('queryJob returns transcript payload for finished jobs', async t => {
|
||||
const payload = TranscriptPayloadSchema.parse({
|
||||
infos: [
|
||||
{
|
||||
url: 'https://example.com/audio-0.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
index: 0,
|
||||
},
|
||||
],
|
||||
normalizedSegments: [
|
||||
{
|
||||
speaker: 'A',
|
||||
startSec: 5,
|
||||
endSec: 9,
|
||||
start: '00:00:05',
|
||||
end: '00:00:09',
|
||||
text: 'Kickoff',
|
||||
},
|
||||
],
|
||||
normalizedTranscript: '00:00:05 A: Kickoff',
|
||||
});
|
||||
const service = new CopilotTranscriptionService(
|
||||
{} as never,
|
||||
{
|
||||
copilotJob: {
|
||||
getWithUser: Sinon.stub().resolves({
|
||||
id: 'job-4',
|
||||
status: AiJobStatus.finished,
|
||||
payload,
|
||||
}),
|
||||
},
|
||||
} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never
|
||||
);
|
||||
|
||||
const result = await service.queryJob('user-1', 'workspace-1', 'job-4');
|
||||
|
||||
t.is(result?.status, AiJobStatus.finished);
|
||||
t.deepEqual(result?.infos, payload.infos);
|
||||
t.is(result?.transcription?.normalizedTranscript, '00:00:05 A: Kickoff');
|
||||
});
|
||||
|
||||
test('createCanonicalPayload keeps sliceManifest undefined when input omits it', async t => {
|
||||
const service = new CopilotTranscriptionService(
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never
|
||||
);
|
||||
|
||||
const payload = await (service as any).createCanonicalPayload('blob-1', [
|
||||
{
|
||||
url: 'https://example.com/audio-0.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
url: 'https://example.com/audio-1.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
index: 1,
|
||||
},
|
||||
]);
|
||||
|
||||
t.is(payload.sliceManifest, undefined);
|
||||
});
|
||||
|
||||
test('transcriptAudio derives manifest-less slice offsets from observed durations', async t => {
|
||||
const event = { emit: Sinon.spy() };
|
||||
let currentPayload: any = {};
|
||||
const service = new CopilotTranscriptionService(
|
||||
event as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never
|
||||
);
|
||||
|
||||
const callTranscript = Sinon.stub(service as any, 'callTranscript');
|
||||
callTranscript.onCall(0).resolves([
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 0,
|
||||
speaker: 'A',
|
||||
startSec: 30,
|
||||
endSec: 45,
|
||||
text: 'Hello, everyone.',
|
||||
},
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 0,
|
||||
speaker: 'B',
|
||||
startSec: 46,
|
||||
endSec: 70,
|
||||
text: 'Hi, thank you for joining the meeting today.',
|
||||
},
|
||||
]);
|
||||
callTranscript.onCall(1).resolves([
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 1,
|
||||
speaker: 'A',
|
||||
startSec: 30,
|
||||
endSec: 45,
|
||||
text: 'Second slice hello.',
|
||||
},
|
||||
{
|
||||
source: 'asr',
|
||||
sliceIndex: 1,
|
||||
speaker: 'B',
|
||||
startSec: 46,
|
||||
endSec: 70,
|
||||
text: 'Second slice response.',
|
||||
},
|
||||
]);
|
||||
Sinon.stub(service as any, 'summarizeMeeting').resolves({
|
||||
title: 'Weekly Sync',
|
||||
durationMinutes: 12,
|
||||
attendees: ['A', 'B'],
|
||||
keyPoints: ['Reviewed launch status'],
|
||||
actionItems: [],
|
||||
decisions: [],
|
||||
openQuestions: [],
|
||||
blockers: [],
|
||||
});
|
||||
Sinon.stub(service as any, 'updatePayload').callsFake(
|
||||
async (...args: any[]) => {
|
||||
const updater = args[1] as (payload: any) => any;
|
||||
currentPayload = await updater(currentPayload);
|
||||
return currentPayload;
|
||||
}
|
||||
);
|
||||
|
||||
await service.transcriptAudio({
|
||||
jobId: 'job-5',
|
||||
modelId: 'model-1',
|
||||
payload: {
|
||||
infos: [
|
||||
{
|
||||
url: 'https://example.com/audio-0.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
index: 0,
|
||||
},
|
||||
{
|
||||
url: 'https://example.com/audio-1.m4a',
|
||||
mimeType: 'audio/m4a',
|
||||
index: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
} as Jobs['copilot.transcript.submit']);
|
||||
|
||||
t.deepEqual(
|
||||
currentPayload.normalizedSegments?.map((segment: any) => segment.start),
|
||||
['00:00:30', '00:00:46', '00:01:40', '00:01:56']
|
||||
);
|
||||
t.is(
|
||||
currentPayload.normalizedTranscript?.split('\n')[2],
|
||||
'00:01:40 A: Second slice hello.'
|
||||
);
|
||||
t.is(currentPayload.sliceManifest, undefined);
|
||||
});
|
||||
Reference in New Issue
Block a user