From 558e84138cb4a224521fb65008837d1cb8b8d049 Mon Sep 17 00:00:00 2001 From: darkskygit Date: Tue, 8 Apr 2025 05:02:30 +0000 Subject: [PATCH] chore: remove lame encoder (#11529) --- Cargo.lock | 31 --- Cargo.toml | 1 - README.md | 1 - .../media-capture-playground/server/encode.ts | 58 +++++ .../media-capture-playground/server/gemini.ts | 4 +- .../media-capture-playground/server/main.ts | 67 +++--- .../web/components/saved-recording-item.tsx | 24 +- .../web/components/saved-recordings.tsx | 2 +- .../media-capture-playground/web/types.ts | 2 +- .../frontend/native/__tests__/audio.spec.mts | 23 -- packages/frontend/native/index.d.ts | 85 ------- packages/frontend/native/index.js | 10 +- .../frontend/native/media_capture/Cargo.toml | 1 - .../frontend/native/media_capture/src/lib.rs | 1 - .../frontend/native/media_capture/src/mp3.rs | 219 ------------------ 15 files changed, 111 insertions(+), 418 deletions(-) create mode 100644 packages/frontend/media-capture-playground/server/encode.ts delete mode 100644 packages/frontend/native/__tests__/audio.spec.mts delete mode 100644 packages/frontend/native/media_capture/src/mp3.rs diff --git a/Cargo.lock b/Cargo.lock index 0b6f8b77e..0fe9be3f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,7 +81,6 @@ dependencies = [ "coreaudio-rs", "dispatch2", "libc", - "mp3lame-encoder", "napi", "napi-build", "napi-derive", @@ -358,15 +357,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" -[[package]] -name = "autotools" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" -dependencies = [ - "cc", -] - [[package]] name = "backtrace" version = "0.3.74" @@ -2212,27 +2202,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "mp3lame-encoder" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc8c8b5cdbe788ccd1098c3d3635298a011cffdebdd3460c9ca5060a7551557b" -dependencies = [ - "libc", - "mp3lame-sys", -] - -[[package]] -name = "mp3lame-sys" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21460ca4d833756cb700430888c67969e40b4560f50c226a0d258de551931ec" -dependencies = [ - "autotools", - "cc", - "libc", -] - [[package]] name = "nanoid" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index a9932cf80..ad2a261e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ homedir = "0.3" infer = { version = "0.19.0" } libc = "0.2" mimalloc = "0.1" -mp3lame-encoder = "0.2" napi = { version = "3.0.0-alpha.31", features = ["async", "chrono_date", "error_anyhow", "napi9", "serde"] } napi-build = { version = "2" } napi-derive = { version = "3.0.0-alpha.28" } diff --git a/README.md b/README.md index 44ec4bfd7..83afa0244 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,6 @@ We would also like to give thanks to open-source projects that make AFFiNE possi - [Jotai](https://github.com/pmndrs/jotai) - Primitive and flexible state management for React. - [async-call-rpc](https://github.com/Jack-Works/async-call-rpc) - A lightweight JSON RPC client & server. - [Vite](https://github.com/vitejs/vite) - Next generation frontend tooling. -- [lame](https://lame.sourceforge.io/) - High quality MPEG Audio Layer III (MP3) encoder. - Other upstream [dependencies](https://github.com/toeverything/AFFiNE/network/dependencies). Thanks a lot to the community for providing such powerful and simple libraries, so that we can focus more on the implementation of the product logic, and we hope that in the future our projects will also provide a more easy-to-use knowledge base for everyone. diff --git a/packages/frontend/media-capture-playground/server/encode.ts b/packages/frontend/media-capture-playground/server/encode.ts new file mode 100644 index 000000000..2fe401b4b --- /dev/null +++ b/packages/frontend/media-capture-playground/server/encode.ts @@ -0,0 +1,58 @@ +export function createWavBuffer( + samples: Float32Array, + options: { + sampleRate: number; + numChannels: number; + } +) { + const { sampleRate = 44100, numChannels = 1 } = options; + const bitsPerSample = 16; + const bytesPerSample = bitsPerSample / 8; + const dataSize = samples.length * bytesPerSample; + const buffer = new ArrayBuffer(44 + dataSize); // WAV header is 44 bytes + const view = new DataView(buffer); + + // Write WAV header + // "RIFF" chunk descriptor + writeString(view, 0, 'RIFF'); + view.setUint32(4, 36 + dataSize, true); // File size - 8 + writeString(view, 8, 'WAVE'); + + // "fmt " sub-chunk + writeString(view, 12, 'fmt '); + view.setUint32(16, 16, true); // Sub-chunk size + view.setUint16(20, 1, true); // Audio format (1 = PCM) + view.setUint16(22, numChannels, true); // Channels + view.setUint32(24, sampleRate, true); // Sample rate + view.setUint32(28, sampleRate * numChannels * bytesPerSample, true); // Byte rate + view.setUint16(32, numChannels * bytesPerSample, true); // Block align + view.setUint16(34, bitsPerSample, true); // Bits per sample + + // "data" sub-chunk + writeString(view, 36, 'data'); + view.setUint32(40, dataSize, true); // Sub-chunk size + + // Write audio data + const offset = 44; + for (let i = 0; i < samples.length; i++) { + // Convert float32 to int16 + const s = Math.max(-1, Math.min(1, samples[i])); + view.setInt16( + offset + i * bytesPerSample, + s < 0 ? s * 0x8000 : s * 0x7fff, + true + ); + } + + return buffer; +} + +function writeString( + view: DataView, + offset: number, + string: string +) { + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +} diff --git a/packages/frontend/media-capture-playground/server/gemini.ts b/packages/frontend/media-capture-playground/server/gemini.ts index b5c3adb11..723a5e742 100644 --- a/packages/frontend/media-capture-playground/server/gemini.ts +++ b/packages/frontend/media-capture-playground/server/gemini.ts @@ -98,8 +98,8 @@ export async function gemini( try { // Upload the audio file uploadResult = await fileManager.uploadFile(audioFilePath, { - mimeType: 'audio/mp3', - displayName: 'audio_transcription.mp3', + mimeType: 'audio/wav', + displayName: 'audio_transcription.wav', }); console.log('File uploaded:', uploadResult.file.uri); diff --git a/packages/frontend/media-capture-playground/server/main.ts b/packages/frontend/media-capture-playground/server/main.ts index 2acc760f7..d8b450aff 100644 --- a/packages/frontend/media-capture-playground/server/main.ts +++ b/packages/frontend/media-capture-playground/server/main.ts @@ -5,8 +5,6 @@ import path from 'node:path'; import { type Application, type AudioTapStream, - Bitrate, - Mp3Encoder, ShareableContent, type TappableApplication, } from '@affine/native'; @@ -19,6 +17,7 @@ import { debounce } from 'lodash-es'; import multer from 'multer'; import { Server } from 'socket.io'; +import { createWavBuffer } from './encode'; import { gemini, type TranscriptionResult } from './gemini'; // Constants @@ -206,36 +205,34 @@ async function saveRecording(recording: Recording): Promise { const recordingDir = path.join(RECORDING_DIR, sanitizedFilename); await fs.ensureDir(recordingDir); - const mp3Filename = path.join(recordingDir, 'recording.mp3'); - const transcriptionMp3Filename = path.join( + const wavFilename = path.join(recordingDir, 'recording.wav'); + const transcriptionWavFilename = path.join( recordingDir, - 'transcription.mp3' + 'transcription.wav' ); const metadataFilename = path.join(recordingDir, 'metadata.json'); const iconFilename = path.join(recordingDir, 'icon.png'); - // Save MP3 file with the actual sample rate from the stream - console.log(`📝 Writing MP3 file to ${mp3Filename}`); - const mp3Encoder = new Mp3Encoder({ - channels: channelCount, - sampleRate: actualSampleRate, - }); - const mp3Data = mp3Encoder.encode(buffer); - await fs.writeFile(mp3Filename, mp3Data); - console.log('✅ MP3 file written successfully'); - - // Save low-quality MP3 file for transcription (8kHz) - console.log( - `📝 Writing transcription MP3 file to ${transcriptionMp3Filename}` + console.log(`📝 Muxing Wav buffer ${wavFilename}`); + const wavBuffer = new Uint8Array( + createWavBuffer(buffer, { + sampleRate: actualSampleRate, + numChannels: channelCount, + }) ); - const transcriptionMp3Encoder = new Mp3Encoder({ - channels: channelCount, - bitrate: Bitrate.Kbps8, - sampleRate: actualSampleRate, - }); - const transcriptionMp3Data = transcriptionMp3Encoder.encode(buffer); - await fs.writeFile(transcriptionMp3Filename, transcriptionMp3Data); - console.log('✅ Transcription MP3 file written successfully'); + + // Save Wav file with the actual sample rate from the stream + console.log(`📝 Writing Wav file to ${wavFilename}`); + await fs.writeFile(wavFilename, wavBuffer); + console.log('✅ Wav file written successfully'); + + // Save low-quality Wav file for transcription (8kHz) + console.log( + `📝 Writing transcription wav file to ${transcriptionWavFilename}` + ); + + await fs.writeFile(transcriptionWavFilename, wavBuffer); + console.log('✅ Transcription Wav file written successfully'); // Save app icon if available if (app?.icon) { @@ -367,7 +364,7 @@ async function stopRecording(processId: number) { // File management async function getRecordings(): Promise< { - mp3: string; + wav: string; metadata?: RecordingMetadata; transcription?: TranscriptionMetadata; }[] @@ -411,7 +408,7 @@ async function getRecordings(): Promise< if (transcriptionExists) { transcription = await fs.readJson(transcriptionPath); } else { - // If transcription.mp3 exists but no transcription.json, it means transcription is available but not started + // If transcription.Wav exists but no transcription.json, it means transcription is available but not started transcription = { transcriptionStartTime: 0, transcriptionEndTime: 0, @@ -423,7 +420,7 @@ async function getRecordings(): Promise< } return { - mp3: dir, + wav: dir, metadata, transcription, }; @@ -473,21 +470,21 @@ async function setupRecordingsWatcher() { // Handle file events fsWatcher .on('add', async path => { - if (path.endsWith('.mp3') || path.endsWith('.json')) { + if (path.endsWith('.wav') || path.endsWith('.json')) { console.log(`📝 File added: ${path}`); const files = await getRecordings(); io.emit('apps:saved', { recordings: files }); } }) .on('change', async path => { - if (path.endsWith('.mp3') || path.endsWith('.json')) { + if (path.endsWith('.wav') || path.endsWith('.json')) { console.log(`📝 File changed: ${path}`); const files = await getRecordings(); io.emit('apps:saved', { recordings: files }); } }) .on('unlink', async path => { - if (path.endsWith('.mp3') || path.endsWith('.json')) { + if (path.endsWith('.wav') || path.endsWith('.json')) { console.log(`🗑️ File removed: ${path}`); const files = await getRecordings(); io.emit('apps:saved', { recordings: files }); @@ -797,11 +794,11 @@ app.post( // Check if directory exists await fs.access(recordingDir); - const transcriptionMp3Path = `${recordingDir}/transcription.mp3`; + const transcriptionWavPath = `${recordingDir}/transcription.wav`; const transcriptionMetadataPath = `${recordingDir}/transcription.json`; // Check if transcription file exists - await fs.access(transcriptionMp3Path); + await fs.access(transcriptionWavPath); // Create initial transcription metadata const initialMetadata: TranscriptionMetadata = { @@ -814,7 +811,7 @@ app.post( // Notify clients that transcription has started io.emit('apps:recording-transcription-start', { filename: foldername }); - const transcription = await gemini(transcriptionMp3Path, { + const transcription = await gemini(transcriptionWavPath, { mode: 'transcript', }); diff --git a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx index 61da0adfd..493651f1f 100644 --- a/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx +++ b/packages/frontend/media-capture-playground/web/components/saved-recording-item.tsx @@ -591,7 +591,7 @@ export function SavedRecordingItem({ const metadata = recording.metadata; // Ensure we have a valid filename, fallback to an empty string if undefined - const fileName = recording.mp3 || ''; + const fileName = recording.wav || ''; const recordingDate = metadata ? new Date(metadata.recordingStartTime).toLocaleString() : 'Unknown date'; @@ -638,7 +638,7 @@ export function SavedRecordingItem({ throw new Error('Invalid recording filename'); } - const response = await fetch(`/api/recordings/${fileName}/recording.mp3`); + const response = await fetch(`/api/recordings/${fileName}/recording.wav`); if (!response.ok) { throw new Error( `Failed to fetch audio file (${response.status}): ${response.statusText}` @@ -754,11 +754,11 @@ export function SavedRecordingItem({ try { // Check if filename is valid - if (!recording.mp3) { + if (!recording.wav) { throw new Error('Invalid recording filename'); } - const response = await fetch(`/api/recordings/${recording.mp3}`, { + const response = await fetch(`/api/recordings/${recording.wav}`, { method: 'DELETE', }); @@ -782,7 +782,7 @@ export function SavedRecordingItem({ } finally { setIsDeleting(false); } - }, [recording.mp3]); + }, [recording.wav]); const handleDeleteClick = React.useCallback(() => { void handleDelete().catch(err => { @@ -796,7 +796,7 @@ export function SavedRecordingItem({ socket.on( 'apps:recording-transcription-start', (data: { filename: string }) => { - if (recording.mp3 && data.filename === recording.mp3) { + if (recording.wav && data.filename === recording.wav) { setTranscriptionError(null); } } @@ -810,7 +810,7 @@ export function SavedRecordingItem({ transcription?: string; error?: string; }) => { - if (recording.mp3 && data.filename === recording.mp3 && !data.success) { + if (recording.wav && data.filename === recording.wav && !data.success) { setTranscriptionError(data.error || 'Transcription failed'); } } @@ -820,17 +820,17 @@ export function SavedRecordingItem({ socket.off('apps:recording-transcription-start'); socket.off('apps:recording-transcription-end'); }; - }, [recording.mp3]); + }, [recording.wav]); const handleTranscribe = React.useCallback(async () => { try { // Check if filename is valid - if (!recording.mp3) { + if (!recording.wav) { throw new Error('Invalid recording filename'); } const response = await fetch( - `/api/recordings/${recording.mp3}/transcribe`, + `/api/recordings/${recording.wav}/transcribe`, { method: 'POST', } @@ -845,7 +845,7 @@ export function SavedRecordingItem({ err instanceof Error ? err.message : 'Failed to start transcription' ); } - }, [recording.mp3]); + }, [recording.wav]); return (
@@ -876,7 +876,7 @@ export function SavedRecordingItem({ />