luulam/AFFiNE
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(electron): adapts to default audio input/output device changes (#11815)

Browse Source 
Enable automatic adoption of new default audio devices upon detection.

1. add `AggregateDeviceManager` to watch for audio device changes, creating & maintaining `AggregateDevice` instances and cleaning up
2. use a workaround of `FastFixedIn` to deal with resampling delay issue (this is low quality and have some artifacts in the resampled audio)

fix AF-2536
This commit is contained in:
pengx17
2025-04-21 08:35:32 +00:00
parent f65c8f8fa6
commit 0442c0e2b6
9 changed files with 890 additions and 289 deletions
Download Patch File Download Diff File Expand all files Collapse all files

269
packages/frontend/media-capture-playground/server/main.ts
View File

@@ -4,13 +4,18 @@ import path from 'node:path';
import {
type Application,
type AudioTapStream,
type AudioCaptureSession,
ShareableContent,
type TappableApplication,
} from '@affine/native';
import type { FSWatcher } from 'chokidar';
import chokidar from 'chokidar';
import express from 'express';
import express, {
type NextFunction,
type Request,
type RequestHandler,
type Response,
} from 'express';
import rateLimit from 'express-rate-limit';
import fs from 'fs-extra';
import { debounce } from 'lodash-es';
@@ -33,7 +38,7 @@ interface Recording {
app: TappableApplication | null;
appGroup: Application | null;
buffers: Float32Array[];
stream: AudioTapStream;
session: AudioCaptureSession;
startTime: number;
isWriting: boolean;
isGlobal?: boolean;
@@ -90,15 +95,19 @@ const io = new Server(httpServer, {
});
// Add CORS headers middleware
app.use((req, res, next) => {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
res.header(
app.use((req: Request, res: Response, next: NextFunction): void => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader(
'Access-Control-Allow-Methods',
'GET, POST, PUT, DELETE, OPTIONS'
);
res.setHeader(
'Access-Control-Allow-Headers',
'Origin, X-Requested-With, Content-Type, Accept'
);
if (req.method === 'OPTIONS') {
return res.sendStatus(200);
res.status(200).end();
return;
}
next();
});
@@ -108,16 +117,18 @@ app.use(express.json());
// Update the static file serving to handle the new folder structure
app.use(
'/recordings',
(req, res, next) => {
(req: Request, res: Response, next: NextFunction): void => {
// Extract the folder name from the path
const parts = req.path.split('/');
if (parts.length < 2) {
return res.status(400).json({ error: 'Invalid request path' });
res.status(400).json({ error: 'Invalid request path' });
return;
}
const folderName = parts[1];
if (!validateAndSanitizeFolderName(folderName)) {
return res.status(400).json({ error: 'Invalid folder name format' });
res.status(400).json({ error: 'Invalid folder name format' });
return;
}
if (req.path.endsWith('.mp3')) {
@@ -141,7 +152,11 @@ const upload = multer({
});
// Recording management
async function saveRecording(recording: Recording): Promise<string | null> {
async function saveRecording(
recording: Recording,
sampleRate: number,
channels: number
): Promise<string | null> {
try {
recording.isWriting = true;
const app = recording.isGlobal ? null : recording.appGroup || recording.app;
@@ -154,9 +169,8 @@ async function saveRecording(recording: Recording): Promise<string | null> {
const recordingEndTime = Date.now();
const recordingDuration = (recordingEndTime - recording.startTime) / 1000;
// Get the actual sample rate from the stream's audio stats
const actualSampleRate = recording.stream.sampleRate;
const channelCount = recording.stream.channels;
const actualSampleRate = sampleRate;
const channelCount = channels;
const expectedSamples = recordingDuration * actualSampleRate;
if (recording.isGlobal) {
@@ -303,7 +317,7 @@ async function startRecording(app: TappableApplication) {
);
const buffers: Float32Array[] = [];
const stream = app.tapAudio((err, samples) => {
const session = app.tapAudio((err, samples) => {
if (err) {
console.error(`❌ Audio stream error for ${rootApp.name}:`, err);
return;
@@ -318,7 +332,7 @@ async function startRecording(app: TappableApplication) {
app,
appGroup: rootApp,
buffers,
stream,
session,
startTime: Date.now(),
isWriting: false,
});
@@ -347,8 +361,22 @@ async function stopRecording(processId: number) {
`⏱️ Recording duration: ${((Date.now() - recording.startTime) / 1000).toFixed(2)}s`
);
recording.stream.stop();
const filename = await saveRecording(recording);
let sampleRate = 0;
let channels = 0;
try {
// Get properties BEFORE stopping the session
sampleRate = recording.session.sampleRate;
channels = recording.session.channels;
} catch (e) {
console.error('❌ Failed to get session properties before stopping:', e);
// Handle error appropriately, maybe use default values or skip saving?
// For now, log and continue, saveRecording might fail later if values are 0.
}
recording.session.stop(); // Stop the session
// Pass the retrieved values to saveRecording
const filename = await saveRecording(recording, sampleRate, channels);
recordingMap.delete(processId);
if (filename) {
@@ -654,11 +682,6 @@ const rateLimiter = rateLimit({
message: { error: 'Too many requests, please try again later.' },
});
app.get('/permissions', (req, res) => {
const permission = shareableContent.checkRecordingPermissions();
res.json({ permission });
});
app.get('/apps', async (_req, res) => {
const apps = await getAllApps();
listenToAppStateChanges(apps);
@@ -691,11 +714,15 @@ function validateAndSanitizeFolderName(folderName: string): string | null {
return sanitized;
}
app.delete('/recordings/:foldername', rateLimiter, async (req, res) => {
app.delete('/recordings/:foldername', rateLimiter, (async (
req: Request,
res: Response
): Promise<void> => {
const foldername = validateAndSanitizeFolderName(req.params.foldername);
if (!foldername) {
console.error('❌ Invalid folder name format:', req.params.foldername);
return res.status(400).json({ error: 'Invalid folder name format' });
res.status(400).json({ error: 'Invalid folder name format' });
return;
}
// Construct the path safely using path.join to avoid path traversal
@@ -712,7 +739,8 @@ app.delete('/recordings/:foldername', rateLimiter, async (req, res) => {
recordingDirPath,
requestedFile: foldername,
});
return res.status(403).json({ error: 'Access denied' });
res.status(403).json({ error: 'Access denied' });
return;
}
console.log(`🗑️ Deleting recording folder: ${foldername}`);
@@ -736,7 +764,7 @@ app.delete('/recordings/:foldername', rateLimiter, async (req, res) => {
});
}
}
});
}) as RequestHandler);
app.get('/apps/:process_id/icon', (req, res) => {
const processId = parseInt(req.params.process_id);
@@ -778,100 +806,101 @@ app.post('/apps/:process_id/stop', async (req, res) => {
});
// Update transcription endpoint to use folder validation
app.post(
'/recordings/:foldername/transcribe',
rateLimiter,
async (req, res) => {
const foldername = validateAndSanitizeFolderName(req.params.foldername);
if (!foldername) {
console.error('❌ Invalid folder name format:', req.params.foldername);
return res.status(400).json({ error: 'Invalid folder name format' });
}
const recordingDir = `${RECORDING_DIR}/${foldername}`;
try {
// Check if directory exists
await fs.access(recordingDir);
const transcriptionWavPath = `${recordingDir}/transcription.wav`;
const transcriptionMetadataPath = `${recordingDir}/transcription.json`;
// Check if transcription file exists
await fs.access(transcriptionWavPath);
// Create initial transcription metadata
const initialMetadata: TranscriptionMetadata = {
transcriptionStartTime: Date.now(),
transcriptionEndTime: 0,
transcriptionStatus: 'pending',
};
await fs.writeJson(transcriptionMetadataPath, initialMetadata);
// Notify clients that transcription has started
io.emit('apps:recording-transcription-start', { filename: foldername });
const transcription = await gemini(transcriptionWavPath, {
mode: 'transcript',
});
// Update transcription metadata with results
const metadata: TranscriptionMetadata = {
transcriptionStartTime: initialMetadata.transcriptionStartTime,
transcriptionEndTime: Date.now(),
transcriptionStatus: 'completed',
transcription: transcription ?? undefined,
};
await fs.writeJson(transcriptionMetadataPath, metadata);
// Notify clients that transcription is complete
io.emit('apps:recording-transcription-end', {
filename: foldername,
success: true,
transcription,
});
res.json({ success: true });
} catch (error) {
console.error('❌ Error during transcription:', error);
// Update transcription metadata with error
const metadata: TranscriptionMetadata = {
transcriptionStartTime: Date.now(),
transcriptionEndTime: Date.now(),
transcriptionStatus: 'error',
error: error instanceof Error ? error.message : 'Unknown error',
};
await fs
.writeJson(`${recordingDir}/transcription.json`, metadata)
.catch(err => {
console.error('❌ Error saving transcription metadata:', err);
});
// Notify clients of transcription error
io.emit('apps:recording-transcription-end', {
filename: foldername,
success: false,
error: error instanceof Error ? error.message : 'Unknown error',
});
res.status(500).json({
error: error instanceof Error ? error.message : 'Unknown error',
});
}
app.post('/recordings/:foldername/transcribe', rateLimiter, (async (
req: Request,
res: Response
): Promise<void> => {
const foldername = validateAndSanitizeFolderName(req.params.foldername);
if (!foldername) {
console.error('❌ Invalid folder name format:', req.params.foldername);
res.status(400).json({ error: 'Invalid folder name format' });
return;
}
);
const recordingDir = `${RECORDING_DIR}/${foldername}`;
try {
// Check if directory exists
await fs.access(recordingDir);
const transcriptionWavPath = `${recordingDir}/transcription.wav`;
const transcriptionMetadataPath = `${recordingDir}/transcription.json`;
// Check if transcription file exists
await fs.access(transcriptionWavPath);
// Create initial transcription metadata
const initialMetadata: TranscriptionMetadata = {
transcriptionStartTime: Date.now(),
transcriptionEndTime: 0,
transcriptionStatus: 'pending',
};
await fs.writeJson(transcriptionMetadataPath, initialMetadata);
// Notify clients that transcription has started
io.emit('apps:recording-transcription-start', { filename: foldername });
const transcription = await gemini(transcriptionWavPath, {
mode: 'transcript',
});
// Update transcription metadata with results
const metadata: TranscriptionMetadata = {
transcriptionStartTime: initialMetadata.transcriptionStartTime,
transcriptionEndTime: Date.now(),
transcriptionStatus: 'completed',
transcription: transcription ?? undefined,
};
await fs.writeJson(transcriptionMetadataPath, metadata);
// Notify clients that transcription is complete
io.emit('apps:recording-transcription-end', {
filename: foldername,
success: true,
transcription,
});
res.json({ success: true });
} catch (error) {
console.error('❌ Error during transcription:', error);
// Update transcription metadata with error
const metadata: TranscriptionMetadata = {
transcriptionStartTime: Date.now(),
transcriptionEndTime: Date.now(),
transcriptionStatus: 'error',
error: error instanceof Error ? error.message : 'Unknown error',
};
await fs
.writeJson(`${recordingDir}/transcription.json`, metadata)
.catch(err => {
console.error('❌ Error saving transcription metadata:', err);
});
// Notify clients of transcription error
io.emit('apps:recording-transcription-end', {
filename: foldername,
success: false,
error: error instanceof Error ? error.message : 'Unknown error',
});
res.status(500).json({
error: error instanceof Error ? error.message : 'Unknown error',
});
}
}) as RequestHandler);
app.post(
'/transcribe',
rateLimiter,
upload.single('audio') as any,
async (req, res) => {
upload.single('audio') as unknown as RequestHandler,
(async (req: Request, res: Response): Promise<void> => {
try {
if (!req.file) {
return res.status(400).json({ error: 'No audio file provided' });
res.status(400).json({ error: 'No audio file provided' });
return;
}
// Notify clients that transcription has started
@@ -896,7 +925,7 @@ app.post(
error: error instanceof Error ? error.message : 'Unknown error',
});
}
}
}) as RequestHandler
);
async function startGlobalRecording() {
@@ -910,7 +939,7 @@ async function startGlobalRecording() {
console.log('🎙️ Starting global recording');
const buffers: Float32Array[] = [];
const stream = ShareableContent.tapGlobalAudio(
const session = ShareableContent.tapGlobalAudio(
null,
(err: Error | null, samples: Float32Array) => {
if (err) {
@@ -928,7 +957,7 @@ async function startGlobalRecording() {
app: null,
appGroup: null,
buffers,
stream,
session,
startTime: Date.now(),
isWriting: false,
isGlobal: true,
@@ -963,9 +992,7 @@ httpServer.listen(PORT, () => {
console.log(`
🎙️ Media Capture Server started successfully:
- Port: ${PORT}
- Recordings directory: ${RECORDING_DIR}
- Sample rate: 44.1kHz
- Channels: Mono
- Recordings directory: ${path.join(process.cwd(), RECORDING_DIR)}
`);
});