From fad49bb070bf9b0dd2b7ead18cee473ec7d12a22 Mon Sep 17 00:00:00 2001 From: pengx17 Date: Thu, 20 Mar 2025 12:46:14 +0000 Subject: [PATCH] feat(editor): audio block (#10947) AudioMedia entity for loading & controlling a single audio media AudioMediaManagerService: Global audio state synchronization across tabs AudioAttachmentService + AudioAttachmentBlock for manipulating AttachmentBlock in affine - e.g., filling transcription (using mock endpoint for now) Added AudioBlock + AudioPlayer for rendering audio block in affine (new transcription block whose renderer is provided in affine) fix AF-2292 fix AF-2337 --- blocksuite/affine/all/src/extensions/store.ts | 2 + blocksuite/affine/all/src/schemas.ts | 2 + .../block-attachment/src/attachment-block.ts | 39 +- .../src/attachment-edgeless-block.ts | 19 - .../src/configs/slash-menu.ts | 6 +- .../blocks/block-attachment/src/styles.ts | 12 - .../src/blocks/attachment/attachment-model.ts | 1 + .../model/src/blocks/callout/callout-model.ts | 1 + blocksuite/affine/model/src/blocks/index.ts | 1 + .../src/blocks/paragraph/paragraph-model.ts | 1 + .../model/src/blocks/transcription/index.ts | 1 + .../transcription/transcription-model.ts | 33 + packages/common/infra/src/index.ts | 1 + packages/common/infra/src/media/index.ts | 1 + packages/common/infra/src/media/types.ts | 72 ++ .../common/infra/src/utils/object-pool.ts | 22 +- .../apps/electron-renderer/package.json | 4 +- .../apps/electron-renderer/src/app.tsx | 331 -------- .../src/{ => app}/app.css.ts | 0 .../apps/electron-renderer/src/app/app.tsx | 65 ++ .../src/app/effects/events.ts | 82 ++ .../src/app/effects/index.ts | 10 + .../src/app/effects/modules.ts | 66 ++ .../src/app/effects/recording.ts | 225 ++++++ .../src/app/effects/store-manager.ts | 62 ++ .../src/app/effects/utils.ts | 23 + .../src/{ => app}/global.css | 0 .../electron-renderer/src/{ => app}/index.tsx | 0 .../electron-renderer/src/{ => app}/setup.ts | 0 .../src/{ => app}/theme-sync.ts | 0 .../apps/electron-renderer/src/shell/setup.ts | 2 +- .../apps/electron-renderer/tsconfig.json | 1 + .../apps/electron-renderer/webpack.config.ts | 2 +- .../apps/electron/src/helper/index.ts | 4 +- .../apps/electron/src/helper/logger.ts | 1 + .../apps/electron/src/helper/main-rpc.ts | 4 +- .../electron/src/helper/recording/encode.ts | 16 - .../apps/electron/src/main/cleanup.ts | 42 +- .../apps/electron/src/main/helper-process.ts | 1 + .../frontend/apps/electron/src/main/index.ts | 4 +- .../frontend/apps/electron/src/main/logger.ts | 1 + .../apps/electron/src/main/protocol.ts | 38 +- .../apps/electron/src/main/recording/index.ts | 94 ++- .../apps/electron/src/main/recording/types.ts | 6 +- .../apps/electron/src/main/tray/index.ts | 6 +- .../apps/electron/src/main/ui/events.ts | 6 + .../apps/electron/src/main/ui/handlers.ts | 7 + .../apps/electron/src/main/ui/subject.ts | 1 + .../src/main/windows-manager/tab-views.ts | 11 +- .../frontend/apps/electron/src/shared/type.ts | 7 - .../src/lit-react/create-component.ts | 17 + packages/frontend/core/package.json | 1 + .../ai-chat-block/ai-transcription-block.ts | 49 ++ .../core/src/blocksuite/ai/effects.ts | 3 + .../src/blocksuite/ai/extensions/enable-ai.ts | 2 + .../attachment-embed-preview.tsx | 25 + .../audio/audio-block.css.ts | 25 + .../attachment-viewer/audio/audio-block.tsx | 116 +++ .../audio/transcription-block.css.ts | 5 + .../audio/transcription-block.tsx | 40 + .../blocksuite/attachment-viewer/error.css.ts | 33 + .../attachment-viewer/error.tsx | 6 +- .../attachment-viewer/index.tsx | 14 +- .../pdf/pdf-viewer-embedded.tsx} | 34 +- .../attachment-viewer/pdf/pdf-viewer.tsx} | 67 +- .../attachment-viewer/pdf}/styles.css.ts | 197 +++-- .../blocksuite/attachment-viewer/pdf/utils.ts | 27 + .../attachment-viewer/types.ts | 4 +- .../src/blocksuite/attachment-viewer/utils.ts | 40 + .../attachment-viewer/viewer.css.ts | 78 ++ .../attachment-viewer/viewer.tsx} | 2 +- .../block-suite-editor/lit-adaper.tsx | 23 +- .../extensions/attachment-embed-view.tsx | 24 +- .../extensions/entry/enable-preview.ts | 2 + .../attachment-viewer/pdf-viewer-embedded.tsx | 13 - .../attachment-viewer/pdf-viewer.tsx | 50 -- .../attachment-viewer/styles.embedded.css.ts | 94 --- .../src/components/attachment-viewer/utils.ts | 75 -- .../audio-player/audio-player.css.ts | 199 +++++ .../components/audio-player/audio-player.tsx | 214 ++++++ .../audio-player/audio-waveform.css.ts | 12 + .../audio-player/audio-waveform.tsx | 182 +++++ .../core/src/components/audio-player/index.ts | 1 + .../lottie/animated-play-icon.css.ts | 3 + .../lottie/animated-play-icon.tsx | 60 ++ .../audio-player/lottie/pausetoplay.json | 715 ++++++++++++++++++ .../audio-player/lottie/playtopause.json | 715 ++++++++++++++++++ .../components/audio-player/use-seek-time.ts | 49 ++ .../components/root-app-sidebar/index.css.ts | 4 + .../src/components/root-app-sidebar/index.tsx | 5 +- .../sidebar-audio-player.css.ts | 7 + .../root-app-sidebar/sidebar-audio-player.tsx | 137 ++++ .../pages/workspace/attachment/index.tsx | 4 +- .../views/sidebar-containers/index.tsx | 22 +- .../desktop-api/service/desktop-api.ts | 20 + .../core/src/modules/feature-flag/constant.ts | 9 + packages/frontend/core/src/modules/index.ts | 2 + .../src/modules/media/audio-sync-diagram.md | 35 + .../media/entities/audio-attachment-block.ts | 191 +++++ .../src/modules/media/entities/audio-media.ts | 443 +++++++++++ .../frontend/core/src/modules/media/index.ts | 46 ++ .../media/providers/global-audio-state.ts | 126 +++ .../media/services/audio-attachment.ts | 41 + .../media/services/audio-media-manager.ts | 357 +++++++++ .../frontend/core/src/modules/media/utils.ts | 70 ++ .../media/views/use-attachment-media.ts | 31 + .../core/src/modules/pdf/entities/pdf.ts | 3 +- .../core/src/modules/pdf/renderer/index.ts | 1 - .../core/src/modules/pdf/renderer/utils.ts | 15 - .../view/attachment-preview/index.tsx | 2 +- .../peek-view/view/peek-view-manager.tsx | 1 - .../services/desktop-state-synchronizer.ts | 6 + .../i18n/src/i18n-completenesses.json | 26 +- packages/frontend/i18n/src/i18n.gen.ts | 18 +- packages/frontend/i18n/src/resources/en.json | 6 +- .../media-capture-playground/package.json | 2 + .../media-capture-playground/server/main.ts | 141 +++- tests/blocksuite/e2e/attachment.spec.ts | 2 +- tools/utils/src/workspace.gen.ts | 1 + yarn.lock | 31 +- 120 files changed, 5407 insertions(+), 950 deletions(-) create mode 100644 blocksuite/affine/model/src/blocks/transcription/index.ts create mode 100644 blocksuite/affine/model/src/blocks/transcription/transcription-model.ts create mode 100644 packages/common/infra/src/media/index.ts create mode 100644 packages/common/infra/src/media/types.ts delete mode 100644 packages/frontend/apps/electron-renderer/src/app.tsx rename packages/frontend/apps/electron-renderer/src/{ => app}/app.css.ts (100%) create mode 100644 packages/frontend/apps/electron-renderer/src/app/app.tsx create mode 100644 packages/frontend/apps/electron-renderer/src/app/effects/events.ts create mode 100644 packages/frontend/apps/electron-renderer/src/app/effects/index.ts create mode 100644 packages/frontend/apps/electron-renderer/src/app/effects/modules.ts create mode 100644 packages/frontend/apps/electron-renderer/src/app/effects/recording.ts create mode 100644 packages/frontend/apps/electron-renderer/src/app/effects/store-manager.ts create mode 100644 packages/frontend/apps/electron-renderer/src/app/effects/utils.ts rename packages/frontend/apps/electron-renderer/src/{ => app}/global.css (100%) rename packages/frontend/apps/electron-renderer/src/{ => app}/index.tsx (100%) rename packages/frontend/apps/electron-renderer/src/{ => app}/setup.ts (100%) rename packages/frontend/apps/electron-renderer/src/{ => app}/theme-sync.ts (100%) delete mode 100644 packages/frontend/apps/electron/src/helper/recording/encode.ts create mode 100644 packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/attachment-embed-preview.tsx create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.css.ts create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/audio/transcription-block.css.ts create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/audio/transcription-block.tsx create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/error.css.ts rename packages/frontend/core/src/{components => blocksuite}/attachment-viewer/error.tsx (98%) rename packages/frontend/core/src/{components => blocksuite}/attachment-viewer/index.tsx (69%) rename packages/frontend/core/src/{components/attachment-viewer/pdf-viewer-embedded-inner.tsx => blocksuite/attachment-viewer/pdf/pdf-viewer-embedded.tsx} (87%) rename packages/frontend/core/src/{components/attachment-viewer/pdf-viewer-inner.tsx => blocksuite/attachment-viewer/pdf/pdf-viewer.tsx} (79%) rename packages/frontend/core/src/{components/attachment-viewer => blocksuite/attachment-viewer/pdf}/styles.css.ts (60%) create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/pdf/utils.ts rename packages/frontend/core/src/{components => blocksuite}/attachment-viewer/types.ts (65%) create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/utils.ts create mode 100644 packages/frontend/core/src/blocksuite/attachment-viewer/viewer.css.ts rename packages/frontend/core/src/{components/attachment-viewer/titlebar.tsx => blocksuite/attachment-viewer/viewer.tsx} (98%) delete mode 100644 packages/frontend/core/src/components/attachment-viewer/pdf-viewer-embedded.tsx delete mode 100644 packages/frontend/core/src/components/attachment-viewer/pdf-viewer.tsx delete mode 100644 packages/frontend/core/src/components/attachment-viewer/styles.embedded.css.ts delete mode 100644 packages/frontend/core/src/components/attachment-viewer/utils.ts create mode 100644 packages/frontend/core/src/components/audio-player/audio-player.css.ts create mode 100644 packages/frontend/core/src/components/audio-player/audio-player.tsx create mode 100644 packages/frontend/core/src/components/audio-player/audio-waveform.css.ts create mode 100644 packages/frontend/core/src/components/audio-player/audio-waveform.tsx create mode 100644 packages/frontend/core/src/components/audio-player/index.ts create mode 100644 packages/frontend/core/src/components/audio-player/lottie/animated-play-icon.css.ts create mode 100644 packages/frontend/core/src/components/audio-player/lottie/animated-play-icon.tsx create mode 100644 packages/frontend/core/src/components/audio-player/lottie/pausetoplay.json create mode 100644 packages/frontend/core/src/components/audio-player/lottie/playtopause.json create mode 100644 packages/frontend/core/src/components/audio-player/use-seek-time.ts create mode 100644 packages/frontend/core/src/components/root-app-sidebar/sidebar-audio-player.css.ts create mode 100644 packages/frontend/core/src/components/root-app-sidebar/sidebar-audio-player.tsx create mode 100644 packages/frontend/core/src/modules/media/audio-sync-diagram.md create mode 100644 packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts create mode 100644 packages/frontend/core/src/modules/media/entities/audio-media.ts create mode 100644 packages/frontend/core/src/modules/media/index.ts create mode 100644 packages/frontend/core/src/modules/media/providers/global-audio-state.ts create mode 100644 packages/frontend/core/src/modules/media/services/audio-attachment.ts create mode 100644 packages/frontend/core/src/modules/media/services/audio-media-manager.ts create mode 100644 packages/frontend/core/src/modules/media/utils.ts create mode 100644 packages/frontend/core/src/modules/media/views/use-attachment-media.ts delete mode 100644 packages/frontend/core/src/modules/pdf/renderer/utils.ts diff --git a/blocksuite/affine/all/src/extensions/store.ts b/blocksuite/affine/all/src/extensions/store.ts index 9c3dab997..db705af05 100644 --- a/blocksuite/affine/all/src/extensions/store.ts +++ b/blocksuite/affine/all/src/extensions/store.ts @@ -29,6 +29,7 @@ import { RootBlockSchemaExtension, SurfaceRefBlockSchemaExtension, TableBlockSchemaExtension, + TranscriptionBlockSchemaExtension, } from '@blocksuite/affine-model'; import { HighlightSelectionExtension, @@ -84,6 +85,7 @@ export const StoreExtensions: ExtensionType[] = [ LatexBlockSchemaExtension, TableBlockSchemaExtension, CalloutBlockSchemaExtension, + TranscriptionBlockSchemaExtension, BlockSelectionExtension, TextSelectionExtension, diff --git a/blocksuite/affine/all/src/schemas.ts b/blocksuite/affine/all/src/schemas.ts index 91dc3e33b..6e5d50686 100644 --- a/blocksuite/affine/all/src/schemas.ts +++ b/blocksuite/affine/all/src/schemas.ts @@ -25,6 +25,7 @@ import { RootBlockSchema, SurfaceRefBlockSchema, TableBlockSchema, + TranscriptionBlockSchema, } from '@blocksuite/affine-model'; import type { BlockSchema } from '@blocksuite/store'; import type { z } from 'zod'; @@ -56,4 +57,5 @@ export const AffineSchemas: z.infer[] = [ LatexBlockSchema, TableBlockSchema, CalloutBlockSchema, + TranscriptionBlockSchema, ]; diff --git a/blocksuite/affine/blocks/block-attachment/src/attachment-block.ts b/blocksuite/affine/blocks/block-attachment/src/attachment-block.ts index dc319c5b7..b7191920c 100644 --- a/blocksuite/affine/blocks/block-attachment/src/attachment-block.ts +++ b/blocksuite/affine/blocks/block-attachment/src/attachment-block.ts @@ -18,7 +18,7 @@ import { humanFileSize } from '@blocksuite/affine-shared/utils'; import { BlockSelection } from '@blocksuite/block-std'; import { Slice } from '@blocksuite/store'; import { html } from 'lit'; -import { property, state } from 'lit/decorators.js'; +import { property } from 'lit/decorators.js'; import { classMap } from 'lit/directives/class-map.js'; import { styleMap } from 'lit/directives/style-map.js'; @@ -34,10 +34,6 @@ import { checkAttachmentBlob, downloadAttachmentBlob } from './utils'; export class AttachmentBlockComponent extends CaptionedBlockComponent { static override styles = styles; - protected _isDragging = false; - - protected _isResizing = false; - blockDraggable = true; protected containerStyleMap = styleMap({ @@ -127,25 +123,6 @@ export class AttachmentBlockComponent extends CaptionedBlockComponent this.requestUpdate()) ); - - // this is required to prevent iframe from capturing pointer events - this.disposables.add( - this.selected$.subscribe(selected => { - this._showOverlay = this._isResizing || this._isDragging || !selected; - }) - ); - // this is required to prevent iframe from capturing pointer events - this.handleEvent('dragStart', () => { - this._isDragging = true; - this._showOverlay = - this._isResizing || this._isDragging || !this.selected$.peek(); - }); - - this.handleEvent('dragEnd', () => { - this._isDragging = false; - this._showOverlay = - this._isResizing || this._isDragging || !this.selected$.peek(); - }); } override disconnectedCallback() { @@ -166,7 +143,9 @@ export class AttachmentBlockComponent extends CaptionedBlockComponent ${embedView} - -
` : html`
{ - this._isResizing = true; - this._showOverlay = true; - }) - ); - - this._disposables.add( - this.slots.elementResizeEnd.subscribe(() => { - this._isResizing = false; - this._showOverlay = - this._isResizing || this._isDragging || !this.selected$.peek(); - }) - ); - } - override onClick(_: MouseEvent) { return; } diff --git a/blocksuite/affine/blocks/block-attachment/src/configs/slash-menu.ts b/blocksuite/affine/blocks/block-attachment/src/configs/slash-menu.ts index 09bdb6ba2..199247527 100644 --- a/blocksuite/affine/blocks/block-attachment/src/configs/slash-menu.ts +++ b/blocksuite/affine/blocks/block-attachment/src/configs/slash-menu.ts @@ -29,7 +29,8 @@ export const attachmentSlashMenuConfig: SlashMenuConfig = { std.host, [file], maxFileSize, - model + model, + 'after' ); if (model.text?.length === 0) { std.store.deleteBlock(model); @@ -60,8 +61,7 @@ export const attachmentSlashMenuConfig: SlashMenuConfig = { [file], maxFileSize, model, - 'after', - true + 'after' ); if (model.text?.length === 0) { std.store.deleteBlock(model); diff --git a/blocksuite/affine/blocks/block-attachment/src/styles.ts b/blocksuite/affine/blocks/block-attachment/src/styles.ts index 71071298e..09869f97c 100644 --- a/blocksuite/affine/blocks/block-attachment/src/styles.ts +++ b/blocksuite/affine/blocks/block-attachment/src/styles.ts @@ -136,16 +136,4 @@ export const styles = css` width: 100%; height: 100%; } - - .affine-attachment-iframe-overlay { - position: absolute; - top: 0; - left: 0; - width: 100%; - height: 100%; - } - - .affine-attachment-iframe-overlay.hide { - display: none; - } `; diff --git a/blocksuite/affine/model/src/blocks/attachment/attachment-model.ts b/blocksuite/affine/model/src/blocks/attachment/attachment-model.ts index b972454e9..848ad030d 100644 --- a/blocksuite/affine/model/src/blocks/attachment/attachment-model.ts +++ b/blocksuite/affine/model/src/blocks/attachment/attachment-model.ts @@ -89,6 +89,7 @@ export const AttachmentBlockSchema = defineBlockSchema({ 'affine:paragraph', 'affine:list', ], + children: ['@attachment-viewer'], }, transformer: transformerConfigs => new AttachmentBlockTransformer(transformerConfigs), diff --git a/blocksuite/affine/model/src/blocks/callout/callout-model.ts b/blocksuite/affine/model/src/blocks/callout/callout-model.ts index a706390d7..f68b88c58 100644 --- a/blocksuite/affine/model/src/blocks/callout/callout-model.ts +++ b/blocksuite/affine/model/src/blocks/callout/callout-model.ts @@ -31,6 +31,7 @@ export const CalloutBlockSchema = defineBlockSchema({ 'affine:paragraph', 'affine:list', 'affine:edgeless-text', + 'affine:transcription', ], children: ['affine:paragraph'], }, diff --git a/blocksuite/affine/model/src/blocks/index.ts b/blocksuite/affine/model/src/blocks/index.ts index 0bc0e23af..269180f3f 100644 --- a/blocksuite/affine/model/src/blocks/index.ts +++ b/blocksuite/affine/model/src/blocks/index.ts @@ -15,3 +15,4 @@ export * from './paragraph/index.js'; export * from './root/index.js'; export * from './surface-ref/index.js'; export * from './table'; +export * from './transcription'; diff --git a/blocksuite/affine/model/src/blocks/paragraph/paragraph-model.ts b/blocksuite/affine/model/src/blocks/paragraph/paragraph-model.ts index 84ed8bb5b..f98f55a11 100644 --- a/blocksuite/affine/model/src/blocks/paragraph/paragraph-model.ts +++ b/blocksuite/affine/model/src/blocks/paragraph/paragraph-model.ts @@ -44,6 +44,7 @@ export const ParagraphBlockSchema = defineBlockSchema({ 'affine:list', 'affine:edgeless-text', 'affine:callout', + 'affine:transcription', ], }, toModel: () => new ParagraphBlockModel(), diff --git a/blocksuite/affine/model/src/blocks/transcription/index.ts b/blocksuite/affine/model/src/blocks/transcription/index.ts new file mode 100644 index 000000000..2875594e6 --- /dev/null +++ b/blocksuite/affine/model/src/blocks/transcription/index.ts @@ -0,0 +1 @@ +export * from './transcription-model.js'; diff --git a/blocksuite/affine/model/src/blocks/transcription/transcription-model.ts b/blocksuite/affine/model/src/blocks/transcription/transcription-model.ts new file mode 100644 index 000000000..5821e8d12 --- /dev/null +++ b/blocksuite/affine/model/src/blocks/transcription/transcription-model.ts @@ -0,0 +1,33 @@ +import { + BlockModel, + BlockSchemaExtension, + defineBlockSchema, +} from '@blocksuite/store'; + +export const TranscriptionBlockFlavour = 'affine:transcription'; + +export const TranscriptionBlockSchema = defineBlockSchema({ + flavour: TranscriptionBlockFlavour, + props: () => ({ + transcription: {}, + jobId: '', + }), + metadata: { + version: 1, + role: 'attachment-viewer', + parent: ['affine:attachment'], + children: ['affine:callout'], + }, + toModel: () => new TranscriptionBlockModel(), +}); + +export type TranscriptionBlockProps = { + transcription: Record; + jobId: string; +}; + +export class TranscriptionBlockModel extends BlockModel {} + +export const TranscriptionBlockSchemaExtension = BlockSchemaExtension( + TranscriptionBlockSchema +); diff --git a/packages/common/infra/src/index.ts b/packages/common/infra/src/index.ts index b7e8174a2..993b105c4 100644 --- a/packages/common/infra/src/index.ts +++ b/packages/common/infra/src/index.ts @@ -2,6 +2,7 @@ export * from './app-config-storage'; export * from './atom'; export * from './framework'; export * from './livedata'; +export * from './media'; export * from './orm'; export * from './storage'; export * from './sync'; diff --git a/packages/common/infra/src/media/index.ts b/packages/common/infra/src/media/index.ts new file mode 100644 index 000000000..fcb073fef --- /dev/null +++ b/packages/common/infra/src/media/index.ts @@ -0,0 +1 @@ +export * from './types'; diff --git a/packages/common/infra/src/media/types.ts b/packages/common/infra/src/media/types.ts new file mode 100644 index 000000000..ca4b8ffc9 --- /dev/null +++ b/packages/common/infra/src/media/types.ts @@ -0,0 +1,72 @@ +/** + * Attachment block audio media. + * blockId/docId/workspaceId are used to identify the source of the media + * to control the exclusivity playback state of audio across the whole application. + */ +export interface AttachmentBlockAudioMedia { + blobId: string; // aka sourceId + blockId: string; + docId: string; + workspaceId: string; +} + +export interface AudioMediaDescriptor { + key: AudioMediaKey; + tabId: string | null; + name: string; + size: number; + blobId: string; // aka sourceId +} + +// workspaceId/docId/blockId/blobId +export type AudioMediaKey = `${string}/${string}/${string}/${string}`; + +export const attachmentBlockAudioMediaKey = ( + media: AttachmentBlockAudioMedia +): AudioMediaKey => { + return `${media.workspaceId}/${media.docId}/${media.blockId}/${media.blobId}`; +}; + +export const parseAudioMediaKey = ( + key: AudioMediaKey +): AttachmentBlockAudioMedia => { + const [workspaceId, docId, blockId, blobId] = key.split('/'); + return { + workspaceId, + docId, + blockId, + blobId, + }; +}; + +type State = 'idle' | 'playing' | 'paused' | 'stopped'; + +export interface MediaStats { + key: AudioMediaKey; + tabId: string | null; + duration: number; + name: string; + size: number; + waveform: number[]; // for drawing waveform, maxmium of 1000 samples +} + +export interface PlaybackState { + key: AudioMediaKey; + tabId: string | null; + state: State; + /** + * Whenever the user seek the media, the startSeekOffset will be updated + */ + seekOffset: number; + /** + * Whenever the media state is updated. + * the updateTime will be updated. It is in milliseconds (unix timestamp). + * The current playback position (0-based, in seconds) is calculated by `seekOffset + (Date.now() - updateTime) / 1000 * rate` + */ + updateTime: number; + /** + * the playback rate + * Not implemented yet. Always 1.0. + */ + // rate: number; +} diff --git a/packages/common/infra/src/utils/object-pool.ts b/packages/common/infra/src/utils/object-pool.ts index a8569ee3e..869cbdbda 100644 --- a/packages/common/infra/src/utils/object-pool.ts +++ b/packages/common/infra/src/utils/object-pool.ts @@ -1,6 +1,6 @@ import { Unreachable } from '@affine/env/constant'; -export interface RcRef { +export interface RcRef extends Disposable { obj: T; release: () => void; } @@ -21,17 +21,19 @@ export class ObjectPool { if (exist) { exist.rc++; let released = false; + const release = () => { + // avoid double release + if (released) { + return; + } + released = true; + exist.rc--; + this.requestGc(); + }; return { obj: exist.obj, - release: () => { - // avoid double release - if (released) { - return; - } - released = true; - exist.rc--; - this.requestGc(); - }, + release, + [Symbol.dispose]: release, }; } return null; diff --git a/packages/frontend/apps/electron-renderer/package.json b/packages/frontend/apps/electron-renderer/package.json index e0aa60b27..7778f113d 100644 --- a/packages/frontend/apps/electron-renderer/package.json +++ b/packages/frontend/apps/electron-renderer/package.json @@ -10,6 +10,7 @@ "dependencies": { "@affine/component": "workspace:*", "@affine/core": "workspace:*", + "@affine/debug": "workspace:*", "@affine/electron-api": "workspace:*", "@affine/i18n": "workspace:*", "@affine/nbstore": "workspace:*", @@ -24,7 +25,8 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "react-router-dom": "^6.28.0", - "uuid": "^11.0.3" + "uuid": "^11.0.3", + "webm-muxer": "^5.0.3" }, "devDependencies": { "@affine-tools/utils": "workspace:*", diff --git a/packages/frontend/apps/electron-renderer/src/app.tsx b/packages/frontend/apps/electron-renderer/src/app.tsx deleted file mode 100644 index 88079245a..000000000 --- a/packages/frontend/apps/electron-renderer/src/app.tsx +++ /dev/null @@ -1,331 +0,0 @@ -import type { DocProps } from '@affine/core/blocksuite/initialization'; -import { AffineContext } from '@affine/core/components/context'; -import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls'; -import { AppContainer } from '@affine/core/desktop/components/app-container'; -import { router } from '@affine/core/desktop/router'; -import { configureCommonModules } from '@affine/core/modules'; -import { configureAppTabsHeaderModule } from '@affine/core/modules/app-tabs-header'; -import { configureDesktopBackupModule } from '@affine/core/modules/backup'; -import { ValidatorProvider } from '@affine/core/modules/cloud'; -import { - configureDesktopApiModule, - DesktopApiService, -} from '@affine/core/modules/desktop-api'; -import { WorkspaceDialogService } from '@affine/core/modules/dialogs'; -import { DocsService } from '@affine/core/modules/doc'; -import { - configureSpellCheckSettingModule, - EditorSettingService, -} from '@affine/core/modules/editor-setting'; -import { configureFindInPageModule } from '@affine/core/modules/find-in-page'; -import { GlobalContextService } from '@affine/core/modules/global-context'; -import { I18nProvider } from '@affine/core/modules/i18n'; -import { JournalService } from '@affine/core/modules/journal'; -import { LifecycleService } from '@affine/core/modules/lifecycle'; -import { - configureElectronStateStorageImpls, - NbstoreProvider, -} from '@affine/core/modules/storage'; -import { - ClientSchemeProvider, - PopupWindowProvider, -} from '@affine/core/modules/url'; -import { - configureDesktopWorkbenchModule, - WorkbenchService, -} from '@affine/core/modules/workbench'; -import { WorkspacesService } from '@affine/core/modules/workspace'; -import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine'; -import createEmotionCache from '@affine/core/utils/create-emotion-cache'; -import { apis, events } from '@affine/electron-api'; -import { StoreManagerClient } from '@affine/nbstore/worker/client'; -import type { AttachmentBlockProps } from '@blocksuite/affine/model'; -import { Text } from '@blocksuite/affine/store'; -import { CacheProvider } from '@emotion/react'; -import { Framework, FrameworkRoot, getCurrentStore } from '@toeverything/infra'; -import { OpClient } from '@toeverything/infra/op'; -import { Suspense } from 'react'; -import { RouterProvider } from 'react-router-dom'; -import { v4 as uuid } from 'uuid'; - -import { DesktopThemeSync } from './theme-sync'; - -const storeManagerClient = createStoreManagerClient(); -window.addEventListener('beforeunload', () => { - storeManagerClient.dispose(); -}); - -const desktopWhiteList = [ - '/open-app/signin-redirect', - '/open-app/url', - '/upgrade-success', - '/ai-upgrade-success', - '/share', - '/oauth', - '/magic-link', -]; -if ( - !BUILD_CONFIG.isElectron && - BUILD_CONFIG.debug && - desktopWhiteList.every(path => !location.pathname.startsWith(path)) -) { - document.body.innerHTML = `

Don't run electron entry in browser.

`; - throw new Error('Wrong distribution'); -} - -const cache = createEmotionCache(); - -const future = { - v7_startTransition: true, -} as const; - -const framework = new Framework(); -configureCommonModules(framework); -configureElectronStateStorageImpls(framework); -configureBrowserWorkspaceFlavours(framework); -configureDesktopWorkbenchModule(framework); -configureAppTabsHeaderModule(framework); -configureFindInPageModule(framework); -configureDesktopApiModule(framework); -configureSpellCheckSettingModule(framework); -configureDesktopBackupModule(framework); -framework.impl(NbstoreProvider, { - openStore(key, options) { - const { store, dispose } = storeManagerClient.open(key, options); - - return { - store, - dispose: () => { - dispose(); - }, - }; - }, -}); -framework.impl(PopupWindowProvider, p => { - const apis = p.get(DesktopApiService).api; - return { - open: (url: string) => { - apis.handler.ui.openExternal(url).catch(e => { - console.error('Failed to open external URL', e); - }); - }, - }; -}); -framework.impl(ClientSchemeProvider, p => { - const appInfo = p.get(DesktopApiService).appInfo; - return { - getClientScheme() { - return appInfo?.scheme; - }, - }; -}); -framework.impl(ValidatorProvider, p => { - const apis = p.get(DesktopApiService).api; - return { - async validate(_challenge, resource) { - const token = await apis.handler.ui.getChallengeResponse(resource); - if (!token) { - throw new Error('Challenge failed'); - } - return token; - }, - }; -}); -const frameworkProvider = framework.provider(); - -// setup application lifecycle events, and emit application start event -window.addEventListener('focus', () => { - frameworkProvider.get(LifecycleService).applicationFocus(); -}); -frameworkProvider.get(LifecycleService).applicationStart(); -window.addEventListener('unload', () => { - frameworkProvider - .get(DesktopApiService) - .api.handler.ui.pingAppLayoutReady(false) - .catch(console.error); -}); - -function getCurrentWorkspace() { - const currentWorkspaceId = frameworkProvider - .get(GlobalContextService) - .globalContext.workspaceId.get(); - const workspacesService = frameworkProvider.get(WorkspacesService); - const workspaceRef = currentWorkspaceId - ? workspacesService.openByWorkspaceId(currentWorkspaceId) - : null; - if (!workspaceRef) { - return; - } - const { workspace, dispose } = workspaceRef; - - return { - workspace, - dispose, - }; -} - -events?.applicationMenu.openAboutPageInSettingModal(() => { - const currentWorkspace = getCurrentWorkspace(); - if (!currentWorkspace) { - return; - } - const { workspace, dispose } = currentWorkspace; - workspace.scope.get(WorkspaceDialogService).open('setting', { - activeTab: 'about', - }); - dispose(); -}); - -events?.applicationMenu.onNewPageAction(type => { - apis?.ui - .isActiveTab() - .then(isActive => { - if (!isActive) { - return; - } - const currentWorkspace = getCurrentWorkspace(); - if (!currentWorkspace) { - return; - } - const { workspace, dispose } = currentWorkspace; - const editorSettingService = frameworkProvider.get(EditorSettingService); - const docsService = workspace.scope.get(DocsService); - const editorSetting = editorSettingService.editorSetting; - - const docProps = { - note: editorSetting.get('affine:note'), - }; - const page = docsService.createDoc({ docProps, primaryMode: type }); - workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); - dispose(); - }) - .catch(err => { - console.error(err); - }); -}); - -events?.recording.onRecordingStatusChanged(status => { - (async () => { - if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') { - const currentWorkspace = getCurrentWorkspace(); - if (!currentWorkspace) { - return; - } - const { workspace, dispose } = currentWorkspace; - const editorSettingService = frameworkProvider.get(EditorSettingService); - const docsService = workspace.scope.get(DocsService); - const editorSetting = editorSettingService.editorSetting; - - const docProps: DocProps = { - note: editorSetting.get('affine:note'), - page: { - title: new Text( - 'Recording ' + - (status.appGroup?.name ?? 'System Audio') + - ' ' + - new Date(status.startTime).toISOString() - ), - }, - onStoreLoad: (doc, { noteId }) => { - (async () => { - const data = await apis?.recording.saveRecording(status.id); - if (!data) { - return; - } - const blob = new Blob([data], { type: 'audio/mp3' }); - const blobId = await doc.workspace.blobSync.set(blob); - const attachmentProps: Partial = { - name: 'Recording', - size: blob.size, - type: 'audio/mp3', - sourceId: blobId, - embed: true, - }; - doc.addBlock('affine:attachment', attachmentProps, noteId); - })().catch(console.error); - }, - }; - const page = docsService.createDoc({ docProps, primaryMode: 'page' }); - workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); - - dispose(); - } - })().catch(console.error); -}); - -events?.applicationMenu.onOpenJournal(() => { - const currentWorkspace = getCurrentWorkspace(); - if (!currentWorkspace) { - return; - } - const { workspace, dispose } = currentWorkspace; - - const workbench = workspace.scope.get(WorkbenchService).workbench; - const journalService = workspace.scope.get(JournalService); - const docId = journalService.ensureJournalByDate(new Date()).id; - workbench.openDoc(docId); - - dispose(); -}); - -export function App() { - return ( - - - - - - - } - router={router} - future={future} - /> - {environment.isWindows && ( -
- -
- )} - - - - - - ); -} - -function createStoreManagerClient() { - const { port1: portForOpClient, port2: portForWorker } = new MessageChannel(); - let portFromWorker: MessagePort | null = null; - let portId = uuid(); - - const handleMessage = (ev: MessageEvent) => { - if ( - ev.data.type === 'electron:worker-connect' && - ev.data.portId === portId - ) { - portFromWorker = ev.ports[0]; - // connect portForWorker and portFromWorker - portFromWorker.addEventListener('message', ev => { - portForWorker.postMessage(ev.data, [...ev.ports]); - }); - portForWorker.addEventListener('message', ev => { - // oxlint-disable-next-line no-non-null-assertion - portFromWorker!.postMessage(ev.data, [...ev.ports]); - }); - portForWorker.start(); - portFromWorker.start(); - } - }; - - window.addEventListener('message', handleMessage); - - // oxlint-disable-next-line no-non-null-assertion - apis!.worker.connectWorker('affine-shared-worker', portId).catch(err => { - console.error('failed to connect worker', err); - }); - - const storeManager = new StoreManagerClient(new OpClient(portForOpClient)); - portForOpClient.start(); - return storeManager; -} diff --git a/packages/frontend/apps/electron-renderer/src/app.css.ts b/packages/frontend/apps/electron-renderer/src/app/app.css.ts similarity index 100% rename from packages/frontend/apps/electron-renderer/src/app.css.ts rename to packages/frontend/apps/electron-renderer/src/app/app.css.ts diff --git a/packages/frontend/apps/electron-renderer/src/app/app.tsx b/packages/frontend/apps/electron-renderer/src/app/app.tsx new file mode 100644 index 000000000..a16f0d612 --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/app/app.tsx @@ -0,0 +1,65 @@ +import { AffineContext } from '@affine/core/components/context'; +import { WindowsAppControls } from '@affine/core/components/pure/header/windows-app-controls'; +import { AppContainer } from '@affine/core/desktop/components/app-container'; +import { router } from '@affine/core/desktop/router'; +import { I18nProvider } from '@affine/core/modules/i18n'; +import createEmotionCache from '@affine/core/utils/create-emotion-cache'; +import { CacheProvider } from '@emotion/react'; +import { FrameworkRoot, getCurrentStore } from '@toeverything/infra'; +import { Suspense } from 'react'; +import { RouterProvider } from 'react-router-dom'; + +import { setupEffects } from './effects'; +import { DesktopThemeSync } from './theme-sync'; + +const { frameworkProvider } = setupEffects(); + +const desktopWhiteList = [ + '/open-app/signin-redirect', + '/open-app/url', + '/upgrade-success', + '/ai-upgrade-success', + '/share', + '/oauth', + '/magic-link', +]; +if ( + !BUILD_CONFIG.isElectron && + BUILD_CONFIG.debug && + desktopWhiteList.every(path => !location.pathname.startsWith(path)) +) { + document.body.innerHTML = `

Don't run electron entry in browser.

`; + throw new Error('Wrong distribution'); +} + +const cache = createEmotionCache(); + +const future = { + v7_startTransition: true, +} as const; + +export function App() { + return ( + + + + + + + } + router={router} + future={future} + /> + {environment.isWindows && ( +
+ +
+ )} + + + + + + ); +} diff --git a/packages/frontend/apps/electron-renderer/src/app/effects/events.ts b/packages/frontend/apps/electron-renderer/src/app/effects/events.ts new file mode 100644 index 000000000..016b67d34 --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/app/effects/events.ts @@ -0,0 +1,82 @@ +import { DesktopApiService } from '@affine/core/modules/desktop-api'; +import { WorkspaceDialogService } from '@affine/core/modules/dialogs'; +import { DocsService } from '@affine/core/modules/doc'; +import { EditorSettingService } from '@affine/core/modules/editor-setting'; +import { JournalService } from '@affine/core/modules/journal'; +import { LifecycleService } from '@affine/core/modules/lifecycle'; +import { WorkbenchService } from '@affine/core/modules/workbench'; +import { apis, events } from '@affine/electron-api'; +import type { FrameworkProvider } from '@toeverything/infra'; + +import { setupRecordingEvents } from './recording'; +import { getCurrentWorkspace } from './utils'; + +export function setupEvents(frameworkProvider: FrameworkProvider) { + // setup application lifecycle events, and emit application start event + window.addEventListener('focus', () => { + frameworkProvider.get(LifecycleService).applicationFocus(); + }); + frameworkProvider.get(LifecycleService).applicationStart(); + window.addEventListener('unload', () => { + frameworkProvider + .get(DesktopApiService) + .api.handler.ui.pingAppLayoutReady(false) + .catch(console.error); + }); + + events?.applicationMenu.openAboutPageInSettingModal(() => { + using currentWorkspace = getCurrentWorkspace(frameworkProvider); + if (!currentWorkspace) { + return; + } + const { workspace } = currentWorkspace; + workspace.scope.get(WorkspaceDialogService).open('setting', { + activeTab: 'about', + }); + }); + + events?.applicationMenu.onNewPageAction(type => { + apis?.ui + .isActiveTab() + .then(isActive => { + if (!isActive) { + return; + } + using currentWorkspace = getCurrentWorkspace(frameworkProvider); + if (!currentWorkspace) { + return; + } + const { workspace } = currentWorkspace; + const editorSettingService = + frameworkProvider.get(EditorSettingService); + const docsService = workspace.scope.get(DocsService); + const editorSetting = editorSettingService.editorSetting; + + const docProps = { + note: editorSetting.get('affine:note'), + }; + const page = docsService.createDoc({ docProps, primaryMode: type }); + workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); + }) + .catch(err => { + console.error(err); + }); + }); + + events?.applicationMenu.onOpenJournal(() => { + using currentWorkspace = getCurrentWorkspace(frameworkProvider); + if (!currentWorkspace) { + return; + } + const { workspace, dispose } = currentWorkspace; + + const workbench = workspace.scope.get(WorkbenchService).workbench; + const journalService = workspace.scope.get(JournalService); + const docId = journalService.ensureJournalByDate(new Date()).id; + workbench.openDoc(docId); + + dispose(); + }); + + setupRecordingEvents(frameworkProvider); +} diff --git a/packages/frontend/apps/electron-renderer/src/app/effects/index.ts b/packages/frontend/apps/electron-renderer/src/app/effects/index.ts new file mode 100644 index 000000000..8e8017329 --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/app/effects/index.ts @@ -0,0 +1,10 @@ +import { setupEvents } from './events'; +import { setupModules } from './modules'; +import { setupStoreManager } from './store-manager'; + +export function setupEffects() { + const { framework, frameworkProvider } = setupModules(); + setupStoreManager(framework); + setupEvents(frameworkProvider); + return { framework, frameworkProvider }; +} diff --git a/packages/frontend/apps/electron-renderer/src/app/effects/modules.ts b/packages/frontend/apps/electron-renderer/src/app/effects/modules.ts new file mode 100644 index 000000000..6da9912ef --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/app/effects/modules.ts @@ -0,0 +1,66 @@ +import { configureCommonModules } from '@affine/core/modules'; +import { configureAppTabsHeaderModule } from '@affine/core/modules/app-tabs-header'; +import { configureDesktopBackupModule } from '@affine/core/modules/backup'; +import { ValidatorProvider } from '@affine/core/modules/cloud'; +import { + configureDesktopApiModule, + DesktopApiService, +} from '@affine/core/modules/desktop-api'; +import { configureSpellCheckSettingModule } from '@affine/core/modules/editor-setting'; +import { configureFindInPageModule } from '@affine/core/modules/find-in-page'; +import { configureElectronStateStorageImpls } from '@affine/core/modules/storage'; +import { + ClientSchemeProvider, + PopupWindowProvider, +} from '@affine/core/modules/url'; +import { configureDesktopWorkbenchModule } from '@affine/core/modules/workbench'; +import { configureBrowserWorkspaceFlavours } from '@affine/core/modules/workspace-engine'; +import { Framework } from '@toeverything/infra'; + +export function setupModules() { + const framework = new Framework(); + configureCommonModules(framework); + configureElectronStateStorageImpls(framework); + configureBrowserWorkspaceFlavours(framework); + configureDesktopWorkbenchModule(framework); + configureAppTabsHeaderModule(framework); + configureFindInPageModule(framework); + configureDesktopApiModule(framework); + configureSpellCheckSettingModule(framework); + configureDesktopBackupModule(framework); + + framework.impl(PopupWindowProvider, p => { + const apis = p.get(DesktopApiService).api; + return { + open: (url: string) => { + apis.handler.ui.openExternal(url).catch(e => { + console.error('Failed to open external URL', e); + }); + }, + }; + }); + framework.impl(ClientSchemeProvider, p => { + const appInfo = p.get(DesktopApiService).appInfo; + return { + getClientScheme() { + return appInfo?.scheme; + }, + }; + }); + framework.impl(ValidatorProvider, p => { + const apis = p.get(DesktopApiService).api; + return { + async validate(_challenge, resource) { + const token = await apis.handler.ui.getChallengeResponse(resource); + if (!token) { + throw new Error('Challenge failed'); + } + return token; + }, + }; + }); + + const frameworkProvider = framework.provider(); + + return { framework, frameworkProvider }; +} diff --git a/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts b/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts new file mode 100644 index 000000000..b15222d51 --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts @@ -0,0 +1,225 @@ +import type { DocProps } from '@affine/core/blocksuite/initialization'; +import { DocsService } from '@affine/core/modules/doc'; +import { EditorSettingService } from '@affine/core/modules/editor-setting'; +import { AudioAttachmentService } from '@affine/core/modules/media/services/audio-attachment'; +import { WorkbenchService } from '@affine/core/modules/workbench'; +import { DebugLogger } from '@affine/debug'; +import { apis, events } from '@affine/electron-api'; +import type { AttachmentBlockModel } from '@blocksuite/affine/model'; +import { Text } from '@blocksuite/affine/store'; +import type { BlobEngine } from '@blocksuite/affine/sync'; +import type { FrameworkProvider } from '@toeverything/infra'; +import { ArrayBufferTarget, Muxer } from 'webm-muxer'; + +import { getCurrentWorkspace } from './utils'; + +const logger = new DebugLogger('electron-renderer:recording'); + +/** + * Encodes raw audio data to Opus in WebM container. + */ +async function encodeRawBufferToOpus({ + filepath, + sampleRate, + numberOfChannels, +}: { + filepath: string; + sampleRate: number; + numberOfChannels: number; +}): Promise { + // Use streams to process audio data incrementally + const response = await fetch(new URL(filepath, location.origin)); + if (!response.body) { + throw new Error('Response body is null'); + } + + // Setup Opus encoder + const encodedChunks: EncodedAudioChunk[] = []; + const encoder = new AudioEncoder({ + output: chunk => { + encodedChunks.push(chunk); + }, + error: err => { + throw new Error(`Encoding error: ${err}`); + }, + }); + + // Configure Opus encoder + encoder.configure({ + codec: 'opus', + sampleRate: sampleRate, + numberOfChannels: numberOfChannels, + bitrate: 96000, // 96 kbps is good for stereo audio + }); + + // Process the stream + const reader = response.body.getReader(); + let offset = 0; + const CHUNK_SIZE = numberOfChannels * 1024; // Process 1024 samples per channel at a time + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + // Convert the chunk to Float32Array + const float32Data = new Float32Array(value.buffer); + + // Process in smaller chunks to avoid large frames + for (let i = 0; i < float32Data.length; i += CHUNK_SIZE) { + const chunkSize = Math.min(CHUNK_SIZE, float32Data.length - i); + const chunk = float32Data.subarray(i, i + chunkSize); + + // Create and encode frame + const frame = new AudioData({ + format: 'f32', + sampleRate: sampleRate, + numberOfFrames: chunk.length / numberOfChannels, + numberOfChannels: numberOfChannels, + timestamp: (offset * 1000000) / sampleRate, // timestamp in microseconds + data: chunk, + }); + + encoder.encode(frame); + frame.close(); + + offset += chunk.length / numberOfChannels; + } + } + } finally { + await encoder.flush(); + encoder.close(); + } + + if (encodedChunks.length === 0) { + throw new Error('No chunks were produced during encoding'); + } + + // Initialize WebM muxer + const target = new ArrayBufferTarget(); + const muxer = new Muxer({ + target, + audio: { + codec: 'A_OPUS', + sampleRate: sampleRate, + numberOfChannels: numberOfChannels, + }, + }); + + // Add all chunks to the muxer + for (const chunk of encodedChunks) { + muxer.addAudioChunk(chunk, {}); + } + + // Finalize and get WebM container + muxer.finalize(); + const { buffer: webmBuffer } = target; + + return new Uint8Array(webmBuffer); +} + +async function saveRecordingBlob( + blobEngine: BlobEngine, + recording: { + id: number; + filepath: string; + sampleRate: number; + numberOfChannels: number; + } +) { + logger.debug('Saving recording', recording.id); + const opusBuffer = await encodeRawBufferToOpus({ + filepath: recording.filepath, + sampleRate: recording.sampleRate, + numberOfChannels: recording.numberOfChannels, + }); + const blob = new Blob([opusBuffer], { + type: 'audio/webm', + }); + const blobId = await blobEngine.set(blob); + logger.debug('Recording saved', blobId); + return { blob, blobId }; +} + +export function setupRecordingEvents(frameworkProvider: FrameworkProvider) { + events?.recording.onRecordingStatusChanged(status => { + (async () => { + if ((await apis?.ui.isActiveTab()) && status?.status === 'stopped') { + using currentWorkspace = getCurrentWorkspace(frameworkProvider); + if (!currentWorkspace) { + return; + } + const { workspace } = currentWorkspace; + const editorSettingService = + frameworkProvider.get(EditorSettingService); + const docsService = workspace.scope.get(DocsService); + const editorSetting = editorSettingService.editorSetting; + + const docProps: DocProps = { + note: editorSetting.get('affine:note'), + page: { + title: new Text( + 'Recording ' + + (status.appGroup?.name ?? 'System Audio') + + ' ' + + new Date(status.startTime).toISOString() + ), + }, + onStoreLoad: (doc, { noteId }) => { + (async () => { + const recording = await apis?.recording.getRecording(status.id); + if (!recording) { + logger.error('Failed to save recording'); + return; + } + + // name + timestamp(readable) + extension + const attachmentName = + (status.appGroup?.name ?? 'System Audio') + + ' ' + + new Date(status.startTime).toISOString() + + '.webm'; + + // add size and sourceId to the attachment later + const attachmentId = doc.addBlock( + 'affine:attachment', + { + name: attachmentName, + type: 'audio/webm', + }, + noteId + ); + + const model = doc.getBlock(attachmentId) + ?.model as AttachmentBlockModel; + + if (model) { + // it takes a while to save the blob, so we show the attachment first + const { blobId, blob } = await saveRecordingBlob( + doc.workspace.blobSync, + recording + ); + + model.props.size = blob.size; + model.props.sourceId = blobId; + model.props.embed = true; + + using currentWorkspace = getCurrentWorkspace(frameworkProvider); + if (!currentWorkspace) { + return; + } + const { workspace } = currentWorkspace; + using audioAttachment = workspace.scope + .get(AudioAttachmentService) + .get(model); + audioAttachment?.obj.transcribe(); + } + })().catch(console.error); + }, + }; + const page = docsService.createDoc({ docProps, primaryMode: 'page' }); + workspace.scope.get(WorkbenchService).workbench.openDoc(page.id); + } + })().catch(console.error); + }); +} diff --git a/packages/frontend/apps/electron-renderer/src/app/effects/store-manager.ts b/packages/frontend/apps/electron-renderer/src/app/effects/store-manager.ts new file mode 100644 index 000000000..884a63005 --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/app/effects/store-manager.ts @@ -0,0 +1,62 @@ +import { NbstoreProvider } from '@affine/core/modules/storage'; +import { apis } from '@affine/electron-api'; +import { StoreManagerClient } from '@affine/nbstore/worker/client'; +import type { Framework } from '@toeverything/infra'; +import { OpClient } from '@toeverything/infra/op'; +import { v4 as uuid } from 'uuid'; + +function createStoreManagerClient() { + const { port1: portForOpClient, port2: portForWorker } = new MessageChannel(); + let portFromWorker: MessagePort | null = null; + let portId = uuid(); + + const handleMessage = (ev: MessageEvent) => { + if ( + ev.data.type === 'electron:worker-connect' && + ev.data.portId === portId + ) { + portFromWorker = ev.ports[0]; + // connect portForWorker and portFromWorker + portFromWorker.addEventListener('message', ev => { + portForWorker.postMessage(ev.data, [...ev.ports]); + }); + portForWorker.addEventListener('message', ev => { + // oxlint-disable-next-line no-non-null-assertion + portFromWorker!.postMessage(ev.data, [...ev.ports]); + }); + portForWorker.start(); + portFromWorker.start(); + } + }; + + window.addEventListener('message', handleMessage); + + // oxlint-disable-next-line no-non-null-assertion + apis!.worker.connectWorker('affine-shared-worker', portId).catch(err => { + console.error('failed to connect worker', err); + }); + + const storeManager = new StoreManagerClient(new OpClient(portForOpClient)); + portForOpClient.start(); + return storeManager; +} + +export function setupStoreManager(framework: Framework) { + const storeManagerClient = createStoreManagerClient(); + window.addEventListener('beforeunload', () => { + storeManagerClient.dispose(); + }); + + framework.impl(NbstoreProvider, { + openStore(key, options) { + const { store, dispose } = storeManagerClient.open(key, options); + + return { + store, + dispose: () => { + dispose(); + }, + }; + }, + }); +} diff --git a/packages/frontend/apps/electron-renderer/src/app/effects/utils.ts b/packages/frontend/apps/electron-renderer/src/app/effects/utils.ts new file mode 100644 index 000000000..ff7424b4f --- /dev/null +++ b/packages/frontend/apps/electron-renderer/src/app/effects/utils.ts @@ -0,0 +1,23 @@ +import { GlobalContextService } from '@affine/core/modules/global-context'; +import { WorkspacesService } from '@affine/core/modules/workspace'; +import type { FrameworkProvider } from '@toeverything/infra'; + +export function getCurrentWorkspace(frameworkProvider: FrameworkProvider) { + const currentWorkspaceId = frameworkProvider + .get(GlobalContextService) + .globalContext.workspaceId.get(); + const workspacesService = frameworkProvider.get(WorkspacesService); + const workspaceRef = currentWorkspaceId + ? workspacesService.openByWorkspaceId(currentWorkspaceId) + : null; + if (!workspaceRef) { + return; + } + const { workspace, dispose } = workspaceRef; + + return { + workspace, + dispose, + [Symbol.dispose]: dispose, + }; +} diff --git a/packages/frontend/apps/electron-renderer/src/global.css b/packages/frontend/apps/electron-renderer/src/app/global.css similarity index 100% rename from packages/frontend/apps/electron-renderer/src/global.css rename to packages/frontend/apps/electron-renderer/src/app/global.css diff --git a/packages/frontend/apps/electron-renderer/src/index.tsx b/packages/frontend/apps/electron-renderer/src/app/index.tsx similarity index 100% rename from packages/frontend/apps/electron-renderer/src/index.tsx rename to packages/frontend/apps/electron-renderer/src/app/index.tsx diff --git a/packages/frontend/apps/electron-renderer/src/setup.ts b/packages/frontend/apps/electron-renderer/src/app/setup.ts similarity index 100% rename from packages/frontend/apps/electron-renderer/src/setup.ts rename to packages/frontend/apps/electron-renderer/src/app/setup.ts diff --git a/packages/frontend/apps/electron-renderer/src/theme-sync.ts b/packages/frontend/apps/electron-renderer/src/app/theme-sync.ts similarity index 100% rename from packages/frontend/apps/electron-renderer/src/theme-sync.ts rename to packages/frontend/apps/electron-renderer/src/app/theme-sync.ts diff --git a/packages/frontend/apps/electron-renderer/src/shell/setup.ts b/packages/frontend/apps/electron-renderer/src/shell/setup.ts index fe6c84404..603787c1c 100644 --- a/packages/frontend/apps/electron-renderer/src/shell/setup.ts +++ b/packages/frontend/apps/electron-renderer/src/shell/setup.ts @@ -1,3 +1,3 @@ import '@affine/core/bootstrap/electron'; import '@affine/component/theme'; -import '../global.css'; +import '../app/global.css'; diff --git a/packages/frontend/apps/electron-renderer/tsconfig.json b/packages/frontend/apps/electron-renderer/tsconfig.json index 422502d62..f8fc509d1 100644 --- a/packages/frontend/apps/electron-renderer/tsconfig.json +++ b/packages/frontend/apps/electron-renderer/tsconfig.json @@ -10,6 +10,7 @@ "references": [ { "path": "../../component" }, { "path": "../../core" }, + { "path": "../../../common/debug" }, { "path": "../../electron-api" }, { "path": "../../i18n" }, { "path": "../../../common/nbstore" }, diff --git a/packages/frontend/apps/electron-renderer/webpack.config.ts b/packages/frontend/apps/electron-renderer/webpack.config.ts index 07fee611f..6d42f48ac 100644 --- a/packages/frontend/apps/electron-renderer/webpack.config.ts +++ b/packages/frontend/apps/electron-renderer/webpack.config.ts @@ -1,6 +1,6 @@ export const config = { entry: { - app: './src/index.tsx', + app: './src/app/index.tsx', shell: './src/shell/index.tsx', backgroundWorker: './src/background-worker/index.ts', }, diff --git a/packages/frontend/apps/electron/src/helper/index.ts b/packages/frontend/apps/electron/src/helper/index.ts index 7fe622259..bda534d3d 100644 --- a/packages/frontend/apps/electron/src/helper/index.ts +++ b/packages/frontend/apps/electron/src/helper/index.ts @@ -12,7 +12,7 @@ function setupRendererConnection(rendererPort: Electron.MessagePortMain) { try { const start = performance.now(); const result = await handler(...args); - logger.info( + logger.debug( '[async-api]', `${namespace}.${name}`, args.filter( @@ -74,7 +74,7 @@ function main() { if (e.data.channel === 'renderer-connect' && e.ports.length === 1) { const rendererPort = e.ports[0]; setupRendererConnection(rendererPort); - logger.info('[helper] renderer connected'); + logger.debug('[helper] renderer connected'); } }); } diff --git a/packages/frontend/apps/electron/src/helper/logger.ts b/packages/frontend/apps/electron/src/helper/logger.ts index 498a18696..cb112a5c9 100644 --- a/packages/frontend/apps/electron/src/helper/logger.ts +++ b/packages/frontend/apps/electron/src/helper/logger.ts @@ -3,3 +3,4 @@ import log from 'electron-log/main'; export const logger = log.scope('helper'); log.transports.file.level = 'info'; +log.transports.console.level = 'info'; diff --git a/packages/frontend/apps/electron/src/helper/main-rpc.ts b/packages/frontend/apps/electron/src/helper/main-rpc.ts index e6026f5d3..7952be587 100644 --- a/packages/frontend/apps/electron/src/helper/main-rpc.ts +++ b/packages/frontend/apps/electron/src/helper/main-rpc.ts @@ -2,7 +2,6 @@ import { AsyncCall } from 'async-call-rpc'; import type { HelperToMain, MainToHelper } from '../shared/type'; import { exposed } from './provide'; -import { encodeToMp3 } from './recording/encode'; const helperToMainServer: HelperToMain = { getMeta: () => { @@ -11,8 +10,6 @@ const helperToMainServer: HelperToMain = { } return exposed; }, - // allow main process encode audio samples to mp3 buffer (because it is slow and blocking) - encodeToMp3, }; export const mainRPC = AsyncCall(helperToMainServer, { @@ -33,4 +30,5 @@ export const mainRPC = AsyncCall(helperToMainServer, { process.parentPort.postMessage(data); }, }, + log: false, }); diff --git a/packages/frontend/apps/electron/src/helper/recording/encode.ts b/packages/frontend/apps/electron/src/helper/recording/encode.ts deleted file mode 100644 index 0aa346487..000000000 --- a/packages/frontend/apps/electron/src/helper/recording/encode.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Mp3Encoder } from '@affine/native'; - -// encode audio samples to mp3 buffer -export function encodeToMp3( - samples: Float32Array, - opts: { - channels?: number; - sampleRate?: number; - } = {} -): Uint8Array { - const mp3Encoder = new Mp3Encoder({ - channels: opts.channels ?? 2, - sampleRate: opts.sampleRate ?? 44100, - }); - return mp3Encoder.encode(samples); -} diff --git a/packages/frontend/apps/electron/src/main/cleanup.ts b/packages/frontend/apps/electron/src/main/cleanup.ts index 1fd5c343b..20ec6c44e 100644 --- a/packages/frontend/apps/electron/src/main/cleanup.ts +++ b/packages/frontend/apps/electron/src/main/cleanup.ts @@ -1,15 +1,22 @@ +import type { MediaStats } from '@toeverything/infra'; import { app } from 'electron'; import { logger } from './logger'; +import { globalStateStorage } from './shared-storage/storage'; -const cleanupRegistry: (() => void)[] = []; +const beforeAppQuitRegistry: (() => void)[] = []; +const beforeTabCloseRegistry: ((tabId: string) => void)[] = []; export function beforeAppQuit(fn: () => void) { - cleanupRegistry.push(fn); + beforeAppQuitRegistry.push(fn); +} + +export function beforeTabClose(fn: (tabId: string) => void) { + beforeTabCloseRegistry.push(fn); } app.on('before-quit', () => { - cleanupRegistry.forEach(fn => { + beforeAppQuitRegistry.forEach(fn => { // some cleanup functions might throw on quit and crash the app try { fn(); @@ -18,3 +25,32 @@ app.on('before-quit', () => { } }); }); + +export function onTabClose(tabId: string) { + beforeTabCloseRegistry.forEach(fn => { + try { + fn(tabId); + } catch (err) { + logger.warn('cleanup error on tab close', err); + } + }); +} + +app.on('ready', () => { + globalStateStorage.set('media:playback-state', null); + globalStateStorage.set('media:stats', null); +}); + +beforeAppQuit(() => { + globalStateStorage.set('media:playback-state', null); + globalStateStorage.set('media:stats', null); +}); + +// set audio play state +beforeTabClose(tabId => { + const stats = globalStateStorage.get('media:stats'); + if (stats && stats.tabId === tabId) { + globalStateStorage.set('media:playback-state', null); + globalStateStorage.set('media:stats', null); + } +}); diff --git a/packages/frontend/apps/electron/src/main/helper-process.ts b/packages/frontend/apps/electron/src/main/helper-process.ts index f009e514b..0918b0fa8 100644 --- a/packages/frontend/apps/electron/src/main/helper-process.ts +++ b/packages/frontend/apps/electron/src/main/helper-process.ts @@ -115,6 +115,7 @@ class HelperProcessManager { unknownMessage: false, }, channel: new MessageEventChannel(this.#process), + log: false, }); } } diff --git a/packages/frontend/apps/electron/src/main/index.ts b/packages/frontend/apps/electron/src/main/index.ts index f67e9dc9d..74a92fb2a 100644 --- a/packages/frontend/apps/electron/src/main/index.ts +++ b/packages/frontend/apps/electron/src/main/index.ts @@ -15,7 +15,7 @@ import { registerHandlers } from './handlers'; import { logger } from './logger'; import { registerProtocol } from './protocol'; import { setupRecording } from './recording'; -import { getTrayState } from './tray'; +import { setupTrayState } from './tray'; import { registerUpdater } from './updater'; import { launch } from './windows-manager/launcher'; import { launchStage } from './windows-manager/stage'; @@ -89,7 +89,7 @@ app .then(launch) .then(setupRecording) .then(createApplicationMenu) - .then(getTrayState) + .then(setupTrayState) .then(registerUpdater) .catch(e => console.error('Failed create window:', e)); diff --git a/packages/frontend/apps/electron/src/main/logger.ts b/packages/frontend/apps/electron/src/main/logger.ts index 53cf6e356..d72681b5e 100644 --- a/packages/frontend/apps/electron/src/main/logger.ts +++ b/packages/frontend/apps/electron/src/main/logger.ts @@ -7,6 +7,7 @@ log.initialize({ }); log.transports.file.level = 'info'; +log.transports.console.level = 'info'; export function getLogFilePath() { return log.transports.file.getFile().path; diff --git a/packages/frontend/apps/electron/src/main/protocol.ts b/packages/frontend/apps/electron/src/main/protocol.ts index b9e6060a7..df7427d3e 100644 --- a/packages/frontend/apps/electron/src/main/protocol.ts +++ b/packages/frontend/apps/electron/src/main/protocol.ts @@ -1,6 +1,6 @@ import { join } from 'node:path'; -import { net, protocol, session } from 'electron'; +import { app, net, protocol, session } from 'electron'; import cookieParser from 'set-cookie-parser'; import { resourcesPath } from '../shared/utils'; @@ -43,8 +43,10 @@ function isNetworkResource(pathname: string) { async function handleFileRequest(request: Request) { const urlObject = new URL(request.url); + const isAbsolutePath = urlObject.host !== '.'; + // Redirect to webpack dev server if defined - if (process.env.DEV_SERVER_URL) { + if (process.env.DEV_SERVER_URL && !isAbsolutePath) { const devServerUrl = new URL( urlObject.pathname, process.env.DEV_SERVER_URL @@ -56,20 +58,30 @@ async function handleFileRequest(request: Request) { }); // this will be file types (in the web-static folder) let filepath = ''; - // if is a file type, load the file in resources - if (urlObject.pathname.split('/').at(-1)?.includes('.')) { - // Sanitize pathname to prevent path traversal attacks - const decodedPath = decodeURIComponent(urlObject.pathname); - const normalizedPath = join(webStaticDir, decodedPath).normalize(); - if (!normalizedPath.startsWith(webStaticDir)) { - // Attempted path traversal - reject by using empty path - filepath = join(webStaticDir, ''); + + // for relative path, load the file in resources + if (!isAbsolutePath) { + if (urlObject.pathname.split('/').at(-1)?.includes('.')) { + // Sanitize pathname to prevent path traversal attacks + const decodedPath = decodeURIComponent(urlObject.pathname); + const normalizedPath = join(webStaticDir, decodedPath).normalize(); + if (!normalizedPath.startsWith(webStaticDir)) { + // Attempted path traversal - reject by using empty path + filepath = join(webStaticDir, ''); + } else { + filepath = normalizedPath; + } } else { - filepath = normalizedPath; + // else, fallback to load the index.html instead + filepath = join(webStaticDir, 'index.html'); } } else { - // else, fallback to load the index.html instead - filepath = join(webStaticDir, 'index.html'); + filepath = decodeURIComponent(urlObject.pathname); + // security check if the filepath is within app.getPath('sessionData') + const sessionDataPath = app.getPath('sessionData'); + if (!filepath.startsWith(sessionDataPath)) { + throw new Error('Invalid filepath'); + } } return net.fetch('file://' + filepath, clonedRequest); } diff --git a/packages/frontend/apps/electron/src/main/recording/index.ts b/packages/frontend/apps/electron/src/main/recording/index.ts index ed9dde168..65295a42f 100644 --- a/packages/frontend/apps/electron/src/main/recording/index.ts +++ b/packages/frontend/apps/electron/src/main/recording/index.ts @@ -1,11 +1,13 @@ +import path from 'node:path'; + import { ShareableContent } from '@affine/native'; -import { nativeImage, Notification } from 'electron'; +import { app, nativeImage, Notification } from 'electron'; +import fs from 'fs-extra'; import { debounce } from 'lodash-es'; import { BehaviorSubject, distinctUntilChanged, groupBy, mergeMap } from 'rxjs'; import { isMacOS } from '../../shared/utils'; import { beforeAppQuit } from '../cleanup'; -import { ensureHelperProcess } from '../helper-process'; import { logger } from '../logger'; import type { NamespaceHandlers } from '../type'; import { getMainWindow } from '../windows-manager'; @@ -18,6 +20,11 @@ import type { const subscribers: Subscriber[] = []; +const SAVED_RECORDINGS_DIR = path.join( + app.getPath('sessionData'), + 'recordings' +); + beforeAppQuit(() => { subscribers.forEach(subscriber => { try { @@ -134,7 +141,9 @@ function setupNewRunningAppGroup() { recordingStatus$.value?.appGroup?.processGroupId === currentGroup.processGroupId ) { - stopRecording(); + stopRecording().catch(err => { + logger.error('failed to stop recording', err); + }); } } }) @@ -142,7 +151,13 @@ function setupNewRunningAppGroup() { } function createRecording(status: RecordingStatus) { - const buffers: Float32Array[] = []; + const bufferedFilePath = path.join( + SAVED_RECORDINGS_DIR, + `${status.appGroup?.bundleIdentifier ?? 'unknown'}-${status.id}-${status.startTime}.raw` + ); + + fs.ensureDirSync(SAVED_RECORDINGS_DIR); + const file = fs.createWriteStream(bufferedFilePath); function tapAudioSamples(err: Error | null, samples: Float32Array) { const recordingStatus = recordingStatus$.getValue(); @@ -157,7 +172,9 @@ function createRecording(status: RecordingStatus) { if (err) { logger.error('failed to get audio samples', err); } else { - buffers.push(new Float32Array(samples)); + // Writing raw Float32Array samples directly to file + // For stereo audio, samples are interleaved [L,R,L,R,...] + file.write(Buffer.from(samples.buffer)); } } @@ -170,44 +187,29 @@ function createRecording(status: RecordingStatus) { startTime: status.startTime, app: status.app, appGroup: status.appGroup, - buffers, + file, stream, }; return recording; } -function concatBuffers(buffers: Float32Array[]): Float32Array { - const totalSamples = buffers.reduce((acc, buf) => acc + buf.length, 0); - const buffer = new Float32Array(totalSamples); - let offset = 0; - buffers.forEach(buf => { - buffer.set(buf, offset); - offset += buf.length; - }); - return buffer; -} - -export async function saveRecording(id: number) { +export async function getRecording(id: number) { const recording = recordings.get(id); if (!recording) { logger.error(`Recording ${id} not found`); return; } - const { buffers } = recording; - const helperProcessManager = await ensureHelperProcess(); - const buffer = concatBuffers(buffers); - const mp3Buffer = await helperProcessManager.rpc?.encodeToMp3(buffer, { - channels: recording.stream.channels, + const rawFilePath = String(recording.file.path); + return { + id, + appGroup: recording.appGroup, + app: recording.app, + startTime: recording.startTime, + filepath: rawFilePath, sampleRate: recording.stream.sampleRate, - }); - - if (!mp3Buffer) { - logger.error('failed to encode audio samples to mp3'); - return; - } - recordings.delete(recording.id); - return mp3Buffer; + numberOfChannels: recording.stream.channels, + }; } function setupRecordingListeners() { @@ -386,9 +388,15 @@ export function resumeRecording() { }); } -export function stopRecording() { +export async function stopRecording() { const recordingStatus = recordingStatus$.value; if (!recordingStatus) { + logger.error('No recording status to stop'); + return; + } + const recording = recordings.get(recordingStatus?.id); + if (!recording) { + logger.error(`Recording ${recordingStatus?.id} not found`); return; } @@ -398,6 +406,15 @@ export function stopRecording() { status: 'stopped', }); + const { file } = recording; + file.end(); + + await new Promise(resolve => { + file.on('finish', () => { + resolve(); + }); + }); + // bring up the window getMainWindow() .then(mainWindow => { @@ -411,8 +428,17 @@ export function stopRecording() { } export const recordingHandlers = { - saveRecording: async (_, id: number) => { - return saveRecording(id); + getRecording: async (_, id: number) => { + return getRecording(id); + }, + deleteCachedRecording: async (_, id: number) => { + const recording = recordings.get(id); + if (recording) { + recording.stream.stop(); + recordings.delete(id); + await fs.unlink(recording.file.path); + } + return true; }, } satisfies NamespaceHandlers; diff --git a/packages/frontend/apps/electron/src/main/recording/types.ts b/packages/frontend/apps/electron/src/main/recording/types.ts index d308b5b11..33f0ada7e 100644 --- a/packages/frontend/apps/electron/src/main/recording/types.ts +++ b/packages/frontend/apps/electron/src/main/recording/types.ts @@ -1,3 +1,5 @@ +import type { WriteStream } from 'node:fs'; + import type { AudioTapStream, TappableApplication } from '@affine/native'; export interface TappableAppInfo { @@ -23,8 +25,8 @@ export interface Recording { // the app may not be available if the user choose to record system audio app?: TappableAppInfo; appGroup?: AppGroupInfo; - // the raw audio buffers that are already accumulated - buffers: Float32Array[]; + // the buffered file that is being recorded streamed to + file: WriteStream; stream: AudioTapStream; startTime: number; } diff --git a/packages/frontend/apps/electron/src/main/tray/index.ts b/packages/frontend/apps/electron/src/main/tray/index.ts index a6d8b21d0..26db9b072 100644 --- a/packages/frontend/apps/electron/src/main/tray/index.ts +++ b/packages/frontend/apps/electron/src/main/tray/index.ts @@ -198,7 +198,9 @@ class TrayState { label: 'Stop', click: () => { logger.info('User action: Stop Recording'); - stopRecording(); + stopRecording().catch(err => { + logger.error('Failed to stop recording:', err); + }); }, }, ], @@ -290,7 +292,7 @@ class TrayState { let _trayState: TrayState | undefined; -export const getTrayState = () => { +export const setupTrayState = () => { if (!_trayState) { _trayState = new TrayState(); _trayState.init(); diff --git a/packages/frontend/apps/electron/src/main/ui/events.ts b/packages/frontend/apps/electron/src/main/ui/events.ts index 3c4dd8b51..31fa44006 100644 --- a/packages/frontend/apps/electron/src/main/ui/events.ts +++ b/packages/frontend/apps/electron/src/main/ui/events.ts @@ -35,6 +35,12 @@ export const uiEvents = { }, onTabsStatusChange, onActiveTabChanged, + onTabGoToRequest: (fn: (opts: { tabId: string; to: string }) => void) => { + const sub = uiSubjects.tabGoToRequest$.subscribe(fn); + return () => { + sub.unsubscribe(); + }; + }, onTabShellViewActiveChange, onAuthenticationRequest: (fn: (state: AuthenticationRequest) => void) => { const sub = uiSubjects.authenticationRequest$.subscribe(fn); diff --git a/packages/frontend/apps/electron/src/main/ui/handlers.ts b/packages/frontend/apps/electron/src/main/ui/handlers.ts index 7480beb86..e1251eb46 100644 --- a/packages/frontend/apps/electron/src/main/ui/handlers.ts +++ b/packages/frontend/apps/electron/src/main/ui/handlers.ts @@ -10,6 +10,7 @@ import { activateView, addTab, closeTab, + ensureTabLoaded, getMainWindow, getOnboardingWindow, getTabsStatus, @@ -187,6 +188,12 @@ export const uiHandlers = { showTab: async (_, ...args: Parameters) => { await showTab(...args); }, + tabGoTo: async (_, tabId: string, to: string) => { + uiSubjects.tabGoToRequest$.next({ tabId, to }); + }, + ensureTabLoaded: async (_, ...args: Parameters) => { + await ensureTabLoaded(...args); + }, closeTab: async (_, ...args: Parameters) => { await closeTab(...args); }, diff --git a/packages/frontend/apps/electron/src/main/ui/subject.ts b/packages/frontend/apps/electron/src/main/ui/subject.ts index ac28173a4..0aab32188 100644 --- a/packages/frontend/apps/electron/src/main/ui/subject.ts +++ b/packages/frontend/apps/electron/src/main/ui/subject.ts @@ -6,6 +6,7 @@ export const uiSubjects = { onMaximized$: new Subject(), onFullScreen$: new Subject(), onToggleRightSidebar$: new Subject(), + tabGoToRequest$: new Subject<{ tabId: string; to: string }>(), authenticationRequest$: new Subject(), // via menu -> close view (CMD+W) onCloseView$: new Subject(), diff --git a/packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts b/packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts index 0b113b4b5..652ad39cc 100644 --- a/packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts +++ b/packages/frontend/apps/electron/src/main/windows-manager/tab-views.ts @@ -24,7 +24,7 @@ import { } from 'rxjs'; import { isMacOS } from '../../shared/utils'; -import { beforeAppQuit } from '../cleanup'; +import { beforeAppQuit, onTabClose } from '../cleanup'; import { mainWindowOrigin, shellViewUrl } from '../constants'; import { ensureHelperProcess } from '../helper-process'; import { logger } from '../logger'; @@ -400,6 +400,8 @@ export class WebContentViewsManager { }); } }, 500); // delay a bit to get rid of the flicker + + onTabClose(id); }; undoCloseTab = async () => { @@ -1052,6 +1054,13 @@ export const loadUrlInActiveTab = async (_url: string) => { // todo: implement throw new Error('loadUrlInActiveTab not implemented'); }; +export const ensureTabLoaded = async (tabId: string) => { + const tab = WebContentViewsManager.instance.tabViewsMap.get(tabId); + if (tab) { + return tab; + } + return WebContentViewsManager.instance.loadTab(tabId); +}; export const showTab = WebContentViewsManager.instance.showTab; export const closeTab = WebContentViewsManager.instance.closeTab; export const undoCloseTab = WebContentViewsManager.instance.undoCloseTab; diff --git a/packages/frontend/apps/electron/src/shared/type.ts b/packages/frontend/apps/electron/src/shared/type.ts index 900a56103..231ee3baf 100644 --- a/packages/frontend/apps/electron/src/shared/type.ts +++ b/packages/frontend/apps/electron/src/shared/type.ts @@ -17,13 +17,6 @@ export interface HelperToRenderer { // helper <-> main export interface HelperToMain { getMeta: () => ExposedMeta; - encodeToMp3: ( - samples: Float32Array, - opts?: { - channels?: number; - sampleRate?: number; - } - ) => Uint8Array; } export type MainToHelper = Pick< diff --git a/packages/frontend/component/src/lit-react/create-component.ts b/packages/frontend/component/src/lit-react/create-component.ts index 08ebc0a79..8d14d4e79 100644 --- a/packages/frontend/component/src/lit-react/create-component.ts +++ b/packages/frontend/component/src/lit-react/create-component.ts @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-3-Clause */ +import type { PropertyDeclaration } from 'lit'; import type React from 'react'; const DEV_MODE = process.env.NODE_ENV !== 'production'; @@ -164,6 +165,7 @@ const setProperty = ( name: string, value: unknown, old: unknown, + elementProperties?: Map, events?: EventNames ) => { const event = events?.[name]; @@ -175,6 +177,15 @@ const setProperty = ( // But don't dirty check properties; elements are assumed to do this. node[name as keyof E] = value as E[keyof E]; + if (elementProperties && elementProperties.has(name)) { + const property = elementProperties.get(name); + if (property?.attribute) { + const attributeName = + property.attribute === true ? name : property.attribute; + node.setAttribute(attributeName, value as string); + } + } + // This block is to replicate React's behavior for attributes of native // elements where `undefined` or `null` values result in attributes being // removed. @@ -302,6 +313,12 @@ export const createComponent = < props[prop], // @ts-expect-error: prop is a key of props prevPropsRef.current ? prevPropsRef.current[prop] : undefined, + 'elementProperties' in elementClass + ? (elementClass.elementProperties as Map< + string, + PropertyDeclaration + >) + : undefined, events ); } diff --git a/packages/frontend/core/package.json b/packages/frontend/core/package.json index 93708d780..fa6c4bd98 100644 --- a/packages/frontend/core/package.json +++ b/packages/frontend/core/package.json @@ -25,6 +25,7 @@ "@emotion/react": "^11.14.0", "@floating-ui/dom": "^1.6.13", "@juggle/resize-observer": "^3.4.0", + "@lit/context": "^1.1.4", "@marsidev/react-turnstile": "^1.1.0", "@preact/signals-core": "^1.8.0", "@radix-ui/react-collapsible": "^1.1.2", diff --git a/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts b/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts new file mode 100644 index 000000000..eeb037c83 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts @@ -0,0 +1,49 @@ +import { + BlockComponent, + BlockViewExtension, +} from '@blocksuite/affine/block-std'; +import type { TranscriptionBlockModel } from '@blocksuite/affine/model'; +import type { ExtensionType } from '@blocksuite/affine/store'; +import { css, type PropertyValues } from 'lit'; +import { property } from 'lit/decorators.js'; +import { literal } from 'lit/static-html.js'; + +export class LitTranscriptionBlock extends BlockComponent { + static override styles = [ + css` + transcription-block { + outline: none; + } + `, + ]; + override render() { + return this.std.host.renderChildren(this.model); + } + + @property({ type: String, attribute: 'data-block-id' }) + override accessor blockId!: string; + + constructor() { + super(); + // questionable: + this.widgets = {}; + + // to allow text selection across paragraphs in the callout block + this.contentEditable = 'true'; + } + + override firstUpdated(changedProperties: PropertyValues) { + super.firstUpdated(changedProperties); + this.disposables.addFromEvent(this, 'click', this.onClick); + } + + protected onClick(event: MouseEvent) { + event.stopPropagation(); + } +} + +export const AITranscriptionBlockSpec: ExtensionType[] = [ + BlockViewExtension('affine:transcription', () => { + return literal`transcription-block`; + }), +]; diff --git a/packages/frontend/core/src/blocksuite/ai/effects.ts b/packages/frontend/core/src/blocksuite/ai/effects.ts index d5c2cc5f4..db797dbe2 100644 --- a/packages/frontend/core/src/blocksuite/ai/effects.ts +++ b/packages/frontend/core/src/blocksuite/ai/effects.ts @@ -2,6 +2,7 @@ import { SpecProvider } from '@blocksuite/affine/shared/utils'; import { AIChatBlockComponent } from './blocks/ai-chat-block/ai-chat-block'; import { EdgelessAIChatBlockComponent } from './blocks/ai-chat-block/ai-chat-edgeless-block'; +import { LitTranscriptionBlock } from './blocks/ai-chat-block/ai-transcription-block'; import { AIChatMessage, AIChatMessages, @@ -151,5 +152,7 @@ export function registerAIEffects() { EdgelessCopilotToolbarEntry ); + customElements.define('transcription-block', LitTranscriptionBlock); + SpecProvider._.extendSpec('store', [AIChatBlockSchemaExtension]); } diff --git a/packages/frontend/core/src/blocksuite/ai/extensions/enable-ai.ts b/packages/frontend/core/src/blocksuite/ai/extensions/enable-ai.ts index 750837471..0a3ad8e57 100644 --- a/packages/frontend/core/src/blocksuite/ai/extensions/enable-ai.ts +++ b/packages/frontend/core/src/blocksuite/ai/extensions/enable-ai.ts @@ -9,6 +9,7 @@ import type { SpecBuilder } from '@blocksuite/affine/shared/utils'; import type { FrameworkProvider } from '@toeverything/infra'; import { AIChatBlockSpec } from '../blocks'; +import { AITranscriptionBlockSpec } from '../blocks/ai-chat-block/ai-transcription-block'; import { AICodeBlockSpec } from './ai-code'; import { createAIEdgelessRootBlockSpec } from './ai-edgeless-root'; import { AIImageBlockSpec } from './ai-image'; @@ -39,4 +40,5 @@ export function enableAIExtension( } specBuilder.extend(AIChatBlockSpec); + specBuilder.extend(AITranscriptionBlockSpec); } diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/attachment-embed-preview.tsx b/packages/frontend/core/src/blocksuite/attachment-viewer/attachment-embed-preview.tsx new file mode 100644 index 000000000..c861ec950 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/attachment-embed-preview.tsx @@ -0,0 +1,25 @@ +import { useMemo } from 'react'; + +import { AudioBlockEmbedded } from './audio/audio-block'; +import { AttachmentPreviewErrorBoundary } from './error'; +import { PDFViewerEmbedded } from './pdf/pdf-viewer-embedded'; +import type { AttachmentViewerProps } from './types'; +import { getAttachmentType } from './utils'; + +// In Embed view +export const AttachmentEmbedPreview = ({ model }: AttachmentViewerProps) => { + const attachmentType = getAttachmentType(model); + const element = useMemo(() => { + switch (attachmentType) { + case 'pdf': + return ; + case 'audio': + return ; + default: + return null; + } + }, [attachmentType, model]); + return ( + {element} + ); +}; diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.css.ts b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.css.ts new file mode 100644 index 000000000..3e2e9d4c2 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.css.ts @@ -0,0 +1,25 @@ +import { cssVar } from '@toeverything/theme'; +import { cssVarV2 } from '@toeverything/theme/v2'; +import { style } from '@vanilla-extract/css'; + +export const root = style({ + display: 'flex', + flexDirection: 'column', + gap: 12, + width: '100%', + height: 'auto', +}); + +export const notesButton = style({ + padding: '4px 8px', + color: cssVarV2('icon/primary'), + fontSize: cssVar('fontXs'), + fontWeight: 500, + userSelect: 'none', +}); + +export const notesButtonIcon = style({ + fontSize: 24, + width: '1em', + height: '1em', +}); diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx new file mode 100644 index 000000000..b9dd1c7c7 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx @@ -0,0 +1,116 @@ +import { Button } from '@affine/component'; +import { AudioPlayer } from '@affine/core/components/audio-player'; +import { useSeekTime } from '@affine/core/components/audio-player/use-seek-time'; +import { useEnableAI } from '@affine/core/components/hooks/affine/use-enable-ai'; +import type { AudioAttachmentBlock } from '@affine/core/modules/media/entities/audio-attachment-block'; +import { useAttachmentMediaBlock } from '@affine/core/modules/media/views/use-attachment-media'; +import { useI18n } from '@affine/i18n'; +import { TranscriptWithAiIcon } from '@blocksuite/icons/rc'; +import { useLiveData } from '@toeverything/infra'; +import { useCallback, useMemo } from 'react'; + +import type { AttachmentViewerProps } from '../types'; +import * as styles from './audio-block.css'; +import { TranscriptionBlock } from './transcription-block'; + +const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => { + const audioMedia = block.audioMedia; + const playbackState = useLiveData(audioMedia.playbackState$); + const stats = useLiveData(audioMedia.stats$); + const loading = useLiveData(audioMedia.loading$); + const expanded = useLiveData(block.expanded$); + const transcribing = useLiveData(block.transcribing$); + const transcribed = useLiveData(block.transcribed$); + const handleClick = useCallback((e: React.MouseEvent) => { + e.stopPropagation(); + }, []); + + const seekTime = useSeekTime(playbackState, stats.duration); + + const handlePlay = useCallback(() => { + audioMedia?.play(); + }, [audioMedia]); + + const handlePause = useCallback(() => { + audioMedia?.pause(); + }, [audioMedia]); + + const handleStop = useCallback(() => { + audioMedia?.stop(); + }, [audioMedia]); + + const handleSeek = useCallback( + (time: number) => { + audioMedia?.seekTo(time); + }, + [audioMedia] + ); + + const t = useI18n(); + + const enableAi = useEnableAI(); + + const notesEntry = useMemo(() => { + if (!enableAi) { + return null; + } + return ( + + ); + }, [enableAi, transcribing, t, transcribed, block, expanded]); + + return ( + + ); +}; + +export const AudioBlockEmbedded = (props: AttachmentViewerProps) => { + const audioAttachmentBlock = useAttachmentMediaBlock(props.model); + const transcriptionBlock = useLiveData( + audioAttachmentBlock?.transcriptionBlock$ + ); + const expanded = useLiveData(audioAttachmentBlock?.expanded$); + return ( +
+ {audioAttachmentBlock && ( + + )} + {transcriptionBlock && expanded && ( + + )} +
+ ); +}; diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/transcription-block.css.ts b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/transcription-block.css.ts new file mode 100644 index 000000000..271d737e7 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/transcription-block.css.ts @@ -0,0 +1,5 @@ +import { style } from '@vanilla-extract/css'; + +export const root = style({ + borderRadius: 6, +}); diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/transcription-block.tsx b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/transcription-block.tsx new file mode 100644 index 000000000..41f534611 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/transcription-block.tsx @@ -0,0 +1,40 @@ +import { createReactComponentFromLit } from '@affine/component'; +import { LitTranscriptionBlock } from '@affine/core/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block'; +import type { TranscriptionBlockModel } from '@blocksuite/affine/model'; +import { LiveData, useLiveData } from '@toeverything/infra'; +import React, { useCallback, useMemo } from 'react'; + +import * as styles from './transcription-block.css'; + +const AdaptedTranscriptionBlock = createReactComponentFromLit({ + react: React, + elementClass: LitTranscriptionBlock, +}); + +export const TranscriptionBlock = ({ + block, +}: { + block: TranscriptionBlockModel; +}) => { + const childMap$ = useMemo( + () => LiveData.fromSignal(block.childMap), + [block.childMap] + ); + const childMap = useLiveData(childMap$); + const onDragStart = useCallback((e: React.DragEvent) => { + e.preventDefault(); + }, []); + if (childMap.size === 0) { + return null; + } + return ( +
+ +
+ ); +}; diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/error.css.ts b/packages/frontend/core/src/blocksuite/attachment-viewer/error.css.ts new file mode 100644 index 000000000..648f92783 --- /dev/null +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/error.css.ts @@ -0,0 +1,33 @@ +import { cssVarV2 } from '@toeverything/theme/v2'; +import { style } from '@vanilla-extract/css'; + +export { viewer } from './viewer.css'; + +export const error = style({ + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + gap: '4px', +}); + +export const errorTitle = style({ + fontSize: '15px', + fontWeight: 500, + lineHeight: '24px', + color: cssVarV2('text/primary'), + marginTop: '12px', +}); + +export const errorMessage = style({ + fontSize: '12px', + fontWeight: 500, + lineHeight: '20px', + color: cssVarV2('text/tertiary'), +}); + +export const errorBtns = style({ + display: 'flex', + flexDirection: 'column', + gap: '10px', + marginTop: '28px', +}); diff --git a/packages/frontend/core/src/components/attachment-viewer/error.tsx b/packages/frontend/core/src/blocksuite/attachment-viewer/error.tsx similarity index 98% rename from packages/frontend/core/src/components/attachment-viewer/error.tsx rename to packages/frontend/core/src/blocksuite/attachment-viewer/error.tsx index 237f6ea60..a50000fad 100644 --- a/packages/frontend/core/src/components/attachment-viewer/error.tsx +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/error.tsx @@ -7,7 +7,7 @@ import type { PropsWithChildren, ReactElement } from 'react'; import { Suspense } from 'react'; import { ErrorBoundary, type FallbackProps } from 'react-error-boundary'; -import * as styles from './styles.css'; +import * as styles from './error.css'; import { download } from './utils'; // https://github.com/toeverything/blocksuite/blob/master/packages/affine/components/src/icons/file-icons.ts @@ -67,7 +67,7 @@ interface ErrorBaseProps { buttons?: ReactElement[]; } -export const ErrorBase = ({ +const ErrorBase = ({ title, subtitle, icon = , @@ -88,7 +88,7 @@ interface ErrorProps { ext: string; } -export const Error = ({ model, ext }: ErrorProps) => { +export const AttachmentFallback = ({ model, ext }: ErrorProps) => { const t = useI18n(); const Icon = FILE_ICONS[model.props.type] ?? FileIcon; const title = t['com.affine.attachment.preview.error.title'](); diff --git a/packages/frontend/core/src/components/attachment-viewer/index.tsx b/packages/frontend/core/src/blocksuite/attachment-viewer/index.tsx similarity index 69% rename from packages/frontend/core/src/components/attachment-viewer/index.tsx rename to packages/frontend/core/src/blocksuite/attachment-viewer/index.tsx index 3bd1e48ac..43d383862 100644 --- a/packages/frontend/core/src/components/attachment-viewer/index.tsx +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/index.tsx @@ -1,11 +1,11 @@ import { ViewBody, ViewHeader } from '@affine/core/modules/workbench'; -import { AttachmentPreviewErrorBoundary, Error } from './error'; -import { PDFViewer } from './pdf-viewer'; -import * as styles from './styles.css'; -import { Titlebar } from './titlebar'; -import type { AttachmentViewerProps, PDFViewerProps } from './types'; +import { AttachmentFallback, AttachmentPreviewErrorBoundary } from './error'; +import { PDFViewer } from './pdf/pdf-viewer'; +import type { AttachmentViewerBaseProps, AttachmentViewerProps } from './types'; import { buildAttachmentProps } from './utils'; +import { Titlebar } from './viewer'; +import * as styles from './viewer.css'; // In Peek view export const AttachmentViewer = ({ model }: AttachmentViewerProps) => { @@ -35,12 +35,12 @@ export const AttachmentViewerView = ({ model }: AttachmentViewerProps) => { ); }; -const AttachmentViewerInner = (props: PDFViewerProps) => { +const AttachmentViewerInner = (props: AttachmentViewerBaseProps) => { return props.model.props.type.endsWith('pdf') ? ( ) : ( - + ); }; diff --git a/packages/frontend/core/src/components/attachment-viewer/pdf-viewer-embedded-inner.tsx b/packages/frontend/core/src/blocksuite/attachment-viewer/pdf/pdf-viewer-embedded.tsx similarity index 87% rename from packages/frontend/core/src/components/attachment-viewer/pdf-viewer-embedded-inner.tsx rename to packages/frontend/core/src/blocksuite/attachment-viewer/pdf/pdf-viewer-embedded.tsx index 664ea53cd..957cee859 100644 --- a/packages/frontend/core/src/components/attachment-viewer/pdf-viewer-embedded-inner.tsx +++ b/packages/frontend/core/src/blocksuite/attachment-viewer/pdf/pdf-viewer-embedded.tsx @@ -28,9 +28,8 @@ import { useState, } from 'react'; +import type { AttachmentViewerProps } from '../types'; import * as styles from './styles.css'; -import * as embeddedStyles from './styles.embedded.css'; -import type { PDFViewerProps } from './types'; function defaultMeta() { return { @@ -40,7 +39,7 @@ function defaultMeta() { }; } -export function PDFViewerEmbeddedInner({ model }: PDFViewerProps) { +export function PDFViewerEmbedded({ model }: AttachmentViewerProps) { const scale = window.devicePixelRatio; const peekView = useService(PeekViewService).peekView; const pdfService = useService(PDFService); @@ -194,8 +193,8 @@ export function PDFViewerEmbeddedInner({ model }: PDFViewerProps) { }, []); return ( -
-
+
+
-
+
} - className={embeddedStyles.pdfControlButton} + className={styles.pdfControlButton} onDoubleClick={stopPropagation} aria-label="Prev" {...navigator.prev} @@ -228,7 +227,7 @@ export function PDFViewerEmbeddedInner({ model }: PDFViewerProps) { } - className={embeddedStyles.pdfControlButton} + className={styles.pdfControlButton} onDoubleClick={stopPropagation} aria-label="Next" {...navigator.next} @@ -236,27 +235,18 @@ export function PDFViewerEmbeddedInner({ model }: PDFViewerProps) { } - className={embeddedStyles.pdfControlButton} + className={styles.pdfControlButton} onDoubleClick={stopPropagation} {...navigator.peek} />
-