From 7c8b977bf977177ff1e410a41c1df7736f3aaf52 Mon Sep 17 00:00:00 2001 From: EYHN Date: Thu, 8 May 2025 19:34:21 +0800 Subject: [PATCH] feat(core): migration for created by and updated by fields (#12171) --- .../get-doc-created-by-updated-by-list.gql | 18 +++ packages/common/graphql/src/graphql/index.ts | 23 +++ packages/common/graphql/src/schema.ts | 35 ++++ .../pages/workspace/all-page-new/all-page.tsx | 2 + .../all-page-new/migration-data.css.ts | 31 ++++ .../workspace/all-page-new/migration-data.tsx | 58 +++++++ .../frontend/core/src/modules/cloud/index.ts | 17 +- .../doc-created-by-updated-by-sync.ts | 150 ++++++++++++++++++ .../stores/doc-created-by-updated-by-sync.ts | 47 ++++++ .../permissions/entities/permission.ts | 3 + .../modules/workspace-engine/impls/cloud.ts | 54 ++++++- packages/frontend/i18n/src/i18n.gen.ts | 18 +++ packages/frontend/i18n/src/resources/en.json | 4 + 13 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 packages/common/graphql/src/graphql/get-doc-created-by-updated-by-list.gql create mode 100644 packages/frontend/core/src/desktop/pages/workspace/all-page-new/migration-data.css.ts create mode 100644 packages/frontend/core/src/desktop/pages/workspace/all-page-new/migration-data.tsx create mode 100644 packages/frontend/core/src/modules/cloud/services/doc-created-by-updated-by-sync.ts create mode 100644 packages/frontend/core/src/modules/cloud/stores/doc-created-by-updated-by-sync.ts diff --git a/packages/common/graphql/src/graphql/get-doc-created-by-updated-by-list.gql b/packages/common/graphql/src/graphql/get-doc-created-by-updated-by-list.gql new file mode 100644 index 000000000..7fe4fe960 --- /dev/null +++ b/packages/common/graphql/src/graphql/get-doc-created-by-updated-by-list.gql @@ -0,0 +1,18 @@ +query getDocCreatedByUpdatedByList($workspaceId: String!, $pagination: PaginationInput!) { + workspace(id: $workspaceId) { + docs(pagination: $pagination) { + totalCount + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + creatorId + lastUpdaterId + } + } + } + } +} \ No newline at end of file diff --git a/packages/common/graphql/src/graphql/index.ts b/packages/common/graphql/src/graphql/index.ts index 262d0620c..c56725cfd 100644 --- a/packages/common/graphql/src/graphql/index.ts +++ b/packages/common/graphql/src/graphql/index.ts @@ -981,6 +981,29 @@ export const getCurrentUserQuery = { deprecations: ["'token' is deprecated: use [/api/auth/sign-in?native=true] instead"], }; +export const getDocCreatedByUpdatedByListQuery = { + id: 'getDocCreatedByUpdatedByListQuery' as const, + op: 'getDocCreatedByUpdatedByList', + query: `query getDocCreatedByUpdatedByList($workspaceId: String!, $pagination: PaginationInput!) { + workspace(id: $workspaceId) { + docs(pagination: $pagination) { + totalCount + pageInfo { + endCursor + hasNextPage + } + edges { + node { + id + creatorId + lastUpdaterId + } + } + } + } +}`, +}; + export const getDocDefaultRoleQuery = { id: 'getDocDefaultRoleQuery' as const, op: 'getDocDefaultRole', diff --git a/packages/common/graphql/src/schema.ts b/packages/common/graphql/src/schema.ts index 6018eb088..232c4f69b 100644 --- a/packages/common/graphql/src/schema.ts +++ b/packages/common/graphql/src/schema.ts @@ -3603,6 +3603,36 @@ export type GetCurrentUserQuery = { } | null; }; +export type GetDocCreatedByUpdatedByListQueryVariables = Exact<{ + workspaceId: Scalars['String']['input']; + pagination: PaginationInput; +}>; + +export type GetDocCreatedByUpdatedByListQuery = { + __typename?: 'Query'; + workspace: { + __typename?: 'WorkspaceType'; + docs: { + __typename?: 'PaginatedDocType'; + totalCount: number; + pageInfo: { + __typename?: 'PageInfo'; + endCursor: string | null; + hasNextPage: boolean; + }; + edges: Array<{ + __typename?: 'DocTypeEdge'; + node: { + __typename?: 'DocType'; + id: string; + creatorId: string | null; + lastUpdaterId: string | null; + }; + }>; + }; + }; +}; + export type GetDocDefaultRoleQueryVariables = Exact<{ workspaceId: Scalars['String']['input']; docId: Scalars['String']['input']; @@ -4736,6 +4766,11 @@ export type Queries = variables: GetCurrentUserQueryVariables; response: GetCurrentUserQuery; } + | { + name: 'getDocCreatedByUpdatedByListQuery'; + variables: GetDocCreatedByUpdatedByListQueryVariables; + response: GetDocCreatedByUpdatedByListQuery; + } | { name: 'getDocDefaultRoleQuery'; variables: GetDocDefaultRoleQueryVariables; diff --git a/packages/frontend/core/src/desktop/pages/workspace/all-page-new/all-page.tsx b/packages/frontend/core/src/desktop/pages/workspace/all-page-new/all-page.tsx index 1ce3915d9..c33f41380 100644 --- a/packages/frontend/core/src/desktop/pages/workspace/all-page-new/all-page.tsx +++ b/packages/frontend/core/src/desktop/pages/workspace/all-page-new/all-page.tsx @@ -15,6 +15,7 @@ import { } from '../../../../modules/workbench'; import { AllDocSidebarTabs } from '../layouts/all-doc-sidebar-tabs'; import * as styles from './all-page.css'; +import { MigrationAllDocsDataNotification } from './migration-data'; export const AllPage = () => { const t = useI18n(); @@ -57,6 +58,7 @@ export const AllPage = () => {
+
{ + const t = useI18n(); + const docCreatedByUpdatedBySyncService = useService( + DocCreatedByUpdatedBySyncService + ); + const needSync = useLiveData(docCreatedByUpdatedBySyncService.needSync$); + const syncing = useLiveData(docCreatedByUpdatedBySyncService.syncing$); + const error = useLiveData(docCreatedByUpdatedBySyncService.error$); + const errorMessage = useMemo(() => { + if (error) { + const userFriendlyError = UserFriendlyError.fromAny(error); + return t[`error.${userFriendlyError.name}`](userFriendlyError.data); + } + return null; + }, [error, t]); + const progress = useLiveData(docCreatedByUpdatedBySyncService.progress$); + + const handleSync = useCallback(() => { + docCreatedByUpdatedBySyncService.sync(); + }, [docCreatedByUpdatedBySyncService]); + + if (!needSync) { + return null; + } + + return ( +
+
+ {t['com.affine.migration-all-docs-notification.title']()} +
+
+ {t['com.affine.migration-all-docs-notification.content']()} +
+ + {error && ( +
+ {t['com.affine.migration-all-docs-notification.error']({ + errorMessage: errorMessage ?? '', + })} +
+ )} + + +
+ ); +}; diff --git a/packages/frontend/core/src/modules/cloud/index.ts b/packages/frontend/core/src/modules/cloud/index.ts index c5e48a8b2..89ec23d29 100644 --- a/packages/frontend/core/src/modules/cloud/index.ts +++ b/packages/frontend/core/src/modules/cloud/index.ts @@ -10,6 +10,7 @@ export { ServerScope } from './scopes/server'; export { AuthService } from './services/auth'; export { CaptchaService } from './services/captcha'; export { DefaultServerService } from './services/default-server'; +export { DocCreatedByUpdatedBySyncService } from './services/doc-created-by-updated-by-sync'; export { EventSourceService } from './services/eventsource'; export { FetchService } from './services/fetch'; export { GraphQLService } from './services/graphql'; @@ -98,6 +99,10 @@ import { UserQuotaStore } from './stores/user-quota'; import { UserSettingsStore } from './stores/user-settings'; import { DocCreatedByService } from './services/doc-created-by'; import { DocUpdatedByService } from './services/doc-updated-by'; +import { DocCreatedByUpdatedBySyncService } from './services/doc-created-by-updated-by-sync'; +import { WorkspacePermissionService } from '../permissions'; +import { DocsService } from '../doc'; +import { DocCreatedByUpdatedBySyncStore } from './stores/doc-created-by-updated-by-sync'; export function configureCloudModule(framework: Framework) { configureDefaultAuthProvider(framework); @@ -181,5 +186,15 @@ export function configureCloudModule(framework: Framework) { .entity(WorkspaceInvoices, [WorkspaceService, WorkspaceServerService]) .service(SelfhostLicenseService, [SelfhostLicenseStore, WorkspaceService]) .store(SelfhostLicenseStore, [WorkspaceServerService]) - .service(BlocksuiteWriterInfoService, [WorkspaceServerService]); + .service(BlocksuiteWriterInfoService, [WorkspaceServerService]) + .service(DocCreatedByUpdatedBySyncService, [ + WorkspaceService, + DocsService, + WorkspacePermissionService, + DocCreatedByUpdatedBySyncStore, + ]) + .store(DocCreatedByUpdatedBySyncStore, [ + WorkspaceServerService, + WorkspaceService, + ]); } diff --git a/packages/frontend/core/src/modules/cloud/services/doc-created-by-updated-by-sync.ts b/packages/frontend/core/src/modules/cloud/services/doc-created-by-updated-by-sync.ts new file mode 100644 index 000000000..9e8682b56 --- /dev/null +++ b/packages/frontend/core/src/modules/cloud/services/doc-created-by-updated-by-sync.ts @@ -0,0 +1,150 @@ +import { + catchErrorInto, + effect, + fromPromise, + LiveData, + onStart, + Service, + throwIfAborted, +} from '@toeverything/infra'; +import { clamp } from 'lodash-es'; +import { combineLatest, exhaustMap, finalize, map } from 'rxjs'; + +import type { DocsService } from '../../doc'; +import type { WorkspacePermissionService } from '../../permissions'; +import type { WorkspaceService } from '../../workspace'; +import type { DocCreatedByUpdatedBySyncStore } from '../stores/doc-created-by-updated-by-sync'; + +/** + * This service is used to sync createdBy and updatedBy data from the cloud to local doc properties. + * + * # When sync is needed + * + * 1. When the user is an owner or admin + * 2. When the root doc sync is complete + * 3. When a doc is missing createdBy data + * 4. When workspace has not been marked as `DocCreatedByUpdatedBySynced` + */ +export class DocCreatedByUpdatedBySyncService extends Service { + constructor( + private readonly workspaceService: WorkspaceService, + private readonly docsService: DocsService, + private readonly workspacePermissionService: WorkspacePermissionService, + private readonly docCreatedByUpdatedBySyncStore: DocCreatedByUpdatedBySyncStore + ) { + super(); + } + + syncing$ = new LiveData(false); + error$ = new LiveData(null); + // sync progress 0.0 - 1.0 + progress$ = new LiveData(0); + + sync = effect( + exhaustMap(() => { + return fromPromise(async (signal?: AbortSignal) => { + let afterCursor: string | null = null; + let finishedCount = 0; + while (true) { + const result = + await this.docCreatedByUpdatedBySyncStore.getDocCreatedByUpdatedByList( + afterCursor + ); + throwIfAborted(signal); + + for (const edge of result.workspace.docs.edges) { + const docId = edge.node.id; + const docRecord = this.docsService.list.doc$(docId).value; + if (docRecord) { + if (edge.node.creatorId) { + docRecord.setCreatedBy(edge.node.creatorId); + } + if (edge.node.lastUpdaterId) { + docRecord.setUpdatedBy(edge.node.lastUpdaterId); + } + } + finishedCount++; + } + this.progress$.value = clamp( + finishedCount / result.workspace.docs.totalCount, + 0, + 1 + ); + if (!result.workspace.docs.pageInfo.hasNextPage) { + break; + } + afterCursor = result.workspace.docs.pageInfo.endCursor; + } + + this.docCreatedByUpdatedBySyncStore.setDocCreatedByUpdatedBySynced( + true + ); + }).pipe( + catchErrorInto(this.error$), + onStart(() => { + this.syncing$.value = true; + this.progress$.value = 0; + this.error$.value = null; + }), + finalize(() => { + this.syncing$.value = false; + }) + ); + }) + ); + + private readonly workspaceRootDocSynced$ = + this.workspaceService.workspace.engine.doc + .docState$(this.workspaceService.workspace.id) + .pipe(map(doc => doc.synced)); + + private readonly isOwnerOrAdmin$ = + this.workspacePermissionService.permission.isOwnerOrAdmin$; + + private readonly missingCreatedBy$ = this.docsService + .propertyValues$('createdBy') + .pipe( + map(allDocsCreatedBy => { + let missingCreatedBy = false; + console.log(allDocsCreatedBy); + for (const createdBy of allDocsCreatedBy.values()) { + if (!createdBy) { + missingCreatedBy = true; + break; + } + } + return missingCreatedBy; + }) + ); + + private readonly markedSynced$ = + this.docCreatedByUpdatedBySyncStore.watchDocCreatedByUpdatedBySynced(); + + needSync$ = LiveData.from( + combineLatest([ + this.workspaceRootDocSynced$, + this.isOwnerOrAdmin$, + this.missingCreatedBy$, + this.markedSynced$, + ]).pipe( + map( + ([ + workspaceRootDocSynced, + isOwnerOrAdmin, + missingCreatedBy, + markedSynced, + ]) => + workspaceRootDocSynced && + isOwnerOrAdmin && + missingCreatedBy && + !markedSynced + ) + ), + false + ); + + override dispose(): void { + super.dispose(); + this.sync.unsubscribe(); + } +} diff --git a/packages/frontend/core/src/modules/cloud/stores/doc-created-by-updated-by-sync.ts b/packages/frontend/core/src/modules/cloud/stores/doc-created-by-updated-by-sync.ts new file mode 100644 index 000000000..9d0ceb679 --- /dev/null +++ b/packages/frontend/core/src/modules/cloud/stores/doc-created-by-updated-by-sync.ts @@ -0,0 +1,47 @@ +import { getDocCreatedByUpdatedByListQuery } from '@affine/graphql'; +import { Store, yjsGetPath } from '@toeverything/infra'; +import type { Observable } from 'rxjs'; + +import type { WorkspaceService } from '../../workspace'; +import type { WorkspaceServerService } from '../services/workspace-server'; + +export class DocCreatedByUpdatedBySyncStore extends Store { + constructor( + private readonly workspaceServerService: WorkspaceServerService, + private readonly workspaceService: WorkspaceService + ) { + super(); + } + + async getDocCreatedByUpdatedByList(afterCursor?: string | null) { + if (!this.workspaceServerService.server) { + throw new Error('Server not found'); + } + + return await this.workspaceServerService.server.gql({ + query: getDocCreatedByUpdatedByListQuery, + variables: { + workspaceId: this.workspaceService.workspace.id, + pagination: { + first: 100, + after: afterCursor, + }, + }, + }); + } + + watchDocCreatedByUpdatedBySynced() { + const rootYDoc = this.workspaceService.workspace.rootYDoc; + return yjsGetPath( + rootYDoc.getMap('affine:workspace-properties'), + 'docCreatedByUpdatedBySynced' + ) as Observable; + } + + setDocCreatedByUpdatedBySynced(synced: boolean) { + const rootYDoc = this.workspaceService.workspace.rootYDoc; + rootYDoc + .getMap('affine:workspace-properties') + .set('docCreatedByUpdatedBySynced', synced); + } +} diff --git a/packages/frontend/core/src/modules/permissions/entities/permission.ts b/packages/frontend/core/src/modules/permissions/entities/permission.ts index 62c43ba0c..bdfd49013 100644 --- a/packages/frontend/core/src/modules/permissions/entities/permission.ts +++ b/packages/frontend/core/src/modules/permissions/entities/permission.ts @@ -20,6 +20,9 @@ export class WorkspacePermission extends Entity { ); isOwner$ = this.cache$.map(cache => cache?.isOwner ?? null); isAdmin$ = this.cache$.map(cache => cache?.isAdmin ?? null); + isOwnerOrAdmin$ = this.cache$.map( + cache => (cache?.isOwner ?? null) || (cache?.isAdmin ?? null) + ); isTeam$ = this.cache$.map(cache => cache?.isTeam ?? null); isRevalidating$ = new LiveData(false); diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts b/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts index 0f9f953b0..3a4f49df9 100644 --- a/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts +++ b/packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts @@ -46,7 +46,13 @@ import { } from '@toeverything/infra'; import { isEqual } from 'lodash-es'; import { map, Observable, switchMap, tap } from 'rxjs'; -import { Doc as YDoc, encodeStateAsUpdate } from 'yjs'; +import { + applyUpdate, + type Array as YArray, + Doc as YDoc, + encodeStateAsUpdate, + type Map as YMap, +} from 'yjs'; import type { Server, ServersService } from '../../cloud'; import { @@ -209,6 +215,13 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { }); } + const accountId = this.authService.session.account$.value?.id; + await this.writeInitialDocProperties( + workspaceId, + docStorage, + accountId ?? '' + ); + docStorage.connection.disconnect(); blobStorage.connection.disconnect(); @@ -533,6 +546,45 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider { }; } + async writeInitialDocProperties( + workspaceId: string, + docStorage: DocStorage, + creatorId: string + ) { + try { + const rootDocBuffer = await docStorage.getDoc(workspaceId); + const rootDoc = new YDoc({ guid: workspaceId }); + if (rootDocBuffer) { + applyUpdate(rootDoc, rootDocBuffer.bin); + } + + const docIds = ( + rootDoc.getMap('meta').get('pages') as YArray> + ) + ?.map(page => page.get('id')) + .filter(Boolean) as string[]; + + const propertiesDBBuffer = await docStorage.getDoc('db$docProperties'); + const propertiesDB = new YDoc({ guid: 'db$docProperties' }); + if (propertiesDBBuffer) { + applyUpdate(propertiesDB, propertiesDBBuffer.bin); + } + + for (const docId of docIds) { + const docProperties = propertiesDB.getMap(docId); + docProperties.set('id', docId); + docProperties.set('createdBy', creatorId); + } + + await docStorage.pushDocUpdate({ + docId: 'db$docProperties', + bin: encodeStateAsUpdate(propertiesDB), + }); + } catch (error) { + logger.error('error to write initial doc properties', error); + } + } + private waitForLoaded() { return this.isRevalidating$.waitFor(loading => !loading); } diff --git a/packages/frontend/i18n/src/i18n.gen.ts b/packages/frontend/i18n/src/i18n.gen.ts index 59f43fddf..9f72efab9 100644 --- a/packages/frontend/i18n/src/i18n.gen.ts +++ b/packages/frontend/i18n/src/i18n.gen.ts @@ -7846,6 +7846,24 @@ export function useAFFiNEI18N(): { * `Stop` */ ["com.affine.recording.stop"](): string; + /** + * `All docs data needs to be migrated in order to fully utilize its features.` + */ + ["com.affine.migration-all-docs-notification.title"](): string; + /** + * `We upgraded the data format, requiring local data migration to utilize features. Created by and Updated by info is now stored locally. Without the update, you will see the actual status. Perform the upgrade under better network conditions.` + */ + ["com.affine.migration-all-docs-notification.content"](): string; + /** + * `Migration failed: {{errorMessage}}` + */ + ["com.affine.migration-all-docs-notification.error"](options: { + readonly errorMessage: string; + }): string; + /** + * `Migrate data` + */ + ["com.affine.migration-all-docs-notification.button"](): string; /** * `An internal error occurred.` */ diff --git a/packages/frontend/i18n/src/resources/en.json b/packages/frontend/i18n/src/resources/en.json index 1b567d46f..b1ac55ba6 100644 --- a/packages/frontend/i18n/src/resources/en.json +++ b/packages/frontend/i18n/src/resources/en.json @@ -1965,6 +1965,10 @@ "com.affine.recording.start": "Start", "com.affine.recording.dismiss": "Dismiss", "com.affine.recording.stop": "Stop", + "com.affine.migration-all-docs-notification.title": "All docs data needs to be migrated in order to fully utilize its features.", + "com.affine.migration-all-docs-notification.content": "We upgraded the data format, requiring local data migration to utilize features. Created by and Updated by info is now stored locally. Without the update, you will see the actual status. Perform the upgrade under better network conditions.", + "com.affine.migration-all-docs-notification.error": "Migration failed: {{errorMessage}}", + "com.affine.migration-all-docs-notification.button": "Migrate data", "error.INTERNAL_SERVER_ERROR": "An internal error occurred.", "error.NETWORK_ERROR": "Network error.", "error.TOO_MANY_REQUEST": "Too many requests.",