diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md index 4f23bb22d..be3918856 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.md @@ -67,7 +67,7 @@ Generated by [AVA](https://avajs.dev). 'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', ], created_at: [ - '2025-03-08T06:04:13.278Z', + Date 2025-03-08 06:04:13 278ms UTC {}, ], doc_id: [ 'docId2', @@ -89,7 +89,7 @@ Generated by [AVA](https://avajs.dev). 'docId1', ], updated_at: [ - '2025-03-08T06:04:13.278Z', + Date 2025-03-08 06:04:13 278ms UTC {}, ], }, highlights: { @@ -119,7 +119,7 @@ Generated by [AVA](https://avajs.dev). 'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', ], created_at: [ - '2025-03-08T06:04:13.278Z', + Date 2025-03-08 06:04:13 278ms UTC {}, ], doc_id: [ 'docId2', @@ -141,7 +141,7 @@ Generated by [AVA](https://avajs.dev). 'docId1', ], updated_at: [ - '2025-03-08T06:04:13.278Z', + Date 2025-03-08 06:04:13 278ms UTC {}, ], }, highlights: { diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap index dc889bfec..b2ff26081 100644 Binary files a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap and b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/elasticsearch.spec.ts.snap differ diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md index 5807a24e7..29990dcf0 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.md @@ -485,7 +485,7 @@ Generated by [AVA](https://avajs.dev). 'title8 hello hello hello hello hello hello hello hello hello hello, hello hello hello hello hello hello hello hello some link https://linear.app/affine-design/issue/AF-1379/slash-commands-%E6%BF%80%E6%B4%BB%E6%8F%92%E5%85%A5-link-%E7%9A%84%E5%BC%B9%E7%AA%97%E9%87%8C%EF%BC%8C%E8%BE%93%E5%85%A5%E9%93%BE%E6%8E%A5%E4%B9%8B%E5%90%8E%E4%B8%8D%E5%BA%94%E8%AF%A5%E7%9B%B4%E6%8E%A5%E5%AF%B9%E9%93%BE%E6%8E%A5%E8%BF%9B%E8%A1%8C%E5%88%86%E8%AF%8D%E6%90%9C%E7%B4%A2', ], created_at: [ - 1741413853, + Date 2025-03-08 06:04:13 UTC {}, ], doc_id: [ 'docId2', @@ -507,7 +507,7 @@ Generated by [AVA](https://avajs.dev). 'docId1', ], updated_at: [ - 1741413853, + Date 2025-03-08 06:04:13 UTC {}, ], }, highlights: { diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap index 95b68a72a..d9d8b475e 100644 Binary files a/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap and b/packages/backend/server/src/plugins/indexer/__tests__/providers/__snapshots__/manticoresearch.spec.ts.snap differ diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts index 0af7ce0fa..4124b70d1 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/elasticsearch.spec.ts @@ -1477,6 +1477,49 @@ test('should not return not exists field:ref_doc_id', async t => { t.snapshot(result.nodes.map(node => omit(node, ['_score']))); }); +test('should created_at and updated_at is date type', async t => { + const result = await searchProvider.search(SearchTable.block, { + _source: ['workspace_id', 'doc_id', 'created_at', 'updated_at'], + sort: [ + '_score', + { + updated_at: 'desc', + }, + 'doc_id', + 'block_id', + ], + query: { + match_all: {}, + }, + fields: ['created_at', 'updated_at'], + size: 2, + }); + + t.truthy(result.total); + t.truthy(result.nodes[0].fields.created_at); + t.truthy(result.nodes[0].fields.updated_at); + t.true( + result.nodes[0].fields.created_at[0] instanceof Date, + 'created_at should be date type, but got ' + + result.nodes[0].fields.created_at[0] + ); + t.true( + result.nodes[0].fields.updated_at[0] instanceof Date, + 'updated_at should be date type, but got ' + + result.nodes[0].fields.updated_at[0] + ); + t.true( + result.nodes[0]._source.created_at instanceof Date, + 'created_at should be date type, but got ' + + result.nodes[0]._source.created_at + ); + t.true( + result.nodes[0]._source.updated_at instanceof Date, + 'updated_at should be date type, but got ' + + result.nodes[0]._source.updated_at + ); +}); + // #endregion // #region aggregate diff --git a/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts b/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts index 0706a8b1e..ae16c76ac 100644 --- a/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts +++ b/packages/backend/server/src/plugins/indexer/__tests__/providers/manticoresearch.spec.ts @@ -1232,6 +1232,49 @@ test('should not return not exists field:ref_doc_id', async t => { t.snapshot(result.nodes.map(node => omit(node, ['_score']))); }); +test('should created_at and updated_at is date type', async t => { + const result = await searchProvider.search(SearchTable.block, { + _source: ['workspace_id', 'doc_id', 'created_at', 'updated_at'], + sort: [ + '_score', + { + updated_at: 'desc', + }, + 'doc_id', + 'block_id', + ], + query: { + match_all: {}, + }, + fields: ['created_at', 'updated_at'], + size: 2, + }); + + t.truthy(result.total); + t.truthy(result.nodes[0].fields.created_at); + t.truthy(result.nodes[0].fields.updated_at); + t.true( + result.nodes[0].fields.created_at[0] instanceof Date, + 'created_at should be date type, but got ' + + result.nodes[0].fields.created_at[0] + ); + t.true( + result.nodes[0].fields.updated_at[0] instanceof Date, + 'updated_at should be date type, but got ' + + result.nodes[0].fields.updated_at[0] + ); + t.true( + result.nodes[0]._source.created_at instanceof Date, + 'created_at should be date type, but got ' + + result.nodes[0]._source.created_at + ); + t.true( + result.nodes[0]._source.updated_at instanceof Date, + 'updated_at should be date type, but got ' + + result.nodes[0]._source.updated_at + ); +}); + // #endregion // #region aggregate diff --git a/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts b/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts index 46eedaa0e..f80d765ce 100644 --- a/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts +++ b/packages/backend/server/src/plugins/indexer/providers/elasticsearch.ts @@ -5,7 +5,7 @@ import { InvalidSearchProviderRequest, } from '../../../base'; import { SearchProviderType } from '../config'; -import { SearchTable, SearchTableUniqueId } from '../tables'; +import { DateFieldNames, SearchTable, SearchTableUniqueId } from '../tables'; import { AggregateQueryDSL, AggregateResult, @@ -161,8 +161,8 @@ export class ElasticsearchProvider extends SearchProvider { nodes: data.hits.hits.map(hit => ({ _id: hit._id, _score: hit._score, - _source: hit._source, - fields: hit.fields, + _source: this.formatDateFields(hit._source), + fields: this.formatDateFields(hit.fields), highlights: hit.highlight, })), }; @@ -187,8 +187,8 @@ export class ElasticsearchProvider extends SearchProvider { nodes: bucket.result.hits.hits.map(hit => ({ _id: hit._id, _score: hit._score, - _source: hit._source, - fields: hit.fields, + _source: this.formatDateFields(hit._source), + fields: this.formatDateFields(hit.fields), highlights: hit.highlight, })), }, @@ -196,6 +196,37 @@ export class ElasticsearchProvider extends SearchProvider { }; } + protected formatDateFields>( + fieldsOrSource: T + ): T { + for (const fieldName of DateFieldNames) { + let values = fieldsOrSource[fieldName]; + if (!values) { + continue; + } + if (Array.isArray(values)) { + // { created_at: ['2025-06-20T03:02:43.442Z'] } => { created_at: [new Date('2025-06-20T03:02:43.442Z')] } + values = values.map(this.formatDateValue); + } else { + // { created_at: '2025-06-20T03:02:43.442Z' } => { created_at: new Date('2025-06-20T03:02:43.442Z') } + values = this.formatDateValue(values); + } + // @ts-expect-error ignore type check + fieldsOrSource[fieldName] = values; + } + return fieldsOrSource; + } + + /** + * elasticsearch return date value as string, we need to convert it to Date object + */ + protected formatDateValue(value: unknown) { + if (value && typeof value === 'string') { + return new Date(value); + } + return value; + } + protected async requestSearch(table: SearchTable, body: Record) { const url = `${this.config.provider.endpoint}/${table}/_search`; const jsonBody = JSON.stringify(body); diff --git a/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts b/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts index 822515a43..cdf0ac2de 100644 --- a/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts +++ b/packages/backend/server/src/plugins/indexer/providers/manticoresearch.ts @@ -141,8 +141,12 @@ export class ManticoresearchProvider extends ElasticsearchProvider { nodes: data.hits.hits.map(hit => ({ _id: hit._id, _score: hit._score, - _source: this.#formatSource(dsl._source, hit._source), - fields: this.#formatFieldsFromSource(dsl.fields, hit._source), + _source: this.formatDateFields( + this.#formatSource(dsl._source, hit._source) + ), + fields: this.formatDateFields( + this.#formatFieldsFromSource(dsl.fields, hit._source) + ), highlights: this.#formatHighlights( dsl.highlight?.fields, hit.highlight @@ -176,8 +180,12 @@ export class ManticoresearchProvider extends ElasticsearchProvider { const node = { _id: hit._id, _score: hit._score, - _source: this.#formatSource(topHits._source, hit._source), - fields: this.#formatFieldsFromSource(topHits.fields, hit._source), + _source: this.formatDateFields( + this.#formatSource(topHits._source, hit._source) + ), + fields: this.formatDateFields( + this.#formatFieldsFromSource(topHits.fields, hit._source) + ), highlights: this.#formatHighlights( topHits.highlight?.fields, hit.highlight @@ -236,6 +244,17 @@ export class ManticoresearchProvider extends ElasticsearchProvider { return data; } + /** + * manticoresearch return date value as timestamp, we need to convert it to Date object + */ + protected override formatDateValue(value: unknown) { + if (value && typeof value === 'number') { + // 1750389254 => new Date(1750389254 * 1000) + return new Date(value * 1000); + } + return value; + } + private parseESQuery( query: Record, options?: { diff --git a/packages/backend/server/src/plugins/indexer/tables/index.ts b/packages/backend/server/src/plugins/indexer/tables/index.ts index 444b46af6..729d11a65 100644 --- a/packages/backend/server/src/plugins/indexer/tables/index.ts +++ b/packages/backend/server/src/plugins/indexer/tables/index.ts @@ -11,5 +11,7 @@ export const SearchTableUniqueId = { [SearchTable.doc]: getDocUniqueId, }; +export const DateFieldNames = ['created_at', 'updated_at']; + export * from './block'; export * from './doc';