luulam/AFFiNE
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(nbstore): add indexer storage (#10953)

Browse Source
This commit is contained in:
EYHN
2025-03-31 12:59:51 +00:00
parent c9e14ac0db
commit 8957d0645f
82 changed files with 3393 additions and 4753 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/common/infra/src/index.ts
View File

@@ -5,5 +5,4 @@ export * from './livedata';
export * from './media'; export * from './media';
export * from './orm'; export * from './orm';
export * from './storage'; export * from './storage';
export * from './sync';
export * from './utils'; export * from './utils';

8
packages/common/infra/src/sync/index.ts
View File

@@ -1,8 +0,0 @@
export * from './indexer';
export {
IndexedDBIndex,
IndexedDBIndexStorage,
} from './indexer/impl/indexeddb';
export { MemoryIndex, MemoryIndexStorage } from './indexer/impl/memory';
export * from './job';
export { IndexedDBJobQueue } from './job/impl/indexeddb';

147
packages/common/infra/src/sync/indexer/README.md
View File

@@ -1,147 +0,0 @@
# index
Search engine abstraction layer for AFFiNE.
## Using
1. Define schema
First, we need to define the shape of the data. Currently, there are the following data types.
- 'Integer'
- 'Boolean'
- 'FullText': for full-text search, it will be tokenized and stemmed.
- 'String': for exact match search, e.g. tags, ids.
```typescript
const schema = defineSchema({
title: 'FullText',
tag: 'String',
size: 'Integer',
});
```
> **Array type**
> All types can contain one or more values, so each field can store an array.
2. Pick a backend
Currently, there are two backends available.
- `MemoryIndex`: in-memory indexer, useful for testing.
- `IndexedDBIndex`: persistent indexer using IndexedDB.
> **Underlying Data Table**
> Some back-end processes need to maintain underlying data tables, including table creation and migration. This operation should be silently executed the first time the indexer is invoked.
> Callers do not need to worry about these details.
>
> This design conforms to the usual conventions of search engine APIs, such as in Elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html
3. Write data
Write data to the indexer. you need to start a write transaction by `await index.write()` first and then complete the batch write through `await writer.commit()`.
> **Transactional**
> Typically, the indexer does not provide transactional guarantees; reliable locking logic needs to be implemented at a higher level.
```typescript
const indexer = new IndexedDBIndex(schema);
const writer = await index.write();
writer.insert(
Document.from('id', {
title: 'hello world',
tag: ['doc', 'page'],
size: '100',
})
);
await writer.commit();
```
4. Search data
To search for content in the indexer, you need to use a specific **query language**. Here are some examples:
```typescript
// match title == 'hello world'
{
type: 'match',
field: 'title',
match: 'hello world',
}
// match title == 'hello world' && tag == 'doc'
{
type: 'boolean',
occur: 'must',
queries: [
{
type: 'match',
field: 'title',
match: 'hello world',
},
{
type: 'match',
field: 'tag',
match: 'doc',
},
],
}
```
There are two ways to perform the search, `index.search()` and `index.aggregate()`.
- **search**: return each matched node and pagination information.
- **aggregate**: aggregate all matched results based on a certain field into buckets, and return the count and score of items in each bucket.
Examples:
```typescript
const result = await index.search({
type: 'match',
field: 'title',
match: 'hello world',
});
// result = {
// nodes: [
// {
// id: '1',
// score: 1,
// },
// ],
// pagination: {
// count: 1,
// hasMore: false,
// limit: 10,
// skip: 0,
// },
// }
```
```typescript
const result = await index.aggregate(
{
type: 'match',
field: 'title',
match: 'affine',
},
'tag'
);
// result = {
// buckets: [
// { key: 'motorcycle', count: 2, score: 1 },
// { key: 'bike', count: 1, score: 1 },
// { key: 'airplane', count: 1, score: 1 },
// ],
// pagination: {
// count: 3,
// hasMore: false,
// limit: 10,
// skip: 0,
// },
// }
```
More uses:
[black-box.spec.ts](./__tests__/black-box.spec.ts)

560
packages/common/infra/src/sync/indexer/__tests__/black-box.spec.ts
View File

@@ -1,560 +0,0 @@
/**
* @vitest-environment happy-dom
*/
import 'fake-indexeddb/auto';
import { map } from 'rxjs';
import { beforeEach, describe, expect, test, vitest } from 'vitest';
import { defineSchema, Document, type Index } from '..';
import { IndexedDBIndex } from '../impl/indexeddb';
import { MemoryIndex } from '../impl/memory';
const schema = defineSchema({
title: 'FullText',
tag: 'String',
size: 'Integer',
});
let index: Index<typeof schema> = null!;
describe.each([
{ name: 'memory', backend: MemoryIndex },
{ name: 'idb', backend: IndexedDBIndex },
])('index tests($name)', ({ backend }) => {
async function writeData(
data: Record<
string,
Partial<Record<keyof typeof schema, string | string[]>>
>
) {
const writer = await index.write();
for (const [id, item] of Object.entries(data)) {
const doc = new Document(id);
for (const [key, value] of Object.entries(item)) {
if (Array.isArray(value)) {
for (const v of value) {
doc.insert(key, v);
}
} else {
doc.insert(key, value);
}
}
writer.insert(doc);
}
await writer.commit();
}
beforeEach(async () => {
index = new backend(schema);
await index.clear();
});
test('basic', async () => {
await writeData({
'1': {
title: 'hello world',
},
});
const result = await index.search({
type: 'match',
field: 'title',
match: 'hello world',
});
expect(result).toEqual({
nodes: [
{
id: '1',
score: expect.anything(),
},
],
pagination: {
count: 1,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
});
test('basic integer', async () => {
await writeData({
'1': {
title: 'hello world',
size: '100',
},
});
const result = await index.search({
type: 'match',
field: 'size',
match: '100',
});
expect(result).toEqual({
nodes: [
{
id: '1',
score: expect.anything(),
},
],
pagination: {
count: 1,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
});
test('fuzz', async () => {
await writeData({
'1': {
title: 'hello world',
},
});
const result = await index.search({
type: 'match',
field: 'title',
match: 'hell',
});
expect(result).toEqual({
nodes: [
{
id: '1',
score: expect.anything(),
},
],
pagination: {
count: 1,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
});
test('highlight', async () => {
await writeData({
'1': {
title: 'hello world',
size: '100',
},
});
const result = await index.search(
{
type: 'match',
field: 'title',
match: 'hello',
},
{
highlights: [
{
field: 'title',
before: '<b>',
end: '</b>',
},
],
}
);
expect(result).toEqual({
nodes: expect.arrayContaining([
{
id: '1',
score: expect.anything(),
highlights: {
title: [expect.stringContaining('<b>hello</b>')],
},
},
]),
pagination: {
count: 1,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
});
test('fields', async () => {
await writeData({
'1': {
title: 'hello world',
tag: ['car', 'bike'],
},
});
const result = await index.search(
{
type: 'match',
field: 'title',
match: 'hello',
},
{
fields: ['title', 'tag'],
}
);
expect(result.nodes[0].fields).toEqual({
title: 'hello world',
tag: expect.arrayContaining(['bike', 'car']),
});
});
test('pagination', async () => {
await writeData(
Array.from({ length: 100 }).reduce((acc: any, _, i) => {
acc['apple' + i] = {
tag: ['apple'],
};
return acc;
}, {}) as any
);
const result = await index.search(
{
type: 'match',
field: 'tag',
match: 'apple',
},
{
pagination: {
skip: 0,
limit: 10,
},
}
);
expect(result).toEqual({
nodes: expect.arrayContaining(
Array.from({ length: 10 }).fill({
id: expect.stringContaining('apple'),
score: expect.anything(),
})
),
pagination: {
count: 100,
hasMore: true,
limit: 10,
skip: 0,
},
});
const result2 = await index.search(
{
type: 'match',
field: 'tag',
match: 'apple',
},
{
pagination: {
skip: 10,
limit: 10,
},
}
);
expect(result2).toEqual({
nodes: expect.arrayContaining(
Array.from({ length: 10 }).fill({
id: expect.stringContaining('apple'),
score: expect.anything(),
})
),
pagination: {
count: 100,
hasMore: true,
limit: 10,
skip: 10,
},
});
});
test('aggr', async () => {
await writeData({
'1': {
title: 'hello world',
tag: ['car', 'bike'],
},
affine1: {
title: 'affine',
tag: ['motorcycle', 'bike'],
},
affine2: {
title: 'affine',
tag: ['motorcycle', 'airplane'],
},
});
const result = await index.aggregate(
{
type: 'match',
field: 'title',
match: 'affine',
},
'tag'
);
expect(result).toEqual({
buckets: expect.arrayContaining([
{ key: 'motorcycle', count: 2, score: expect.anything() },
{ key: 'bike', count: 1, score: expect.anything() },
{ key: 'airplane', count: 1, score: expect.anything() },
]),
pagination: {
count: 3,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
});
test('hits', async () => {
await writeData(
Array.from({ length: 100 }).reduce((acc: any, _, i) => {
acc['apple' + i] = {
title: 'apple',
tag: ['apple', 'fruit'],
};
return acc;
}, {}) as any
);
const result = await index.aggregate(
{
type: 'match',
field: 'title',
match: 'apple',
},
'tag',
{
hits: {
pagination: {
skip: 0,
limit: 5,
},
highlights: [
{
field: 'title',
before: '<b>',
end: '</b>',
},
],
fields: ['title', 'tag'],
},
}
);
expect(result).toEqual({
buckets: expect.arrayContaining([
{
key: 'apple',
count: 100,
score: expect.anything(),
hits: {
pagination: {
count: 100,
hasMore: true,
limit: 5,
skip: 0,
},
nodes: expect.arrayContaining(
Array.from({ length: 5 }).fill({
id: expect.stringContaining('apple'),
score: expect.anything(),
highlights: {
title: [expect.stringContaining('<b>apple</b>')],
},
fields: {
title: expect.stringContaining('apple'),
tag: expect.arrayContaining(['apple', 'fruit']),
},
})
),
},
},
{
key: 'fruit',
count: 100,
score: expect.anything(),
hits: {
pagination: {
count: 100,
hasMore: true,
limit: 5,
skip: 0,
},
nodes: expect.arrayContaining(
Array.from({ length: 5 }).fill({
id: expect.stringContaining('apple'),
score: expect.anything(),
highlights: {
title: [expect.stringContaining('<b>apple</b>')],
},
fields: {
title: expect.stringContaining('apple'),
tag: expect.arrayContaining(['apple', 'fruit']),
},
})
),
},
},
]),
pagination: {
count: 2,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
});
test('exists', async () => {
await writeData({
'1': {
title: 'hello world',
tag: '111',
},
'2': {
tag: '222',
},
'3': {
title: 'hello world',
tag: '333',
},
});
const result = await index.search({
type: 'exists',
field: 'title',
});
expect(result).toEqual({
nodes: expect.arrayContaining([
{
id: '1',
score: expect.anything(),
},
{
id: '3',
score: expect.anything(),
},
]),
pagination: {
count: 2,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
});
test(
'subscribe',
{
timeout: 30000,
},
async () => {
await writeData({
'1': {
title: 'hello world',
},
});
let value = null as any;
index
.search$({
type: 'match',
field: 'title',
match: 'hello world',
})
.pipe(map(v => (value = v)))
.subscribe();
await vitest.waitFor(
() => {
expect(value).toEqual({
nodes: [
{
id: '1',
score: expect.anything(),
},
],
pagination: {
count: 1,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
},
{
timeout: 10000,
}
);
await writeData({
'2': {
title: 'hello world',
},
});
await vitest.waitFor(
() => {
expect(value).toEqual({
nodes: [
{
id: '1',
score: expect.anything(),
},
{
id: '2',
score: expect.anything(),
},
],
pagination: {
count: 2,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
},
{
timeout: 10000,
}
);
const writer = await index.write();
writer.delete('1');
await writer.commit();
await vitest.waitFor(
() => {
expect(value).toEqual({
nodes: [
{
id: '2',
score: expect.anything(),
},
],
pagination: {
count: 1,
hasMore: false,
limit: expect.anything(),
skip: 0,
},
});
},
{
timeout: 10000,
}
);
}
);
});

1
packages/common/infra/src/sync/indexer/field-type.ts
View File

@@ -1 +0,0 @@
export type FieldType = 'Integer' | 'FullText' | 'String' | 'Boolean';

551
packages/common/infra/src/sync/indexer/impl/indexeddb/data-struct.ts
View File

@@ -1,551 +0,0 @@
import { DebugLogger } from '@affine/debug';
import {
type DBSchema,
type IDBPDatabase,
type IDBPTransaction,
openDB,
type StoreNames,
} from 'idb';
import {
type AggregateOptions,
type AggregateResult,
Document,
type Query,
type Schema,
type SearchOptions,
type SearchResult,
} from '../../';
import { highlighter } from './highlighter';
import {
BooleanInvertedIndex,
FullTextInvertedIndex,
IntegerInvertedIndex,
type InvertedIndex,
StringInvertedIndex,
} from './inverted-index';
import { Match } from './match';
const logger = new DebugLogger('indexeddb');
export interface IndexDB extends DBSchema {
kvMetadata: {
key: string;
value: {
key: string;
value: any;
};
};
records: {
key: number;
value: {
id: string;
data: Map<string, string[]>;
};
indexes: { id: string };
};
invertedIndex: {
key: number;
value: {
nid: number;
pos?: {
i: number /* index */;
l: number /* length */;
rs: [number, number][] /* ranges: [start, end] */;
};
key: ArrayBuffer;
};
indexes: { key: ArrayBuffer; nid: number };
};
}
export type DataStructRWTransaction = IDBPTransaction<
IndexDB,
ArrayLike<StoreNames<IndexDB>>,
'readwrite'
>;
export type DataStructROTransaction = IDBPTransaction<
IndexDB,
ArrayLike<StoreNames<IndexDB>>,
'readonly' | 'readwrite'
>;
export class DataStruct {
private initializePromise: Promise<void> | null = null;
database: IDBPDatabase<IndexDB> = null as any;
invertedIndex = new Map<string, InvertedIndex>();
constructor(
readonly databaseName: string,
readonly schema: Schema
) {
for (const [key, type] of Object.entries(schema)) {
const typeInfo = typeof type === 'string' ? { type } : type;
if (typeInfo.index === false) {
// If index is false, we don't need to create an inverted index for this field.
continue;
}
if (typeInfo.type === 'String') {
this.invertedIndex.set(key, new StringInvertedIndex(key));
} else if (typeInfo.type === 'Integer') {
this.invertedIndex.set(key, new IntegerInvertedIndex(key));
} else if (typeInfo.type === 'FullText') {
this.invertedIndex.set(key, new FullTextInvertedIndex(key));
} else if (type === 'Boolean') {
this.invertedIndex.set(key, new BooleanInvertedIndex(key));
} else {
throw new Error(`Field type '${type}' not supported`);
}
}
}
private async insert(trx: DataStructRWTransaction, document: Document) {
const exists = await trx
.objectStore('records')
.index('id')
.get(document.id);
if (exists) {
throw new Error('Document already exists');
}
const dataMap = new Map();
for (const [key, values] of document.fields) {
const type = this.schema[key as string];
if (!type) {
continue;
}
const typeInfo = typeof type === 'string' ? { type } : type;
if (typeInfo.store !== false) {
// If store is false, the field will not be stored
dataMap.set(key, values);
}
}
const nid = await trx.objectStore('records').put({
id: document.id,
data: dataMap,
});
for (const [key, values] of document.fields) {
const iidx = this.invertedIndex.get(key as string);
if (!iidx) {
continue;
}
await iidx.insert(trx, nid, values);
}
}
private async delete(trx: DataStructRWTransaction, id: string) {
const nid = await trx.objectStore('records').index('id').getKey(id);
if (nid) {
await trx.objectStore('records').delete(nid);
} else {
return;
}
const indexIds = await trx
.objectStore('invertedIndex')
.index('nid')
.getAllKeys(nid);
for (const indexId of indexIds) {
await trx.objectStore('invertedIndex').delete(indexId);
}
}
async batchWrite(
trx: DataStructRWTransaction,
deletes: string[],
inserts: Document[]
) {
const startTime = performance.now();
try {
for (const del of deletes) {
await this.delete(trx, del);
}
for (const inst of inserts) {
await this.insert(trx, inst);
}
} finally {
const endTime = performance.now();
if (BUILD_CONFIG.debug) {
performance.measure(
`[IndexedDB Indexer] Batch Write (${this.databaseName})`,
{
start: startTime,
end: endTime,
}
);
}
logger.debug(
`[indexer ${this.databaseName}] batchWrite`,
endTime - startTime,
'ms'
);
}
}
async matchAll(trx: DataStructROTransaction): Promise<Match> {
const allNids = await trx.objectStore('records').getAllKeys();
const match = new Match();
for (const nid of allNids) {
match.addScore(nid, 1);
}
return match;
}
private async queryRaw(
trx: DataStructROTransaction,
query: Query<any>
): Promise<Match> {
if (query.type === 'match') {
const iidx = this.invertedIndex.get(query.field as string);
if (!iidx) {
return new Match();
}
return await iidx.match(trx, query.match);
} else if (query.type === 'boolean') {
const weights = [];
for (const q of query.queries) {
weights.push(await this.queryRaw(trx, q));
}
if (query.occur === 'must') {
return weights.reduce((acc, w) => acc.and(w));
} else if (query.occur === 'must_not') {
const total = weights.reduce((acc, w) => acc.and(w));
return (await this.matchAll(trx)).exclude(total);
} else if (query.occur === 'should') {
return weights.reduce((acc, w) => acc.or(w));
}
} else if (query.type === 'all') {
return await this.matchAll(trx);
} else if (query.type === 'boost') {
return (await this.queryRaw(trx, query.query)).boost(query.boost);
} else if (query.type === 'exists') {
const iidx = this.invertedIndex.get(query.field as string);
if (!iidx) {
return new Match();
}
return await iidx.all(trx);
}
throw new Error(`Query type '${query.type}' not supported`);
}
async clear(trx: DataStructRWTransaction) {
await trx.objectStore('records').clear();
await trx.objectStore('invertedIndex').clear();
await trx.objectStore('kvMetadata').clear();
}
async search(
trx: DataStructROTransaction,
query: Query<any>,
options: SearchOptions<any>
): Promise<SearchResult<any, any>> {
const startTime = performance.now();
try {
const pagination = {
skip: options.pagination?.skip ?? 0,
limit: options.pagination?.limit ?? 100,
};
const match = await this.queryRaw(trx, query);
const nids = match
.toArray()
.slice(pagination.skip, pagination.skip + pagination.limit);
const nodes = [];
for (const nid of nids) {
const record = await trx.objectStore('records').get(nid);
if (!record) {
continue;
}
nodes.push(this.resultNode(record, options, match, nid));
}
return {
pagination: {
count: match.size(),
hasMore: match.size() > pagination.limit + pagination.skip,
limit: pagination.limit,
skip: pagination.skip,
},
nodes: nodes,
};
} finally {
const endTime = performance.now();
if (BUILD_CONFIG.debug) {
performance.measure(
`[IndexedDB Indexer] Search (${this.databaseName})`,
{
detail: { query, options },
start: startTime,
end: endTime,
}
);
}
logger.debug(
`[indexer ${this.databaseName}] search`,
endTime - startTime,
'ms',
query
);
}
}
async aggregate(
trx: DataStructROTransaction,
query: Query<any>,
field: string,
options: AggregateOptions<any>
): Promise<AggregateResult<any, any>> {
const startTime = performance.now();
try {
const pagination = {
skip: options.pagination?.skip ?? 0,
limit: options.pagination?.limit ?? 100,
};
const hitPagination = options.hits
? {
skip: options.hits.pagination?.skip ?? 0,
limit: options.hits.pagination?.limit ?? 3,
}
: {
skip: 0,
limit: 0,
};
const match = await this.queryRaw(trx, query);
const nids = match.toArray();
const buckets: {
key: string;
nids: number[];
hits: SearchResult<any, any>['nodes'];
}[] = [];
for (const nid of nids) {
const record = await trx.objectStore('records').get(nid);
if (!record) {
continue;
}
const values = record.data.get(field);
for (const value of values ?? []) {
let bucket;
let bucketIndex = buckets.findIndex(b => b.key === value);
if (bucketIndex === -1) {
bucket = { key: value, nids: [], hits: [] };
buckets.push(bucket);
bucketIndex = buckets.length - 1;
} else {
bucket = buckets[bucketIndex];
}
if (
bucketIndex >= pagination.skip &&
bucketIndex < pagination.skip + pagination.limit
) {
bucket.nids.push(nid);
if (
bucket.nids.length - 1 >= hitPagination.skip &&
bucket.nids.length - 1 < hitPagination.skip + hitPagination.limit
) {
bucket.hits.push(
this.resultNode(record, options.hits ?? {}, match, nid)
);
}
}
}
}
return {
buckets: buckets
.slice(pagination.skip, pagination.skip + pagination.limit)
.map(bucket => {
const result = {
key: bucket.key,
score: match.getScore(bucket.nids[0]),
count: bucket.nids.length,
} as AggregateResult<any, any>['buckets'][number];
if (options.hits) {
(result as any).hits = {
pagination: {
count: bucket.nids.length,
hasMore:
bucket.nids.length >
hitPagination.limit + hitPagination.skip,
limit: hitPagination.limit,
skip: hitPagination.skip,
},
nodes: bucket.hits,
} as SearchResult<any, any>;
}
return result;
}),
pagination: {
count: buckets.length,
hasMore: buckets.length > pagination.limit + pagination.skip,
limit: pagination.limit,
skip: pagination.skip,
},
};
} finally {
const endTime = performance.now();
if (BUILD_CONFIG.debug) {
performance.measure(
`[IndexedDB Indexer] Aggregate (${this.databaseName})`,
{
detail: { query, field, options },
start: startTime,
end: endTime,
}
);
}
logger.debug(
`[indexer ${this.databaseName}] aggregate`,
endTime - startTime,
'ms'
);
}
}
async getAll(
trx: DataStructROTransaction,
ids?: string[]
): Promise<Document[]> {
const docs = [];
if (ids) {
for (const id of ids) {
const record = await trx.objectStore('records').index('id').get(id);
if (record) {
docs.push(Document.from(record.id, record.data));
}
}
} else {
const records = await trx.objectStore('records').getAll();
for (const record of records) {
docs.push(Document.from(record.id, record.data));
}
}
return docs;
}
async has(trx: DataStructROTransaction, id: string): Promise<boolean> {
const nid = await trx.objectStore('records').index('id').getKey(id);
return nid !== undefined;
}
async readonly() {
await this.ensureInitialized();
return this.database.transaction(
['records', 'invertedIndex', 'kvMetadata'],
'readonly',
{
durability: 'relaxed',
}
);
}
async readwrite() {
await this.ensureInitialized();
return this.database.transaction(
['records', 'invertedIndex', 'kvMetadata'],
'readwrite',
{
durability: 'relaxed',
}
);
}
private async ensureInitialized() {
if (this.database) {
return;
}
this.initializePromise ??= this.initialize();
await this.initializePromise;
}
private async initialize() {
this.database = await openDB<IndexDB>(this.databaseName, 1, {
upgrade(database) {
database.createObjectStore('kvMetadata', {
keyPath: 'key',
});
const recordsStore = database.createObjectStore('records', {
autoIncrement: true,
});
recordsStore.createIndex('id', 'id', {
unique: true,
});
const invertedIndexStore = database.createObjectStore('invertedIndex', {
autoIncrement: true,
});
invertedIndexStore.createIndex('key', 'key', { unique: false });
invertedIndexStore.createIndex('nid', 'nid', { unique: false });
},
});
}
private resultNode(
record: { id: string; data: Map<string, string[]> },
options: SearchOptions<any>,
match?: Match,
nid?: number
): SearchResult<any, any>['nodes'][number] {
const node = {
id: record.id,
score: match && nid ? match.getScore(nid) : 1,
} as any;
if (options.fields) {
const fields = {} as Record<string, string | string[]>;
for (const field of options.fields as string[]) {
fields[field] = record.data.get(field) ?? [''];
if (fields[field].length === 1) {
fields[field] = fields[field][0];
}
}
node.fields = fields;
}
if (match && nid && options.highlights) {
const highlights = {} as Record<string, string[]>;
for (const { field, before, end } of options.highlights) {
const highlightValues = match.getHighlighters(nid, field);
if (highlightValues) {
const rawValues = record.data.get(field) ?? [];
highlights[field] = Array.from(highlightValues)
.map(([index, ranges]) => {
const raw = rawValues[index];
if (raw) {
return (
highlighter(raw, before, end, ranges, {
maxPrefix: 20,
maxLength: 50,
}) ?? ''
);
}
return '';
})
.filter(Boolean);
}
}
node.highlights = highlights;
}
return node;
}
}

182
packages/common/infra/src/sync/indexer/impl/indexeddb/index.ts
View File

@@ -1,182 +0,0 @@
import { DebugLogger } from '@affine/debug';
import type { Observable } from 'rxjs';
import { merge, of, Subject, throttleTime } from 'rxjs';
import { backoffRetry, fromPromise } from '../../../../livedata';
import { exhaustMapWithTrailing } from '../../../../utils/';
import {
type AggregateOptions,
type AggregateResult,
type Document,
type Index,
type IndexStorage,
type IndexWriter,
type Query,
type Schema,
type SearchOptions,
type SearchResult,
} from '../../';
import { DataStruct, type DataStructRWTransaction } from './data-struct';
const logger = new DebugLogger('IndexedDBIndex');
export class IndexedDBIndex<S extends Schema> implements Index<S> {
data: DataStruct = new DataStruct(this.databaseName, this.schema);
broadcast$ = new Subject();
constructor(
private readonly schema: S,
private readonly databaseName: string = 'indexer'
) {
const channel = new BroadcastChannel(this.databaseName + ':indexer');
channel.onmessage = () => {
this.broadcast$.next(1);
};
}
async get(id: string): Promise<Document<S> | null> {
return (await this.getAll([id]))[0] ?? null;
}
async getAll(ids: string[]): Promise<Document<S>[]> {
const trx = await this.data.readonly();
return this.data.getAll(trx, ids);
}
async write(): Promise<IndexWriter<S>> {
return new IndexedDBIndexWriter(this.data, await this.data.readwrite());
}
async has(id: string): Promise<boolean> {
const trx = await this.data.readonly();
return this.data.has(trx, id);
}
async search(
query: Query<any>,
options: SearchOptions<any> = {}
): Promise<SearchResult<any, SearchOptions<any>>> {
const trx = await this.data.readonly();
return this.data.search(trx, query, options);
}
search$(
query: Query<any>,
options: SearchOptions<any> = {}
): Observable<SearchResult<any, SearchOptions<any>>> {
return merge(of(1), this.broadcast$).pipe(
throttleTime(3000, undefined, { leading: true, trailing: true }),
exhaustMapWithTrailing(() => {
return fromPromise(async () => {
try {
const trx = await this.data.readonly();
return await this.data.search(trx, query, options);
} catch (error) {
logger.error('search error', error);
throw error;
}
}).pipe(backoffRetry());
})
);
}
async aggregate(
query: Query<any>,
field: string,
options: AggregateOptions<any> = {}
): Promise<AggregateResult<any, AggregateOptions<any>>> {
const trx = await this.data.readonly();
return this.data.aggregate(trx, query, field, options);
}
aggregate$(
query: Query<any>,
field: string,
options: AggregateOptions<any> = {}
): Observable<AggregateResult<S, AggregateOptions<any>>> {
return merge(of(1), this.broadcast$).pipe(
throttleTime(3000, undefined, { leading: true, trailing: true }),
exhaustMapWithTrailing(() => {
return fromPromise(async () => {
try {
const trx = await this.data.readonly();
return await this.data.aggregate(trx, query, field, options);
} catch (error) {
logger.error('aggregate error', error);
throw error;
}
}).pipe(backoffRetry());
})
);
}
async clear(): Promise<void> {
const trx = await this.data.readwrite();
return this.data.clear(trx);
}
}
export class IndexedDBIndexWriter<S extends Schema> implements IndexWriter<S> {
inserts: Document[] = [];
deletes: string[] = [];
channel = new BroadcastChannel(this.data.databaseName + ':indexer');
constructor(
private readonly data: DataStruct,
private readonly trx: DataStructRWTransaction
) {}
async get(id: string): Promise<Document<S> | null> {
return (await this.getAll([id]))[0] ?? null;
}
async getAll(ids?: string[]): Promise<Document<S>[]> {
const trx = await this.data.readonly();
return this.data.getAll(trx, ids);
}
insert(document: Document): void {
this.inserts.push(document);
}
delete(id: string): void {
this.deletes.push(id);
}
put(document: Document): void {
this.delete(document.id);
this.insert(document);
}
async commit(): Promise<void> {
await this.data.batchWrite(this.trx, this.deletes, this.inserts);
this.trx.commit();
this.channel.postMessage(1);
}
rollback(): void {}
has(id: string): Promise<boolean> {
return this.data.has(this.trx, id);
}
async search(
query: Query<any>,
options: SearchOptions<any> = {}
): Promise<SearchResult<any, SearchOptions<any>>> {
return this.data.search(this.trx, query, options);
}
async aggregate(
query: Query<any>,
field: string,
options: AggregateOptions<any> = {}
): Promise<AggregateResult<any, AggregateOptions<any>>> {
return this.data.aggregate(this.trx, query, field, options);
}
}
export class IndexedDBIndexStorage implements IndexStorage {
constructor(private readonly databaseName: string) {}
getIndex<S extends Schema>(name: string, s: S): Index<S> {
return new IndexedDBIndex(s, this.databaseName + ':' + name);
}
}

290
packages/common/infra/src/sync/indexer/impl/memory/data-struct.ts
View File

@@ -1,290 +0,0 @@
import {
type AggregateOptions,
type AggregateResult,
Document,
type Query,
type Schema,
type SearchOptions,
type SearchResult,
} from '../../';
import {
BooleanInvertedIndex,
FullTextInvertedIndex,
IntegerInvertedIndex,
type InvertedIndex,
StringInvertedIndex,
} from './inverted-index';
import { Match } from './match';
type DataRecord = {
id: string;
data: Map<string, string[]>;
deleted: boolean;
};
export class DataStruct {
records: DataRecord[] = [];
idMap = new Map<string, number>();
invertedIndex = new Map<string, InvertedIndex>();
constructor(schema: Schema) {
for (const [key, type] of Object.entries(schema)) {
const typeInfo = typeof type === 'string' ? { type } : type;
if (typeInfo.type === 'String') {
this.invertedIndex.set(key, new StringInvertedIndex(key));
} else if (typeInfo.type === 'Integer') {
this.invertedIndex.set(key, new IntegerInvertedIndex(key));
} else if (typeInfo.type === 'FullText') {
this.invertedIndex.set(key, new FullTextInvertedIndex(key));
} else if (typeInfo.type === 'Boolean') {
this.invertedIndex.set(key, new BooleanInvertedIndex(key));
} else {
throw new Error(`Field type '${type}' not supported`);
}
}
}
getAll(ids?: string[]): Document[] {
if (ids) {
return ids
.map(id => {
const nid = this.idMap.get(id);
if (nid === undefined) {
return undefined;
}
return Document.from(id, this.records[nid].data);
})
.filter((v): v is Document => v !== undefined);
} else {
return this.records
.filter(record => !record.deleted)
.map(record => Document.from(record.id, record.data));
}
}
insert(document: Document) {
if (this.idMap.has(document.id)) {
throw new Error('Document already exists');
}
this.records.push({
id: document.id,
data: document.fields as Map<string, string[]>,
deleted: false,
});
const nid = this.records.length - 1;
this.idMap.set(document.id, nid);
for (const [key, values] of document.fields) {
for (const value of values) {
const iidx = this.invertedIndex.get(key as string);
if (!iidx) {
throw new Error(
`Inverted index '${key.toString()}' not found, document not match schema`
);
}
iidx.insert(nid, value);
}
}
}
delete(id: string) {
const nid = this.idMap.get(id);
if (nid === undefined) {
throw new Error('Document not found');
}
this.records[nid].deleted = true;
this.records[nid].data = new Map();
}
matchAll(): Match {
const weight = new Match();
for (let i = 0; i < this.records.length; i++) {
weight.addScore(i, 1);
}
return weight;
}
clear() {
this.records = [];
this.idMap.clear();
this.invertedIndex.forEach(v => v.clear());
}
private queryRaw(query: Query<any>): Match {
if (query.type === 'match') {
const iidx = this.invertedIndex.get(query.field as string);
if (!iidx) {
throw new Error(`Field '${query.field as string}' not found`);
}
return iidx.match(query.match);
} else if (query.type === 'boolean') {
const weights = query.queries.map(q => this.queryRaw(q));
if (query.occur === 'must') {
return weights.reduce((acc, w) => acc.and(w));
} else if (query.occur === 'must_not') {
const total = weights.reduce((acc, w) => acc.and(w));
return this.matchAll().exclude(total);
} else if (query.occur === 'should') {
return weights.reduce((acc, w) => acc.or(w));
}
} else if (query.type === 'all') {
return this.matchAll();
} else if (query.type === 'boost') {
return this.queryRaw(query.query).boost(query.boost);
} else if (query.type === 'exists') {
const iidx = this.invertedIndex.get(query.field as string);
if (!iidx) {
throw new Error(`Field '${query.field as string}' not found`);
}
return iidx.all();
}
throw new Error(`Query type '${query.type}' not supported`);
}
query(query: Query<any>): Match {
return this.queryRaw(query).filter(id => !this.records[id].deleted);
}
search(
query: Query<any>,
options: SearchOptions<any> = {}
): SearchResult<any, any> {
const pagination = {
skip: options.pagination?.skip ?? 0,
limit: options.pagination?.limit ?? 100,
};
const match = this.query(query);
const nids = match
.toArray()
.slice(pagination.skip, pagination.skip + pagination.limit);
return {
pagination: {
count: match.size(),
hasMore: match.size() > pagination.limit + pagination.skip,
limit: pagination.limit,
skip: pagination.skip,
},
nodes: nids.map(nid => this.resultNode(match, nid, options)),
};
}
aggregate(
query: Query<any>,
field: string,
options: AggregateOptions<any> = {}
): AggregateResult<any, any> {
const pagination = {
skip: options.pagination?.skip ?? 0,
limit: options.pagination?.limit ?? 100,
};
const match = this.query(query);
const nids = match.toArray();
const buckets: { key: string; nids: number[] }[] = [];
for (const nid of nids) {
for (const value of this.records[nid].data.get(field) ?? []) {
let bucket = buckets.find(b => b.key === value);
if (!bucket) {
bucket = { key: value, nids: [] };
buckets.push(bucket);
}
bucket.nids.push(nid);
}
}
return {
buckets: buckets
.slice(pagination.skip, pagination.skip + pagination.limit)
.map(bucket => {
const result = {
key: bucket.key,
score: match.getScore(bucket.nids[0]),
count: bucket.nids.length,
} as AggregateResult<any, any>['buckets'][number];
if (options.hits) {
const hitsOptions = options.hits;
const pagination = {
skip: options.hits.pagination?.skip ?? 0,
limit: options.hits.pagination?.limit ?? 3,
};
const hits = bucket.nids.slice(
pagination.skip,
pagination.skip + pagination.limit
);
(result as any).hits = {
pagination: {
count: bucket.nids.length,
hasMore:
bucket.nids.length > pagination.limit + pagination.skip,
limit: pagination.limit,
skip: pagination.skip,
},
nodes: hits.map(nid => this.resultNode(match, nid, hitsOptions)),
} as SearchResult<any, any>;
}
return result;
}),
pagination: {
count: buckets.length,
hasMore: buckets.length > pagination.limit + pagination.skip,
limit: pagination.limit,
skip: pagination.skip,
},
};
}
has(id: string): boolean {
return this.idMap.has(id);
}
private resultNode(
match: Match,
nid: number,
options: SearchOptions<any>
): SearchResult<any, any>['nodes'][number] {
const node = {
id: this.records[nid].id,
score: match.getScore(nid),
} as any;
if (options.fields) {
const fields = {} as Record<string, string | string[]>;
for (const field of options.fields as string[]) {
fields[field] = this.records[nid].data.get(field) ?? [''];
if (fields[field].length === 1) {
fields[field] = fields[field][0];
}
}
node.fields = fields;
}
if (options.highlights) {
const highlights = {} as Record<string, string[]>;
for (const { field, before, end } of options.highlights) {
highlights[field] = match
.getHighlighters(nid, field)
.flatMap(highlighter => {
return highlighter(before, end);
});
}
node.highlights = highlights;
}
return node;
}
}

141
packages/common/infra/src/sync/indexer/impl/memory/index.ts
View File

@@ -1,141 +0,0 @@
import { map, merge, type Observable, of, Subject, throttleTime } from 'rxjs';
import type {
AggregateOptions,
AggregateResult,
Document,
Index,
IndexStorage,
IndexWriter,
Query,
Schema,
SearchOptions,
SearchResult,
} from '../../';
import { DataStruct } from './data-struct';
export class MemoryIndex<S extends Schema> implements Index<S> {
private readonly data: DataStruct = new DataStruct(this.schema);
broadcast$ = new Subject<number>();
constructor(private readonly schema: Schema) {}
write(): Promise<IndexWriter<S>> {
return Promise.resolve(new MemoryIndexWriter(this.data, this.broadcast$));
}
async get(id: string): Promise<Document<S> | null> {
return (await this.getAll([id]))[0] ?? null;
}
getAll(ids?: string[]): Promise<Document<S>[]> {
return Promise.resolve(this.data.getAll(ids));
}
has(id: string): Promise<boolean> {
return Promise.resolve(this.data.has(id));
}
async search(
query: Query<any>,
options: SearchOptions<any> = {}
): Promise<SearchResult<any, any>> {
return this.data.search(query, options);
}
search$(
query: Query<any>,
options: SearchOptions<any> = {}
): Observable<SearchResult<any, any>> {
return merge(of(1), this.broadcast$).pipe(
throttleTime(500, undefined, { leading: false, trailing: true }),
map(() => this.data.search(query, options))
);
}
async aggregate(
query: Query<any>,
field: string,
options: AggregateOptions<any> = {}
): Promise<AggregateResult<any, any>> {
return this.data.aggregate(query, field, options);
}
aggregate$(
query: Query<any>,
field: string,
options: AggregateOptions<any> = {}
): Observable<AggregateResult<S, AggregateOptions<any>>> {
return merge(of(1), this.broadcast$).pipe(
throttleTime(500, undefined, { leading: false, trailing: true }),
map(() => this.data.aggregate(query, field, options))
);
}
clear(): Promise<void> {
this.data.clear();
return Promise.resolve();
}
}
export class MemoryIndexWriter<S extends Schema> implements IndexWriter<S> {
inserts: Document[] = [];
deletes: string[] = [];
constructor(
private readonly data: DataStruct,
private readonly broadcast$: Subject<number>
) {}
async get(id: string): Promise<Document<S> | null> {
return (await this.getAll([id]))[0] ?? null;
}
getAll(ids: string[]): Promise<Document<S>[]> {
return Promise.resolve(this.data.getAll(ids));
}
insert(document: Document): void {
this.inserts.push(document);
}
delete(id: string): void {
this.deletes.push(id);
}
put(document: Document): void {
this.delete(document.id);
this.insert(document);
}
async search(
query: Query<any>,
options: SearchOptions<any> = {}
): Promise<SearchResult<any, any>> {
return this.data.search(query, options);
}
async aggregate(
query: Query<any>,
field: string,
options: AggregateOptions<any> = {}
): Promise<AggregateResult<any, any>> {
return this.data.aggregate(query, field, options);
}
commit(): Promise<void> {
for (const del of this.deletes) {
this.data.delete(del);
}
for (const inst of this.inserts) {
this.data.insert(inst);
}
this.broadcast$.next(1);
return Promise.resolve();
}
rollback(): void {}
has(id: string): Promise<boolean> {
return Promise.resolve(this.data.has(id));
}
}
export class MemoryIndexStorage implements IndexStorage {
getIndex<S extends Schema>(_: string, schema: S): Index<S> {
return new MemoryIndex(schema);
}
}

220
packages/common/infra/src/sync/indexer/impl/memory/inverted-index.ts
View File

@@ -1,220 +0,0 @@
import Fuse from 'fuse.js';
import { Match } from './match';
export interface InvertedIndex {
fieldKey: string;
match(term: string): Match;
all(): Match;
insert(id: number, term: string): void;
clear(): void;
}
export class StringInvertedIndex implements InvertedIndex {
index: Map<string, number[]> = new Map();
constructor(readonly fieldKey: string) {}
match(term: string): Match {
const match = new Match();
for (const id of this.index.get(term) ?? []) {
match.addScore(id, 1);
}
return match;
}
all(): Match {
const match = new Match();
for (const [_term, ids] of this.index) {
for (const id of ids) {
if (match.getScore(id) === 0) {
match.addScore(id, 1);
}
}
}
return match;
}
insert(id: number, term: string): void {
const ids = this.index.get(term) ?? [];
ids.push(id);
this.index.set(term, ids);
}
clear(): void {
this.index.clear();
}
}
export class IntegerInvertedIndex implements InvertedIndex {
index: Map<string, number[]> = new Map();
constructor(readonly fieldKey: string) {}
// eslint-disable-next-line sonarjs/no-identical-functions
match(term: string): Match {
const match = new Match();
for (const id of this.index.get(term) ?? []) {
match.addScore(id, 1);
}
return match;
}
// eslint-disable-next-line sonarjs/no-identical-functions
all(): Match {
const match = new Match();
for (const [_term, ids] of this.index) {
for (const id of ids) {
if (match.getScore(id) === 0) {
match.addScore(id, 1);
}
}
}
return match;
}
// eslint-disable-next-line sonarjs/no-identical-functions
insert(id: number, term: string): void {
const ids = this.index.get(term) ?? [];
ids.push(id);
this.index.set(term, ids);
}
clear(): void {
this.index.clear();
}
}
export class BooleanInvertedIndex implements InvertedIndex {
index: Map<boolean, number[]> = new Map();
constructor(readonly fieldKey: string) {}
// eslint-disable-next-line sonarjs/no-identical-functions
match(term: string): Match {
const match = new Match();
for (const id of this.index.get(term === 'true') ?? []) {
match.addScore(id, 1);
}
return match;
}
// eslint-disable-next-line sonarjs/no-identical-functions
all(): Match {
const match = new Match();
for (const [_term, ids] of this.index) {
for (const id of ids) {
if (match.getScore(id) === 0) {
match.addScore(id, 1);
}
}
}
return match;
}
// eslint-disable-next-line sonarjs/no-identical-functions
insert(id: number, term: string): void {
const ids = this.index.get(term === 'true') ?? [];
ids.push(id);
this.index.set(term === 'true', ids);
}
clear(): void {
this.index.clear();
}
}
export class FullTextInvertedIndex implements InvertedIndex {
records = [] as { id: number; v: string }[];
index = Fuse.createIndex(['v'], [] as { id: number; v: string }[]);
constructor(readonly fieldKey: string) {}
match(term: string): Match {
const searcher = new Fuse(
this.records,
{
includeScore: true,
includeMatches: true,
shouldSort: true,
keys: ['v'],
},
this.index
);
const result = searcher.search(term);
const match = new Match();
for (const value of result) {
match.addScore(value.item.id, 1 - (value.score ?? 1));
match.addHighlighter(value.item.id, this.fieldKey, (before, after) => {
const matches = value.matches;
if (!matches || matches.length === 0) {
return [''];
}
const firstMatch = matches[0];
const text = firstMatch.value;
if (!text) {
return [''];
}
let result = '';
let pointer = 0;
for (const match of matches) {
for (const [start, end] of match.indices) {
result += text.substring(pointer, start);
result += `${before}${text.substring(start, end + 1)}${after}`;
pointer = end + 1;
}
}
result += text.substring(pointer);
return [result];
});
}
return match;
}
// eslint-disable-next-line sonarjs/no-identical-functions
all(): Match {
const match = new Match();
for (const { id } of this.records) {
if (match.getScore(id) === 0) {
match.addScore(id, 1);
}
}
return match;
}
insert(id: number, term: string): void {
this.index.add({ id, v: term });
this.records.push({ id, v: term });
}
clear(): void {
this.records = [];
this.index = Fuse.createIndex(['v'], [] as { id: number; v: string }[]);
}
}

108
packages/common/infra/src/sync/indexer/impl/memory/match.ts
View File

@@ -1,108 +0,0 @@
export class Match {
scores = new Map<number, number>();
highlighters = new Map<
number,
Map<string, ((before: string, after: string) => string[])[]>
>();
constructor() {}
size() {
return this.scores.size;
}
getScore(id: number) {
return this.scores.get(id) ?? 0;
}
addScore(id: number, score: number) {
const currentScore = this.scores.get(id) || 0;
this.scores.set(id, currentScore + score);
}
getHighlighters(id: number, field: string) {
return this.highlighters.get(id)?.get(field) ?? [];
}
addHighlighter(
id: number,
field: string,
highlighter: (before: string, after: string) => string[]
) {
const fields = this.highlighters.get(id) || new Map();
const highlighters = fields.get(field) || [];
highlighters.push(highlighter);
fields.set(field, highlighters);
this.highlighters.set(id, fields);
}
and(other: Match) {
const newWeight = new Match();
for (const [id, score] of this.scores) {
if (other.scores.has(id)) {
newWeight.addScore(id, score + (other.scores.get(id) ?? 0));
newWeight.copyExtData(this, id);
newWeight.copyExtData(other, id);
}
}
return newWeight;
}
or(other: Match) {
const newWeight = new Match();
for (const [id, score] of this.scores) {
newWeight.addScore(id, score);
newWeight.copyExtData(this, id);
}
for (const [id, score] of other.scores) {
newWeight.addScore(id, score);
newWeight.copyExtData(other, id);
}
return newWeight;
}
exclude(other: Match) {
const newWeight = new Match();
for (const [id, score] of this.scores) {
if (!other.scores.has(id)) {
newWeight.addScore(id, score);
newWeight.copyExtData(this, id);
}
}
return newWeight;
}
boost(boost: number) {
const newWeight = new Match();
for (const [id, score] of this.scores) {
newWeight.addScore(id, score * boost);
newWeight.copyExtData(this, id);
}
return newWeight;
}
toArray() {
return Array.from(this.scores.entries())
.sort((a, b) => b[1] - a[1])
.map(e => e[0]);
}
filter(predicate: (id: number) => boolean) {
const newWeight = new Match();
for (const [id, score] of this.scores) {
if (predicate(id)) {
newWeight.addScore(id, score);
newWeight.copyExtData(this, id);
}
}
return newWeight;
}
private copyExtData(from: Match, id: number) {
for (const [field, highlighters] of from.highlighters.get(id) ?? []) {
for (const highlighter of highlighters) {
this.addHighlighter(id, field, highlighter);
}
}
}
}

6
packages/common/infra/src/sync/indexer/index.ts
View File

@@ -1,6 +0,0 @@
export * from './document';
export * from './field-type';
export * from './indexer';
export * from './query';
export * from './schema';
export * from './searcher';

41
packages/common/infra/src/sync/indexer/indexer.ts
View File

@@ -1,41 +0,0 @@
import type { Document } from './document';
import type { Schema } from './schema';
import type { Searcher, Subscriber } from './searcher';
export interface Index<S extends Schema>
extends IndexReader<S>,
Searcher<S>,
Subscriber<S> {
write(): Promise<IndexWriter<S>>;
clear(): Promise<void>;
}
export interface IndexWriter<S extends Schema>
extends IndexReader<S>,
Searcher<S> {
insert(document: Document<S>): void;
put(document: Document<S>): void;
delete(id: string): void;
// TODO(@eyhn)
// deleteByQuery(query: Query<S>): void;
commit(): Promise<void>;
rollback(): void;
}
export interface IndexReader<S extends Schema> {
get(id: string): Promise<Document<S> | null>;
getAll(ids?: string[]): Promise<Document<S>[]>;
has(id: string): Promise<boolean>;
}
export interface IndexStorage {
getIndex<S extends Schema>(name: string, schema: S): Index<S>;
}

35
packages/common/infra/src/sync/indexer/query.ts
View File

@@ -1,35 +0,0 @@
import type { Schema } from './schema';
export type MatchQuery<S extends Schema> = {
type: 'match';
field: keyof S;
match: string;
};
export type BoostQuery = {
type: 'boost';
query: Query<any>;
boost: number;
};
export type BooleanQuery<S extends Schema> = {
type: 'boolean';
occur: 'should' | 'must' | 'must_not';
queries: Query<S>[];
};
export type ExistsQuery<S extends Schema> = {
type: 'exists';
field: keyof S;
};
export type AllQuery = {
type: 'all';
};
export type Query<S extends Schema> =
| BooleanQuery<S>
| MatchQuery<S>
| AllQuery
| ExistsQuery<S>
| BoostQuery;

25
packages/common/infra/src/sync/indexer/schema.ts
View File

@@ -1,25 +0,0 @@
import type { FieldType } from './field-type';
export type Schema = Record<
string,
| FieldType
| {
type: FieldType;
/**
* If false, the field will not be indexed, and thus not searchable.
*
* default: true
*/
index?: boolean;
/**
* If false, the field will not be stored, and not included in the search result.
*
* default: true
*/
store?: boolean;
}
>;
export function defineSchema<T extends Schema>(schema: T): T {
return schema;
}

83
packages/common/infra/src/sync/indexer/searcher.ts
View File

@@ -1,83 +0,0 @@
import type { Observable } from 'rxjs';
import type { Query } from './query';
import type { Schema } from './schema';
type HighlightAbleField<S extends Schema> = {
[K in keyof S]: S[K] extends 'FullText' ? K : never;
}[keyof S];
export interface Searcher<S extends Schema = any> {
search<const O extends SearchOptions<S>>(
query: Query<S>,
options?: O
): Promise<SearchResult<S, O>>;
aggregate<const O extends AggregateOptions<S>>(
query: Query<S>,
field: keyof S,
options?: O
): Promise<AggregateResult<S, O>>;
}
export interface Subscriber<S extends Schema = any> {
search$<const O extends SearchOptions<S>>(
query: Query<S>,
options?: O
): Observable<SearchResult<S, O>>;
aggregate$<const O extends AggregateOptions<S>>(
query: Query<S>,
field: keyof S,
options?: O
): Observable<AggregateResult<S, O>>;
}
type ResultPagination = {
count: number;
limit: number;
skip: number;
hasMore: boolean;
};
type PaginationOption = {
limit?: number;
skip?: number;
};
export type SearchOptions<S extends Schema> = {
pagination?: PaginationOption;
highlights?: {
field: HighlightAbleField<S>;
before: string;
end: string;
}[];
fields?: (keyof S)[];
};
export type SearchResult<S extends Schema, O extends SearchOptions<S>> = {
pagination: ResultPagination;
nodes: ({
id: string;
score: number;
} & (O['fields'] extends any[]
? { fields: { [key in O['fields'][number]]: string | string[] } }
: unknown) &
(O['highlights'] extends any[]
? { highlights: { [key in O['highlights'][number]['field']]: string[] } }
: unknown))[];
};
export interface AggregateOptions<S extends Schema> {
pagination?: PaginationOption;
hits?: SearchOptions<S>;
}
export type AggregateResult<S extends Schema, O extends AggregateOptions<S>> = {
pagination: ResultPagination;
buckets: ({
key: string;
score: number;
count: number;
} & (O['hits'] extends object
? { hits: SearchResult<S, O['hits']> }
: unknown))[];
};

47
packages/common/infra/src/sync/job/README.md
View File

@@ -1,47 +0,0 @@
# job
Job system abstraction for AFFiNE. Currently, only `IndexedDBJobQueue` is implemented; more backends will be implemented in the future.
Run background jobs in browser & distributed environment. `runners` can consume tasks simultaneously without additional communication.
# Basic Usage
```ts
const queue = new IndexedDBJobQueue('my-queue');
await queue.enqueue([
{
batchKey: '1',
payload: { a: 'hello' },
},
{
batchKey: '2',
payload: { a: 'world' },
},
]);
const runner = new JobRunner(queue, job => {
console.log(job);
});
runner.start();
// Output:
// { batchKey: '1', payload: { a: 'hello' } }
// { batchKey: '2', payload: { a: 'world' } }
```
## `batchKey`
Each job has a `batchKey`, and jobs with the same `batchKey` are handed over to one `runner` for execution at once.
Additionally, if there are ongoing jobs with the same `batchKey`, other `runners` will not take on jobs with this `batchKey`, ensuring exclusive resource locking.
> In the future, `batchKey` will be used to implement priority.
## `timeout`
If the job execution time exceeds 30 seconds, it will be considered a timeout and reassigned to another `runner`.
## Error Handling
If an error is thrown during job execution, will log an error, but the job will be considered complete.

231
packages/common/infra/src/sync/job/__tests__/black-box.spec.ts
View File

@@ -1,231 +0,0 @@
/**
* @vitest-environment happy-dom
*/
import 'fake-indexeddb/auto';
import { afterEach, beforeEach, describe, expect, test, vitest } from 'vitest';
import { IndexedDBJobQueue } from '../impl/indexeddb';
import type { JobQueue } from '../queue';
let queue: JobQueue<{
a: string;
}> = null!;
describe.each([{ name: 'idb', backend: IndexedDBJobQueue }])(
'impl tests($name)',
({ backend }) => {
beforeEach(async () => {
queue = new backend();
await queue.clear();
vitest.useFakeTimers({
toFake: ['Date'],
});
});
afterEach(() => {
vitest.useRealTimers();
});
test('basic', async () => {
await queue.enqueue([
{
batchKey: '1',
payload: { a: 'hello' },
},
{
batchKey: '2',
payload: { a: 'world' },
},
]);
const job1 = await queue.accept();
const job2 = await queue.accept();
expect([job1!, job2!]).toEqual([
[
{
id: expect.any(String),
batchKey: '1',
payload: { a: 'hello' },
},
],
[
{
id: expect.any(String),
batchKey: '2',
payload: { a: 'world' },
},
],
]);
const job3 = await queue.accept();
expect(job3).toBeNull();
await queue.return(job1!);
await queue.return(job2!);
});
test('batch', async () => {
await queue.enqueue([
{
batchKey: '1',
payload: { a: 'hello' },
},
{
batchKey: '1',
payload: { a: 'world' },
},
]);
const job1 = await queue.accept();
expect(job1).toEqual(
expect.arrayContaining([
{
id: expect.any(String),
batchKey: '1',
payload: { a: 'hello' },
},
{
id: expect.any(String),
batchKey: '1',
payload: { a: 'world' },
},
])
);
});
test('timeout', async () => {
await queue.enqueue([
{
batchKey: '1',
payload: { a: 'hello' },
},
]);
{
const job = await queue.accept();
expect(job).toEqual([
{
id: expect.any(String),
batchKey: '1',
payload: { a: 'hello' },
},
]);
}
{
const job = await queue.accept();
expect(job).toBeNull();
}
vitest.advanceTimersByTime(1000 * 60 * 60);
{
const job = await queue.accept();
expect(job).toEqual([
{
id: expect.any(String),
batchKey: '1',
payload: { a: 'hello' },
},
]);
}
});
test('waitForAccept', async () => {
const abort = new AbortController();
let result = null as any;
queue.waitForAccept(abort.signal).then(jobs => (result = jobs));
await new Promise(resolve => setTimeout(resolve, 100));
expect(result).toBeNull();
await queue.enqueue([
{
batchKey: '1',
payload: { a: 'hello' },
},
]);
await vitest.waitFor(() => {
expect(result).toEqual([
{
id: expect.any(String),
batchKey: '1',
payload: { a: 'hello' },
},
]);
});
});
test('waitForAccept race', async () => {
const abort = new AbortController();
let result1 = null as any;
let result2 = null as any;
queue.waitForAccept(abort.signal).then(jobs => (result1 = jobs));
queue.waitForAccept(abort.signal).then(jobs => (result2 = jobs));
await new Promise(resolve => setTimeout(resolve, 100));
expect(result1).toBeNull();
expect(result2).toBeNull();
await queue.enqueue([
{
batchKey: '1',
payload: { a: 'hello' },
},
]);
await new Promise(resolve => setTimeout(resolve, 100));
expect([result1, result2]).toEqual(
expect.arrayContaining([
[
{
id: expect.any(String),
batchKey: '1',
payload: { a: 'hello' },
},
],
null,
])
);
await queue.enqueue([
{
batchKey: '2',
payload: { a: 'world' },
},
]);
await vitest.waitFor(() => {
expect([result1, result2]).toEqual(
expect.arrayContaining([
[
{
id: expect.any(String),
batchKey: '1',
payload: { a: 'hello' },
},
],
[
{
id: expect.any(String),
batchKey: '2',
payload: { a: 'world' },
},
],
])
);
});
});
}
);

257
packages/common/infra/src/sync/job/impl/indexeddb/index.ts
View File

@@ -1,257 +0,0 @@
import type { DBSchema, IDBPDatabase } from 'idb';
import { openDB } from 'idb';
import { merge, Observable, of, throttleTime } from 'rxjs';
import { fromPromise } from '../../../../livedata';
import { throwIfAborted } from '../../../../utils';
import { exhaustMapWithTrailing } from '../../../../utils/';
import type { Job, JobParams, JobQueue } from '../../';
interface IndexDB extends DBSchema {
jobs: {
key: number;
value: JobRecord;
indexes: {
batchKey: string;
};
};
}
interface JobRecord {
batchKey: string;
startTime: number | null;
payload: any;
}
export class IndexedDBJobQueue<J> implements JobQueue<J> {
database: IDBPDatabase<IndexDB> = null as any;
broadcast = new BroadcastChannel('idb-job-queue:' + this.databaseName);
constructor(private readonly databaseName: string = 'jobs') {}
async enqueue(jobs: JobParams[]): Promise<void> {
await this.ensureInitialized();
const trx = this.database.transaction(['jobs'], 'readwrite');
for (const job of jobs) {
await trx.objectStore('jobs').add({
batchKey: job.batchKey,
payload: job.payload,
startTime: null,
});
}
trx.commit();
// send broadcast to notify new jobs
this.broadcast.postMessage('new-jobs');
}
async accept(): Promise<Job[] | null> {
await this.ensureInitialized();
const jobs = [];
const trx = this.database.transaction(['jobs'], 'readwrite', {
durability: 'relaxed',
});
// if no priority jobs
if (jobs.length === 0) {
const batchKeys = trx.objectStore('jobs').index('batchKey').iterate();
let currentBatchKey: string = null as any;
let currentBatchJobs = [];
let skipCurrentBatch = false;
for await (const item of batchKeys) {
if (item.value.batchKey !== currentBatchKey) {
if (!skipCurrentBatch && currentBatchJobs.length > 0) {
break;
}
currentBatchKey = item.value.batchKey;
currentBatchJobs = [];
skipCurrentBatch = false;
}
if (skipCurrentBatch) {
continue;
}
if (this.isAcceptable(item.value)) {
currentBatchJobs.push({
id: item.primaryKey,
job: item.value,
});
} else {
skipCurrentBatch = true;
}
}
if (skipCurrentBatch === false && currentBatchJobs.length > 0) {
jobs.push(...currentBatchJobs);
}
}
for (const { id, job } of jobs) {
const startTime = Date.now();
await trx.objectStore('jobs').put({ ...job, startTime }, id);
}
if (jobs.length === 0) {
return null;
}
return jobs.map(({ id, job }) => ({
id: id.toString(),
batchKey: job.batchKey,
payload: job.payload,
}));
}
async waitForAccept(signal: AbortSignal): Promise<Job<J>[]> {
const broadcast = new BroadcastChannel(
'idb-job-queue:' + this.databaseName
);
try {
let deferred = Promise.withResolvers<void>();
broadcast.onmessage = () => {
deferred.resolve();
};
while (throwIfAborted(signal)) {
const jobs = await this.accept();
if (jobs !== null) {
return jobs;
}
await Promise.race([
deferred.promise,
new Promise(resolve => {
setTimeout(resolve, 5000);
}),
new Promise((_, reject) => {
// exit if manually stopped
if (signal?.aborted) {
reject(signal.reason);
}
signal?.addEventListener('abort', () => {
reject(signal.reason);
});
}),
]);
deferred = Promise.withResolvers<void>();
}
return [];
} finally {
broadcast.close();
}
}
async complete(jobs: Job[]): Promise<void> {
await this.ensureInitialized();
const trx = this.database.transaction(['jobs'], 'readwrite', {
durability: 'relaxed',
});
for (const { id } of jobs) {
await trx
.objectStore('jobs')
.delete(typeof id === 'string' ? parseInt(id) : id);
}
trx.commit();
this.broadcast.postMessage('job-completed');
}
async return(jobs: Job[], retry: boolean = false): Promise<void> {
await this.ensureInitialized();
const trx = this.database.transaction(['jobs'], 'readwrite', {
durability: 'relaxed',
});
for (const { id } of jobs) {
if (retry) {
const nid = typeof id === 'string' ? parseInt(id) : id;
const job = await trx.objectStore('jobs').get(nid);
if (job) {
await trx.objectStore('jobs').put({ ...job, startTime: null }, nid);
}
} else {
await trx
.objectStore('jobs')
.delete(typeof id === 'string' ? parseInt(id) : id);
}
}
trx.commit();
this.broadcast.postMessage('job-completed');
}
async clear(): Promise<void> {
await this.ensureInitialized();
const trx = this.database.transaction(['jobs'], 'readwrite', {
durability: 'relaxed',
});
await trx.objectStore('jobs').clear();
}
private async ensureInitialized(): Promise<void> {
if (!this.database) {
await this.initialize();
}
}
private async initialize(): Promise<void> {
if (this.database) {
return;
}
this.database = await openDB(this.databaseName, 1, {
upgrade(database) {
const jobs = database.createObjectStore('jobs', {
autoIncrement: true,
});
jobs.createIndex('batchKey', 'batchKey');
},
});
}
TIMEOUT = 1000 * 30 /* 30 seconds */;
private isTimeout(job: JobRecord) {
return job.startTime !== null && job.startTime + this.TIMEOUT < Date.now();
}
private isAcceptable(job: JobRecord) {
return job.startTime === null || this.isTimeout(job);
}
get status$() {
return merge(
of(1),
new Observable(subscriber => {
const broadcast = new BroadcastChannel(
'idb-job-queue:' + this.databaseName
);
broadcast.onmessage = () => {
subscriber.next(1);
};
return () => {
broadcast.close();
};
})
).pipe(
throttleTime(300, undefined, { leading: true, trailing: true }),
exhaustMapWithTrailing(() =>
fromPromise(async () => {
await this.ensureInitialized();
const trx = this.database.transaction(['jobs'], 'readonly');
const remaining = await trx.objectStore('jobs').count();
return { remaining };
})
)
);
}
}

2
packages/common/infra/src/sync/job/index.ts
View File

@@ -1,2 +0,0 @@
export * from './queue';
export * from './runner';

28
packages/common/infra/src/sync/job/queue.ts
View File

@@ -1,28 +0,0 @@
import type { Observable } from 'rxjs';
export interface JobParams<Payload = any> {
batchKey: string;
payload: Payload;
}
export interface Job<Payload = any> extends JobParams<Payload> {
id: string;
}
export interface JobQueueStatus {
remaining: number;
}
export interface JobQueue<Payload> {
enqueue(jobs: JobParams<Payload>[]): Promise<void>;
accept(): Promise<Job<Payload>[] | null>;
waitForAccept(signal: AbortSignal): Promise<Job<Payload>[]>;
return(jobs: Job<Payload>[], retry?: boolean): Promise<void>;
clear(): Promise<void>;
status$: Observable<JobQueueStatus>;
}

63
packages/common/infra/src/sync/job/runner.ts
View File

@@ -1,63 +0,0 @@
import { DebugLogger } from '@affine/debug';
import { MANUALLY_STOP, throwIfAborted } from '../../utils';
import type { Job, JobQueue } from './queue';
const logger = new DebugLogger('job-runner');
export class JobRunner<J> {
abort: AbortController | null = null;
constructor(
private readonly queue: JobQueue<J>,
private readonly worker: (
jobs: Job<J>[],
signal: AbortSignal
) => Promise<void>,
private readonly interval: () => Promise<void> = async () => {}
) {}
start() {
this.stop();
this.abort = new AbortController();
this.loop(this.abort.signal).catch(err => {
if (err === MANUALLY_STOP) {
return;
}
logger.error(err);
});
}
stop() {
this.abort?.abort(MANUALLY_STOP);
this.abort = null;
}
async loop(signal: AbortSignal) {
while (throwIfAborted(signal)) {
const jobs = await this.queue.waitForAccept(signal);
if (jobs !== null) {
try {
await this.worker(jobs, signal);
await this.queue.return(jobs);
} catch (err) {
if (err === MANUALLY_STOP) {
await this.queue.return(jobs, true);
} else {
// TODO: retry logic
await this.queue.return(jobs);
}
logger.error(
'Error processing jobs',
err instanceof Error ? (err.stack ?? err.message) : err
);
}
} else {
await new Promise(resolve => setTimeout(resolve, 1000));
}
await this.interval();
}
}
}

8
packages/common/nbstore/package.json
View File

@@ -13,12 +13,16 @@
"./idb/v1": "./src/impls/idb/v1/index.ts", "./idb/v1": "./src/impls/idb/v1/index.ts",
"./cloud": "./src/impls/cloud/index.ts", "./cloud": "./src/impls/cloud/index.ts",
"./sqlite": "./src/impls/sqlite/index.ts", "./sqlite": "./src/impls/sqlite/index.ts",
"./sqlite/v1": "./src/impls/sqlite/v1/index.ts" "./sqlite/v1": "./src/impls/sqlite/v1/index.ts",
"./sync": "./src/sync/index.ts",
"./frontend": "./src/frontend/index.ts"
}, },
"dependencies": { "dependencies": {
"@datastructures-js/binary-search-tree": "^5.3.2", "@datastructures-js/binary-search-tree": "^5.3.2",
"@toeverything/infra": "workspace:*", "@toeverything/infra": "workspace:*",
"eventemitter2": "^6.4.9", "eventemitter2": "^6.4.9",
"graphemer": "^1.4.0",
"lib0": "^0.2.99",
"lodash-es": "^4.17.21", "lodash-es": "^4.17.21",
"nanoid": "^5.0.9", "nanoid": "^5.0.9",
"rxjs": "^7.8.1", "rxjs": "^7.8.1",
@@ -29,6 +33,7 @@
"devDependencies": { "devDependencies": {
"@affine/error": "workspace:*", "@affine/error": "workspace:*",
"@affine/graphql": "workspace:*", "@affine/graphql": "workspace:*",
"@blocksuite/affine": "workspace:*",
"fake-indexeddb": "^6.0.0", "fake-indexeddb": "^6.0.0",
"idb": "^8.0.0", "idb": "^8.0.0",
"socket.io-client": "^4.8.1", "socket.io-client": "^4.8.1",
@@ -37,6 +42,7 @@
"peerDependencies": { "peerDependencies": {
"@affine/error": "workspace:*", "@affine/error": "workspace:*",
"@affine/graphql": "workspace:*", "@affine/graphql": "workspace:*",
"@blocksuite/affine": "workspace:*",
"idb": "^8.0.0", "idb": "^8.0.0",
"socket.io-client": "^4.7.5" "socket.io-client": "^4.7.5"
} }

1
packages/common/nbstore/src/frontend/index.ts
View File

@@ -1,3 +1,4 @@
export * from './awareness'; export * from './awareness';
export * from './blob'; export * from './blob';
export * from './doc'; export * from './doc';
export * from './indexer';

45
packages/common/nbstore/src/frontend/indexer.ts Normal file
View File

@@ -0,0 +1,45 @@
import type { IndexerStorage } from '../storage';
import type { IndexerSync } from '../sync/indexer';
export class IndexerFrontend {
constructor(
public readonly storage: IndexerStorage,
public readonly sync: IndexerSync
) {}
get state$() {
return this.sync.state$;
}
docState$(docId: string) {
return this.sync.docState$(docId);
}
search = this.storage.search.bind(this.storage);
aggregate = this.storage.aggregate.bind(this.storage);
// eslint-disable-next-line rxjs/finnish
search$ = this.storage.search$.bind(this.storage);
// eslint-disable-next-line rxjs/finnish
aggregate$ = this.storage.aggregate$.bind(this.storage);
addPriority(docId: string, priority: number) {
return this.sync.addPriority(docId, priority);
}
waitForCompleted(signal?: AbortSignal) {
return this.sync.waitForCompleted(signal);
}
waitForDocCompleted(docId: string, signal?: AbortSignal) {
return this.sync.waitForDocCompleted(docId, signal);
}
waitForDocCompletedWithPriority(
docId: string,
priority: number,
signal?: AbortSignal
) {
const undo = this.addPriority(docId, priority);
return this.sync.waitForDocCompleted(docId, signal).finally(() => undo());
}
}

11
packages/common/nbstore/src/impls/idb/index.ts
View File

@@ -3,15 +3,26 @@ import { IndexedDBBlobStorage } from './blob';
import { IndexedDBBlobSyncStorage } from './blob-sync'; import { IndexedDBBlobSyncStorage } from './blob-sync';
import { IndexedDBDocStorage } from './doc'; import { IndexedDBDocStorage } from './doc';
import { IndexedDBDocSyncStorage } from './doc-sync'; import { IndexedDBDocSyncStorage } from './doc-sync';
import { IndexedDBIndexerStorage } from './indexer';
import { IndexedDBIndexerSyncStorage } from './indexer-sync';
export * from './blob'; export * from './blob';
export * from './blob-sync'; export * from './blob-sync';
export * from './doc'; export * from './doc';
export * from './doc-sync'; export * from './doc-sync';
export * from './indexer';
export * from './indexer-sync';
export const idbStorages = [ export const idbStorages = [
IndexedDBDocStorage, IndexedDBDocStorage,
IndexedDBBlobStorage, IndexedDBBlobStorage,
IndexedDBDocSyncStorage, IndexedDBDocSyncStorage,
IndexedDBBlobSyncStorage, IndexedDBBlobSyncStorage,
IndexedDBIndexerStorage,
IndexedDBIndexerSyncStorage,
] satisfies StorageConstructor[];
export const idbStoragesIndexerOnly = [
IndexedDBIndexerStorage,
IndexedDBIndexerSyncStorage,
] satisfies StorageConstructor[]; ] satisfies StorageConstructor[];

38
packages/common/nbstore/src/impls/idb/indexer-sync.ts Normal file
View File

@@ -0,0 +1,38 @@
import { share } from '../../connection';
import type { DocClock } from '../../storage/doc';
import { IndexerSyncStorageBase } from '../../storage/indexer-sync';
import { IDBConnection, type IDBConnectionOptions } from './db';
export class IndexedDBIndexerSyncStorage extends IndexerSyncStorageBase {
static readonly identifier = 'IndexedDBIndexerSyncStorage';
readonly connection = share(new IDBConnection(this.options));
constructor(private readonly options: IDBConnectionOptions) {
super();
}
async getDocIndexedClock(docId: string): Promise<DocClock | null> {
const tx = this.connection.inner.db.transaction('indexerSync', 'readonly');
const store = tx.store;
const result = await store.get(docId);
return result
? { docId: result.docId, timestamp: result.indexedClock }
: null;
}
async setDocIndexedClock(docClock: DocClock): Promise<void> {
const tx = this.connection.inner.db.transaction('indexerSync', 'readwrite');
const store = tx.store;
await store.put({
docId: docClock.docId,
indexedClock: docClock.timestamp,
});
}
async clearDocIndexedClock(docId: string): Promise<void> {
const tx = this.connection.inner.db.transaction('indexerSync', 'readwrite');
const store = tx.store;
await store.delete(docId);
}
}

0
packages/common/infra/src/sync/indexer/impl/indexeddb/__tests__/bm25.spec.ts → packages/common/nbstore/src/impls/idb/indexer/__tests__/bm25.spec.ts
View File

0
packages/common/infra/src/sync/indexer/impl/indexeddb/__tests__/highlighter.spec.ts → packages/common/nbstore/src/impls/idb/indexer/__tests__/highlighter.spec.ts
View File

0
packages/common/infra/src/sync/indexer/impl/indexeddb/__tests__/tokenizer.spec.ts → packages/common/nbstore/src/impls/idb/indexer/__tests__/tokenizer.spec.ts
View File

0
packages/common/infra/src/sync/indexer/impl/indexeddb/bm25.ts → packages/common/nbstore/src/impls/idb/indexer/bm25.ts
View File

493
packages/common/nbstore/src/impls/idb/indexer/data-struct.ts Normal file
View File

@@ -0,0 +1,493 @@
import { type IDBPDatabase, type IDBPTransaction, type StoreNames } from 'idb';
import {
type AggregateOptions,
type AggregateResult,
type IndexerDocument,
type IndexerFieldSchema,
IndexerSchema,
type Query,
type SearchOptions,
type SearchResult,
} from '../../../storage';
import type { DocStorageSchema } from '../schema';
import { highlighter } from './highlighter';
import {
BooleanInvertedIndex,
FullTextInvertedIndex,
IntegerInvertedIndex,
type InvertedIndex,
StringInvertedIndex,
} from './inverted-index';
import { Match } from './match';
export type DataStructRWTransaction = IDBPTransaction<
DocStorageSchema,
ArrayLike<StoreNames<DocStorageSchema>>,
'readwrite'
>;
export type DataStructROTransaction = IDBPTransaction<
DocStorageSchema,
ArrayLike<StoreNames<DocStorageSchema>>,
'readonly' | 'readwrite'
>;
export class DataStruct {
database: IDBPDatabase<DocStorageSchema> = null as any;
invertedIndex = new Map<string, Map<string, InvertedIndex>>();
constructor() {
for (const [tableName, table] of Object.entries(IndexerSchema)) {
const tableInvertedIndex = new Map<string, InvertedIndex>();
for (const [fieldName, type] of Object.entries(table)) {
const typeInfo: IndexerFieldSchema =
typeof type === 'string' ? { type } : type;
if ('index' in typeInfo && typeInfo.index === false) {
// If index is false, we don't need to create an inverted index for this field.
continue;
}
if (typeInfo.type === 'String') {
tableInvertedIndex.set(
fieldName,
new StringInvertedIndex(tableName, fieldName)
);
} else if (typeInfo.type === 'Integer') {
tableInvertedIndex.set(
fieldName,
new IntegerInvertedIndex(tableName, fieldName)
);
} else if (typeInfo.type === 'FullText') {
tableInvertedIndex.set(
fieldName,
new FullTextInvertedIndex(tableName, fieldName)
);
} else if (typeInfo.type === 'Boolean') {
tableInvertedIndex.set(
fieldName,
new BooleanInvertedIndex(tableName, fieldName)
);
} else {
throw new Error(`Field type '${typeInfo.type}' not supported`);
}
}
this.invertedIndex.set(tableName, tableInvertedIndex);
}
}
private async update(
trx: DataStructRWTransaction,
table: keyof IndexerSchema,
document: IndexerDocument
) {
const existsNid = await trx
.objectStore('indexerRecords')
.index('id')
.getKey([table, document.id]);
const exists = existsNid
? await trx.objectStore('indexerRecords').get(existsNid)
: null;
if (!existsNid || !exists) {
// if not exists, return
return;
}
// delete exists one
await this.deleteByNid(trx, existsNid);
const dataMap = new Map([...exists.data, ...document.fields]); // merge exists data with new data
const nid = await trx
.objectStore('indexerRecords')
.put({ table, id: document.id, data: dataMap });
for (const [key, values] of dataMap) {
const type = IndexerSchema[table][
key as keyof IndexerSchema[typeof table]
] as IndexerFieldSchema;
if (!type) {
continue;
}
const typeInfo = typeof type === 'string' ? { type } : type;
if (typeInfo.index !== false) {
// If index is false, the field will not be indexed
const iidx = this.invertedIndex.get(table)?.get(key);
if (!iidx) {
continue;
}
await iidx.insert(trx, nid, values);
}
}
}
private async insert(
trx: DataStructRWTransaction,
table: keyof IndexerSchema,
document: IndexerDocument
) {
const existsNid = await trx
.objectStore('indexerRecords')
.index('id')
.getKey([table, document.id]);
if (existsNid) {
// delete exists one
await this.deleteByNid(trx, existsNid);
}
const dataMap = document.fields;
const nid = await trx
.objectStore('indexerRecords')
.put({ table, id: document.id, data: dataMap });
for (const [key, values] of dataMap) {
const type = IndexerSchema[table][
key as keyof IndexerSchema[typeof table]
] as IndexerFieldSchema;
if (!type) {
continue;
}
const typeInfo = typeof type === 'string' ? { type } : type;
if (typeInfo.index !== false) {
// If index is false, the field will not be indexed
const iidx = this.invertedIndex.get(table)?.get(key);
if (!iidx) {
continue;
}
await iidx.insert(trx, nid, values);
}
}
}
private async deleteByNid(trx: DataStructRWTransaction, nid: number) {
await trx.objectStore('indexerRecords').delete(nid);
const indexIds = await trx
.objectStore('invertedIndex')
.index('nid')
.getAllKeys(nid);
for (const indexId of indexIds) {
await trx.objectStore('invertedIndex').delete(indexId);
}
}
private async delete(
trx: DataStructRWTransaction,
table: keyof IndexerSchema,
id: string
) {
const nid = await trx
.objectStore('indexerRecords')
.index('id')
.getKey([table, id]);
if (nid) {
await this.deleteByNid(trx, nid);
} else {
return;
}
}
async deleteByQuery(
trx: DataStructRWTransaction,
table: keyof IndexerSchema,
query: Query<any>
) {
const match = await this.queryRaw(trx, table, query);
for (const nid of match.scores.keys()) {
await this.deleteByNid(trx, nid);
}
}
async batchWrite(
trx: DataStructRWTransaction,
table: keyof IndexerSchema,
deleteByQueries: Query<any>[],
deletes: string[],
inserts: IndexerDocument<any>[],
updates: IndexerDocument<any>[]
) {
for (const query of deleteByQueries) {
await this.deleteByQuery(trx, table, query);
}
for (const del of deletes) {
await this.delete(trx, table, del);
}
for (const inst of inserts) {
await this.insert(trx, table, inst);
}
for (const update of updates) {
await this.update(trx, table, update);
}
}
async matchAll(
trx: DataStructROTransaction,
table: keyof IndexerSchema
): Promise<Match> {
const allNids = await trx
.objectStore('indexerRecords')
.index('table')
.getAllKeys(table);
const match = new Match();
for (const nid of allNids) {
match.addScore(nid, 1);
}
return match;
}
async queryRaw(
trx: DataStructROTransaction,
table: keyof IndexerSchema,
query: Query<any>
): Promise<Match> {
if (query.type === 'match') {
const iidx = this.invertedIndex.get(table)?.get(query.field as string);
if (!iidx) {
return new Match();
}
return await iidx.match(trx, query.match);
} else if (query.type === 'boolean') {
const weights = [];
for (const q of query.queries) {
weights.push(await this.queryRaw(trx, table, q));
}
if (query.occur === 'must') {
return weights.reduce((acc, w) => acc.and(w));
} else if (query.occur === 'must_not') {
const total = weights.reduce((acc, w) => acc.and(w));
return (await this.matchAll(trx, table)).exclude(total);
} else if (query.occur === 'should') {
return weights.reduce((acc, w) => acc.or(w));
}
} else if (query.type === 'all') {
return await this.matchAll(trx, table);
} else if (query.type === 'boost') {
return (await this.queryRaw(trx, table, query.query)).boost(query.boost);
} else if (query.type === 'exists') {
const iidx = this.invertedIndex.get(table)?.get(query.field as string);
if (!iidx) {
return new Match();
}
return await iidx.all(trx);
}
throw new Error(`Query type '${query.type}' not supported`);
}
async clear(trx: DataStructRWTransaction) {
await trx.objectStore('indexerRecords').clear();
await trx.objectStore('invertedIndex').clear();
await trx.objectStore('indexerMetadata').clear();
}
async search(
trx: DataStructROTransaction,
table: keyof IndexerSchema,
query: Query<any>,
options: SearchOptions<any> = {}
): Promise<SearchResult<any, any>> {
const pagination = {
skip: options.pagination?.skip ?? 0,
limit: options.pagination?.limit ?? 100,
};
const match = await this.queryRaw(trx, table, query);
const nids = match
.toArray()
.slice(pagination.skip, pagination.skip + pagination.limit);
const nodes = [];
for (const nid of nids) {
const record = await trx.objectStore('indexerRecords').get(nid);
if (!record) {
continue;
}
nodes.push(this.resultNode(record, options, match, nid));
}
return {
pagination: {
count: match.size(),
hasMore: match.size() > pagination.limit + pagination.skip,
limit: pagination.limit,
skip: pagination.skip,
},
nodes: nodes,
};
}
async aggregate(
trx: DataStructROTransaction,
table: keyof IndexerSchema,
query: Query<any>,
field: string,
options: AggregateOptions<any> = {}
): Promise<AggregateResult<any, any>> {
const pagination = {
skip: options.pagination?.skip ?? 0,
limit: options.pagination?.limit ?? 100,
};
const hitPagination = options.hits
? {
skip: options.hits.pagination?.skip ?? 0,
limit: options.hits.pagination?.limit ?? 3,
}
: { skip: 0, limit: 0 };
const match = await this.queryRaw(trx, table, query);
const nids = match.toArray();
const buckets: {
key: string;
nids: number[];
hits: SearchResult<any, any>['nodes'];
}[] = [];
for (const nid of nids) {
const record = await trx.objectStore('indexerRecords').get(nid);
if (!record) {
continue;
}
const values = record.data.get(field);
for (const value of values ?? []) {
let bucket;
let bucketIndex = buckets.findIndex(b => b.key === value);
if (bucketIndex === -1) {
bucket = { key: value, nids: [], hits: [] };
buckets.push(bucket);
bucketIndex = buckets.length - 1;
} else {
bucket = buckets[bucketIndex];
}
if (
bucketIndex >= pagination.skip &&
bucketIndex < pagination.skip + pagination.limit
) {
bucket.nids.push(nid);
if (
bucket.nids.length - 1 >= hitPagination.skip &&
bucket.nids.length - 1 < hitPagination.skip + hitPagination.limit
) {
bucket.hits.push(
this.resultNode(record, options.hits ?? {}, match, nid)
);
}
}
}
}
return {
buckets: buckets
.slice(pagination.skip, pagination.skip + pagination.limit)
.map(bucket => {
const result = {
key: bucket.key,
score: match.getScore(bucket.nids[0]),
count: bucket.nids.length,
} as AggregateResult<any, any>['buckets'][number];
if (options.hits) {
(result as any).hits = {
pagination: {
count: bucket.nids.length,
hasMore:
bucket.nids.length > hitPagination.limit + hitPagination.skip,
limit: hitPagination.limit,
skip: hitPagination.skip,
},
nodes: bucket.hits,
} as SearchResult<any, any>;
}
return result;
}),
pagination: {
count: buckets.length,
hasMore: buckets.length > pagination.limit + pagination.skip,
limit: pagination.limit,
skip: pagination.skip,
},
};
}
async readonly(database: IDBPDatabase<DocStorageSchema>) {
return database.transaction(
['indexerRecords', 'invertedIndex', 'indexerMetadata'],
'readonly',
{ durability: 'relaxed' }
);
}
async readwrite(database: IDBPDatabase<DocStorageSchema>) {
return database.transaction(
['indexerRecords', 'invertedIndex', 'indexerMetadata'],
'readwrite',
{ durability: 'relaxed' }
);
}
private resultNode(
record: { id: string; data: Map<string, string[]> },
options: SearchOptions<any>,
match?: Match,
nid?: number
): SearchResult<any, any>['nodes'][number] {
const node = {
id: record.id,
score: match && nid ? match.getScore(nid) : 1,
} as any;
if (options.fields) {
const fields = {} as Record<string, string | string[]>;
for (const field of options.fields as string[]) {
fields[field] = record.data.get(field) ?? [''];
if (fields[field].length === 1) {
fields[field] = fields[field][0];
}
}
node.fields = fields;
}
if (match && nid && options.highlights) {
const highlights = {} as Record<string, string[]>;
for (const { field, before, end } of options.highlights) {
const highlightValues = match.getHighlighters(nid, field);
if (highlightValues) {
const rawValues = record.data.get(field) ?? [];
highlights[field] = Array.from(highlightValues)
.map(([index, ranges]) => {
const raw = rawValues[index];
if (raw) {
return (
highlighter(raw, before, end, ranges, {
maxPrefix: 20,
maxLength: 50,
}) ?? ''
);
}
return '';
})
.filter(Boolean);
}
}
node.highlights = highlights;
}
return node;
}
}

0
packages/common/infra/src/sync/indexer/impl/indexeddb/highlighter.ts → packages/common/nbstore/src/impls/idb/indexer/highlighter.ts
View File

203
packages/common/nbstore/src/impls/idb/indexer/index.ts Normal file
View File

@@ -0,0 +1,203 @@
import { merge, Observable, of, Subject, throttleTime } from 'rxjs';
import type {
AggregateOptions,
AggregateResult,
IndexerDocument,
IndexerSchema,
Query,
SearchOptions,
SearchResult,
} from '../../../storage';
import { IndexerStorageBase } from '../../../storage';
import { IDBConnection, type IDBConnectionOptions } from '../db';
import { DataStruct } from './data-struct';
import { backoffRetry, exhaustMapWithTrailing, fromPromise } from './utils';
export class IndexedDBIndexerStorage extends IndexerStorageBase {
static readonly identifier = 'IndexedDBIndexerStorage';
readonly connection = new IDBConnection(this.options);
override isReadonly = false;
private readonly data = new DataStruct();
private readonly tableUpdate$ = new Subject<string>();
/**
* The write operations of IndexedDBIndexerStorage are first cached in pendingUpdates,
* and then committed to IndexedDB in a batch through the refresh method.
*/
private readonly pendingUpdates: Record<
keyof IndexerSchema,
{
deleteByQueries: Query<any>[];
deletes: string[];
inserts: IndexerDocument[];
updates: IndexerDocument[];
}
> = {
doc: { deleteByQueries: [], deletes: [], inserts: [], updates: [] },
block: { deleteByQueries: [], deletes: [], inserts: [], updates: [] },
};
get channel() {
return this.connection.inner.channel;
}
get database() {
return this.connection.inner.db;
}
constructor(private readonly options: IDBConnectionOptions) {
super();
}
override async search<
T extends keyof IndexerSchema,
const O extends SearchOptions<T>,
>(table: T, query: Query<T>, options?: O): Promise<SearchResult<T, O>> {
const trx = await this.data.readonly(this.database);
return this.data.search(trx, table, query, options);
}
override async aggregate<
T extends keyof IndexerSchema,
const O extends AggregateOptions<T>,
>(
table: T,
query: Query<T>,
field: keyof IndexerSchema[T],
options?: O
): Promise<AggregateResult<T, O>> {
const trx = await this.data.readonly(this.database);
return this.data.aggregate(trx, table, query, field as string, options);
}
override search$<
T extends keyof IndexerSchema,
const O extends SearchOptions<T>,
>(table: T, query: Query<T>, options?: O): Observable<SearchResult<T, O>> {
return merge(of(1), this.watchTableUpdated(table)).pipe(
throttleTime(3000, undefined, { leading: true, trailing: true }),
exhaustMapWithTrailing(() => {
return fromPromise(async () => {
try {
const trx = await this.data.readonly(this.database);
return await this.data.search(trx, table, query, options);
} catch (error) {
console.error('search error', error);
throw error;
}
}).pipe(backoffRetry());
})
);
}
override aggregate$<
T extends keyof IndexerSchema,
const O extends AggregateOptions<T>,
>(
table: T,
query: Query<T>,
field: keyof IndexerSchema[T],
options?: O
): Observable<AggregateResult<T, O>> {
return merge(of(1), this.watchTableUpdated(table)).pipe(
throttleTime(3000, undefined, { leading: true, trailing: true }),
exhaustMapWithTrailing(() => {
return fromPromise(async () => {
try {
const trx = await this.data.readonly(this.database);
return await this.data.aggregate(
trx,
table,
query,
field as string,
options
);
} catch (error) {
console.error('aggregate error', error);
throw error;
}
}).pipe(backoffRetry());
})
);
}
override async deleteByQuery<T extends keyof IndexerSchema>(
table: T,
query: Query<T>
): Promise<void> {
this.pendingUpdates[table].deleteByQueries.push(query);
}
override insert<T extends keyof IndexerSchema>(
table: T,
document: IndexerDocument
): Promise<void> {
this.pendingUpdates[table].inserts.push(document);
return Promise.resolve();
}
override delete<T extends keyof IndexerSchema>(
table: T,
id: string
): Promise<void> {
this.pendingUpdates[table].deletes.push(id);
return Promise.resolve();
}
override update<T extends keyof IndexerSchema>(
table: T,
document: IndexerDocument
): Promise<void> {
this.pendingUpdates[table].updates.push(document);
return Promise.resolve();
}
override async refresh<T extends keyof IndexerSchema>(
table: T
): Promise<void> {
const trx = await this.data.readwrite(this.database);
const tables = table ? [table] : (['doc', 'block'] as const);
for (const table of tables) {
await this.data.batchWrite(
trx,
table,
this.pendingUpdates[table].deleteByQueries,
this.pendingUpdates[table].deletes,
this.pendingUpdates[table].inserts,
this.pendingUpdates[table].updates
);
this.pendingUpdates[table] = {
deleteByQueries: [],
deletes: [],
inserts: [],
updates: [],
};
}
this.emitTableUpdated(table);
}
private watchTableUpdated(table: keyof IndexerSchema) {
return new Observable(subscriber => {
const listener = (ev: MessageEvent) => {
if (ev.data.type === 'indexer-updated' && ev.data.table === table) {
subscriber.next(1);
}
};
const subscription = this.tableUpdate$.subscribe(updatedTable => {
if (updatedTable === table) {
subscriber.next(1);
}
});
this.channel.addEventListener('message', listener);
return () => {
this.channel.removeEventListener('message', listener);
subscription.unsubscribe();
};
});
}
emitTableUpdated(table: keyof IndexerSchema) {
this.tableUpdate$.next(table);
this.channel.postMessage({ type: 'indexer-updated', table });
}
}

103
packages/common/infra/src/sync/indexer/impl/indexeddb/inverted-index.ts → packages/common/nbstore/src/impls/idb/indexer/inverted-index.ts
View File

@@ -22,16 +22,18 @@ export interface InvertedIndex {
export class StringInvertedIndex implements InvertedIndex { export class StringInvertedIndex implements InvertedIndex {
constructor( constructor(
readonly fieldKey: string, readonly table: string,
readonly index: boolean = true, readonly fieldKey: string
readonly store: boolean = true
) {} ) {}
async match(trx: DataStructROTransaction, term: string): Promise<Match> { async match(trx: DataStructROTransaction, term: string): Promise<Match> {
const objs = await trx const objs = await trx
.objectStore('invertedIndex') .objectStore('invertedIndex')
.index('key') .index('key')
.getAll(InvertedIndexKey.forString(this.fieldKey, term).buffer()); .getAll([
this.table,
InvertedIndexKey.forString(this.fieldKey, term).buffer(),
]);
const match = new Match(); const match = new Match();
for (const obj of objs) { for (const obj of objs) {
match.addScore(obj.nid, 1); match.addScore(obj.nid, 1);
@@ -45,8 +47,11 @@ export class StringInvertedIndex implements InvertedIndex {
.index('key') .index('key')
.getAll( .getAll(
IDBKeyRange.bound( IDBKeyRange.bound(
InvertedIndexKey.forPrefix(this.fieldKey).buffer(), [this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer() [
this.table,
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
]
) )
); );
@@ -65,6 +70,7 @@ export class StringInvertedIndex implements InvertedIndex {
async insert(trx: DataStructRWTransaction, id: number, terms: string[]) { async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
for (const term of terms) { for (const term of terms) {
await trx.objectStore('invertedIndex').put({ await trx.objectStore('invertedIndex').put({
table: this.table,
key: InvertedIndexKey.forString(this.fieldKey, term).buffer(), key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
nid: id, nid: id,
}); });
@@ -74,16 +80,18 @@ export class StringInvertedIndex implements InvertedIndex {
export class IntegerInvertedIndex implements InvertedIndex { export class IntegerInvertedIndex implements InvertedIndex {
constructor( constructor(
readonly fieldKey: string, readonly table: string,
readonly index: boolean = true, readonly fieldKey: string
readonly store: boolean = true
) {} ) {}
async match(trx: DataStructROTransaction, term: string): Promise<Match> { async match(trx: DataStructROTransaction, term: string): Promise<Match> {
const objs = await trx const objs = await trx
.objectStore('invertedIndex') .objectStore('invertedIndex')
.index('key') .index('key')
.getAll(InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer()); .getAll([
this.table,
InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(),
]);
const match = new Match(); const match = new Match();
for (const obj of objs) { for (const obj of objs) {
match.addScore(obj.nid, 1); match.addScore(obj.nid, 1);
@@ -98,8 +106,11 @@ export class IntegerInvertedIndex implements InvertedIndex {
.index('key') .index('key')
.getAll( .getAll(
IDBKeyRange.bound( IDBKeyRange.bound(
InvertedIndexKey.forPrefix(this.fieldKey).buffer(), [this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer() [
this.table,
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
]
) )
); );
@@ -118,6 +129,7 @@ export class IntegerInvertedIndex implements InvertedIndex {
async insert(trx: DataStructRWTransaction, id: number, terms: string[]) { async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
for (const term of terms) { for (const term of terms) {
await trx.objectStore('invertedIndex').put({ await trx.objectStore('invertedIndex').put({
table: this.table,
key: InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(), key: InvertedIndexKey.forInt64(this.fieldKey, BigInt(term)).buffer(),
nid: id, nid: id,
}); });
@@ -127,9 +139,8 @@ export class IntegerInvertedIndex implements InvertedIndex {
export class BooleanInvertedIndex implements InvertedIndex { export class BooleanInvertedIndex implements InvertedIndex {
constructor( constructor(
readonly fieldKey: string, readonly table: string,
readonly index: boolean = true, readonly fieldKey: string
readonly store: boolean = true
) {} ) {}
// eslint-disable-next-line sonarjs/no-identical-functions // eslint-disable-next-line sonarjs/no-identical-functions
@@ -139,8 +150,11 @@ export class BooleanInvertedIndex implements InvertedIndex {
.index('key') .index('key')
.getAll( .getAll(
IDBKeyRange.bound( IDBKeyRange.bound(
InvertedIndexKey.forPrefix(this.fieldKey).buffer(), [this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer() [
this.table,
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
]
) )
); );
@@ -160,9 +174,10 @@ export class BooleanInvertedIndex implements InvertedIndex {
const objs = await trx const objs = await trx
.objectStore('invertedIndex') .objectStore('invertedIndex')
.index('key') .index('key')
.getAll( .getAll([
InvertedIndexKey.forBoolean(this.fieldKey, term === 'true').buffer() this.table,
); InvertedIndexKey.forBoolean(this.fieldKey, term === 'true').buffer(),
]);
const match = new Match(); const match = new Match();
for (const obj of objs) { for (const obj of objs) {
match.addScore(obj.nid, 1); match.addScore(obj.nid, 1);
@@ -173,6 +188,7 @@ export class BooleanInvertedIndex implements InvertedIndex {
async insert(trx: DataStructRWTransaction, id: number, terms: string[]) { async insert(trx: DataStructRWTransaction, id: number, terms: string[]) {
for (const term of terms) { for (const term of terms) {
await trx.objectStore('invertedIndex').put({ await trx.objectStore('invertedIndex').put({
table: this.table,
key: InvertedIndexKey.forBoolean( key: InvertedIndexKey.forBoolean(
this.fieldKey, this.fieldKey,
term === 'true' term === 'true'
@@ -185,9 +201,8 @@ export class BooleanInvertedIndex implements InvertedIndex {
export class FullTextInvertedIndex implements InvertedIndex { export class FullTextInvertedIndex implements InvertedIndex {
constructor( constructor(
readonly fieldKey: string, readonly table: string,
readonly index: boolean = true, readonly fieldKey: string
readonly store: boolean = true
) {} ) {}
async match(trx: DataStructROTransaction, term: string): Promise<Match> { async match(trx: DataStructROTransaction, term: string): Promise<Match> {
@@ -205,8 +220,8 @@ export class FullTextInvertedIndex implements InvertedIndex {
const avgFieldLength = const avgFieldLength =
( (
await trx await trx
.objectStore('kvMetadata') .objectStore('indexerMetadata')
.get(`full-text:avg-field-length:${this.fieldKey}`) .get(`full-text:avg-field-length:${this.table}:${this.fieldKey}`)
)?.value ?? 0; )?.value ?? 0;
for (const token of queryTokens) { for (const token of queryTokens) {
const key = InvertedIndexKey.forString(this.fieldKey, token.term); const key = InvertedIndexKey.forString(this.fieldKey, token.term);
@@ -214,7 +229,12 @@ export class FullTextInvertedIndex implements InvertedIndex {
.objectStore('invertedIndex') .objectStore('invertedIndex')
.index('key') .index('key')
.getAll( .getAll(
IDBKeyRange.bound(key.buffer(), key.add1().buffer(), false, true) IDBKeyRange.bound(
[this.table, key.buffer()],
[this.table, key.add1().buffer()],
false,
true
)
); );
const submatched: { const submatched: {
nid: number; nid: number;
@@ -316,8 +336,11 @@ export class FullTextInvertedIndex implements InvertedIndex {
.index('key') .index('key')
.getAll( .getAll(
IDBKeyRange.bound( IDBKeyRange.bound(
InvertedIndexKey.forPrefix(this.fieldKey).buffer(), [this.table, InvertedIndexKey.forPrefix(this.fieldKey).buffer()],
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer() [
this.table,
InvertedIndexKey.forPrefix(this.fieldKey).add1().buffer(),
]
) )
); );
@@ -348,6 +371,7 @@ export class FullTextInvertedIndex implements InvertedIndex {
for (const [term, tokens] of tokenMap) { for (const [term, tokens] of tokenMap) {
await trx.objectStore('invertedIndex').put({ await trx.objectStore('invertedIndex').put({
table: this.table,
key: InvertedIndexKey.forString(this.fieldKey, term).buffer(), key: InvertedIndexKey.forString(this.fieldKey, term).buffer(),
nid: id, nid: id,
pos: { pos: {
@@ -358,23 +382,26 @@ export class FullTextInvertedIndex implements InvertedIndex {
}); });
} }
const kvMetadataStore = trx.objectStore('kvMetadata'); const indexerMetadataStore = trx.objectStore('indexerMetadata');
// update avg-field-length // update avg-field-length
const totalCount = const totalCount =
(await kvMetadataStore.get(`full-text:field-count:${this.fieldKey}`))
?.value ?? 0;
const avgFieldLength =
( (
await kvMetadataStore.get( await indexerMetadataStore.get(
`full-text:avg-field-length:${this.fieldKey}` `full-text:field-count:${this.table}:${this.fieldKey}`
) )
)?.value ?? 0; )?.value ?? 0;
await kvMetadataStore.put({ const avgFieldLength =
key: `full-text:field-count:${this.fieldKey}`, (
await indexerMetadataStore.get(
`full-text:avg-field-length:${this.table}:${this.fieldKey}`
)
)?.value ?? 0;
await indexerMetadataStore.put({
key: `full-text:field-count:${this.table}:${this.fieldKey}`,
value: totalCount + 1, value: totalCount + 1,
}); });
await kvMetadataStore.put({ await indexerMetadataStore.put({
key: `full-text:avg-field-length:${this.fieldKey}`, key: `full-text:avg-field-length:${this.table}:${this.fieldKey}`,
value: value:
avgFieldLength + avgFieldLength +
(terms.reduce((acc, term) => acc + term.length, 0) - avgFieldLength) / (terms.reduce((acc, term) => acc + term.length, 0) - avgFieldLength) /

0
packages/common/infra/src/sync/indexer/impl/indexeddb/match.ts → packages/common/nbstore/src/impls/idb/indexer/match.ts
View File

0
packages/common/infra/src/sync/indexer/impl/indexeddb/tokenizer.ts → packages/common/nbstore/src/impls/idb/indexer/tokenizer.ts
View File

104
packages/common/nbstore/src/impls/idb/indexer/utils.ts Normal file
View File

@@ -0,0 +1,104 @@
import {
asyncScheduler,
defer,
exhaustMap,
finalize,
Observable,
type ObservableInput,
type OperatorFunction,
retry,
scheduled,
Subject,
throttle,
throwError,
timer,
} from 'rxjs';
import { MANUALLY_STOP } from '../../../utils/throw-if-aborted';
/**
* Like exhaustMap, but also includes the trailing value emitted from the source observable while waiting for the preceding inner observable to complete
*
* Original code adapted from https://github.com/ReactiveX/rxjs/issues/5004
* @param {function<T, K>(value: T, ?index: number): ObservableInput<K>} project - A function that, when applied to an item emitted by the
* source Observable, returns a projected Observable.
*/
export function exhaustMapWithTrailing<T, R>(
project: (value: T, index: number) => ObservableInput<R>
): OperatorFunction<T, R> {
return (source$): Observable<R> =>
defer(() => {
const release$ = new Subject<void>();
return source$.pipe(
throttle(() => release$, {
leading: true,
trailing: true,
}),
exhaustMap((value, index) =>
scheduled(project(value, index), asyncScheduler).pipe(
finalize(() => {
release$.next();
})
)
)
);
});
}
/**
* Convert a promise to an observable.
*
* like `from` but support `AbortSignal`.
*/
export function fromPromise<T>(
promise: Promise<T> | ((signal: AbortSignal) => Promise<T>)
): Observable<T> {
return new Observable(subscriber => {
const abortController = new AbortController();
const rawPromise =
promise instanceof Function ? promise(abortController.signal) : promise;
rawPromise
.then(value => {
subscriber.next(value);
subscriber.complete();
})
.catch(error => {
subscriber.error(error);
});
return () => abortController.abort(MANUALLY_STOP);
});
}
/**
* An operator that retries the source observable when an error occurs.
*
* https://en.wikipedia.org/wiki/Exponential_backoff
*/
export function backoffRetry<T>({
when,
count = 3,
delay = 200,
maxDelay = 15000,
}: {
when?: (err: any) => boolean;
count?: number;
delay?: number;
maxDelay?: number;
} = {}) {
return (obs$: Observable<T>) =>
obs$.pipe(
retry({
count,
delay: (err, retryIndex) => {
if (when && !when(err)) {
return throwError(() => err);
}
const d = Math.pow(2, retryIndex - 1) * delay;
return timer(Math.min(d, maxDelay));
},
})
);
}

69
packages/common/nbstore/src/impls/idb/schema.ts
View File

@@ -37,6 +37,11 @@ Table(PeerClocks)
|------|-------|-----------|-----------| |------|-------|-----------|-----------|
| str | str | Date | Date | | str | str | Date | Date |
Table(IndexerSync)
| docId | clock |
|-------|-------|
| str | Date |
Table(BlobSync) Table(BlobSync)
| peer | key | uploadedAt | | peer | key | uploadedAt |
|------|-----|------------| |------|-----|------------|
@@ -124,6 +129,43 @@ export interface DocStorageSchema extends DBSchema {
lock: Date; lock: Date;
}; };
}; };
indexerSync: {
key: string;
value: {
docId: string;
indexedClock: Date;
};
};
indexerMetadata: {
key: string;
value: {
key: string;
value: any;
};
};
indexerRecords: {
key: number;
value: {
table: string;
id: string;
data: Map<string, string[]>;
};
indexes: { table: string; id: [string, string] };
};
invertedIndex: {
key: number;
value: {
table: string;
nid: number;
pos?: {
i: number /* index */;
l: number /* length */;
rs: [number, number][] /* ranges: [start, end] */;
};
key: ArrayBuffer;
};
indexes: { key: [string, ArrayBuffer]; nid: number };
};
} }
const migrate: OpenDBCallbacks<DocStorageSchema>['upgrade'] = ( const migrate: OpenDBCallbacks<DocStorageSchema>['upgrade'] = (
@@ -199,11 +241,36 @@ const initBlobSync: Migrate = db => {
blobSync.createIndex('peer', 'peer', { unique: false }); blobSync.createIndex('peer', 'peer', { unique: false });
}; };
const initIndexer: Migrate = db => {
db.createObjectStore('indexerMetadata', {
keyPath: 'key',
});
const indexRecordsStore = db.createObjectStore('indexerRecords', {
autoIncrement: true,
});
indexRecordsStore.createIndex('table', 'table', {
unique: false,
});
indexRecordsStore.createIndex('id', ['table', 'id'], {
unique: true,
});
const invertedIndexStore = db.createObjectStore('invertedIndex', {
autoIncrement: true,
});
invertedIndexStore.createIndex('key', ['table', 'key'], {
unique: false,
});
invertedIndexStore.createIndex('nid', 'nid', { unique: false });
db.createObjectStore('indexerSync', {
keyPath: 'docId',
autoIncrement: false,
});
};
// END REGION // END REGION
// 1. all schema changed should be put in migrations // 1. all schema changed should be put in migrations
// 2. order matters // 2. order matters
const migrations: Migrate[] = [init, initBlobSync]; const migrations: Migrate[] = [init, initBlobSync, initIndexer];
export const migrator = { export const migrator = {
version: migrations.length, version: migrations.length,

4
packages/common/nbstore/src/index.ts
View File

@@ -1,5 +1,5 @@
export * from './connection'; export * from './connection';
export * from './frontend'; export type * from './frontend'; // // Only export types. For implementation, please import from '@affine/nbstore/frontend'
export * from './storage'; export * from './storage';
export * from './sync'; export type * from './sync'; // Only export types. For implementation, please import from '@affine/nbstore/sync'
export * from './utils/universal-id'; export * from './utils/universal-id';

3
packages/common/nbstore/src/storage/doc.ts
View File

@@ -47,6 +47,7 @@ export interface DocStorageOptions {
export interface DocStorage extends Storage { export interface DocStorage extends Storage {
readonly storageType: 'doc'; readonly storageType: 'doc';
readonly isReadonly: boolean; readonly isReadonly: boolean;
readonly spaceId: string;
/** /**
* Get a doc record with latest binary. * Get a doc record with latest binary.
*/ */
@@ -103,7 +104,7 @@ export abstract class DocStorageBase<Opts = {}> implements DocStorage {
readonly storageType = 'doc'; readonly storageType = 'doc';
abstract readonly connection: Connection; abstract readonly connection: Connection;
protected readonly locker: Locker = new SingletonLocker(); protected readonly locker: Locker = new SingletonLocker();
protected readonly spaceId = this.options.id; readonly spaceId = this.options.id;
constructor(protected readonly options: Opts & DocStorageOptions) {} constructor(protected readonly options: Opts & DocStorageOptions) {}

16
packages/common/nbstore/src/storage/dummy/indexer-sync.ts Normal file
View File

@@ -0,0 +1,16 @@
import { DummyConnection } from '../../connection';
import type { DocClock } from '../doc';
import { IndexerSyncStorageBase } from '../indexer-sync';
export class DummyIndexerSyncStorage extends IndexerSyncStorageBase {
override connection = new DummyConnection();
override getDocIndexedClock(_docId: string): Promise<DocClock | null> {
return Promise.resolve(null);
}
override setDocIndexedClock(_docClock: DocClock): Promise<void> {
return Promise.resolve();
}
override clearDocIndexedClock(_docId: string): Promise<void> {
return Promise.resolve();
}
}

88
packages/common/nbstore/src/storage/dummy/indexer.ts Normal file
View File

@@ -0,0 +1,88 @@
import { NEVER, type Observable } from 'rxjs';
import { DummyConnection } from '../../connection';
import {
type AggregateOptions,
type AggregateResult,
type IndexerDocument,
type IndexerSchema,
IndexerStorageBase,
type Query,
type SearchOptions,
type SearchResult,
} from '../indexer';
export class DummyIndexerStorage extends IndexerStorageBase {
readonly isReadonly = true;
readonly connection = new DummyConnection();
override search<
T extends keyof IndexerSchema,
const O extends SearchOptions<T>,
>(_table: T, _query: Query<T>, _options?: O): Promise<SearchResult<T, O>> {
return Promise.resolve({
pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
nodes: [],
});
}
override aggregate<
T extends keyof IndexerSchema,
const O extends AggregateOptions<T>,
>(
_table: T,
_query: Query<T>,
_field: keyof IndexerSchema[T],
_options?: O
): Promise<AggregateResult<T, O>> {
return Promise.resolve({
pagination: { count: 0, limit: 0, skip: 0, hasMore: false },
buckets: [],
});
}
override search$<
T extends keyof IndexerSchema,
const O extends SearchOptions<T>,
>(_table: T, _query: Query<T>, _options?: O): Observable<SearchResult<T, O>> {
return NEVER;
}
override aggregate$<
T extends keyof IndexerSchema,
const O extends AggregateOptions<T>,
>(
_table: T,
_query: Query<T>,
_field: keyof IndexerSchema[T],
_options?: O
): Observable<AggregateResult<T, O>> {
return NEVER;
}
override deleteByQuery<T extends keyof IndexerSchema>(
_table: T,
_query: Query<T>
): Promise<void> {
return Promise.resolve();
}
override insert<T extends keyof IndexerSchema>(
_table: T,
_document: IndexerDocument<T>
): Promise<void> {
return Promise.resolve();
}
override delete<T extends keyof IndexerSchema>(
_table: T,
_id: string
): Promise<void> {
return Promise.resolve();
}
override update<T extends keyof IndexerSchema>(
_table: T,
_document: IndexerDocument<T>
): Promise<void> {
return Promise.resolve();
}
override refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
return Promise.resolve();
}
}

11
packages/common/nbstore/src/storage/index.ts
View File

@@ -10,6 +10,10 @@ import { DummyBlobStorage } from './dummy/blob';
import { DummyBlobSyncStorage } from './dummy/blob-sync'; import { DummyBlobSyncStorage } from './dummy/blob-sync';
import { DummyDocStorage } from './dummy/doc'; import { DummyDocStorage } from './dummy/doc';
import { DummyDocSyncStorage } from './dummy/doc-sync'; import { DummyDocSyncStorage } from './dummy/doc-sync';
import { DummyIndexerStorage } from './dummy/indexer';
import { DummyIndexerSyncStorage } from './dummy/indexer-sync';
import type { IndexerStorage } from './indexer';
import type { IndexerSyncStorage } from './indexer-sync';
import type { StorageType } from './storage'; import type { StorageType } from './storage';
type Storages = type Storages =
@@ -17,7 +21,9 @@ type Storages =
| BlobStorage | BlobStorage
| BlobSyncStorage | BlobSyncStorage
| DocSyncStorage | DocSyncStorage
| AwarenessStorage; | AwarenessStorage
| IndexerStorage
| IndexerSyncStorage;
export type SpaceStorageOptions = { export type SpaceStorageOptions = {
[K in StorageType]?: Storages & { storageType: K }; [K in StorageType]?: Storages & { storageType: K };
@@ -37,6 +43,8 @@ export class SpaceStorage {
blobSync: storages.blobSync ?? new DummyBlobSyncStorage(), blobSync: storages.blobSync ?? new DummyBlobSyncStorage(),
doc: storages.doc ?? new DummyDocStorage(), doc: storages.doc ?? new DummyDocStorage(),
docSync: storages.docSync ?? new DummyDocSyncStorage(), docSync: storages.docSync ?? new DummyDocSyncStorage(),
indexer: storages.indexer ?? new DummyIndexerStorage(),
indexerSync: storages.indexerSync ?? new DummyIndexerSyncStorage(),
}; };
} }
@@ -83,4 +91,5 @@ export * from './doc';
export * from './doc-sync'; export * from './doc-sync';
export * from './errors'; export * from './errors';
export * from './history'; export * from './history';
export * from './indexer';
export * from './storage'; export * from './storage';

21
packages/common/nbstore/src/storage/indexer-sync.ts Normal file
View File

@@ -0,0 +1,21 @@
import type { Connection } from '../connection';
import type { DocClock } from './doc';
import type { Storage } from './storage';
export interface IndexerSyncStorage extends Storage {
readonly storageType: 'indexerSync';
getDocIndexedClock(docId: string): Promise<DocClock | null>;
setDocIndexedClock(docClock: DocClock): Promise<void>;
clearDocIndexedClock(docId: string): Promise<void>;
}
export abstract class IndexerSyncStorageBase implements IndexerSyncStorage {
readonly storageType = 'indexerSync';
abstract connection: Connection<any>;
abstract getDocIndexedClock(docId: string): Promise<DocClock | null>;
abstract setDocIndexedClock(docClock: DocClock): Promise<void>;
abstract clearDocIndexedClock(docId: string): Promise<void>;
}

176
packages/common/nbstore/src/storage/indexer.ts Normal file
View File

@@ -0,0 +1,176 @@
export * from './indexer/document';
export * from './indexer/field-type';
export * from './indexer/query';
export * from './indexer/schema';
import type { Observable } from 'rxjs';
import type { Connection } from '../connection';
import type { IndexerDocument } from './indexer/document';
import type { Query } from './indexer/query';
import type { IndexerSchema } from './indexer/schema';
import type { Storage } from './storage';
export interface IndexerStorage extends Storage {
readonly storageType: 'indexer';
readonly isReadonly: boolean;
search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
table: T,
query: Query<T>,
options?: O
): Promise<SearchResult<T, O>>;
aggregate<T extends keyof IndexerSchema, const O extends AggregateOptions<T>>(
table: T,
query: Query<T>,
field: keyof IndexerSchema[T],
options?: O
): Promise<AggregateResult<T, O>>;
search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
table: T,
query: Query<T>,
options?: O
): Observable<SearchResult<T, O>>;
aggregate$<
T extends keyof IndexerSchema,
const O extends AggregateOptions<T>,
>(
table: T,
query: Query<T>,
field: keyof IndexerSchema[T],
options?: O
): Observable<AggregateResult<T, O>>;
deleteByQuery<T extends keyof IndexerSchema>(
table: T,
query: Query<T>
): Promise<void>;
insert<T extends keyof IndexerSchema>(
table: T,
document: IndexerDocument<T>
): Promise<void>;
delete<T extends keyof IndexerSchema>(table: T, id: string): Promise<void>;
update<T extends keyof IndexerSchema>(
table: T,
document: IndexerDocument<T>
): Promise<void>;
refresh<T extends keyof IndexerSchema>(table: T): Promise<void>;
}
type ResultPagination = {
count: number;
limit: number;
skip: number;
hasMore: boolean;
};
type PaginationOption = { limit?: number; skip?: number };
type HighlightAbleField<T extends keyof IndexerSchema> = {
[K in keyof IndexerSchema[T]]: IndexerSchema[T][K] extends 'FullText'
? K
: never;
}[keyof IndexerSchema[T]];
export type SearchOptions<T extends keyof IndexerSchema> = {
pagination?: PaginationOption;
highlights?: { field: HighlightAbleField<T>; before: string; end: string }[];
fields?: (keyof IndexerSchema[T])[];
};
export type SearchResult<
T extends keyof IndexerSchema,
O extends SearchOptions<T>,
> = {
pagination: ResultPagination;
nodes: ({ id: string; score: number } & (O['fields'] extends any[]
? { fields: { [key in O['fields'][number]]: string | string[] } }
: unknown) &
(O['highlights'] extends any[]
? { highlights: { [key in O['highlights'][number]['field']]: string[] } }
: unknown))[];
};
export interface AggregateOptions<T extends keyof IndexerSchema> {
pagination?: PaginationOption;
hits?: SearchOptions<T>;
}
export type AggregateResult<
T extends keyof IndexerSchema,
O extends AggregateOptions<T>,
> = {
pagination: ResultPagination;
buckets: ({
key: string;
score: number;
count: number;
} & (O['hits'] extends object
? { hits: SearchResult<T, O['hits']> }
: unknown))[];
};
export abstract class IndexerStorageBase implements IndexerStorage {
readonly storageType = 'indexer';
abstract readonly connection: Connection;
abstract readonly isReadonly: boolean;
abstract search<
T extends keyof IndexerSchema,
const O extends SearchOptions<T>,
>(table: T, query: Query<T>, options?: O): Promise<SearchResult<T, O>>;
abstract aggregate<
T extends keyof IndexerSchema,
const O extends AggregateOptions<T>,
>(
table: T,
query: Query<T>,
field: keyof IndexerSchema[T],
options?: O
): Promise<AggregateResult<T, O>>;
abstract search$<
T extends keyof IndexerSchema,
const O extends SearchOptions<T>,
>(table: T, query: Query<T>, options?: O): Observable<SearchResult<T, O>>;
abstract aggregate$<
T extends keyof IndexerSchema,
const O extends AggregateOptions<T>,
>(
table: T,
query: Query<T>,
field: keyof IndexerSchema[T],
options?: O
): Observable<AggregateResult<T, O>>;
abstract deleteByQuery<T extends keyof IndexerSchema>(
table: T,
query: Query<T>
): Promise<void>;
abstract insert<T extends keyof IndexerSchema>(
table: T,
document: IndexerDocument<T>
): Promise<void>;
abstract delete<T extends keyof IndexerSchema>(
table: T,
id: string
): Promise<void>;
abstract update<T extends keyof IndexerSchema>(
table: T,
document: IndexerDocument<T>
): Promise<void>;
abstract refresh<T extends keyof IndexerSchema>(table: T): Promise<void>;
}

27
packages/common/infra/src/sync/indexer/document.ts → packages/common/nbstore/src/storage/indexer/document.ts
View File

@@ -1,11 +1,16 @@
import type { Schema } from './schema'; import type { IndexerSchema } from './schema';
export class Document<S extends Schema = any> { export class IndexerDocument<
S extends keyof IndexerSchema = keyof IndexerSchema,
> {
constructor(public readonly id: string) {} constructor(public readonly id: string) {}
fields = new Map<keyof S, string[]>(); fields = new Map<keyof IndexerSchema[S], string[]>();
public insert<F extends keyof S>(field: F, value: string | string[]) { public insert<F extends keyof IndexerSchema[S]>(
field: F,
value: string | string[]
) {
const values = this.fields.get(field) ?? []; const values = this.fields.get(field) ?? [];
if (Array.isArray(value)) { if (Array.isArray(value)) {
values.push(...value); values.push(...value);
@@ -15,7 +20,9 @@ export class Document<S extends Schema = any> {
this.fields.set(field, values); this.fields.set(field, values);
} }
get<F extends keyof S>(field: F): string[] | string | undefined { get<F extends keyof IndexerSchema[S]>(
field: F
): string[] | string | undefined {
const values = this.fields.get(field); const values = this.fields.get(field);
if (values === undefined) { if (values === undefined) {
return undefined; return undefined;
@@ -26,13 +33,13 @@ export class Document<S extends Schema = any> {
} }
} }
static from<S extends Schema>( static from<S extends keyof IndexerSchema>(
id: string, id: string,
map: map:
| Partial<Record<keyof S, string | string[]>> | Partial<Record<keyof IndexerSchema[S], string | string[]>>
| Map<keyof S, string | string[]> | Map<keyof IndexerSchema[S], string | string[]>
): Document<S> { ): IndexerDocument<S> {
const doc = new Document(id); const doc = new IndexerDocument<S>(id);
if (map instanceof Map) { if (map instanceof Map) {
for (const [key, value] of map) { for (const [key, value] of map) {

1
packages/common/nbstore/src/storage/indexer/field-type.ts Normal file
View File

@@ -0,0 +1 @@
export type IndexFieldType = 'Integer' | 'FullText' | 'String' | 'Boolean';

35
packages/common/nbstore/src/storage/indexer/query.ts Normal file
View File

@@ -0,0 +1,35 @@
import type { IndexerSchema } from './schema';
export type MatchQuery<T extends keyof IndexerSchema> = {
type: 'match';
field: keyof IndexerSchema[T];
match: string;
};
export type BoostQuery = {
type: 'boost';
query: Query<any>;
boost: number;
};
export type BooleanQuery<T extends keyof IndexerSchema> = {
type: 'boolean';
occur: 'should' | 'must' | 'must_not';
queries: Query<T>[];
};
export type ExistsQuery<T extends keyof IndexerSchema> = {
type: 'exists';
field: keyof IndexerSchema[T];
};
export type AllQuery = {
type: 'all';
};
export type Query<T extends keyof IndexerSchema> =
| BooleanQuery<T>
| MatchQuery<T>
| AllQuery
| ExistsQuery<T>
| BoostQuery;

51
packages/common/nbstore/src/storage/indexer/schema.ts Normal file
View File

@@ -0,0 +1,51 @@
import type { IndexFieldType } from './field-type';
export const IndexerSchema = {
doc: {
docId: 'String',
title: 'FullText',
// summary of the doc, used for preview
summary: { type: 'String', index: false },
},
block: {
docId: 'String',
blockId: 'String',
content: 'FullText',
flavour: 'String',
blob: 'String',
// reference doc id
// ['xxx','yyy']
refDocId: 'String',
// reference info, used for backlink to specific block
// [{"docId":"xxx","mode":"page","blockIds":["gt5Yfq1maYvgNgpi13rIq"]},{"docId":"yyy","mode":"edgeless","blockIds":["k5prpOlDF-9CzfatmO0W7"]}]
ref: { type: 'String', index: false },
// parent block flavour
parentFlavour: 'String',
// parent block id
parentBlockId: 'String',
// additional info
// { "databaseName": "xxx", "displayMode": "page/edgeless", "noteBlockId": "xxx" }
additional: { type: 'String', index: false },
markdownPreview: { type: 'String', index: false },
},
} satisfies Record<string, Record<string, IndexerFieldSchema>>;
export type IndexerFieldSchema =
| IndexFieldType
| {
type: IndexFieldType;
/**
* If false, the field will not be indexed, and thus not searchable.
*
* default: true
*/
index?: boolean;
/**
* If false, the field will not be stored, and not included in the search result.
*
* default: true
*/
store?: boolean;
};
export type IndexerSchema = typeof IndexerSchema;

9
packages/common/nbstore/src/storage/storage.ts
View File

@@ -1,6 +1,13 @@
import type { Connection } from '../connection'; import type { Connection } from '../connection';
export type StorageType = 'blob' | 'blobSync' | 'doc' | 'docSync' | 'awareness'; export type StorageType =
| 'blob'
| 'blobSync'
| 'doc'
| 'docSync'
| 'awareness'
| 'indexer'
| 'indexerSync';
export interface Storage { export interface Storage {
readonly storageType: StorageType; readonly storageType: StorageType;

6
packages/common/nbstore/src/sync/blob/peer.ts
View File

@@ -420,7 +420,11 @@ class BlobSyncPeerStatus {
return () => { return () => {
dispose.unsubscribe(); dispose.unsubscribe();
}; };
}); }).pipe(
share({
connector: () => new ReplaySubject(1),
})
);
} }
private readonly statusUpdatedSubject$ = new Subject<string | true>(); private readonly statusUpdatedSubject$ = new Subject<string | true>();

8
packages/common/nbstore/src/sync/index.ts
View File

@@ -4,10 +4,12 @@ import type { SpaceStorage } from '../storage';
import { AwarenessSyncImpl } from './awareness'; import { AwarenessSyncImpl } from './awareness';
import { BlobSyncImpl } from './blob'; import { BlobSyncImpl } from './blob';
import { DocSyncImpl, type DocSyncState } from './doc'; import { DocSyncImpl, type DocSyncState } from './doc';
import { IndexerSyncImpl } from './indexer';
import type { PeerStorageOptions } from './types'; import type { PeerStorageOptions } from './types';
export type { BlobSyncState } from './blob'; export type { BlobSyncState } from './blob';
export type { DocSyncDocState, DocSyncState } from './doc'; export type { DocSyncDocState, DocSyncState } from './doc';
export type { IndexerDocSyncState, IndexerSyncState } from './indexer';
export interface SyncState { export interface SyncState {
doc?: DocSyncState; doc?: DocSyncState;
@@ -17,6 +19,7 @@ export class Sync {
readonly doc: DocSyncImpl; readonly doc: DocSyncImpl;
readonly blob: BlobSyncImpl; readonly blob: BlobSyncImpl;
readonly awareness: AwarenessSyncImpl; readonly awareness: AwarenessSyncImpl;
readonly indexer: IndexerSyncImpl;
readonly state$: Observable<SyncState>; readonly state$: Observable<SyncState>;
@@ -26,6 +29,8 @@ export class Sync {
const docSync = storages.local.get('docSync'); const docSync = storages.local.get('docSync');
const blobSync = storages.local.get('blobSync'); const blobSync = storages.local.get('blobSync');
const awareness = storages.local.get('awareness'); const awareness = storages.local.get('awareness');
const indexer = storages.local.get('indexer');
const indexerSync = storages.local.get('indexerSync');
this.doc = new DocSyncImpl( this.doc = new DocSyncImpl(
{ {
@@ -60,6 +65,7 @@ export class Sync {
]) ])
), ),
}); });
this.indexer = new IndexerSyncImpl(doc, indexer, indexerSync);
this.state$ = this.doc.state$.pipe(map(doc => ({ doc }))); this.state$ = this.doc.state$.pipe(map(doc => ({ doc })));
} }
@@ -67,10 +73,12 @@ export class Sync {
start() { start() {
this.doc?.start(); this.doc?.start();
this.blob?.start(); this.blob?.start();
this.indexer?.start();
} }
stop() { stop() {
this.doc?.stop(); this.doc?.stop();
this.blob?.stop(); this.blob?.stop();
this.indexer?.stop();
} }
} }

862
packages/common/nbstore/src/sync/indexer/crawler.ts Normal file
View File

@@ -0,0 +1,862 @@
import { defaultBlockMarkdownAdapterMatchers } from '@blocksuite/affine/adapters';
import { Container } from '@blocksuite/affine/global/di';
import {
InlineDeltaToMarkdownAdapterExtensions,
MarkdownInlineToDeltaAdapterExtensions,
} from '@blocksuite/affine/inlines/preset';
import type {
AttachmentBlockModel,
BookmarkBlockModel,
EmbedBlockModel,
ImageBlockModel,
TableBlockModel,
} from '@blocksuite/affine/model';
import { AffineSchemas } from '@blocksuite/affine/schemas';
import { MarkdownAdapter } from '@blocksuite/affine/shared/adapters';
import type { AffineTextAttributes } from '@blocksuite/affine/shared/types';
import {
createYProxy,
type DeltaInsert,
type DraftModel,
Schema,
Transformer,
type TransformerMiddleware,
type YBlock,
} from '@blocksuite/affine/store';
import { uniq } from 'lodash-es';
import {
Array as YArray,
type Doc as YDoc,
Map as YMap,
Text as YText,
} from 'yjs';
import { IndexerDocument } from '../../storage';
const blocksuiteSchema = new Schema();
blocksuiteSchema.register([...AffineSchemas]);
interface BlockDocumentInfo {
docId: string;
blockId: string;
content?: string | string[];
flavour: string;
blob?: string[];
refDocId?: string[];
ref?: string[];
parentFlavour?: string;
parentBlockId?: string;
additional?: {
databaseName?: string;
displayMode?: string;
noteBlockId?: string;
};
yblock: YMap<any>;
markdownPreview?: string;
}
const bookmarkFlavours = new Set([
'affine:bookmark',
'affine:embed-youtube',
'affine:embed-figma',
'affine:embed-github',
'affine:embed-loom',
]);
function generateMarkdownPreviewBuilder(
yRootDoc: YDoc,
workspaceId: string,
blocks: BlockDocumentInfo[]
) {
function yblockToDraftModal(yblock: YBlock): DraftModel | null {
const flavour = yblock.get('sys:flavour') as string;
const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour);
if (!blockSchema) {
return null;
}
const keys = Array.from(yblock.keys())
.filter(key => key.startsWith('prop:'))
.map(key => key.substring(5));
const props = Object.fromEntries(
keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))])
);
return {
props,
id: yblock.get('sys:id') as string,
flavour,
children: [],
role: blockSchema.model.role,
version: (yblock.get('sys:version') as number) ?? blockSchema.version,
keys: Array.from(yblock.keys())
.filter(key => key.startsWith('prop:'))
.map(key => key.substring(5)),
} as unknown as DraftModel;
}
const titleMiddleware: TransformerMiddleware = ({ adapterConfigs }) => {
const pages = yRootDoc.getMap('meta').get('pages');
if (!(pages instanceof YArray)) {
return;
}
for (const meta of pages.toArray()) {
adapterConfigs.set(
'title:' + meta.get('id'),
meta.get('title')?.toString() ?? 'Untitled'
);
}
};
const baseUrl = `/workspace/${workspaceId}`;
function getDocLink(docId: string, blockId: string) {
const searchParams = new URLSearchParams();
searchParams.set('blockIds', blockId);
return `${baseUrl}/${docId}?${searchParams.toString()}`;
}
const docLinkBaseURLMiddleware: TransformerMiddleware = ({
adapterConfigs,
}) => {
adapterConfigs.set('docLinkBaseUrl', baseUrl);
};
const container = new Container();
[
...MarkdownInlineToDeltaAdapterExtensions,
...defaultBlockMarkdownAdapterMatchers,
...InlineDeltaToMarkdownAdapterExtensions,
].forEach(ext => {
ext.setup(container);
});
const provider = container.provider();
const markdownAdapter = new MarkdownAdapter(
new Transformer({
schema: blocksuiteSchema,
blobCRUD: {
delete: () => Promise.resolve(),
get: () => Promise.resolve(null),
list: () => Promise.resolve([]),
set: () => Promise.resolve(''),
},
docCRUD: {
create: () => {
throw new Error('Not implemented');
},
get: () => null,
delete: () => {},
},
middlewares: [docLinkBaseURLMiddleware, titleMiddleware],
}),
provider
);
const markdownPreviewCache = new WeakMap<BlockDocumentInfo, string | null>();
function trimCodeBlock(markdown: string) {
const lines = markdown.split('\n').filter(line => line.trim() !== '');
if (lines.length > 5) {
return [...lines.slice(0, 4), '...', lines.at(-1), ''].join('\n');
}
return [...lines, ''].join('\n');
}
function trimParagraph(markdown: string) {
const lines = markdown.split('\n').filter(line => line.trim() !== '');
if (lines.length > 3) {
return [...lines.slice(0, 3), '...', lines.at(-1), ''].join('\n');
}
return [...lines, ''].join('\n');
}
function getListDepth(block: BlockDocumentInfo) {
let parentBlockCount = 0;
let currentBlock: BlockDocumentInfo | undefined = block;
do {
currentBlock = blocks.find(
b => b.blockId === currentBlock?.parentBlockId
);
// reach the root block. do not count it.
if (!currentBlock || currentBlock.flavour !== 'affine:list') {
break;
}
parentBlockCount++;
} while (currentBlock);
return parentBlockCount;
}
// only works for list block
function indentMarkdown(markdown: string, depth: number) {
if (depth <= 0) {
return markdown;
}
return (
markdown
.split('\n')
.map(line => ' '.repeat(depth) + line)
.join('\n') + '\n'
);
}
const generateDatabaseMarkdownPreview = (block: BlockDocumentInfo) => {
const isDatabaseBlock = (block: BlockDocumentInfo) => {
return block.flavour === 'affine:database';
};
const model = yblockToDraftModal(block.yblock);
if (!model) {
return null;
}
let dbBlock: BlockDocumentInfo | null = null;
if (isDatabaseBlock(block)) {
dbBlock = block;
} else {
const parentBlock = blocks.find(b => b.blockId === block.parentBlockId);
if (parentBlock && isDatabaseBlock(parentBlock)) {
dbBlock = parentBlock;
}
}
if (!dbBlock) {
return null;
}
const url = getDocLink(block.docId, dbBlock.blockId);
const title = dbBlock.additional?.databaseName;
return `[database · ${title || 'Untitled'}][](${url})\n`;
};
const generateImageMarkdownPreview = (block: BlockDocumentInfo) => {
const isImageModel = (
model: DraftModel | null
): model is DraftModel<ImageBlockModel> => {
return model?.flavour === 'affine:image';
};
const model = yblockToDraftModal(block.yblock);
if (!isImageModel(model)) {
return null;
}
const info = ['an image block'];
if (model.props.sourceId) {
info.push(`file id ${model.props.sourceId}`);
}
if (model.props.caption) {
info.push(`with caption ${model.props.caption}`);
}
return info.join(', ') + '\n';
};
const generateEmbedMarkdownPreview = (block: BlockDocumentInfo) => {
const isEmbedModel = (
model: DraftModel | null
): model is DraftModel<EmbedBlockModel> => {
return (
model?.flavour === 'affine:embed-linked-doc' ||
model?.flavour === 'affine:embed-synced-doc'
);
};
const draftModel = yblockToDraftModal(block.yblock);
if (!isEmbedModel(draftModel)) {
return null;
}
const url = getDocLink(block.docId, draftModel.id);
return `[](${url})\n`;
};
const generateLatexMarkdownPreview = (block: BlockDocumentInfo) => {
let content =
typeof block.content === 'string'
? block.content.trim()
: block.content?.join('').trim();
content = content?.split('\n').join(' ') ?? '';
return `LaTeX, with value ${content}\n`;
};
const generateBookmarkMarkdownPreview = (block: BlockDocumentInfo) => {
const isBookmarkModel = (
model: DraftModel | null
): model is DraftModel<BookmarkBlockModel> => {
return bookmarkFlavours.has(model?.flavour ?? '');
};
const draftModel = yblockToDraftModal(block.yblock);
if (!isBookmarkModel(draftModel)) {
return null;
}
const title = draftModel.props.title;
const url = draftModel.props.url;
return `[${title}](${url})\n`;
};
const generateAttachmentMarkdownPreview = (block: BlockDocumentInfo) => {
const isAttachmentModel = (
model: DraftModel | null
): model is DraftModel<AttachmentBlockModel> => {
return model?.flavour === 'affine:attachment';
};
const draftModel = yblockToDraftModal(block.yblock);
if (!isAttachmentModel(draftModel)) {
return null;
}
return `[${draftModel.props.name}](${draftModel.props.sourceId})\n`;
};
const generateTableMarkdownPreview = (block: BlockDocumentInfo) => {
const isTableModel = (
model: DraftModel | null
): model is DraftModel<TableBlockModel> => {
return model?.flavour === 'affine:table';
};
const draftModel = yblockToDraftModal(block.yblock);
if (!isTableModel(draftModel)) {
return null;
}
const url = getDocLink(block.docId, draftModel.id);
return `[table][](${url})\n`;
};
const generateMarkdownPreview = async (block: BlockDocumentInfo) => {
if (markdownPreviewCache.has(block)) {
return markdownPreviewCache.get(block);
}
const flavour = block.flavour;
let markdown: string | null = null;
if (
flavour === 'affine:paragraph' ||
flavour === 'affine:list' ||
flavour === 'affine:code'
) {
const draftModel = yblockToDraftModal(block.yblock);
markdown =
block.parentFlavour === 'affine:database'
? generateDatabaseMarkdownPreview(block)
: ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null)
?.file ?? null);
if (markdown) {
if (flavour === 'affine:code') {
markdown = trimCodeBlock(markdown);
} else if (flavour === 'affine:paragraph') {
markdown = trimParagraph(markdown);
}
}
} else if (flavour === 'affine:database') {
markdown = generateDatabaseMarkdownPreview(block);
} else if (
flavour === 'affine:embed-linked-doc' ||
flavour === 'affine:embed-synced-doc'
) {
markdown = generateEmbedMarkdownPreview(block);
} else if (flavour === 'affine:attachment') {
markdown = generateAttachmentMarkdownPreview(block);
} else if (flavour === 'affine:image') {
markdown = generateImageMarkdownPreview(block);
} else if (flavour === 'affine:surface' || flavour === 'affine:page') {
// skip
} else if (flavour === 'affine:latex') {
markdown = generateLatexMarkdownPreview(block);
} else if (bookmarkFlavours.has(flavour)) {
markdown = generateBookmarkMarkdownPreview(block);
} else if (flavour === 'affine:table') {
markdown = generateTableMarkdownPreview(block);
} else {
console.warn(`unknown flavour: ${flavour}`);
}
if (markdown && flavour === 'affine:list') {
const blockDepth = getListDepth(block);
markdown = indentMarkdown(markdown, Math.max(0, blockDepth));
}
markdownPreviewCache.set(block, markdown);
return markdown;
};
return generateMarkdownPreview;
}
// remove the indent of the first line of list
// e.g.,
// ```
// - list item 1
// - list item 2
// ```
// becomes
// ```
// - list item 1
// - list item 2
// ```
function unindentMarkdown(markdown: string) {
const lines = markdown.split('\n');
const res: string[] = [];
let firstListFound = false;
let baseIndent = 0;
for (let current of lines) {
const indent = current.match(/^\s*/)?.[0]?.length ?? 0;
if (indent > 0) {
if (!firstListFound) {
// For the first list item, remove all indentation
firstListFound = true;
baseIndent = indent;
current = current.trimStart();
} else {
// For subsequent list items, maintain relative indentation
current =
' '.repeat(Math.max(0, indent - baseIndent)) + current.trimStart();
}
}
res.push(current);
}
return res.join('\n');
}
export async function crawlingDocData({
ydoc,
rootYDoc,
spaceId,
docId,
}: {
ydoc: YDoc;
rootYDoc: YDoc;
spaceId: string;
docId: string;
}): Promise<{
blocks: IndexerDocument<'block'>[];
preview?: string;
}> {
let docTitle = '';
let summaryLenNeeded = 1000;
let summary = '';
const blockDocuments: BlockDocumentInfo[] = [];
const generateMarkdownPreview = generateMarkdownPreviewBuilder(
rootYDoc,
spaceId,
blockDocuments
);
const blocks = ydoc.getMap<any>('blocks');
// build a parent map for quick lookup
// for each block, record its parent id
const parentMap: Record<string, string | null> = {};
for (const [id, block] of blocks.entries()) {
const children = block.get('sys:children') as YArray<string> | undefined;
if (children instanceof YArray && children.length) {
for (const child of children) {
parentMap[child] = id;
}
}
}
if (blocks.size === 0) {
return { blocks: [] };
}
// find the nearest block that satisfies the predicate
const nearest = (
blockId: string,
predicate: (block: YMap<any>) => boolean
) => {
let current: string | null = blockId;
while (current) {
const block = blocks.get(current);
if (block && predicate(block)) {
return block;
}
current = parentMap[current] ?? null;
}
return null;
};
const nearestByFlavour = (blockId: string, flavour: string) =>
nearest(blockId, block => block.get('sys:flavour') === flavour);
let rootBlockId: string | null = null;
for (const block of blocks.values()) {
const flavour = block.get('sys:flavour')?.toString();
const blockId = block.get('sys:id')?.toString();
if (flavour === 'affine:page' && blockId) {
rootBlockId = blockId;
}
}
if (!rootBlockId) {
return { blocks: [] };
}
const queue: { parent?: string; id: string }[] = [{ id: rootBlockId }];
const visited = new Set<string>(); // avoid loop
const pushChildren = (id: string, block: YMap<any>) => {
const children = block.get('sys:children');
if (children instanceof YArray && children.length) {
for (let i = children.length - 1; i >= 0; i--) {
const childId = children.get(i);
if (childId && !visited.has(childId)) {
queue.push({ parent: id, id: childId });
visited.add(childId);
}
}
}
};
// #region first loop - generate block base info
while (queue.length) {
const next = queue.pop();
if (!next) {
break;
}
const { parent: parentBlockId, id: blockId } = next;
const block = blockId ? blocks.get(blockId) : null;
const parentBlock = parentBlockId ? blocks.get(parentBlockId) : null;
if (!block) {
break;
}
const flavour = block.get('sys:flavour')?.toString();
const parentFlavour = parentBlock?.get('sys:flavour')?.toString();
const noteBlock = nearestByFlavour(blockId, 'affine:note');
// display mode:
// - both: page and edgeless -> fallback to page
// - page: only page -> page
// - edgeless: only edgeless -> edgeless
// - undefined: edgeless (assuming it is a normal element on the edgeless)
let displayMode = noteBlock?.get('prop:displayMode') ?? 'edgeless';
if (displayMode === 'both') {
displayMode = 'page';
}
const noteBlockId: string | undefined = noteBlock
?.get('sys:id')
?.toString();
pushChildren(blockId, block);
const commonBlockProps = {
docId: ydoc.guid,
flavour,
blockId,
yblock: block,
additional: { displayMode, noteBlockId },
};
if (flavour === 'affine:page') {
docTitle = block.get('prop:title').toString();
blockDocuments.push({ ...commonBlockProps, content: docTitle });
} else if (
flavour === 'affine:paragraph' ||
flavour === 'affine:list' ||
flavour === 'affine:code'
) {
const text = block.get('prop:text') as YText;
if (!text) {
continue;
}
const deltas: DeltaInsert<AffineTextAttributes>[] = text.toDelta();
const refs = uniq(
deltas
.flatMap(delta => {
if (
delta.attributes &&
delta.attributes.reference &&
delta.attributes.reference.pageId
) {
const { pageId: refDocId, params = {} } =
delta.attributes.reference;
return {
refDocId,
ref: JSON.stringify({ docId: refDocId, ...params }),
};
}
return null;
})
.filter(ref => !!ref)
);
const databaseName =
flavour === 'affine:paragraph' && parentFlavour === 'affine:database' // if block is a database row
? parentBlock?.get('prop:title')?.toString()
: undefined;
blockDocuments.push({
...commonBlockProps,
content: text.toString(),
...refs.reduce<{ refDocId: string[]; ref: string[] }>(
(prev, curr) => {
prev.refDocId.push(curr.refDocId);
prev.ref.push(curr.ref);
return prev;
},
{ refDocId: [], ref: [] }
),
parentFlavour,
parentBlockId,
additional: { ...commonBlockProps.additional, databaseName },
});
if (summaryLenNeeded > 0) {
summary += text.toString();
summaryLenNeeded -= text.length;
}
} else if (
flavour === 'affine:embed-linked-doc' ||
flavour === 'affine:embed-synced-doc'
) {
const pageId = block.get('prop:pageId');
if (typeof pageId === 'string') {
// reference info
const params = block.get('prop:params') ?? {};
blockDocuments.push({
...commonBlockProps,
refDocId: [pageId],
ref: [JSON.stringify({ docId: pageId, ...params })],
parentFlavour,
parentBlockId,
});
}
} else if (flavour === 'affine:attachment' || flavour === 'affine:image') {
const blobId = block.get('prop:sourceId');
if (typeof blobId === 'string') {
blockDocuments.push({
...commonBlockProps,
blob: [blobId],
parentFlavour,
parentBlockId,
});
}
} else if (flavour === 'affine:surface') {
const texts = [];
const elementsObj = block.get('prop:elements');
if (
!(
elementsObj instanceof YMap &&
elementsObj.get('type') === '$blocksuite:internal:native$'
)
) {
continue;
}
const elements = elementsObj.get('value') as YMap<any>;
if (!(elements instanceof YMap)) {
continue;
}
for (const element of elements.values()) {
if (!(element instanceof YMap)) {
continue;
}
const text = element.get('text') as YText;
if (!text) {
continue;
}
texts.push(text.toString());
}
blockDocuments.push({
...commonBlockProps,
content: texts,
parentFlavour,
parentBlockId,
});
} else if (flavour === 'affine:database') {
const texts = [];
const columnsObj = block.get('prop:columns');
const databaseTitle = block.get('prop:title');
if (databaseTitle instanceof YText) {
texts.push(databaseTitle.toString());
}
if (columnsObj instanceof YArray) {
for (const column of columnsObj) {
if (!(column instanceof YMap)) {
continue;
}
if (typeof column.get('name') === 'string') {
texts.push(column.get('name'));
}
const data = column.get('data');
if (!(data instanceof YMap)) {
continue;
}
const options = data.get('options');
if (!(options instanceof YArray)) {
continue;
}
for (const option of options) {
if (!(option instanceof YMap)) {
continue;
}
const value = option.get('value');
if (typeof value === 'string') {
texts.push(value);
}
}
}
}
blockDocuments.push({
...commonBlockProps,
content: texts,
additional: {
...commonBlockProps.additional,
databaseName: databaseTitle?.toString(),
},
});
} else if (flavour === 'affine:latex') {
blockDocuments.push({
...commonBlockProps,
content: block.get('prop:latex')?.toString() ?? '',
});
} else if (flavour === 'affine:table') {
const contents = Array.from<string>(block.keys())
.map(key => {
if (key.startsWith('prop:cells.') && key.endsWith('.text')) {
return block.get(key)?.toString() ?? '';
}
return '';
})
.filter(Boolean);
blockDocuments.push({
...commonBlockProps,
content: contents,
});
} else if (bookmarkFlavours.has(flavour)) {
blockDocuments.push({ ...commonBlockProps });
}
}
// #endregion
// #region second loop - generate markdown preview
const TARGET_PREVIEW_CHARACTER = 500;
const TARGET_PREVIOUS_BLOCK = 1;
const TARGET_FOLLOW_BLOCK = 4;
for (const block of blockDocuments) {
if (block.ref?.length) {
const target = block;
// should only generate the markdown preview belong to the same affine:note
const noteBlock = nearestByFlavour(block.blockId, 'affine:note');
const sameNoteBlocks = noteBlock
? blockDocuments.filter(
candidate =>
nearestByFlavour(candidate.blockId, 'affine:note') === noteBlock
)
: [];
// only generate markdown preview for reference blocks
let previewText = (await generateMarkdownPreview(target)) ?? '';
let previousBlock = 0;
let followBlock = 0;
let previousIndex = sameNoteBlocks.findIndex(
block => block.blockId === target.blockId
);
let followIndex = previousIndex;
while (
!(
(
previewText.length > TARGET_PREVIEW_CHARACTER || // stop if preview text reaches the limit
((previousBlock >= TARGET_PREVIOUS_BLOCK || previousIndex < 0) &&
(followBlock >= TARGET_FOLLOW_BLOCK ||
followIndex >= sameNoteBlocks.length))
) // stop if no more blocks, or preview block reaches the limit
)
) {
if (previousBlock < TARGET_PREVIOUS_BLOCK) {
previousIndex--;
const block =
previousIndex >= 0 ? sameNoteBlocks.at(previousIndex) : null;
const markdown = block ? await generateMarkdownPreview(block) : null;
if (
markdown &&
!previewText.startsWith(
markdown
) /* A small hack to skip blocks with the same content */
) {
previewText = markdown + '\n' + previewText;
previousBlock++;
}
}
if (followBlock < TARGET_FOLLOW_BLOCK) {
followIndex++;
const block = sameNoteBlocks.at(followIndex);
const markdown = block ? await generateMarkdownPreview(block) : null;
if (
markdown &&
!previewText.endsWith(
markdown
) /* A small hack to skip blocks with the same content */
) {
previewText = previewText + '\n' + markdown;
followBlock++;
}
}
}
block.markdownPreview = unindentMarkdown(previewText);
}
}
// #endregion
return {
blocks: blockDocuments.map(block =>
IndexerDocument.from<'block'>(`${docId}:${block.blockId}`, {
docId: block.docId,
blockId: block.blockId,
content: block.content,
flavour: block.flavour,
blob: block.blob,
refDocId: block.refDocId,
ref: block.ref,
parentFlavour: block.parentFlavour,
parentBlockId: block.parentBlockId,
additional: block.additional
? JSON.stringify(block.additional)
: undefined,
markdownPreview: block.markdownPreview,
})
),
preview: summary,
};
}

579
packages/common/nbstore/src/sync/indexer/index.ts Normal file
View File

@@ -0,0 +1,579 @@
import {
filter,
first,
Observable,
ReplaySubject,
share,
Subject,
throttleTime,
} from 'rxjs';
import {
applyUpdate,
type Array as YArray,
Doc as YDoc,
type Map as YMap,
} from 'yjs';
import {
type DocStorage,
IndexerDocument,
type IndexerStorage,
} from '../../storage';
import type { IndexerSyncStorage } from '../../storage/indexer-sync';
import { AsyncPriorityQueue } from '../../utils/async-priority-queue';
import { takeUntilAbort } from '../../utils/take-until-abort';
import { MANUALLY_STOP, throwIfAborted } from '../../utils/throw-if-aborted';
import { crawlingDocData } from './crawler';
export interface IndexerSyncState {
/**
* Number of documents currently in the indexing queue
*/
indexing: number;
/**
* Indicates whether all documents have been successfully indexed
*
* This is only for UI display purposes. For logical operations, please use `waitForCompleted()`
*/
completed: boolean;
/**
* Total number of documents in the workspace
*/
total: number;
errorMessage: string | null;
}
export interface IndexerDocSyncState {
/**
* Indicates whether this document is currently in the indexing queue
*/
indexing: boolean;
/**
* Indicates whether this document has been successfully indexed
*
* This is only for UI display purposes. For logical operations, please use `waitForDocCompleted()`
*/
completed: boolean;
}
export interface IndexerSync {
state$: Observable<IndexerSyncState>;
docState$(docId: string): Observable<IndexerDocSyncState>;
addPriority(docId: string, priority: number): () => void;
waitForCompleted(signal?: AbortSignal): Promise<void>;
waitForDocCompleted(docId: string, signal?: AbortSignal): Promise<void>;
}
export class IndexerSyncImpl implements IndexerSync {
private abort: AbortController | null = null;
private readonly rootDocId = this.doc.spaceId;
private readonly status = new IndexerSyncStatus(this.rootDocId);
state$ = this.status.state$.pipe(
// throttle the state to 1 second to avoid spamming the UI
throttleTime(1000)
);
docState$(docId: string) {
return this.status.docState$(docId).pipe(
// throttle the state to 1 second to avoid spamming the UI
throttleTime(1000)
);
}
waitForCompleted(signal?: AbortSignal) {
return new Promise<void>((resolve, reject) => {
this.status.state$
.pipe(
filter(state => state.completed),
takeUntilAbort(signal),
first()
)
.subscribe({
next: () => {
resolve();
},
error: err => {
reject(err);
},
});
});
}
waitForDocCompleted(docId: string, signal?: AbortSignal) {
return new Promise<void>((resolve, reject) => {
this.status
.docState$(docId)
.pipe(
filter(state => state.completed),
takeUntilAbort(signal),
first()
)
.subscribe({
next: () => {
resolve();
},
error: err => {
reject(err);
},
});
});
}
readonly interval = () =>
new Promise<void>(resolve =>
requestIdleCallback(() => resolve(), {
timeout: 200,
})
);
constructor(
readonly doc: DocStorage,
readonly indexer: IndexerStorage,
readonly indexerSync: IndexerSyncStorage
) {}
start() {
if (this.abort) {
this.abort.abort(MANUALLY_STOP);
}
const abort = new AbortController();
this.abort = abort;
this.mainLoop(abort.signal).catch(error => {
if (error === MANUALLY_STOP) {
return;
}
console.error('index error', error);
});
}
stop() {
this.abort?.abort(MANUALLY_STOP);
this.abort = null;
}
addPriority(id: string, priority: number) {
return this.status.addPriority(id, priority);
}
private async mainLoop(signal?: AbortSignal) {
if (this.indexer.isReadonly) {
return;
}
while (true) {
try {
await this.retryLoop(signal);
} catch (error) {
if (signal?.aborted) {
return;
}
console.error('index error, retry in 5s', error);
this.status.errorMessage =
error instanceof Error ? error.message : `${error}`;
this.status.statusUpdatedSubject$.next(true);
} finally {
// reset all status
this.status.reset();
// wait for 5s before next retry
await Promise.race([
new Promise<void>(resolve => {
setTimeout(resolve, 5000);
}),
new Promise((_, reject) => {
// exit if manually stopped
if (signal?.aborted) {
reject(signal.reason);
}
signal?.addEventListener('abort', () => {
reject(signal.reason);
});
}),
]);
}
}
}
private async retryLoop(signal?: AbortSignal) {
await Promise.race([
Promise.all([
this.doc.connection.waitForConnected(signal),
this.indexer.connection.waitForConnected(signal),
this.indexerSync.connection.waitForConnected(signal),
]),
new Promise<void>((_, reject) => {
setTimeout(() => {
reject(new Error('Connect to remote timeout'));
}, 1000 * 30);
}),
new Promise((_, reject) => {
signal?.addEventListener('abort', reason => {
reject(reason);
});
}),
]);
this.status.errorMessage = null;
this.status.statusUpdatedSubject$.next(true);
console.log('indexer sync start');
const unsubscribe = this.doc.subscribeDocUpdate(update => {
if (!this.status.rootDocReady) {
return;
}
if (update.docId === this.rootDocId) {
applyUpdate(this.status.rootDoc, update.bin);
const allDocs = this.getAllDocsFromRootDoc();
for (const [docId, { title }] of allDocs) {
const existingDoc = this.status.docsInRootDoc.get(docId);
if (!existingDoc) {
this.status.scheduleJob(docId);
this.status.docsInRootDoc.set(docId, { title });
this.status.statusUpdatedSubject$.next(docId);
} else {
if (existingDoc.title !== title) {
this.status.docsInRootDoc.set(docId, { title });
this.status.statusUpdatedSubject$.next(docId);
}
}
}
for (const docId of this.status.docsInRootDoc.keys()) {
if (!allDocs.has(docId)) {
this.status.docsInRootDoc.delete(docId);
this.status.statusUpdatedSubject$.next(docId);
}
}
this.status.scheduleJob(this.rootDocId);
} else {
const docId = update.docId;
const existingDoc = this.status.docsInRootDoc.get(docId);
if (existingDoc) {
this.status.scheduleJob(docId);
}
}
});
try {
const rootDocBin = (await this.doc.getDoc(this.rootDocId))?.bin;
if (rootDocBin) {
applyUpdate(this.status.rootDoc, rootDocBin);
}
this.status.scheduleJob(this.rootDocId);
const allDocs = this.getAllDocsFromRootDoc();
this.status.docsInRootDoc = allDocs;
this.status.statusUpdatedSubject$.next(true);
for (const docId of allDocs.keys()) {
this.status.scheduleJob(docId);
}
this.status.rootDocReady = true;
this.status.statusUpdatedSubject$.next(true);
const allIndexedDocs = await this.getAllDocsFromIndexer();
this.status.docsInIndexer = allIndexedDocs;
this.status.statusUpdatedSubject$.next(true);
while (true) {
throwIfAborted(signal);
const docId = await this.status.acceptJob(signal);
if (docId === this.rootDocId) {
// #region crawl root doc
for (const [docId, { title }] of this.status.docsInRootDoc) {
const existingDoc = this.status.docsInIndexer.get(docId);
if (existingDoc) {
if (existingDoc.title !== title) {
// need update
await this.indexer.update(
'doc',
IndexerDocument.from(docId, {
docId,
title,
})
);
this.status.docsInIndexer.set(docId, { title });
this.status.statusUpdatedSubject$.next(docId);
}
} else {
// need add
await this.indexer.insert(
'doc',
IndexerDocument.from(docId, {
docId,
title,
})
);
this.status.docsInIndexer.set(docId, { title });
this.status.statusUpdatedSubject$.next(docId);
}
}
for (const docId of this.status.docsInIndexer.keys()) {
if (!this.status.docsInRootDoc.has(docId)) {
await this.indexer.delete('doc', docId);
await this.indexer.deleteByQuery('block', {
type: 'match',
field: 'docId',
match: docId,
});
await this.indexerSync.clearDocIndexedClock(docId);
this.status.docsInIndexer.delete(docId);
this.status.statusUpdatedSubject$.next(docId);
}
}
await this.indexer.refresh('block');
await this.indexer.refresh('doc');
// #endregion
} else {
// #region crawl doc
const existingDoc = this.status.docsInIndexer.get(docId);
if (!existingDoc) {
// doc is deleted, just skip
continue;
}
const docClock = await this.doc.getDocTimestamp(docId);
if (!docClock) {
// doc is deleted, just skip
continue;
}
const docIndexedClock =
await this.indexerSync.getDocIndexedClock(docId);
if (
docIndexedClock &&
docIndexedClock.timestamp.getTime() === docClock.timestamp.getTime()
) {
// doc is already indexed, just skip
continue;
}
const docBin = await this.doc.getDoc(docId);
if (!docBin) {
// doc is deleted, just skip
continue;
}
const docYDoc = new YDoc({ guid: docId });
applyUpdate(docYDoc, docBin.bin);
let blocks: IndexerDocument<'block'>[] = [];
let preview: string | undefined;
try {
const result = await crawlingDocData({
ydoc: docYDoc,
rootYDoc: this.status.rootDoc,
spaceId: this.status.rootDocId,
docId,
});
blocks = result.blocks;
preview = result.preview;
} catch (error) {
console.error('error crawling doc', error);
}
await this.indexer.deleteByQuery('block', {
type: 'match',
field: 'docId',
match: docId,
});
for (const block of blocks) {
await this.indexer.insert('block', block);
}
await this.indexer.refresh('block');
if (preview) {
await this.indexer.update(
'doc',
IndexerDocument.from(docId, {
summary: preview,
})
);
await this.indexer.refresh('doc');
}
await this.indexerSync.setDocIndexedClock({
docId,
timestamp: docClock.timestamp,
});
// #endregion
}
this.status.completeJob();
}
} finally {
unsubscribe();
}
}
/**
* Get all docs from the root doc, without deleted docs
*/
private getAllDocsFromRootDoc() {
const docs = this.status.rootDoc.getMap('meta').get('pages') as
| YArray<YMap<any>>
| undefined;
const availableDocs = new Map<string, { title: string | undefined }>();
if (docs) {
for (const page of docs) {
const docId = page.get('id');
if (typeof docId !== 'string') {
continue;
}
const inTrash = page.get('trash') ?? false;
const title = page.get('title');
if (!inTrash) {
availableDocs.set(docId, { title });
}
}
}
return availableDocs;
}
private async getAllDocsFromIndexer() {
const docs = await this.indexer.search(
'doc',
{
type: 'all',
},
{
pagination: {
limit: Infinity,
},
fields: ['docId', 'title'],
}
);
return new Map(
docs.nodes.map(node => {
const title = node.fields.title;
return [
node.id,
{
title: typeof title === 'string' ? title : title.at(0),
},
];
})
);
}
}
class IndexerSyncStatus {
prioritySettings = new Map<string, number>();
jobs = new AsyncPriorityQueue();
rootDoc = new YDoc({ guid: this.rootDocId });
rootDocReady = false;
docsInIndexer = new Map<string, { title: string | undefined }>();
docsInRootDoc = new Map<string, { title: string | undefined }>();
currentJob: string | null = null;
errorMessage: string | null = null;
statusUpdatedSubject$ = new Subject<string | true>();
state$ = new Observable<IndexerSyncState>(subscribe => {
const next = () => {
subscribe.next({
indexing: this.jobs.length() + (this.currentJob ? 1 : 0),
total: this.docsInRootDoc.size + 1,
errorMessage: this.errorMessage,
completed: this.rootDocReady && this.jobs.length() === 0,
});
};
next();
const dispose = this.statusUpdatedSubject$.subscribe(() => {
next();
});
return () => {
dispose.unsubscribe();
};
}).pipe(
share({
connector: () => new ReplaySubject(1),
})
);
docState$(docId: string) {
return new Observable<IndexerDocSyncState>(subscribe => {
const next = () => {
subscribe.next({
indexing: this.jobs.has(docId),
completed: this.docsInIndexer.has(docId) && !this.jobs.has(docId),
});
};
next();
const dispose = this.statusUpdatedSubject$.subscribe(updatedDocId => {
if (updatedDocId === docId || updatedDocId === true) {
next();
}
});
return () => {
dispose.unsubscribe();
};
}).pipe(
share({
connector: () => new ReplaySubject(1),
})
);
}
constructor(readonly rootDocId: string) {
this.prioritySettings.set(this.rootDocId, Infinity);
}
scheduleJob(docId: string) {
const priority = this.prioritySettings.get(docId) ?? 0;
this.jobs.push(docId, priority);
this.statusUpdatedSubject$.next(docId);
}
async acceptJob(abort?: AbortSignal) {
const job = await this.jobs.asyncPop(abort);
this.currentJob = job;
this.statusUpdatedSubject$.next(job);
return job;
}
completeJob() {
const job = this.currentJob;
this.currentJob = null;
this.statusUpdatedSubject$.next(job ?? true);
}
addPriority(id: string, priority: number) {
const oldPriority = this.prioritySettings.get(id) ?? 0;
this.prioritySettings.set(id, priority);
this.jobs.setPriority(id, oldPriority + priority);
return () => {
const currentPriority = this.prioritySettings.get(id) ?? 0;
this.prioritySettings.set(id, currentPriority - priority);
this.jobs.setPriority(id, currentPriority - priority);
};
}
reset() {
// reset all state, except prioritySettings
this.jobs.clear();
this.docsInRootDoc.clear();
this.docsInIndexer.clear();
this.rootDoc = new YDoc();
this.rootDocReady = false;
this.currentJob = null;
this.statusUpdatedSubject$.next(true);
}
}

6
packages/common/nbstore/src/utils/priority-queue.ts
View File

@@ -52,6 +52,10 @@ export class PriorityQueue {
return removed; return removed;
} }
has(id: string) {
return this.priorityMap.has(id);
}
clear() { clear() {
this.tree.clear(); this.tree.clear();
this.priorityMap.clear(); this.priorityMap.clear();
@@ -64,6 +68,6 @@ export class PriorityQueue {
} }
get length() { get length() {
return this.tree.count; return this.tree.count.bind(this.tree);
} }
} }

42
packages/common/nbstore/src/utils/take-until-abort.ts Normal file
View File

@@ -0,0 +1,42 @@
import { Observable, type OperatorFunction } from 'rxjs';
/**
* Creates an operator that takes values from the source Observable until the given AbortSignal aborts.
* When the signal aborts, the Observable completes.
*
* @param signal - The AbortSignal that will trigger completion when aborted
* @returns An operator function that takes values until the signal aborts
*/
export function takeUntilAbort<T>(
signal?: AbortSignal
): OperatorFunction<T, T> {
return (source$: Observable<T>) => {
return new Observable<T>(subscriber => {
if (signal?.aborted) {
subscriber.error(signal.reason);
return;
}
const abortHandler = () => {
subscriber.error(signal?.reason);
};
if (signal) {
signal.addEventListener('abort', abortHandler);
}
const subscription = source$.subscribe({
next: value => subscriber.next(value),
error: err => subscriber.error(err),
complete: () => subscriber.complete(),
});
return () => {
if (signal) {
signal.removeEventListener('abort', abortHandler);
}
subscription.unsubscribe();
};
});
};
}

170
packages/common/nbstore/src/worker/client.ts
View File

@@ -1,20 +1,35 @@
import { OpClient, transfer } from '@toeverything/infra/op'; import { OpClient, transfer } from '@toeverything/infra/op';
import type { Observable } from 'rxjs';
import { v4 as uuid } from 'uuid'; import { v4 as uuid } from 'uuid';
import { DummyConnection } from '../connection'; import { DummyConnection } from '../connection';
import { AwarenessFrontend, BlobFrontend, DocFrontend } from '../frontend';
import { import {
AwarenessFrontend,
BlobFrontend,
DocFrontend,
IndexerFrontend,
} from '../frontend';
import {
type AggregateOptions,
type AggregateResult,
type AwarenessRecord, type AwarenessRecord,
type BlobRecord, type BlobRecord,
type BlobStorage, type BlobStorage,
type DocRecord, type DocRecord,
type DocStorage, type DocStorage,
type DocUpdate, type DocUpdate,
type IndexerDocument,
type IndexerSchema,
type IndexerStorage,
type ListedBlobRecord, type ListedBlobRecord,
type Query,
type SearchOptions,
type SearchResult,
} from '../storage'; } from '../storage';
import type { AwarenessSync } from '../sync/awareness'; import type { AwarenessSync } from '../sync/awareness';
import type { BlobSync } from '../sync/blob'; import type { BlobSync } from '../sync/blob';
import type { DocSync } from '../sync/doc'; import type { DocSync } from '../sync/doc';
import type { IndexerSync } from '../sync/indexer';
import type { StoreInitOptions, WorkerManagerOps, WorkerOps } from './ops'; import type { StoreInitOptions, WorkerManagerOps, WorkerOps } from './ops';
export type { StoreInitOptions as WorkerInitOptions } from './ops'; export type { StoreInitOptions as WorkerInitOptions } from './ops';
@@ -85,6 +100,12 @@ export class StoreClient {
this.docFrontend = new DocFrontend(this.docStorage, this.docSync); this.docFrontend = new DocFrontend(this.docStorage, this.docSync);
this.blobFrontend = new BlobFrontend(this.blobStorage, this.blobSync); this.blobFrontend = new BlobFrontend(this.blobStorage, this.blobSync);
this.awarenessFrontend = new AwarenessFrontend(this.awarenessSync); this.awarenessFrontend = new AwarenessFrontend(this.awarenessSync);
this.indexerStorage = new WorkerIndexerStorage(this.client);
this.indexerSync = new WorkerIndexerSync(this.client);
this.indexerFrontend = new IndexerFrontend(
this.indexerStorage,
this.indexerSync
);
} }
private readonly docStorage: WorkerDocStorage; private readonly docStorage: WorkerDocStorage;
@@ -92,14 +113,18 @@ export class StoreClient {
private readonly docSync: WorkerDocSync; private readonly docSync: WorkerDocSync;
private readonly blobSync: WorkerBlobSync; private readonly blobSync: WorkerBlobSync;
private readonly awarenessSync: WorkerAwarenessSync; private readonly awarenessSync: WorkerAwarenessSync;
private readonly indexerStorage: WorkerIndexerStorage;
private readonly indexerSync: WorkerIndexerSync;
readonly docFrontend: DocFrontend; readonly docFrontend: DocFrontend;
readonly blobFrontend: BlobFrontend; readonly blobFrontend: BlobFrontend;
readonly awarenessFrontend: AwarenessFrontend; readonly awarenessFrontend: AwarenessFrontend;
readonly indexerFrontend: IndexerFrontend;
} }
class WorkerDocStorage implements DocStorage { class WorkerDocStorage implements DocStorage {
constructor(private readonly client: OpClient<WorkerOps>) {} constructor(private readonly client: OpClient<WorkerOps>) {}
spaceId = '';
readonly storageType = 'doc'; readonly storageType = 'doc';
readonly isReadonly = false; readonly isReadonly = false;
@@ -316,3 +341,146 @@ class WorkerAwarenessSync implements AwarenessSync {
}; };
} }
} }
class WorkerIndexerStorage implements IndexerStorage {
constructor(private readonly client: OpClient<WorkerOps>) {}
readonly storageType = 'indexer';
readonly isReadonly = true;
connection = new DummyConnection();
search<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
table: T,
query: Query<T>,
options?: O
): Promise<SearchResult<T, O>> {
return this.client.call('indexerStorage.search', { table, query, options });
}
aggregate<T extends keyof IndexerSchema, const O extends AggregateOptions<T>>(
table: T,
query: Query<T>,
field: keyof IndexerSchema[T],
options?: O
): Promise<AggregateResult<T, O>> {
return this.client.call('indexerStorage.aggregate', {
table,
query,
field: field as string,
options,
});
}
search$<T extends keyof IndexerSchema, const O extends SearchOptions<T>>(
table: T,
query: Query<T>,
options?: O
): Observable<SearchResult<T, O>> {
return this.client.ob$('indexerStorage.subscribeSearch', {
table,
query,
options,
});
}
aggregate$<
T extends keyof IndexerSchema,
const O extends AggregateOptions<T>,
>(
table: T,
query: Query<T>,
field: keyof IndexerSchema[T],
options?: O
): Observable<AggregateResult<T, O>> {
return this.client.ob$('indexerStorage.subscribeAggregate', {
table,
query,
field: field as string,
options,
});
}
deleteByQuery<T extends keyof IndexerSchema>(
_table: T,
_query: Query<T>
): Promise<void> {
throw new Error('Method not implemented.');
}
insert<T extends keyof IndexerSchema>(
_table: T,
_document: IndexerDocument<T>
): Promise<void> {
throw new Error('Method not implemented.');
}
delete<T extends keyof IndexerSchema>(_table: T, _id: string): Promise<void> {
throw new Error('Method not implemented.');
}
update<T extends keyof IndexerSchema>(
_table: T,
_document: IndexerDocument<T>
): Promise<void> {
throw new Error('Method not implemented.');
}
refresh<T extends keyof IndexerSchema>(_table: T): Promise<void> {
throw new Error('Method not implemented.');
}
}
class WorkerIndexerSync implements IndexerSync {
constructor(private readonly client: OpClient<WorkerOps>) {}
waitForCompleted(signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
const abortListener = () => {
reject(signal?.reason);
subscription.unsubscribe();
};
signal?.addEventListener('abort', abortListener);
const subscription = this.client
.ob$('indexerSync.waitForCompleted')
.subscribe({
complete() {
signal?.removeEventListener('abort', abortListener);
resolve();
},
error(err) {
signal?.removeEventListener('abort', abortListener);
reject(err);
},
});
});
}
waitForDocCompleted(docId: string, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
const abortListener = () => {
reject(signal?.reason);
subscription.unsubscribe();
};
signal?.addEventListener('abort', abortListener);
const subscription = this.client
.ob$('indexerSync.waitForDocCompleted', docId)
.subscribe({
complete() {
signal?.removeEventListener('abort', abortListener);
resolve();
},
error(err) {
signal?.removeEventListener('abort', abortListener);
reject(err);
},
});
});
}
get state$() {
return this.client.ob$('indexerSync.state');
}
docState$(docId: string) {
return this.client.ob$('indexerSync.docState', docId);
}
addPriority(docId: string, priority: number) {
const subscription = this.client
.ob$('indexerSync.addPriority', { docId, priority })
.subscribe();
return () => {
subscription.unsubscribe();
};
}
}

52
packages/common/nbstore/src/worker/consumer.ts
View File

@@ -1,4 +1,3 @@
import { MANUALLY_STOP } from '@toeverything/infra';
import { OpConsumer } from '@toeverything/infra/op'; import { OpConsumer } from '@toeverything/infra/op';
import { Observable } from 'rxjs'; import { Observable } from 'rxjs';
@@ -7,6 +6,7 @@ import { SpaceStorage } from '../storage';
import type { AwarenessRecord } from '../storage/awareness'; import type { AwarenessRecord } from '../storage/awareness';
import { Sync } from '../sync'; import { Sync } from '../sync';
import type { PeerStorageOptions } from '../sync/types'; import type { PeerStorageOptions } from '../sync/types';
import { MANUALLY_STOP } from '../utils/throw-if-aborted';
import type { StoreInitOptions, WorkerManagerOps, WorkerOps } from './ops'; import type { StoreInitOptions, WorkerManagerOps, WorkerOps } from './ops';
export type { WorkerManagerOps }; export type { WorkerManagerOps };
@@ -57,6 +57,14 @@ class StoreConsumer {
return this.ensureSync.awareness; return this.ensureSync.awareness;
} }
get indexerStorage() {
return this.ensureLocal.get('indexer');
}
get indexerSync() {
return this.ensureSync.indexer;
}
constructor( constructor(
private readonly availableStorageImplementations: StorageConstructor[], private readonly availableStorageImplementations: StorageConstructor[],
init: StoreInitOptions init: StoreInitOptions
@@ -262,6 +270,48 @@ class StoreConsumer {
}), }),
'awarenessSync.collect': ({ collectId, awareness }) => 'awarenessSync.collect': ({ collectId, awareness }) =>
collectJobs.get(collectId)?.(awareness), collectJobs.get(collectId)?.(awareness),
'indexerStorage.aggregate': ({ table, query, field, options }) =>
this.indexerStorage.aggregate(table, query, field, options),
'indexerStorage.search': ({ table, query, options }) =>
this.indexerStorage.search(table, query, options),
'indexerStorage.subscribeSearch': ({ table, query, options }) =>
this.indexerStorage.search$(table, query, options),
'indexerStorage.subscribeAggregate': ({ table, query, field, options }) =>
this.indexerStorage.aggregate$(table, query, field, options),
'indexerSync.state': () => this.indexerSync.state$,
'indexerSync.docState': (docId: string) =>
this.indexerSync.docState$(docId),
'indexerSync.addPriority': ({ docId, priority }) =>
new Observable(() => {
const undo = this.indexerSync.addPriority(docId, priority);
return () => undo();
}),
'indexerSync.waitForCompleted': () =>
new Observable(subscriber => {
this.indexerSync
.waitForCompleted()
.then(() => {
subscriber.next();
subscriber.complete();
})
.catch(error => {
subscriber.error(error);
});
}),
'indexerSync.waitForDocCompleted': (docId: string) =>
new Observable(subscriber => {
const abortController = new AbortController();
this.indexerSync
.waitForDocCompleted(docId, abortController.signal)
.then(() => {
subscriber.next();
subscriber.complete();
})
.catch(error => {
subscriber.error(error);
});
return () => abortController.abort(MANUALLY_STOP);
}),
}); });
} }
} }

43
packages/common/nbstore/src/worker/ops.ts
View File

@@ -1,5 +1,7 @@
import type { AvailableStorageImplementations } from '../impls'; import type { AvailableStorageImplementations } from '../impls';
import type { import type {
AggregateOptions,
AggregateResult,
BlobRecord, BlobRecord,
DocClock, DocClock,
DocClocks, DocClocks,
@@ -7,11 +9,15 @@ import type {
DocRecord, DocRecord,
DocUpdate, DocUpdate,
ListedBlobRecord, ListedBlobRecord,
Query,
SearchOptions,
SearchResult,
StorageType, StorageType,
} from '../storage'; } from '../storage';
import type { AwarenessRecord } from '../storage/awareness'; import type { AwarenessRecord } from '../storage/awareness';
import type { BlobSyncBlobState, BlobSyncState } from '../sync/blob'; import type { BlobSyncBlobState, BlobSyncState } from '../sync/blob';
import type { DocSyncDocState, DocSyncState } from '../sync/doc'; import type { DocSyncDocState, DocSyncState } from '../sync/doc';
import type { IndexerDocSyncState, IndexerSyncState } from '../sync/indexer';
type StorageInitOptions = Values<{ type StorageInitOptions = Values<{
[key in keyof AvailableStorageImplementations]: { [key in keyof AvailableStorageImplementations]: {
@@ -61,6 +67,35 @@ interface GroupedWorkerOps {
collect: [{ collectId: string; awareness: AwarenessRecord }, void]; collect: [{ collectId: string; awareness: AwarenessRecord }, void];
}; };
indexerStorage: {
search: [
{ table: string; query: Query<any>; options?: SearchOptions<any> },
SearchResult<any, any>,
];
aggregate: [
{
table: string;
query: Query<any>;
field: string;
options?: AggregateOptions<any>;
},
AggregateResult<any, any>,
];
subscribeSearch: [
{ table: string; query: Query<any>; options?: SearchOptions<any> },
SearchResult<any, any>,
];
subscribeAggregate: [
{
table: string;
query: Query<any>;
field: string;
options?: AggregateOptions<any>;
},
AggregateResult<any, any>,
];
};
docSync: { docSync: {
state: [void, DocSyncState]; state: [void, DocSyncState];
docState: [string, DocSyncDocState]; docState: [string, DocSyncDocState];
@@ -91,6 +126,14 @@ interface GroupedWorkerOps {
]; ];
collect: [{ collectId: string; awareness: AwarenessRecord }, void]; collect: [{ collectId: string; awareness: AwarenessRecord }, void];
}; };
indexerSync: {
state: [void, IndexerSyncState];
docState: [string, IndexerDocSyncState];
addPriority: [{ docId: string; priority: number }, boolean];
waitForCompleted: [void, void];
waitForDocCompleted: [string, void];
};
} }
type Values<T> = T extends { [k in keyof T]: any } ? T[keyof T] : never; type Values<T> = T extends { [k in keyof T]: any } ? T[keyof T] : never;

3
packages/common/nbstore/tsconfig.json
View File

@@ -9,6 +9,7 @@
"references": [ "references": [
{ "path": "../infra" }, { "path": "../infra" },
{ "path": "../error" }, { "path": "../error" },
{ "path": "../graphql" } { "path": "../graphql" },
{ "path": "../../../blocksuite/affine/all" }
] ]
} }

2
packages/frontend/apps/electron-renderer/src/background-worker/index.ts
View File

@@ -3,6 +3,7 @@ import '@affine/core/bootstrap/electron';
import { apis } from '@affine/electron-api'; import { apis } from '@affine/electron-api';
import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel'; import { broadcastChannelStorages } from '@affine/nbstore/broadcast-channel';
import { cloudStorages } from '@affine/nbstore/cloud'; import { cloudStorages } from '@affine/nbstore/cloud';
import { idbStoragesIndexerOnly } from '@affine/nbstore/idb';
import { bindNativeDBApis, sqliteStorages } from '@affine/nbstore/sqlite'; import { bindNativeDBApis, sqliteStorages } from '@affine/nbstore/sqlite';
import { import {
bindNativeDBV1Apis, bindNativeDBV1Apis,
@@ -20,6 +21,7 @@ bindNativeDBApis(apis!.nbstore);
bindNativeDBV1Apis(apis!.db); bindNativeDBV1Apis(apis!.db);
const storeManager = new StoreManagerConsumer([ const storeManager = new StoreManagerConsumer([
...idbStoragesIndexerOnly,
...sqliteStorages, ...sqliteStorages,
...sqliteV1Storages, ...sqliteV1Storages,
...broadcastChannelStorages, ...broadcastChannelStorages,

2
packages/frontend/apps/ios/src/worker.ts
View File

@@ -5,6 +5,7 @@ import {
cloudStorages, cloudStorages,
configureSocketAuthMethod, configureSocketAuthMethod,
} from '@affine/nbstore/cloud'; } from '@affine/nbstore/cloud';
import { idbStoragesIndexerOnly } from '@affine/nbstore/idb';
import { import {
bindNativeDBApis, bindNativeDBApis,
type NativeDBApis, type NativeDBApis,
@@ -61,6 +62,7 @@ const consumer = new OpConsumer<WorkerManagerOps>(
); );
const storeManager = new StoreManagerConsumer([ const storeManager = new StoreManagerConsumer([
...idbStoragesIndexerOnly,
...sqliteStorages, ...sqliteStorages,
...broadcastChannelStorages, ...broadcastChannelStorages,
...cloudStorages, ...cloudStorages,

1
packages/frontend/core/src/bootstrap/polyfill/set-difference.ts Normal file
View File

@@ -0,0 +1 @@
import 'core-js/es/set/difference.js';

7
packages/frontend/core/src/components/page-list/page-content-preview.tsx
View File

@@ -1,6 +1,6 @@
import { DocsSearchService } from '@affine/core/modules/docs-search'; import { DocsSearchService } from '@affine/core/modules/docs-search';
import { LiveData, useLiveData, useService } from '@toeverything/infra'; import { LiveData, useLiveData, useService } from '@toeverything/infra';
import { type ReactNode, useMemo } from 'react'; import { type ReactNode, useEffect, useMemo } from 'react';
interface PagePreviewProps { interface PagePreviewProps {
pageId: string; pageId: string;
@@ -21,6 +21,11 @@ const PagePreviewInner = ({
) )
); );
useEffect(() => {
const undo = docSummary.indexer.addPriority(pageId, 100);
return undo;
}, [docSummary, pageId]);
const res = const res =
summary === null ? fallback : summary === '' ? emptyFallback : summary; summary === null ? fallback : summary === '' ? emptyFallback : summary;
return res; return res;

27
packages/frontend/core/src/mobile/components/explorer/nodes/doc/index.tsx
View File

@@ -10,6 +10,7 @@ import { GlobalContextService } from '@affine/core/modules/global-context';
import { useI18n } from '@affine/i18n'; import { useI18n } from '@affine/i18n';
import { import {
LiveData, LiveData,
MANUALLY_STOP,
useLiveData, useLiveData,
useService, useService,
useServices, useServices,
@@ -78,19 +79,23 @@ export const ExplorerDocNode = ({
) )
); );
const indexerLoading = useLiveData(
docsSearchService.indexer.status$.map(
v => v.remaining === undefined || v.remaining > 0
)
);
const [referencesLoading, setReferencesLoading] = useState(true); const [referencesLoading, setReferencesLoading] = useState(true);
useLayoutEffect(() => { useLayoutEffect(() => {
setReferencesLoading( const abortController = new AbortController();
prev => docsSearchService.indexer
prev && .waitForDocCompletedWithPriority(docId, 100, abortController.signal)
indexerLoading /* after loading becomes false, it never becomes true */ .then(() => {
); setReferencesLoading(false);
}, [indexerLoading]); })
.catch(err => {
if (err !== MANUALLY_STOP) {
console.error(err);
}
});
return () => {
abortController.abort(MANUALLY_STOP);
};
}, [docId, docsSearchService]);
const workspaceDialogService = useService(WorkspaceDialogService); const workspaceDialogService = useService(WorkspaceDialogService);
const option = useMemo( const option = useMemo(

19
packages/frontend/core/src/modules/blob-management/entity/unused-blobs.ts
View File

@@ -9,7 +9,6 @@ import {
} from '@toeverything/infra'; } from '@toeverything/infra';
import { fileTypeFromBuffer } from 'file-type'; import { fileTypeFromBuffer } from 'file-type';
import { import {
combineLatest,
EMPTY, EMPTY,
filter, filter,
firstValueFrom, firstValueFrom,
@@ -88,16 +87,9 @@ export class UnusedBlobs extends Entity {
async getUnusedBlobs(abortSignal?: AbortSignal) { async getUnusedBlobs(abortSignal?: AbortSignal) {
// Wait for both sync and indexing to complete // Wait for both sync and indexing to complete
const ready$ = combineLatest([ const ready$ = this.workspaceService.workspace.engine.doc.state$
this.workspaceService.workspace.engine.doc.state$.pipe( .pipe(filter(state => state.syncing === 0 && !state.syncRetrying))
filter(state => state.syncing === 0 && !state.syncRetrying) .pipe(map(() => true));
),
this.docsSearchService.indexer.status$.pipe(
filter(
status => status.remaining === undefined || status.remaining === 0
)
),
]).pipe(map(() => true));
await firstValueFrom( await firstValueFrom(
abortSignal abortSignal
@@ -105,6 +97,8 @@ export class UnusedBlobs extends Entity {
: ready$ : ready$
); );
await this.docsSearchService.indexer.waitForCompleted(abortSignal);
const [blobs, usedBlobs] = await Promise.all([ const [blobs, usedBlobs] = await Promise.all([
this.listBlobs(), this.listBlobs(),
this.getUsedBlobs(), this.getUsedBlobs(),
@@ -121,7 +115,8 @@ export class UnusedBlobs extends Entity {
} }
private async getUsedBlobs(): Promise<string[]> { private async getUsedBlobs(): Promise<string[]> {
const result = await this.docsSearchService.indexer.blockIndex.aggregate( const result = await this.docsSearchService.indexer.aggregate(
'block',
{ {
type: 'boolean', type: 'boolean',
occur: 'must', occur: 'must',

301
packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts
View File

@@ -1,301 +0,0 @@
import { DebugLogger } from '@affine/debug';
import type { Job, JobQueue } from '@toeverything/infra';
import {
Entity,
IndexedDBIndexStorage,
IndexedDBJobQueue,
JobRunner,
LiveData,
} from '@toeverything/infra';
import { map } from 'rxjs';
import { WorkspaceDBService } from '../../db';
import type { WorkspaceLocalState, WorkspaceService } from '../../workspace';
import { blockIndexSchema, docIndexSchema } from '../schema';
import { createWorker, type IndexerWorker } from '../worker/out-worker';
export function isEmptyUpdate(binary: Uint8Array) {
return (
binary.byteLength === 0 ||
(binary.byteLength === 2 && binary[0] === 0 && binary[1] === 0)
);
}
const logger = new DebugLogger('crawler');
const WORKSPACE_DOCS_INDEXER_VERSION_KEY = 'docs-indexer-version';
interface IndexerJobPayload {
docId: string;
}
export class DocsIndexer extends Entity {
/**
* increase this number to re-index all docs
*/
static INDEXER_VERSION = 19;
private readonly jobQueue: JobQueue<IndexerJobPayload> =
new IndexedDBJobQueue<IndexerJobPayload>(
'jq:' + this.workspaceService.workspace.id
);
private readonly runner = new JobRunner(
this.jobQueue,
(jobs, signal) => this.execJob(jobs, signal),
() =>
new Promise<void>(resolve =>
requestIdleCallback(() => resolve(), {
timeout: 200,
})
)
);
private readonly indexStorage = new IndexedDBIndexStorage(
'idx:' + this.workspaceService.workspace.id
);
readonly docIndex = this.indexStorage.getIndex('doc', docIndexSchema);
readonly blockIndex = this.indexStorage.getIndex('block', blockIndexSchema);
private readonly workspaceEngine = this.workspaceService.workspace.engine;
private readonly workspaceId = this.workspaceService.workspace.id;
private worker: IndexerWorker | null = null;
readonly status$ = LiveData.from<{ remaining?: number }>(
this.jobQueue.status$.pipe(
map(status => ({
remaining: status.remaining,
}))
),
{}
);
constructor(
private readonly workspaceService: WorkspaceService,
private readonly workspaceLocalState: WorkspaceLocalState
) {
super();
}
setupListener() {
this.workspaceEngine.doc.storage.connection
.waitForConnected()
.then(() => {
this.disposables.push(
this.workspaceEngine.doc.storage.subscribeDocUpdate(updated => {
if (WorkspaceDBService.isDBDocId(updated.docId)) {
// skip db doc
return;
}
this.jobQueue
.enqueue([
{
batchKey: updated.docId,
payload: { docId: updated.docId },
},
])
.catch(err => {
console.error('Error enqueueing job', err);
});
})
);
})
.catch(err => {
console.error('Error waiting for doc storage connection', err);
});
}
async execJob(jobs: Job<IndexerJobPayload>[], signal: AbortSignal) {
if (jobs.length === 0) {
return;
}
const dbVersion = this.getVersion();
if (dbVersion > DocsIndexer.INDEXER_VERSION) {
// stop if db version is higher then self
this.runner.stop();
throw new Error('Indexer is outdated');
}
const isUpgrade = dbVersion < DocsIndexer.INDEXER_VERSION;
// jobs should have the same storage docId, so we just pick the first one
const docId = jobs[0].payload.docId;
const worker = await this.ensureWorker(signal);
const startTime = performance.now();
logger.debug('Start crawling job for docId:', docId);
let workerOutput;
if (docId === this.workspaceId) {
const rootDocBuffer = (
await this.workspaceEngine.doc.storage.getDoc(this.workspaceId)
)?.bin;
if (!rootDocBuffer) {
return;
}
const allIndexedDocs = (await this.docIndex.getAll()).map(d => d.id);
workerOutput = await worker.run({
type: 'rootDoc',
allIndexedDocs,
rootDocBuffer,
reindexAll: isUpgrade,
rootDocId: this.workspaceId,
});
} else {
const rootDocBuffer = (
await this.workspaceEngine.doc.storage.getDoc(this.workspaceId)
)?.bin;
const docBuffer =
(await this.workspaceEngine.doc.storage.getDoc(docId))?.bin ??
new Uint8Array(0);
if (!rootDocBuffer) {
return;
}
workerOutput = await worker.run({
type: 'doc',
docBuffer,
docId,
rootDocBuffer,
rootDocId: this.workspaceId,
});
}
if (workerOutput.deletedDoc || workerOutput.addedDoc) {
if (workerOutput.deletedDoc) {
const docIndexWriter = await this.docIndex.write();
for (const docId of workerOutput.deletedDoc) {
docIndexWriter.delete(docId);
}
await docIndexWriter.commit();
const blockIndexWriter = await this.blockIndex.write();
for (const docId of workerOutput.deletedDoc) {
const oldBlocks = await blockIndexWriter.search(
{
type: 'match',
field: 'docId',
match: docId,
},
{
pagination: {
limit: Number.MAX_SAFE_INTEGER,
},
}
);
for (const block of oldBlocks.nodes) {
blockIndexWriter.delete(block.id);
}
}
await blockIndexWriter.commit();
}
if (workerOutput.addedDoc) {
const docIndexWriter = await this.docIndex.write();
for (const { doc } of workerOutput.addedDoc) {
docIndexWriter.put(doc);
}
await docIndexWriter.commit();
const blockIndexWriter = await this.blockIndex.write();
for (const { id, blocks } of workerOutput.addedDoc) {
// delete old blocks
const oldBlocks = await blockIndexWriter.search(
{
type: 'match',
field: 'docId',
match: id,
},
{
pagination: {
limit: Number.MAX_SAFE_INTEGER,
},
}
);
for (const block of oldBlocks.nodes) {
blockIndexWriter.delete(block.id);
}
for (const block of blocks) {
blockIndexWriter.insert(block);
}
}
await blockIndexWriter.commit();
}
}
if (workerOutput.reindexDoc) {
await this.jobQueue.enqueue(
workerOutput.reindexDoc.map(({ docId }) => ({
batchKey: docId,
payload: { docId },
}))
);
}
if (isUpgrade) {
this.setVersion();
}
const duration = performance.now() - startTime;
logger.debug(
'Finish crawling job for docId:' + docId + ' in ' + duration + 'ms '
);
}
startCrawling() {
this.runner.start();
this.jobQueue
.enqueue([
{
batchKey: this.workspaceId,
payload: { docId: this.workspaceId },
},
])
.catch(err => {
console.error('Error enqueueing job', err);
});
}
async ensureWorker(signal: AbortSignal): Promise<IndexerWorker> {
if (!this.worker) {
this.worker = await createWorker(signal);
}
return this.worker;
}
getVersion() {
const version = this.workspaceLocalState.get<number>(
WORKSPACE_DOCS_INDEXER_VERSION_KEY
);
if (typeof version !== 'number') {
return -1;
} else {
return version;
}
}
setVersion(version = DocsIndexer.INDEXER_VERSION) {
if (this.getVersion() >= version) {
return;
}
return this.workspaceLocalState.set(
WORKSPACE_DOCS_INDEXER_VERSION_KEY,
version
);
}
override dispose(): void {
super.dispose();
this.runner.stop();
this.worker?.dispose();
}
}

11
packages/frontend/core/src/modules/docs-search/index.ts
View File

@@ -2,17 +2,12 @@ export { DocsSearchService } from './services/docs-search';
import { type Framework } from '@toeverything/infra'; import { type Framework } from '@toeverything/infra';
import { import { DocsService } from '../doc';
WorkspaceLocalState, import { WorkspaceScope, WorkspaceService } from '../workspace';
WorkspaceScope,
WorkspaceService,
} from '../workspace';
import { DocsIndexer } from './entities/docs-indexer';
import { DocsSearchService } from './services/docs-search'; import { DocsSearchService } from './services/docs-search';
export function configureDocsSearchModule(framework: Framework) { export function configureDocsSearchModule(framework: Framework) {
framework framework
.scope(WorkspaceScope) .scope(WorkspaceScope)
.service(DocsSearchService, [WorkspaceService]) .service(DocsSearchService, [WorkspaceService, DocsService]);
.entity(DocsIndexer, [WorkspaceService, WorkspaceLocalState]);
} }

35
packages/frontend/core/src/modules/docs-search/schema.ts
View File

@@ -1,35 +0,0 @@
import { defineSchema } from '@toeverything/infra';
export const docIndexSchema = defineSchema({
docId: 'String',
title: 'FullText',
// summary of the doc, used for preview
summary: { type: 'String', index: false },
journal: 'String',
});
export type DocIndexSchema = typeof docIndexSchema;
export const blockIndexSchema = defineSchema({
docId: 'String',
blockId: 'String',
content: 'FullText',
flavour: 'String',
blob: 'String',
// reference doc id
// ['xxx','yyy']
refDocId: 'String',
// reference info, used for backlink to specific block
// [{"docId":"xxx","mode":"page","blockIds":["gt5Yfq1maYvgNgpi13rIq"]},{"docId":"yyy","mode":"edgeless","blockIds":["k5prpOlDF-9CzfatmO0W7"]}]
ref: { type: 'String', index: false },
// parent block flavour
parentFlavour: 'String',
// parent block id
parentBlockId: 'String',
// additional info
// { "databaseName": "xxx", "displayMode": "page/edgeless", "noteBlockId": "xxx" }
additional: { type: 'String', index: false },
markdownPreview: { type: 'String', index: false },
});
export type BlockIndexSchema = typeof blockIndexSchema;

135
packages/frontend/core/src/modules/docs-search/services/docs-search.ts
View File

@@ -1,31 +1,31 @@
import { toURLSearchParams } from '@affine/core/modules/navigation'; import { toURLSearchParams } from '@affine/core/modules/navigation';
import type { IndexerSyncState } from '@affine/nbstore';
import type { ReferenceParams } from '@blocksuite/affine/model'; import type { ReferenceParams } from '@blocksuite/affine/model';
import { fromPromise, OnEvent, Service } from '@toeverything/infra'; import { fromPromise, LiveData, Service } from '@toeverything/infra';
import { isEmpty, omit } from 'lodash-es'; import { isEmpty, omit } from 'lodash-es';
import { map, type Observable, of, switchMap } from 'rxjs'; import { map, type Observable, of, switchMap } from 'rxjs';
import { z } from 'zod'; import { z } from 'zod';
import type { DocsService } from '../../doc/services/docs';
import type { WorkspaceService } from '../../workspace'; import type { WorkspaceService } from '../../workspace';
import { WorkspaceEngineBeforeStart } from '../../workspace';
import { DocsIndexer } from '../entities/docs-indexer';
@OnEvent(WorkspaceEngineBeforeStart, s => s.handleWorkspaceEngineBeforeStart)
export class DocsSearchService extends Service { export class DocsSearchService extends Service {
readonly indexer = this.framework.createEntity(DocsIndexer); constructor(
private readonly workspaceService: WorkspaceService,
constructor(private readonly workspaceService: WorkspaceService) { private readonly docsService: DocsService
) {
super(); super();
} }
handleWorkspaceEngineBeforeStart() { get indexer() {
// skip if in shared mode return this.workspaceService.workspace.engine.indexer;
if (this.workspaceService.workspace.openOptions.isSharedMode) {
return;
}
this.indexer.setupListener();
this.indexer.startCrawling();
} }
readonly indexerState$ = LiveData.from(this.indexer.state$, {
indexing: 0,
errorMessage: null,
} as IndexerSyncState);
search$(query: string): Observable< search$(query: string): Observable<
{ {
docId: string; docId: string;
@@ -35,8 +35,9 @@ export class DocsSearchService extends Service {
blockContent?: string; blockContent?: string;
}[] }[]
> { > {
return this.indexer.blockIndex return this.indexer
.aggregate$( .aggregate$(
'block',
{ {
type: 'boolean', type: 'boolean',
occur: 'must', occur: 'must',
@@ -89,47 +90,40 @@ export class DocsSearchService extends Service {
} }
) )
.pipe( .pipe(
switchMap(({ buckets }) => { map(({ buckets }) => {
return fromPromise(async () => { const result = [];
const docData = await this.indexer.docIndex.getAll(
buckets.map(bucket => bucket.key)
);
const result = []; for (const bucket of buckets) {
const firstMatchFlavour = bucket.hits.nodes[0]?.fields.flavour;
for (const bucket of buckets) { if (firstMatchFlavour === 'affine:page') {
const firstMatchFlavour = bucket.hits.nodes[0]?.fields.flavour; // is title match
if (firstMatchFlavour === 'affine:page') { const blockContent = bucket.hits.nodes[1]?.highlights.content[0]; // try to get block content
// is title match result.push({
const blockContent = docId: bucket.key,
bucket.hits.nodes[1]?.highlights.content[0]; // try to get block content title: bucket.hits.nodes[0].highlights.content[0],
result.push({ score: bucket.score,
docId: bucket.key, blockContent,
title: bucket.hits.nodes[0].highlights.content[0], });
score: bucket.score, } else {
blockContent, const title =
}); this.docsService.list.doc$(bucket.key).value?.title$.value ??
} else { '';
const title = const matchedBlockId = bucket.hits.nodes[0]?.fields.blockId;
docData.find(doc => doc.id === bucket.key)?.get('title') ?? // is block match
''; result.push({
const matchedBlockId = bucket.hits.nodes[0]?.fields.blockId; docId: bucket.key,
// is block match title: title,
result.push({ blockId:
docId: bucket.key, typeof matchedBlockId === 'string'
title: typeof title === 'string' ? title : title[0], ? matchedBlockId
blockId: : matchedBlockId[0],
typeof matchedBlockId === 'string' score: bucket.score,
? matchedBlockId blockContent: bucket.hits.nodes[0]?.highlights.content[0],
: matchedBlockId[0], });
score: bucket.score,
blockContent: bucket.hits.nodes[0]?.highlights.content[0],
});
}
} }
}
return result; return result;
});
}) })
); );
} }
@@ -140,8 +134,9 @@ export class DocsSearchService extends Service {
return of([]); return of([]);
} }
return this.indexer.blockIndex return this.indexer
.search$( .search$(
'block',
{ {
type: 'boolean', type: 'boolean',
occur: 'must', occur: 'must',
@@ -185,18 +180,12 @@ export class DocsSearchService extends Service {
).values() ).values()
); );
const docData = await this.indexer.docIndex.getAll(
Array.from(new Set(refs.map(ref => ref.docId)))
);
return refs return refs
.flatMap(ref => { .flatMap(ref => {
const doc = docData.find(doc => doc.id === ref.docId); const doc = this.docsService.list.doc$(ref.docId).value;
if (!doc) return null; if (!doc) return null;
const titles = doc.get('title'); const title = doc.title$.value;
const title =
(Array.isArray(titles) ? titles[0] : titles) ?? '';
const params = omit(ref, ['docId']); const params = omit(ref, ['docId']);
return { return {
@@ -214,8 +203,9 @@ export class DocsSearchService extends Service {
} }
watchRefsTo(docId: string) { watchRefsTo(docId: string) {
return this.indexer.blockIndex return this.indexer
.aggregate$( .aggregate$(
'block',
{ {
type: 'boolean', type: 'boolean',
occur: 'must', occur: 'must',
@@ -262,13 +252,10 @@ export class DocsSearchService extends Service {
.pipe( .pipe(
switchMap(({ buckets }) => { switchMap(({ buckets }) => {
return fromPromise(async () => { return fromPromise(async () => {
const docData = await this.indexer.docIndex.getAll(
buckets.map(bucket => bucket.key)
);
return buckets.flatMap(bucket => { return buckets.flatMap(bucket => {
const title = const title =
docData.find(doc => doc.id === bucket.key)?.get('title') ?? ''; this.docsService.list.doc$(bucket.key).value?.title$.value ??
'';
return bucket.hits.nodes.map(node => { return bucket.hits.nodes.map(node => {
const blockId = node.fields.blockId ?? ''; const blockId = node.fields.blockId ?? '';
@@ -297,7 +284,7 @@ export class DocsSearchService extends Service {
return { return {
docId: bucket.key, docId: bucket.key,
blockId: typeof blockId === 'string' ? blockId : blockId[0], blockId: typeof blockId === 'string' ? blockId : blockId[0],
title: typeof title === 'string' ? title : title[0], title: title,
markdownPreview: markdownPreview:
typeof markdownPreview === 'string' typeof markdownPreview === 'string'
? markdownPreview ? markdownPreview
@@ -330,8 +317,9 @@ export class DocsSearchService extends Service {
const DatabaseAdditionalSchema = z.object({ const DatabaseAdditionalSchema = z.object({
databaseName: z.string().optional(), databaseName: z.string().optional(),
}); });
return this.indexer.blockIndex return this.indexer
.search$( .search$(
'block',
{ {
type: 'boolean', type: 'boolean',
occur: 'must', occur: 'must',
@@ -397,8 +385,9 @@ export class DocsSearchService extends Service {
} }
watchDocSummary(docId: string) { watchDocSummary(docId: string) {
return this.indexer.docIndex return this.indexer
.search$( .search$(
'doc',
{ {
type: 'match', type: 'match',
field: 'docId', field: 'docId',
@@ -422,8 +411,4 @@ export class DocsSearchService extends Service {
}) })
); );
} }
override dispose(): void {
this.indexer.dispose();
}
} }

1037
packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
View File

File diff suppressed because it is too large Load Diff

105
packages/frontend/core/src/modules/docs-search/worker/out-worker.ts
View File

@@ -1,105 +0,0 @@
import { DebugLogger } from '@affine/debug';
import { MANUALLY_STOP, throwIfAborted } from '@toeverything/infra';
import type {
WorkerIngoingMessage,
WorkerInput,
WorkerOutgoingMessage,
WorkerOutput,
} from './types';
const logger = new DebugLogger('affine:indexer-worker');
export async function createWorker(abort: AbortSignal) {
let worker: Worker | null = null;
while (throwIfAborted(abort)) {
try {
worker = await new Promise<Worker>((resolve, reject) => {
// @TODO(@forehalo): need to make a general worker
const worker = new Worker(
/* webpackChunkName: "worker" */ new URL(
'./in-worker.ts',
import.meta.url
)
);
worker.addEventListener('error', reject);
worker.addEventListener('message', event => {
if (event.data.type === 'init') {
resolve(worker);
}
});
worker.postMessage({ type: 'init', msgId: 0 } as WorkerIngoingMessage);
setTimeout(() => {
reject('timeout');
}, 1000 * 30 /* 30 sec */);
});
} catch (err) {
logger.debug(
`Indexer worker init failed, ${err}, will retry in 5 seconds.`
);
await new Promise(resolve => setTimeout(resolve, 5000));
}
if (worker) {
break;
}
}
if (!worker) {
// never reach here
throw new Error('Worker is not created');
}
const terminateAbort = new AbortController();
let msgId = 1;
return {
run: async (input: WorkerInput) => {
const dispose: (() => void)[] = [];
return new Promise<WorkerOutput>((resolve, reject) => {
const currentMsgId = msgId++;
const msgHandler = (event: MessageEvent<WorkerOutgoingMessage>) => {
if (event.data.msgId === currentMsgId) {
if (event.data.type === 'done') {
resolve(event.data.output);
} else if (event.data.type === 'failed') {
reject(new Error(event.data.error));
} else {
reject(new Error('Unknown message type'));
}
}
};
const abortHandler = (reason: any) => {
reject(reason);
};
worker.addEventListener('message', msgHandler);
dispose.push(() => {
worker?.removeEventListener('message', msgHandler);
});
terminateAbort.signal.addEventListener('abort', abortHandler);
dispose.push(() => {
terminateAbort.signal.removeEventListener('abort', abortHandler);
});
worker.postMessage({
type: 'run',
msgId: currentMsgId,
input,
} as WorkerIngoingMessage);
}).finally(() => {
for (const d of dispose) {
d();
}
});
},
dispose: () => {
terminateAbort.abort(MANUALLY_STOP);
worker.terminate();
},
};
}
export type IndexerWorker = Awaited<ReturnType<typeof createWorker>>;

53
packages/frontend/core/src/modules/docs-search/worker/types.ts
View File

@@ -1,53 +0,0 @@
import type { Document } from '@toeverything/infra';
import type { BlockIndexSchema, DocIndexSchema } from '../schema';
export type WorkerIngoingMessage = (
| {
type: 'init';
}
| {
type: 'run';
input: WorkerInput;
}
) & { msgId: number };
export type WorkerOutgoingMessage = (
| {
type: 'init';
}
| {
type: 'done';
output: WorkerOutput;
}
| {
type: 'failed';
error: string;
}
) & { msgId: number };
export type WorkerInput =
| {
type: 'rootDoc';
rootDocBuffer: Uint8Array;
rootDocId: string;
allIndexedDocs: string[];
reindexAll?: boolean;
}
| {
type: 'doc';
docId: string;
rootDocId: string;
rootDocBuffer: Uint8Array;
docBuffer: Uint8Array;
};
export interface WorkerOutput {
reindexDoc?: { docId: string }[];
addedDoc?: {
id: string;
blocks: Document<BlockIndexSchema>[];
doc: Document<DocIndexSchema>;
}[];
deletedDoc?: string[];
}

27
packages/frontend/core/src/modules/explorer/views/nodes/doc/index.tsx
View File

@@ -19,6 +19,7 @@ import { useI18n } from '@affine/i18n';
import { track } from '@affine/track'; import { track } from '@affine/track';
import { import {
LiveData, LiveData,
MANUALLY_STOP,
useLiveData, useLiveData,
useService, useService,
useServices, useServices,
@@ -98,19 +99,23 @@ export const ExplorerDocNode = ({
); );
const searching = children === null; const searching = children === null;
const indexerLoading = useLiveData(
docsSearchService.indexer.status$.map(
v => v.remaining === undefined || v.remaining > 0
)
);
const [referencesLoading, setReferencesLoading] = useState(true); const [referencesLoading, setReferencesLoading] = useState(true);
useLayoutEffect(() => { useLayoutEffect(() => {
setReferencesLoading( const abortController = new AbortController();
prev => docsSearchService.indexer
prev && .waitForDocCompletedWithPriority(docId, 100, abortController.signal)
indexerLoading /* after loading becomes false, it never becomes true */ .then(() => {
); setReferencesLoading(false);
}, [indexerLoading]); })
.catch(err => {
if (err !== MANUALLY_STOP) {
console.error(err);
}
});
return () => {
abortController.abort(MANUALLY_STOP);
};
}, [docId, docsSearchService]);
const dndData = useMemo(() => { const dndData = useMemo(() => {
return { return {

9
packages/frontend/core/src/modules/quicksearch/impls/docs.ts
View File

@@ -33,10 +33,11 @@ export class DocsQuickSearchSession
super(); super();
} }
private readonly isIndexerLoading$ = private readonly isIndexerLoading$ = this.docsSearchService.indexerState$.map(
this.docsSearchService.indexer.status$.map(({ remaining }) => { ({ completed }) => {
return remaining === undefined || remaining > 0; return !completed;
}); }
);
private readonly isQueryLoading$ = new LiveData(false); private readonly isQueryLoading$ = new LiveData(false);

16
packages/frontend/core/src/modules/search-menu/services/index.ts
View File

@@ -17,7 +17,7 @@ import { cssVarV2 } from '@toeverything/theme/v2';
import Fuse, { type FuseResultMatch } from 'fuse.js'; import Fuse, { type FuseResultMatch } from 'fuse.js';
import { html } from 'lit'; import { html } from 'lit';
import { unsafeHTML } from 'lit/directives/unsafe-html.js'; import { unsafeHTML } from 'lit/directives/unsafe-html.js';
import { map, takeWhile } from 'rxjs'; import { map } from 'rxjs';
import type { CollectionService } from '../../collection'; import type { CollectionService } from '../../collection';
import type { DocDisplayMetaService } from '../../doc-display-meta'; import type { DocDisplayMetaService } from '../../doc-display-meta';
@@ -119,15 +119,6 @@ export class SearchMenuService extends Service {
[] []
); );
const { signal: isIndexerLoading, cleanup: cleanupIndexerLoading } =
createSignalFromObservable(
this.docsSearch.indexer.status$.pipe(
map(status => status.remaining !== undefined && status.remaining > 0),
takeWhile(isLoading => isLoading, true)
),
false
);
const overflowText = computed(() => { const overflowText = computed(() => {
const overflowCount = docsSignal.value.length - MAX_DOCS; const overflowCount = docsSignal.value.length - MAX_DOCS;
return I18n.t('com.affine.editor.at-menu.more-docs-hint', { return I18n.t('com.affine.editor.at-menu.more-docs-hint', {
@@ -137,14 +128,12 @@ export class SearchMenuService extends Service {
abortSignal.addEventListener('abort', () => { abortSignal.addEventListener('abort', () => {
cleanupDocs(); cleanupDocs();
cleanupIndexerLoading();
}); });
return { return {
name: I18n.t('com.affine.editor.at-menu.link-to-doc', { name: I18n.t('com.affine.editor.at-menu.link-to-doc', {
query, query,
}), }),
loading: isIndexerLoading,
items: docsSignal, items: docsSignal,
maxDisplay: MAX_DOCS, maxDisplay: MAX_DOCS,
overflowText, overflowText,
@@ -153,8 +142,9 @@ export class SearchMenuService extends Service {
// only search docs by title, excluding blocks // only search docs by title, excluding blocks
private searchDocs$(query: string) { private searchDocs$(query: string) {
return this.docsSearch.indexer.docIndex return this.docsSearch.indexer
.aggregate$( .aggregate$(
'doc',
{ {
type: 'boolean', type: 'boolean',
occur: 'must', occur: 'must',

16
packages/frontend/core/src/modules/workspace-engine/impls/cloud.ts
View File

@@ -460,6 +460,22 @@ class CloudWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
id: `${this.flavour}:${workspaceId}`, id: `${this.flavour}:${workspaceId}`,
}, },
}, },
indexer: {
name: 'IndexedDBIndexerStorage',
opts: {
flavour: this.flavour,
type: 'workspace',
id: workspaceId,
},
},
indexerSync: {
name: 'IndexedDBIndexerSyncStorage',
opts: {
flavour: this.flavour,
type: 'workspace',
id: workspaceId,
},
},
}, },
remotes: { remotes: {
[`cloud:${this.flavour}`]: { [`cloud:${this.flavour}`]: {

16
packages/frontend/core/src/modules/workspace-engine/impls/local.ts
View File

@@ -349,6 +349,22 @@ class LocalWorkspaceFlavourProvider implements WorkspaceFlavourProvider {
id: workspaceId, id: workspaceId,
}, },
}, },
indexer: {
name: 'IndexedDBIndexerStorage',
opts: {
flavour: this.flavour,
type: 'workspace',
id: workspaceId,
},
},
indexerSync: {
name: 'IndexedDBIndexerSyncStorage',
opts: {
flavour: this.flavour,
type: 'workspace',
id: workspaceId,
},
},
}, },
remotes: { remotes: {
v1: { v1: {

7
packages/frontend/core/src/modules/workspace/entities/engine.ts
View File

@@ -36,6 +36,13 @@ export class WorkspaceEngine extends Entity<{
return this.client.blobFrontend; return this.client.blobFrontend;
} }
get indexer() {
if (!this.client) {
throw new Error('Engine is not initialized');
}
return this.client.indexerFrontend;
}
get awareness() { get awareness() {
if (!this.client) { if (!this.client) {
throw new Error('Engine is not initialized'); throw new Error('Engine is not initialized');

1
tools/utils/src/workspace.gen.ts
View File

@@ -903,6 +903,7 @@ export const PackageList = [
'packages/common/infra', 'packages/common/infra',
'packages/common/error', 'packages/common/error',
'packages/common/graphql', 'packages/common/graphql',
'blocksuite/affine/all',
], ],
}, },
{ {

4
yarn.lock
View File

@@ -808,11 +808,14 @@ __metadata:
dependencies: dependencies:
"@affine/error": "workspace:*" "@affine/error": "workspace:*"
"@affine/graphql": "workspace:*" "@affine/graphql": "workspace:*"
"@blocksuite/affine": "workspace:*"
"@datastructures-js/binary-search-tree": "npm:^5.3.2" "@datastructures-js/binary-search-tree": "npm:^5.3.2"
"@toeverything/infra": "workspace:*" "@toeverything/infra": "workspace:*"
eventemitter2: "npm:^6.4.9" eventemitter2: "npm:^6.4.9"
fake-indexeddb: "npm:^6.0.0" fake-indexeddb: "npm:^6.0.0"
graphemer: "npm:^1.4.0"
idb: "npm:^8.0.0" idb: "npm:^8.0.0"
lib0: "npm:^0.2.99"
lodash-es: "npm:^4.17.21" lodash-es: "npm:^4.17.21"
nanoid: "npm:^5.0.9" nanoid: "npm:^5.0.9"
rxjs: "npm:^7.8.1" rxjs: "npm:^7.8.1"
@@ -824,6 +827,7 @@ __metadata:
peerDependencies: peerDependencies:
"@affine/error": "workspace:*" "@affine/error": "workspace:*"
"@affine/graphql": "workspace:*" "@affine/graphql": "workspace:*"
"@blocksuite/affine": "workspace:*"
idb: ^8.0.0 idb: ^8.0.0
socket.io-client: ^4.7.5 socket.io-client: ^4.7.5
languageName: unknown languageName: unknown