luulam/AFFiNE
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: impl text delta support (#14132)

Browse Source
This commit is contained in:
DarkSky
2025-12-22 03:16:16 +08:00
committed by GitHub
parent efbdee5508
commit 321965a424
31 changed files with 2299 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
packages/common/native/Cargo.toml
View File

@@ -1,7 +1,8 @@
[package]
edition = "2021"
name = "affine_common"
version = "0.1.0"
edition = "2021"
license-file = "LICENSE"
name = "affine_common"
version = "0.1.0"
[features]
default = []

44
packages/common/native/LICENSE Normal file
View File

@@ -0,0 +1,44 @@
The AFFiNE Enterprise Edition (EE) license (the “EE License”)
Copyright (c) 2022-present TOEVERYTHING PTE. LTD. and its affiliates.
With regard to the AFFiNE Software:
This software and associated documentation files (the "Software") may only be
used in production, if you (and any entity that you represent) have agreed to,
and are in compliance with, the AFFiNE Subscription Terms of Service, available
at https://affine.pro/terms/#subscription (the “EE Terms”), or other
agreement governing the use of the Software, as agreed by you and AFFiNE,
and otherwise have a valid AFFiNE Enterprise Edition subscription for the
correct number of user seats. Subject to the foregoing sentence, you are free to
modify this Software and publish patches to the Software. You agree that AFFiNE
and/or its licensors (as applicable) retain all right, title and interest in and
to all such modifications and/or patches, and all such modifications and/or
patches may only be used, copied, modified, displayed, distributed, or otherwise
exploited with a valid AFFiNE Enterprise Edition subscription for the correct
number of user seats. Notwithstanding the foregoing, you may copy and modify
the Software for development and testing purposes, without requiring a
subscription. You agree that AFFiNE and/or its licensors (as applicable) retain
all right, title and interest in and to all such modifications. You are not
granted any other rights beyond what is expressly stated herein. Subject to the
foregoing, it is forbidden to copy, merge, publish, distribute, sublicense,
and/or sell the Software.
This EE License applies only to the part of this Software that is not
distributed as part of AFFiNE Community Edition (CE). Any part of this Software
distributed as part of AFFiNE CE or is served client-side as an image, font,
cascading stylesheet (CSS), file which produces or is compiled, arranged,
augmented, or combined into client-side JavaScript, in whole or in part, is
copyrighted under the MPL2.0 license. The full text of this EE License shall
be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
For all third party components incorporated into the AFFiNE Software, those
components are licensed under the original license provided by the owner of the
applicable component.

2
packages/common/native/src/doc_loader/loader/source/parser.rs
View File

@@ -110,7 +110,7 @@ fn get_language_parser(language: &Language) -> Parser {
};
parser
.set_language(&lang.into())
.unwrap_or_else(|_| panic!("Error loading grammar for language: {:?}", language));
.unwrap_or_else(|_| panic!("Error loading grammar for language: {language:?}"));
parser
}

633
packages/common/native/src/doc_parser.rs → packages/common/native/src/doc_parser/affine.rs
View File

@@ -1,10 +1,22 @@
use std::collections::{HashMap, HashSet};
use serde::{Deserialize, Serialize};
use serde_json::{Map as JsonMap, Value as JsonValue};
use thiserror::Error;
use y_octo::{Any, DocOptions, JwstCodecError, Map, Value};
use super::{
blocksuite::{
collect_child_ids, get_block_id, get_flavour, get_list_depth, get_string, nearest_by_flavour,
DocContext,
},
delta_markdown::{
delta_value_to_inline_markdown, extract_inline_references, text_to_inline_markdown,
text_to_markdown, DeltaToMdOptions,
},
value::{
any_as_string, any_truthy, build_reference_payload, params_value_to_json, value_to_string,
},
};
const SUMMARY_LIMIT: usize = 1000;
const PAGE_FLAVOUR: &str = "affine:page";
const NOTE_FLAVOUR: &str = "affine:note";
@@ -89,6 +101,7 @@ pub fn parse_doc_to_markdown(
doc_bin: Vec<u8>,
doc_id: String,
ai_editable: bool,
doc_url_prefix: Option<String>,
) -> Result<MarkdownResult, ParseError> {
if doc_bin.is_empty() || doc_bin == [0, 0] {
return Err(ParseError::InvalidBinary);
@@ -107,36 +120,16 @@ pub fn parse_doc_to_markdown(
});
}
let mut block_pool: HashMap<String, Map> = HashMap::new();
let mut parent_lookup: HashMap<String, String> = HashMap::new();
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map() {
if let Some(block_id) = get_block_id(&block_map) {
for child_id in collect_child_ids(&block_map) {
parent_lookup.insert(child_id, block_id.clone());
}
block_pool.insert(block_id, block_map);
}
}
}
let root_block_id = block_pool
.iter()
.find_map(|(id, block)| {
get_flavour(block)
.filter(|flavour| flavour == PAGE_FLAVOUR)
.map(|_| id.clone())
})
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
let mut queue: Vec<(Option<String>, String)> = vec![(None, root_block_id.clone())];
let mut visited: HashSet<String> = HashSet::from([root_block_id.clone()]);
let root_block_id = context.root_block_id.clone();
let mut walker = context.walker();
let mut doc_title = String::from("Untitled");
let mut markdown = String::new();
let md_options = DeltaToMdOptions::new(doc_url_prefix);
while let Some((parent_block_id, block_id)) = queue.pop() {
let block = match block_pool.get(&block_id) {
while let Some((parent_block_id, block_id)) = walker.next() {
let block = match context.block_pool.get(&block_id) {
Some(block) => block,
None => continue,
};
@@ -146,9 +139,9 @@ pub fn parse_doc_to_markdown(
None => continue,
};
let parent_id = parent_lookup.get(&block_id);
let parent_id = context.parent_lookup.get(&block_id);
let parent_flavour = parent_id
.and_then(|id| block_pool.get(id))
.and_then(|id| context.block_pool.get(id))
.and_then(get_flavour);
if parent_flavour.as_deref() == Some("affine:database") {
@@ -156,12 +149,7 @@ pub fn parse_doc_to_markdown(
}
// enqueue children first to keep traversal order similar to JS implementation
let mut child_ids = collect_child_ids(block);
for child_id in child_ids.drain(..).rev() {
if visited.insert(child_id.clone()) {
queue.push((Some(block_id.clone()), child_id));
}
}
walker.enqueue_children(&block_id, block);
if flavour == PAGE_FLAVOUR {
let title = get_string(block, "prop:title").unwrap_or_default();
@@ -171,109 +159,103 @@ pub fn parse_doc_to_markdown(
if flavour == "affine:database" {
let title = get_string(block, "prop:title").unwrap_or_default();
markdown.push_str(&format!("\n### {}\n", title));
markdown.push_str(&format!("\n### {title}\n"));
let columns_array = block.get("prop:columns").and_then(|v| v.to_array());
let columns = parse_database_columns(block);
let cells_map = block.get("prop:cells").and_then(|v| v.to_map());
if let (Some(columns_array), Some(cells_map)) = (columns_array, cells_map) {
let mut columns = Vec::new();
for col_val in columns_array.iter() {
if let Some(col_map) = col_val.to_map() {
let id = get_string(&col_map, "id").unwrap_or_default();
let name = get_string(&col_map, "name").unwrap_or_default();
let type_ = get_string(&col_map, "type").unwrap_or_default();
let data = col_map.get("data").and_then(|v| v.to_map());
columns.push((id, name, type_, data));
}
}
if let (Some(columns), Some(cells_map)) = (columns, cells_map) {
let escape_table = |s: &str| s.replace('|', "\\|").replace('\n', "<br>");
let mut table = String::new();
markdown.push('|');
for (_, name, _, _) in &columns {
markdown.push_str(&escape_table(name));
markdown.push('|');
table.push('|');
for column in &columns {
table.push_str(&escape_table(column.name.as_deref().unwrap_or_default()));
table.push('|');
}
markdown.push('\n');
table.push('\n');
markdown.push('|');
table.push('|');
for _ in &columns {
markdown.push_str("---|");
table.push_str("---|");
}
markdown.push('\n');
table.push('\n');
let child_ids = collect_child_ids(block);
for child_id in child_ids {
markdown.push('|');
table.push('|');
let row_cells = cells_map.get(&child_id).and_then(|v| v.to_map());
for (col_id, _, col_type, col_data) in &columns {
for column in &columns {
let mut cell_text = String::new();
if col_type == "title" {
if let Some(child_block) = block_pool.get(&child_id) {
if let Some((text, _)) = text_content(child_block, "prop:text") {
if column.col_type == "title" {
if let Some(child_block) = context.block_pool.get(&child_id) {
if let Some(text_md) =
text_to_inline_markdown(child_block, "prop:text", &md_options)
{
cell_text = text_md;
} else if let Some((text, _)) = text_content(child_block, "prop:text") {
cell_text = text;
}
}
} else if let Some(row_cells) = &row_cells {
if let Some(cell_val) = row_cells.get(col_id).and_then(|v| v.to_map()) {
if let Some(value) = cell_val.get("value").and_then(|v| v.to_any()) {
cell_text = format_cell_value(&value, col_type, col_data.as_ref());
if let Some(cell_val) = row_cells.get(&column.id).and_then(|v| v.to_map()) {
if let Some(value) = cell_val.get("value") {
if let Some(text_md) = delta_value_to_inline_markdown(&value, &md_options) {
cell_text = text_md;
} else {
cell_text = format_cell_value(&value, column);
}
}
}
}
markdown.push_str(&escape_table(&cell_text));
markdown.push('|');
table.push_str(&escape_table(&cell_text));
table.push('|');
}
markdown.push('\n');
table.push('\n');
}
append_table_block(&mut markdown, &table);
}
continue;
}
if flavour == "affine:table" {
let contents = gather_table_contents(block);
markdown.push_str(&contents.join("|"));
markdown.push('\n');
let table = contents.join("|");
append_table_block(&mut markdown, &table);
continue;
}
if ai_editable && parent_block_id.as_ref() == Some(&root_block_id) {
markdown.push_str(&format!(
"<!-- block_id={} flavour={} -->\n",
block_id, flavour
));
markdown.push_str(&format!("<!-- block_id={block_id} flavour={flavour} -->\n"));
}
if flavour == "affine:paragraph" {
if let Some((text, _)) = text_content(block, "prop:text") {
let type_ = get_string(block, "prop:type").unwrap_or_default();
let prefix = match type_.as_str() {
"h1" => "# ",
"h2" => "## ",
"h3" => "### ",
"h4" => "#### ",
"h5" => "##### ",
"h6" => "###### ",
"quote" => "> ",
_ => "",
};
markdown.push_str(prefix);
markdown.push_str(&text);
markdown.push('\n');
let type_ = get_string(block, "prop:type").unwrap_or_default();
let prefix = paragraph_prefix(type_.as_str());
if let Some(text_md) = text_to_markdown(block, "prop:text", &md_options) {
append_paragraph(&mut markdown, prefix, &text_md);
} else if let Some((text, _)) = text_content(block, "prop:text") {
append_paragraph(&mut markdown, prefix, &text);
}
continue;
}
if flavour == "affine:list" {
if let Some((text, _)) = text_content(block, "prop:text") {
let depth = get_list_depth(&block_id, &parent_lookup, &block_pool);
let indent = " ".repeat(depth);
markdown.push_str(&indent);
markdown.push_str("- ");
markdown.push_str(&text);
markdown.push('\n');
let type_ = get_string(block, "prop:type").unwrap_or_default();
let checked = block
.get("prop:checked")
.and_then(|value| value.to_any())
.as_ref()
.map(any_truthy)
.unwrap_or(false);
let prefix = list_prefix(type_.as_str(), checked);
let depth = get_list_depth(&block_id, &context.parent_lookup, &context.block_pool);
let indent = list_indent(depth);
if let Some(text_md) = text_to_markdown(block, "prop:text", &md_options) {
append_list_item(&mut markdown, &indent, prefix, &text_md);
} else if let Some((text, _)) = text_content(block, "prop:text") {
append_list_item(&mut markdown, &indent, prefix, &text);
}
continue;
}
@@ -281,11 +263,7 @@ pub fn parse_doc_to_markdown(
if flavour == "affine:code" {
if let Some((text, _)) = text_content(block, "prop:text") {
let lang = get_string(block, "prop:language").unwrap_or_default();
markdown.push_str("```");
markdown.push_str(&lang);
markdown.push('\n');
markdown.push_str(&text);
markdown.push_str("\n```\n");
append_code_block(&mut markdown, &lang, &text);
}
continue;
}
@@ -297,27 +275,6 @@ pub fn parse_doc_to_markdown(
})
}
fn get_list_depth(
block_id: &str,
parent_lookup: &HashMap<String, String>,
blocks: &HashMap<String, Map>,
) -> usize {
let mut depth = 0;
let mut current_id = block_id.to_string();
while let Some(parent_id) = parent_lookup.get(&current_id) {
if let Some(parent_block) = blocks.get(parent_id) {
if get_flavour(parent_block).as_deref() == Some("affine:list") {
depth += 1;
current_id = parent_id.clone();
continue;
}
}
break;
}
depth
}
pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlResult, ParseError> {
if doc_bin.is_empty() || doc_bin == [0, 0] {
return Err(ParseError::InvalidBinary);
@@ -333,38 +290,16 @@ pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlRe
return Err(ParseError::ParserError("blocks map is empty".into()));
}
let mut block_pool: HashMap<String, Map> = HashMap::new();
let mut parent_lookup: HashMap<String, String> = HashMap::new();
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map() {
if let Some(block_id) = get_block_id(&block_map) {
for child_id in collect_child_ids(&block_map) {
parent_lookup.insert(child_id, block_id.clone());
}
block_pool.insert(block_id, block_map);
}
}
}
let root_block_id = block_pool
.iter()
.find_map(|(id, block)| {
get_flavour(block)
.filter(|flavour| flavour == PAGE_FLAVOUR)
.map(|_| id.clone())
})
let context = DocContext::from_blocks_map(&blocks_map, PAGE_FLAVOUR)
.ok_or_else(|| ParseError::ParserError("root block not found".into()))?;
let mut queue: Vec<(Option<String>, String)> = vec![(None, root_block_id.clone())];
let mut visited: HashSet<String> = HashSet::from([root_block_id.clone()]);
let mut blocks: Vec<BlockInfo> = Vec::with_capacity(block_pool.len());
let mut walker = context.walker();
let mut blocks: Vec<BlockInfo> = Vec::with_capacity(context.block_pool.len());
let mut doc_title = String::new();
let mut summary = String::new();
let mut summary_remaining = SUMMARY_LIMIT as isize;
while let Some((parent_block_id, block_id)) = queue.pop() {
let block = match block_pool.get(&block_id) {
while let Some((parent_block_id, block_id)) = walker.next() {
let block = match context.block_pool.get(&block_id) {
Some(block) => block,
None => continue,
};
@@ -374,20 +309,22 @@ pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlRe
None => continue,
};
let parent_block = parent_block_id.as_ref().and_then(|id| block_pool.get(id));
let parent_block = parent_block_id
.as_ref()
.and_then(|id| context.block_pool.get(id));
let parent_flavour = parent_block.and_then(get_flavour);
let note_block = nearest_by_flavour(&block_id, NOTE_FLAVOUR, &parent_lookup, &block_pool);
let note_block = nearest_by_flavour(
&block_id,
NOTE_FLAVOUR,
&context.parent_lookup,
&context.block_pool,
);
let note_block_id = note_block.as_ref().and_then(get_block_id);
let display_mode = determine_display_mode(note_block.as_ref());
// enqueue children first to keep traversal order similar to JS implementation
let mut child_ids = collect_child_ids(block);
for child_id in child_ids.drain(..).rev() {
if visited.insert(child_id.clone()) {
queue.push((Some(block_id.clone()), child_id));
}
}
walker.enqueue_children(&block_id, block);
let build_block = |database_name: Option<&String>| {
BlockInfo::base(
@@ -412,7 +349,7 @@ pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlRe
flavour.as_str(),
"affine:paragraph" | "affine:list" | "affine:code"
) {
if let Some((text, text_len)) = text_content(block, "prop:text") {
if let Some(text) = block.get("prop:text").and_then(|value| value.to_text()) {
let database_name = if flavour == "affine:paragraph"
&& parent_flavour.as_deref() == Some("affine:database")
{
@@ -421,10 +358,18 @@ pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlRe
None
};
let content = text.to_string();
let text_len = text.len() as usize;
let refs = extract_inline_references(&text.to_delta());
let mut info = build_block(database_name.as_ref());
info.content = Some(vec![text.clone()]);
info.content = Some(vec![content.clone()]);
if !refs.is_empty() {
info.ref_doc_id = Some(refs.iter().map(|r| r.doc_id.clone()).collect());
info.ref_info = Some(refs.into_iter().map(|r| r.payload).collect());
}
blocks.push(info);
append_summary(&mut summary, &mut summary_remaining, text_len, &text);
append_summary(&mut summary, &mut summary_remaining, text_len, &content);
}
continue;
}
@@ -435,10 +380,8 @@ pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlRe
) {
if let Some(page_id) = get_string(block, "prop:pageId") {
let mut info = build_block(None);
info.ref_doc_id = Some(vec![page_id.clone()]);
if let Some(payload) = embed_ref_payload(block, &page_id) {
info.ref_info = Some(vec![payload]);
}
let payload = embed_ref_payload(block, &page_id);
apply_doc_ref(&mut info, page_id, payload);
blocks.push(info);
}
continue;
@@ -447,8 +390,8 @@ pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlRe
if flavour == "affine:attachment" {
if let Some(blob_id) = get_string(block, "prop:sourceId") {
let mut info = build_block(None);
info.blob = Some(vec![blob_id]);
info.content = Some(vec![get_string(block, "prop:name").unwrap_or_default()]);
let name = get_string(block, "prop:name").unwrap_or_default();
apply_blob_info(&mut info, blob_id, name);
blocks.push(info);
}
continue;
@@ -457,8 +400,8 @@ pub fn parse_doc_from_binary(doc_bin: Vec<u8>, doc_id: String) -> Result<CrawlRe
if flavour == "affine:image" {
if let Some(blob_id) = get_string(block, "prop:sourceId") {
let mut info = build_block(None);
info.blob = Some(vec![blob_id]);
info.content = Some(vec![get_string(block, "prop:caption").unwrap_or_default()]);
let caption = get_string(block, "prop:caption").unwrap_or_default();
apply_blob_info(&mut info, blob_id, caption);
blocks.push(info);
}
continue;
@@ -566,29 +509,73 @@ pub fn get_doc_ids_from_binary(
Ok(doc_ids)
}
fn collect_child_ids(block: &Map) -> Vec<String> {
block
.get("sys:children")
.and_then(|value| value.to_array())
.map(|array| {
array
.iter()
.filter_map(|value| value_to_string(&value))
.collect::<Vec<_>>()
})
.unwrap_or_default()
fn paragraph_prefix(type_: &str) -> &'static str {
match type_ {
"h1" => "# ",
"h2" => "## ",
"h3" => "### ",
"h4" => "#### ",
"h5" => "##### ",
"h6" => "###### ",
"quote" => "> ",
_ => "",
}
}
fn get_block_id(block: &Map) -> Option<String> {
get_string(block, "sys:id")
fn list_prefix(type_: &str, checked: bool) -> &'static str {
match type_ {
"bulleted" => "* ",
"todo" => {
if checked {
"- [x] "
} else {
"- [ ] "
}
}
_ => "1. ",
}
}
fn get_flavour(block: &Map) -> Option<String> {
get_string(block, "sys:flavour")
fn list_indent(depth: usize) -> String {
" ".repeat(depth)
}
fn get_string(block: &Map, key: &str) -> Option<String> {
block.get(key).and_then(|value| value_to_string(&value))
fn append_paragraph(markdown: &mut String, prefix: &str, text: &str) {
markdown.push_str(prefix);
markdown.push_str(text);
if !text.ends_with('\n') {
markdown.push('\n');
}
markdown.push('\n');
}
fn append_list_item(markdown: &mut String, indent: &str, prefix: &str, text: &str) {
markdown.push_str(indent);
markdown.push_str(prefix);
markdown.push_str(text);
if !text.ends_with('\n') {
markdown.push('\n');
}
}
fn append_code_block(markdown: &mut String, lang: &str, text: &str) {
markdown.push_str("```");
markdown.push_str(lang);
markdown.push('\n');
markdown.push_str(text);
markdown.push_str("\n```\n\n");
}
fn append_table_block(markdown: &mut String, table: &str) {
if table.is_empty() {
markdown.push('\n');
return;
}
markdown.push_str(table);
if !table.ends_with('\n') {
markdown.push('\n');
}
markdown.push('\n');
}
fn text_content(block: &Map, key: &str) -> Option<(String, usize)> {
@@ -601,24 +588,6 @@ fn text_content(block: &Map, key: &str) -> Option<(String, usize)> {
})
}
fn nearest_by_flavour(
start: &str,
flavour: &str,
parent_lookup: &HashMap<String, String>,
blocks: &HashMap<String, Map>,
) -> Option<Map> {
let mut cursor = Some(start.to_string());
while let Some(node) = cursor {
if let Some(block) = blocks.get(&node) {
if get_flavour(block).as_deref() == Some(flavour) {
return Some(block.clone());
}
}
cursor = parent_lookup.get(&node).cloned();
}
None
}
fn determine_display_mode(note_block: Option<&Map>) -> String {
match note_block.and_then(|block| get_string(block, "prop:displayMode")) {
Some(mode) if mode == "both" => "page".into(),
@@ -646,19 +615,24 @@ fn compose_additional(
Some(JsonValue::Object(payload).to_string())
}
fn embed_ref_payload(block: &Map, page_id: &str) -> Option<String> {
let mut payload = JsonMap::new();
payload.insert("docId".into(), JsonValue::String(page_id.to_string()));
fn apply_blob_info(info: &mut BlockInfo, blob_id: String, content: String) {
info.blob = Some(vec![blob_id]);
info.content = Some(vec![content]);
}
if let Some(params_value) = block.get("prop:params") {
if let Ok(JsonValue::Object(params)) = serde_json::to_value(&params_value) {
for (key, value) in params.into_iter() {
payload.insert(key, value);
}
}
fn apply_doc_ref(info: &mut BlockInfo, page_id: String, payload: Option<String>) {
info.ref_doc_id = Some(vec![page_id]);
if let Some(payload) = payload {
info.ref_info = Some(vec![payload]);
}
}
Some(JsonValue::Object(payload).to_string())
fn embed_ref_payload(block: &Map, page_id: &str) -> Option<String> {
let params = block
.get("prop:params")
.as_ref()
.and_then(params_value_to_json);
Some(build_reference_payload(page_id, params))
}
fn gather_surface_texts(block: &Map) -> Vec<String> {
@@ -698,22 +672,14 @@ fn gather_database_texts(block: &Map) -> (Vec<String>, Option<String>) {
texts.push(title.clone());
}
if let Some(columns) = block.get("prop:columns").and_then(|value| value.to_array()) {
for column_value in columns.iter() {
if let Some(column) = column_value.to_map() {
if let Some(name) = get_string(&column, "name") {
texts.push(name);
}
if let Some(data) = column.get("data").and_then(|value| value.to_map()) {
if let Some(options) = data.get("options").and_then(|value| value.to_array()) {
for option_value in options.iter() {
if let Some(option) = option_value.to_map() {
if let Some(value) = get_string(&option, "value") {
texts.push(value);
}
}
}
}
if let Some(columns) = parse_database_columns(block) {
for column in columns.iter() {
if let Some(name) = column.name.as_ref() {
texts.push(name.clone());
}
for option in column.options.iter() {
if let Some(value) = option.value.as_ref() {
texts.push(value.clone());
}
}
}
@@ -736,79 +702,118 @@ fn gather_table_contents(block: &Map) -> Vec<String> {
contents
}
fn format_cell_value(value: &Any, col_type: &str, col_data: Option<&Map>) -> String {
match col_type {
struct DatabaseOption {
id: Option<String>,
value: Option<String>,
color: Option<String>,
}
struct DatabaseColumn {
id: String,
name: Option<String>,
col_type: String,
options: Vec<DatabaseOption>,
}
fn parse_database_columns(block: &Map) -> Option<Vec<DatabaseColumn>> {
let columns = block
.get("prop:columns")
.and_then(|value| value.to_array())?;
let mut parsed = Vec::new();
for column_value in columns.iter() {
if let Some(column) = column_value.to_map() {
let id = get_string(&column, "id").unwrap_or_default();
let name = get_string(&column, "name");
let col_type = get_string(&column, "type").unwrap_or_default();
let options = parse_database_options(&column);
parsed.push(DatabaseColumn {
id,
name,
col_type,
options,
});
}
}
Some(parsed)
}
fn parse_database_options(column: &Map) -> Vec<DatabaseOption> {
let Some(data) = column.get("data").and_then(|value| value.to_map()) else {
return Vec::new();
};
let Some(options) = data.get("options").and_then(|value| value.to_array()) else {
return Vec::new();
};
let mut parsed = Vec::new();
for option_value in options.iter() {
if let Some(option) = option_value.to_map() {
parsed.push(DatabaseOption {
id: get_string(&option, "id"),
value: get_string(&option, "value"),
color: get_string(&option, "color"),
});
}
}
parsed
}
fn format_option_tag(option: &DatabaseOption) -> String {
let id = option.id.as_deref().unwrap_or_default();
let value = option.value.as_deref().unwrap_or_default();
let color = option.color.as_deref().unwrap_or_default();
format!(
"<span data-affine-option data-value=\"{id}\" data-option-color=\"{color}\">{value}</span>"
)
}
fn format_cell_value(value: &Value, column: &DatabaseColumn) -> String {
match column.col_type.as_str() {
"select" => {
if let Any::String(id) = value {
if let Some(options) = col_data
.and_then(|d| d.get("options"))
.and_then(|v| v.to_array())
{
for opt in options.iter() {
if let Some(opt_map) = opt.to_map() {
if let Some(opt_id) = get_string(&opt_map, "id") {
if opt_id == *id {
return get_string(&opt_map, "value").unwrap_or_default();
}
}
}
let id = match value {
Value::Any(any) => any_as_string(any).map(str::to_string),
Value::Text(text) => Some(text.to_string()),
_ => None,
};
if let Some(id) = id {
for option in column.options.iter() {
if option.id.as_deref() == Some(id.as_str()) {
return format_option_tag(option);
}
}
}
String::new()
}
"multi-select" => {
if let Any::Array(ids) = value {
let mut selected = Vec::new();
if let Some(options) = col_data
.and_then(|d| d.get("options"))
.and_then(|v| v.to_array())
{
for id_val in ids.iter() {
if let Any::String(id) = id_val {
for opt in options.iter() {
if let Some(opt_map) = opt.to_map() {
if let Some(opt_id) = get_string(&opt_map, "id") {
if opt_id == *id {
selected.push(get_string(&opt_map, "value").unwrap_or_default());
}
}
}
}
}
let ids: Vec<String> = match value {
Value::Any(Any::Array(ids)) => ids
.iter()
.filter_map(any_as_string)
.map(str::to_string)
.collect(),
Value::Array(array) => array
.iter()
.filter_map(|id_val| value_to_string(&id_val))
.collect(),
_ => Vec::new(),
};
if ids.is_empty() {
return String::new();
}
let mut selected = Vec::new();
for id in ids.iter() {
for option in column.options.iter() {
if option.id.as_deref() == Some(id.as_str()) {
selected.push(format_option_tag(option));
}
}
return selected.join(", ");
}
String::new()
selected.join("")
}
_ => any_to_string(value).unwrap_or_default(),
}
}
fn value_to_string(value: &Value) -> Option<String> {
if let Some(text) = value.to_text() {
return Some(text.to_string());
}
if let Some(any) = value.to_any() {
return any_to_string(&any);
}
None
}
fn any_to_string(any: &Any) -> Option<String> {
match any {
Any::String(value) => Some(value.to_string()),
Any::Integer(value) => Some(value.to_string()),
Any::Float32(value) => Some(value.0.to_string()),
Any::Float64(value) => Some(value.0.to_string()),
Any::BigInt64(value) => Some(value.to_string()),
Any::True => Some("true".into()),
Any::False => Some("false".into()),
Any::Null | Any::Undefined => None,
Any::Array(_) | Any::Object(_) | Any::Binary(_) => serde_json::to_string(any).ok(),
_ => value_to_string(value).unwrap_or_default(),
}
}
@@ -825,8 +830,8 @@ mod tests {
#[test]
fn test_parse_doc_from_binary() {
let json = include_bytes!("../fixtures/demo.ydoc.json");
let doc_bin = include_bytes!("../fixtures/demo.ydoc").to_vec();
let json = include_bytes!("../../fixtures/demo.ydoc.json");
let doc_bin = include_bytes!("../../fixtures/demo.ydoc").to_vec();
let doc_id = "dYpV7PPhk8amRkY5IAcVO".to_string();
let result = parse_doc_from_binary(doc_bin, doc_id).unwrap();
@@ -838,4 +843,44 @@ mod tests {
config
);
}
#[test]
fn test_paragraph_newlines() {
let mut markdown = String::new();
append_paragraph(&mut markdown, "# ", "Title\n");
assert_eq!(markdown, "# Title\n\n");
markdown.clear();
append_paragraph(&mut markdown, "", "Plain");
assert_eq!(markdown, "Plain\n\n");
}
#[test]
fn test_list_newlines() {
let mut markdown = String::new();
append_list_item(&mut markdown, " ", "* ", "Item\n");
assert_eq!(markdown, " * Item\n");
markdown.clear();
append_list_item(&mut markdown, "", "- [ ] ", "Task");
assert_eq!(markdown, "- [ ] Task\n");
}
#[test]
fn test_code_block_newlines() {
let mut markdown = String::new();
append_code_block(&mut markdown, "rs", "fn main() {}");
assert_eq!(markdown, "```rs\nfn main() {}\n```\n\n");
}
#[test]
fn test_table_newlines() {
let mut markdown = String::new();
append_table_block(&mut markdown, "|a|b|\n|---|---|\n|1|2|\n");
assert_eq!(markdown, "|a|b|\n|---|---|\n|1|2|\n\n");
markdown.clear();
append_table_block(&mut markdown, "|a|b|");
assert_eq!(markdown, "|a|b|\n\n");
}
}

164
packages/common/native/src/doc_parser/blocksuite.rs Normal file
View File

@@ -0,0 +1,164 @@
use std::collections::{HashMap, HashSet};
use y_octo::Map;
use super::value::value_to_string;
pub(super) struct BlockIndex {
pub(super) block_pool: HashMap<String, Map>,
pub(super) parent_lookup: HashMap<String, String>,
}
pub(super) struct DocContext {
pub(super) block_pool: HashMap<String, Map>,
pub(super) parent_lookup: HashMap<String, String>,
pub(super) root_block_id: String,
}
pub(super) struct BlockWalker {
queue: Vec<(Option<String>, String)>,
visited: HashSet<String>,
}
pub(super) fn build_block_index(blocks_map: &Map) -> BlockIndex {
let mut block_pool: HashMap<String, Map> = HashMap::new();
let mut parent_lookup: HashMap<String, String> = HashMap::new();
for (_, value) in blocks_map.iter() {
if let Some(block_map) = value.to_map() {
if let Some(block_id) = get_block_id(&block_map) {
for child_id in collect_child_ids(&block_map) {
parent_lookup.insert(child_id, block_id.clone());
}
block_pool.insert(block_id, block_map);
}
}
}
BlockIndex {
block_pool,
parent_lookup,
}
}
impl DocContext {
pub(super) fn from_blocks_map(blocks_map: &Map, root_flavour: &str) -> Option<Self> {
let BlockIndex {
block_pool,
parent_lookup,
} = build_block_index(blocks_map);
let root_block_id = find_block_id_by_flavour(&block_pool, root_flavour)?;
Some(Self {
block_pool,
parent_lookup,
root_block_id,
})
}
pub(super) fn walker(&self) -> BlockWalker {
BlockWalker::new(&self.root_block_id)
}
}
impl BlockWalker {
fn new(root_block_id: &str) -> Self {
let mut visited = HashSet::new();
visited.insert(root_block_id.to_string());
Self {
queue: vec![(None, root_block_id.to_string())],
visited,
}
}
pub(super) fn next(&mut self) -> Option<(Option<String>, String)> {
self.queue.pop()
}
pub(super) fn enqueue_children(&mut self, parent_block_id: &str, block: &Map) {
let mut child_ids = collect_child_ids(block);
for child_id in child_ids.drain(..).rev() {
if self.visited.insert(child_id.clone()) {
self
.queue
.push((Some(parent_block_id.to_string()), child_id));
}
}
}
}
pub(super) fn find_block_id_by_flavour(
block_pool: &HashMap<String, Map>,
flavour: &str,
) -> Option<String> {
block_pool.iter().find_map(|(id, block)| {
get_flavour(block)
.filter(|block_flavour| block_flavour == flavour)
.map(|_| id.clone())
})
}
pub(super) fn collect_child_ids(block: &Map) -> Vec<String> {
block
.get("sys:children")
.and_then(|value| value.to_array())
.map(|array| {
array
.iter()
.filter_map(|value| value_to_string(&value))
.collect::<Vec<_>>()
})
.unwrap_or_default()
}
pub(super) fn get_block_id(block: &Map) -> Option<String> {
get_string(block, "sys:id")
}
pub(super) fn get_flavour(block: &Map) -> Option<String> {
get_string(block, "sys:flavour")
}
pub(super) fn get_string(block: &Map, key: &str) -> Option<String> {
block.get(key).and_then(|value| value_to_string(&value))
}
pub(super) fn get_list_depth(
block_id: &str,
parent_lookup: &HashMap<String, String>,
blocks: &HashMap<String, Map>,
) -> usize {
let mut depth = 0;
let mut current_id = block_id.to_string();
while let Some(parent_id) = parent_lookup.get(&current_id) {
if let Some(parent_block) = blocks.get(parent_id) {
if get_flavour(parent_block).as_deref() == Some("affine:list") {
depth += 1;
current_id = parent_id.clone();
continue;
}
}
break;
}
depth
}
pub(super) fn nearest_by_flavour(
start: &str,
flavour: &str,
parent_lookup: &HashMap<String, String>,
blocks: &HashMap<String, Map>,
) -> Option<Map> {
let mut cursor = Some(start.to_string());
while let Some(node) = cursor {
if let Some(block) = blocks.get(&node) {
if get_flavour(block).as_deref() == Some(flavour) {
return Some(block.clone());
}
}
cursor = parent_lookup.get(&node).cloned();
}
None
}

791
packages/common/native/src/doc_parser/delta_markdown.rs Normal file
View File

@@ -0,0 +1,791 @@
use std::{
cell::RefCell,
collections::{HashMap, HashSet},
rc::{Rc, Weak},
};
use y_octo::{AHashMap, Any, Map, Text, TextAttributes, TextDeltaOp, TextInsert, Value};
use super::value::{
any_as_string, any_as_u64, any_truthy, build_reference_payload, params_any_map_to_json,
value_to_any,
};
#[derive(Debug, Clone)]
struct InlineReference {
ref_type: Option<String>,
page_id: String,
title: Option<String>,
params: Option<AHashMap<String, Any>>,
mode: Option<String>,
}
#[derive(Debug, Clone)]
pub(super) struct InlineReferencePayload {
pub(super) doc_id: String,
pub(super) payload: String,
}
#[derive(Debug, Clone)]
pub(super) struct DeltaToMdOptions {
doc_url_prefix: Option<String>,
}
impl DeltaToMdOptions {
pub(super) fn new(doc_url_prefix: Option<String>) -> Self {
Self { doc_url_prefix }
}
fn build_reference_link(&self, reference: &InlineReference) -> (String, String) {
let title = reference.title.clone().unwrap_or_default();
if let Some(prefix) = self.doc_url_prefix.as_deref() {
let prefix = prefix.trim_end_matches('/');
return (title, format!("{}/{}", prefix, reference.page_id));
}
let mut parts = Vec::new();
parts.push(
reference
.ref_type
.clone()
.unwrap_or_else(|| "LinkedPage".into()),
);
parts.push(reference.page_id.clone());
if let Some(mode) = reference.mode.as_ref() {
parts.push(mode.clone());
}
(title, parts.join(":"))
}
}
pub(super) fn text_to_markdown(
block: &Map,
key: &str,
options: &DeltaToMdOptions,
) -> Option<String> {
block
.get(key)
.and_then(|value| value.to_text())
.map(|text| delta_to_markdown(&text, options))
}
pub(super) fn text_to_inline_markdown(
block: &Map,
key: &str,
options: &DeltaToMdOptions,
) -> Option<String> {
block
.get(key)
.and_then(|value| value.to_text())
.map(|text| delta_to_inline_markdown(&text, options))
}
pub(super) fn extract_inline_references(delta: &[TextDeltaOp]) -> Vec<InlineReferencePayload> {
let mut refs = Vec::new();
let mut seen: HashSet<(String, String)> = HashSet::new();
for op in delta {
let attrs = match op {
TextDeltaOp::Insert {
format: Some(format),
..
} => format,
_ => continue,
};
let reference = match attrs.get("reference").and_then(parse_inline_reference) {
Some(reference) => reference,
None => continue,
};
let payload = match inline_reference_payload(&reference) {
Some(payload) => payload,
None => continue,
};
let key = (reference.page_id.clone(), payload.clone());
if seen.insert(key.clone()) {
refs.push(InlineReferencePayload {
doc_id: key.0,
payload: key.1,
});
}
}
refs
}
fn parse_inline_reference(value: &Any) -> Option<InlineReference> {
let map = match value {
Any::Object(map) => map,
_ => return None,
};
let page_id = map
.get("pageId")
.and_then(any_as_string)
.map(str::to_string)?;
let title = map.get("title").and_then(any_as_string).map(str::to_string);
let ref_type = map.get("type").and_then(any_as_string).map(str::to_string);
let params = map.get("params").and_then(|value| match value {
Any::Object(map) => Some(map.clone()),
_ => None,
});
let mode = params
.as_ref()
.and_then(|params| params.get("mode"))
.and_then(any_as_string)
.map(str::to_string);
Some(InlineReference {
ref_type,
page_id,
title,
params,
mode,
})
}
fn inline_reference_payload(reference: &InlineReference) -> Option<String> {
let params = reference.params.as_ref().map(params_any_map_to_json);
Some(build_reference_payload(&reference.page_id, params))
}
fn delta_to_markdown(text: &Text, options: &DeltaToMdOptions) -> String {
delta_to_markdown_with_options(&text.to_delta(), options, true)
}
fn delta_to_inline_markdown(text: &Text, options: &DeltaToMdOptions) -> String {
delta_to_markdown_with_options(&text.to_delta(), options, false)
}
fn delta_to_markdown_with_options(
delta: &[TextDeltaOp],
options: &DeltaToMdOptions,
trailing_newline: bool,
) -> String {
let ops = build_delta_ops(delta);
delta_ops_to_markdown_with_options(&ops, options, trailing_newline)
}
fn delta_ops_to_markdown_with_options(
ops: &[DeltaOp],
options: &DeltaToMdOptions,
trailing_newline: bool,
) -> String {
let root = convert_delta_ops(ops, options);
let mut rendered = render_node(&root);
rendered = rendered.trim_end().to_string();
if trailing_newline {
rendered.push('\n');
}
rendered
}
#[derive(Debug, Clone)]
struct DeltaOp {
insert: DeltaInsert,
attributes: TextAttributes,
}
#[derive(Debug, Clone)]
enum DeltaInsert {
Text(String),
Embed(Vec<Any>),
}
fn delta_ops_from_any(value: &Any) -> Option<Vec<DeltaOp>> {
let map = match value {
Any::Object(map) => map,
_ => return None,
};
match map.get("$blocksuite:internal:text$") {
Some(Any::True) => {}
_ => return None,
}
let delta = map.get("delta")?;
let entries = match delta {
Any::Array(entries) => entries,
_ => return None,
};
let mut ops = Vec::new();
for entry in entries {
if let Some(op) = delta_op_from_any(entry) {
ops.push(op);
}
}
Some(ops)
}
fn delta_op_from_any(value: &Any) -> Option<DeltaOp> {
let map = match value {
Any::Object(map) => map,
_ => return None,
};
let insert_value = map.get("insert")?;
let insert = match insert_value {
Any::String(text) => DeltaInsert::Text(text.clone()),
Any::Array(values) => DeltaInsert::Embed(values.clone()),
_ => DeltaInsert::Embed(vec![insert_value.clone()]),
};
let attributes = map
.get("attributes")
.and_then(any_to_attributes)
.unwrap_or_default();
Some(DeltaOp { insert, attributes })
}
fn any_to_attributes(value: &Any) -> Option<TextAttributes> {
let map = match value {
Any::Object(map) => map,
_ => return None,
};
let mut attrs = TextAttributes::new();
for (key, value) in map.iter() {
attrs.insert(key.clone(), value.clone());
}
Some(attrs)
}
fn delta_any_to_inline_markdown(value: &Any, options: &DeltaToMdOptions) -> Option<String> {
delta_ops_from_any(value).map(|ops| delta_ops_to_markdown_with_options(&ops, options, false))
}
pub(super) fn delta_value_to_inline_markdown(
value: &Value,
options: &DeltaToMdOptions,
) -> Option<String> {
if let Some(text) = value.to_text() {
return Some(delta_to_inline_markdown(&text, options));
}
let any = value_to_any(value)?;
delta_any_to_inline_markdown(&any, options)
}
fn build_delta_ops(delta: &[TextDeltaOp]) -> Vec<DeltaOp> {
let mut ops = Vec::new();
for op in delta {
let (insert, attrs) = match op {
TextDeltaOp::Insert { insert, format } => (insert, format.clone().unwrap_or_default()),
_ => continue,
};
match insert {
TextInsert::Text(text) => ops.push(DeltaOp {
insert: DeltaInsert::Text(text.clone()),
attributes: attrs,
}),
TextInsert::Embed(values) => ops.push(DeltaOp {
insert: DeltaInsert::Embed(values.clone()),
attributes: attrs,
}),
}
}
ops
}
#[derive(Debug)]
struct Group {
node: Rc<RefCell<Node>>,
kind: String,
distance: usize,
count: usize,
}
fn convert_delta_ops(ops: &[DeltaOp], options: &DeltaToMdOptions) -> Rc<RefCell<Node>> {
let root = Node::new_root();
let mut group: Option<Group> = None;
let mut active_inline: HashMap<String, Any> = HashMap::new();
let mut beginning_of_line = false;
let mut line = Node::new_line();
let mut el = line.clone();
Node::append(&root, line.clone());
for index in 0..ops.len() {
let op = &ops[index];
let next_attrs = ops.get(index + 1).map(|next| &next.attributes);
match &op.insert {
DeltaInsert::Embed(values) => {
apply_inline_attributes(&mut el, &op.attributes, None, &mut active_inline, options);
for value in values {
match value {
Any::Object(map) => {
for (key, value) in map.iter() {
match key.as_str() {
"image" => {
if let Some(src) = any_as_string(value) {
let url = encode_link(src);
Node::append(&el, Node::new_text(&format!("![]({url})")));
}
}
"thematic_break" => {
let current_open = el.borrow().open.clone();
el.borrow_mut().open = format!("\n---\n{current_open}");
}
_ => {}
}
}
}
Any::String(value) => {
Node::append(&el, Node::new_text(value));
}
_ => {}
}
}
}
DeltaInsert::Text(text) => {
let lines: Vec<&str> = text.split('\n').collect();
if has_block_level_attribute(&op.attributes) {
for _ in 1..lines.len() {
for (attr, value) in op.attributes.iter() {
match attr.as_str() {
"header" => {
if let Some(level) = any_as_u64(value) {
let prefix = "#".repeat(level as usize);
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("{prefix} {current_open}");
new_line(&root, &mut line, &mut el, &mut active_inline);
break;
}
}
"blockquote" => {
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("> {current_open}");
new_line(&root, &mut line, &mut el, &mut active_inline);
break;
}
"list" => {
if group.as_ref().is_some_and(|g| g.kind != attr.as_str()) {
group = None;
}
if group.is_none() {
let group_node = Node::new_line();
Node::append(&root, group_node.clone());
group = Some(Group {
node: group_node,
kind: attr.to_string(),
distance: 0,
count: 0,
});
}
if let Some(group) = group.as_mut() {
Node::append(&group.node, line.clone());
group.distance = 0;
match any_as_string(value) {
Some("bullet") => {
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("- {current_open}");
}
Some("checked") => {
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("- [x] {current_open}");
}
Some("unchecked") => {
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("- [ ] {current_open}");
}
Some("ordered") => {
group.count += 1;
let current_open = line.borrow().open.clone();
line.borrow_mut().open = format!("{}. {}", group.count, current_open);
}
_ => {}
}
}
new_line(&root, &mut line, &mut el, &mut active_inline);
break;
}
_ => {}
}
}
}
beginning_of_line = true;
} else {
for (line_index, segment) in lines.iter().enumerate() {
if (line_index > 0 || beginning_of_line) && group.is_some() {
let reset_group = group.as_mut().map(|group| {
group.distance += 1;
group.distance >= 2
});
if reset_group == Some(true) {
group = None;
}
}
apply_inline_attributes(
&mut el,
&op.attributes,
next_attrs,
&mut active_inline,
options,
);
Node::append(&el, Node::new_text(segment));
if line_index + 1 < lines.len() {
new_line(&root, &mut line, &mut el, &mut active_inline);
}
}
beginning_of_line = false;
}
}
}
}
root
}
fn apply_inline_attributes(
el: &mut Rc<RefCell<Node>>,
attrs: &TextAttributes,
next: Option<&TextAttributes>,
active_inline: &mut HashMap<String, Any>,
options: &DeltaToMdOptions,
) {
let mut first = Vec::new();
let mut then = Vec::new();
let mut seen: HashSet<String> = HashSet::new();
let mut tag = el.clone();
loop {
let format = match tag.borrow().format.clone() {
Some(format) => format,
None => break,
};
seen.insert(format.clone());
let should_close = match attrs.get(&format) {
Some(value) => !any_truthy(value) || tag.borrow().open != tag.borrow().close,
None => true,
};
if should_close {
for key in seen.iter() {
active_inline.remove(key);
}
let parent = {
let tag_ref = tag.borrow();
tag_ref.parent.as_ref().and_then(|p| p.upgrade())
};
if let Some(parent) = parent {
*el = parent.clone();
tag = parent;
continue;
}
break;
}
let parent = {
let tag_ref = tag.borrow();
tag_ref.parent.as_ref().and_then(|p| p.upgrade())
};
if let Some(parent) = parent {
tag = parent;
} else {
break;
}
}
for (attr, value) in attrs.iter() {
if !is_inline_attribute(attr) || !any_truthy(value) {
continue;
}
if let Some(active) = active_inline.get(attr) {
if active == value {
continue;
}
}
let next_matches = next
.and_then(|next_attrs| next_attrs.get(attr))
.map(|next_value| next_value == value)
.unwrap_or(false);
if next_matches {
first.push(attr.clone());
} else {
then.push(attr.clone());
}
active_inline.insert(attr.clone(), value.clone());
}
for attr in first.into_iter().chain(then) {
if let Some(node) = inline_node_for_attr(&attr, attrs, options) {
node.borrow_mut().format = Some(attr.clone());
Node::append(el, node.clone());
*el = node;
}
}
}
fn inline_node_for_attr(
attr: &str,
attrs: &TextAttributes,
options: &DeltaToMdOptions,
) -> Option<Rc<RefCell<Node>>> {
match attr {
"italic" => Some(Node::new_inline("_", "_")),
"bold" => Some(Node::new_inline("**", "**")),
"link" => attrs
.get(attr)
.and_then(any_as_string)
.map(|url| Node::new_inline("[", &format!("]({url})"))),
"reference" => attrs
.get(attr)
.and_then(parse_inline_reference)
.map(|reference| {
let (title, link) = options.build_reference_link(&reference);
Node::new_inline("[", &format!("{title}]({link})"))
}),
"strike" => Some(Node::new_inline("~~", "~~")),
"code" => Some(Node::new_inline("`", "`")),
_ => None,
}
}
fn has_block_level_attribute(attrs: &TextAttributes) -> bool {
attrs.contains_key("header") || attrs.contains_key("blockquote") || attrs.contains_key("list")
}
fn is_inline_attribute(attr: &str) -> bool {
matches!(
attr,
"italic" | "bold" | "link" | "reference" | "strike" | "code"
)
}
fn encode_link(link: &str) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
#[inline]
fn push_pct(out: &mut String, b: u8) {
out.push('%');
out.push(HEX[(b >> 4) as usize] as char);
out.push(HEX[(b & 0x0f) as usize] as char);
}
#[inline]
fn is_allowed(b: u8) -> bool {
matches!(
b,
b'A'..=b'Z'
| b'a'..=b'z'
| b'0'..=b'9'
| b'-'
| b'_'
| b'.'
| b'!'
| b'~'
| b'*'
| b'\''
| b';'
| b','
| b'/'
| b'?'
| b':'
| b'@'
| b'&'
| b'='
| b'+'
| b'$'
| b'#'
)
}
let mut out = String::with_capacity(link.len());
for &b in link.as_bytes() {
match b {
b'(' | b')' => push_pct(&mut out, b),
b if is_allowed(b) => out.push(b as char),
b => push_pct(&mut out, b),
}
}
if let Some(i) = out.find("?response-content-disposition=attachment") {
out.truncate(i);
} else if let Some(i) = out.find("&response-content-disposition=attachment") {
out.truncate(i);
}
out
}
#[derive(Debug)]
struct Node {
open: String,
close: String,
text: String,
children: Vec<Rc<RefCell<Node>>>,
parent: Option<Weak<RefCell<Node>>>,
format: Option<String>,
}
impl Node {
fn new_root() -> Rc<RefCell<Node>> {
Rc::new(RefCell::new(Node {
open: String::new(),
close: String::new(),
text: String::new(),
children: Vec::new(),
parent: None,
format: None,
}))
}
fn new_inline(open: &str, close: &str) -> Rc<RefCell<Node>> {
Rc::new(RefCell::new(Node {
open: open.to_string(),
close: close.to_string(),
text: String::new(),
children: Vec::new(),
parent: None,
format: None,
}))
}
fn new_text(text: &str) -> Rc<RefCell<Node>> {
Rc::new(RefCell::new(Node {
open: String::new(),
close: String::new(),
text: text.to_string(),
children: Vec::new(),
parent: None,
format: None,
}))
}
fn new_line() -> Rc<RefCell<Node>> {
Rc::new(RefCell::new(Node {
open: String::new(),
close: "\n".to_string(),
text: String::new(),
children: Vec::new(),
parent: None,
format: None,
}))
}
fn append(parent: &Rc<RefCell<Node>>, child: Rc<RefCell<Node>>) {
if let Some(old_parent) = child.borrow().parent.as_ref().and_then(|p| p.upgrade()) {
let mut old_parent = old_parent.borrow_mut();
old_parent
.children
.retain(|existing| !Rc::ptr_eq(existing, &child));
}
child.borrow_mut().parent = Some(Rc::downgrade(parent));
parent.borrow_mut().children.push(child);
}
}
fn render_node(node: &Rc<RefCell<Node>>) -> String {
let node_ref = node.borrow();
let mut inner = node_ref.text.clone();
for child in node_ref.children.iter() {
inner.push_str(&render_node(child));
}
if inner.trim().is_empty()
&& node_ref.open == node_ref.close
&& !node_ref.open.is_empty()
&& !node_ref.close.is_empty()
{
return String::new();
}
let wrapped = !node_ref.open.is_empty() && !node_ref.close.is_empty();
let empty_inner = inner.trim().is_empty();
let mut fragments = Vec::new();
if inner.starts_with(' ') && !empty_inner && wrapped {
fragments.push(" ".to_string());
}
if !node_ref.open.is_empty() {
fragments.push(node_ref.open.clone());
}
fragments.push(if wrapped {
inner.trim().to_string()
} else {
inner.clone()
});
if !node_ref.close.is_empty() {
fragments.push(node_ref.close.clone());
}
if inner.ends_with(' ') && !empty_inner && wrapped {
fragments.push(" ".to_string());
}
fragments.join("")
}
fn new_line(
root: &Rc<RefCell<Node>>,
line: &mut Rc<RefCell<Node>>,
el: &mut Rc<RefCell<Node>>,
active_inline: &mut HashMap<String, Any>,
) {
*line = Node::new_line();
*el = line.clone();
Node::append(root, line.clone());
active_inline.clear();
}
#[cfg(test)]
mod tests {
use serde_json::Value;
use super::*;
#[test]
fn test_delta_to_inline_markdown_link() {
let mut attrs = TextAttributes::new();
attrs.insert("link".into(), Any::String("https://example.com".into()));
let delta = vec![TextDeltaOp::Insert {
insert: TextInsert::Text("AFFiNE".into()),
format: Some(attrs),
}];
let options = DeltaToMdOptions::new(None);
let rendered = delta_to_markdown_with_options(&delta, &options, false);
assert_eq!(rendered, "[AFFiNE](https://example.com)");
}
#[test]
fn test_extract_inline_references_payload() {
let mut ref_map = AHashMap::default();
ref_map.insert("pageId".into(), Any::String("doc123".into()));
ref_map.insert("title".into(), Any::String("Doc Title".into()));
ref_map.insert("type".into(), Any::String("LinkedPage".into()));
let mut attrs = TextAttributes::new();
attrs.insert("reference".into(), Any::Object(ref_map));
let delta = vec![TextDeltaOp::Insert {
insert: TextInsert::Text("Doc Title".into()),
format: Some(attrs),
}];
let refs = extract_inline_references(&delta);
assert_eq!(refs.len(), 1);
assert_eq!(refs[0].doc_id, "doc123");
let payload: Value = serde_json::from_str(&refs[0].payload).unwrap();
assert_eq!(payload, serde_json::json!({ "docId": "doc123" }));
}
}

9
packages/common/native/src/doc_parser/mod.rs Normal file
View File

@@ -0,0 +1,9 @@
mod affine;
mod blocksuite;
mod delta_markdown;
mod value;
pub use affine::{
get_doc_ids_from_binary, parse_doc_from_binary, parse_doc_to_markdown, BlockInfo, CrawlResult,
MarkdownResult, ParseError,
};

121
packages/common/native/src/doc_parser/value.rs Normal file
View File

@@ -0,0 +1,121 @@
use serde_json::{Map as JsonMap, Value as JsonValue};
use y_octo::{AHashMap, Any, Value};
pub(super) fn any_truthy(value: &Any) -> bool {
match value {
Any::True => true,
Any::False | Any::Null | Any::Undefined => false,
Any::String(value) => !value.is_empty(),
Any::Integer(value) => *value != 0,
Any::Float32(value) => value.0 != 0.0,
Any::Float64(value) => value.0 != 0.0,
Any::BigInt64(value) => *value != 0,
Any::Object(_) | Any::Array(_) | Any::Binary(_) => true,
}
}
pub(super) fn any_as_string(value: &Any) -> Option<&str> {
match value {
Any::String(value) => Some(value),
_ => None,
}
}
pub(super) fn any_as_u64(value: &Any) -> Option<u64> {
match value {
Any::Integer(value) if *value >= 0 => Some(*value as u64),
Any::Float32(value) if value.0 >= 0.0 => Some(value.0 as u64),
Any::Float64(value) if value.0 >= 0.0 => Some(value.0 as u64),
Any::BigInt64(value) if *value >= 0 => Some(*value as u64),
_ => None,
}
}
pub(super) fn value_to_string(value: &Value) -> Option<String> {
if let Some(text) = value.to_text() {
return Some(text.to_string());
}
if let Some(any) = value.to_any() {
return any_to_string(&any);
}
None
}
pub(super) fn value_to_any(value: &Value) -> Option<Any> {
if let Some(any) = value.to_any() {
return Some(any);
}
if let Some(text) = value.to_text() {
return Some(Any::String(text.to_string()));
}
if let Some(array) = value.to_array() {
let mut values = Vec::new();
for item in array.iter() {
if let Some(any) = value_to_any(&item) {
values.push(any);
} else if let Some(text) = value_to_string(&item) {
values.push(Any::String(text));
}
}
return Some(Any::Array(values));
}
if let Some(map) = value.to_map() {
let mut values = AHashMap::default();
for key in map.keys() {
if let Some(entry) = map.get(key) {
if let Some(any) = value_to_any(&entry) {
values.insert(key.to_string(), any);
} else if let Some(text) = value_to_string(&entry) {
values.insert(key.to_string(), Any::String(text));
}
}
}
return Some(Any::Object(values));
}
None
}
pub(super) fn any_to_string(any: &Any) -> Option<String> {
match any {
Any::String(value) => Some(value.to_string()),
Any::Integer(value) => Some(value.to_string()),
Any::Float32(value) => Some(value.0.to_string()),
Any::Float64(value) => Some(value.0.to_string()),
Any::BigInt64(value) => Some(value.to_string()),
Any::True => Some("true".into()),
Any::False => Some("false".into()),
Any::Null | Any::Undefined => None,
Any::Array(_) | Any::Object(_) | Any::Binary(_) => serde_json::to_string(any).ok(),
}
}
pub(super) fn params_any_map_to_json(params: &AHashMap<String, Any>) -> JsonValue {
let mut values = JsonMap::new();
for (key, value) in params.iter() {
if let Ok(value) = serde_json::to_value(value) {
values.insert(key.clone(), value);
}
}
JsonValue::Object(values)
}
pub(super) fn params_value_to_json(params: &Value) -> Option<JsonValue> {
serde_json::to_value(params).ok()
}
pub(super) fn build_reference_payload(doc_id: &str, params: Option<JsonValue>) -> String {
let mut payload = JsonMap::new();
payload.insert("docId".into(), JsonValue::String(doc_id.to_string()));
if let Some(JsonValue::Object(params)) = params {
for (key, value) in params.into_iter() {
payload.insert(key, value);
}
}
JsonValue::Object(payload).to_string()
}

4
packages/common/native/src/hashcash.rs
View File

@@ -85,10 +85,10 @@ impl Stamp {
let hex_digits = ((bits as f32) / 4.).ceil() as usize;
let zeros = String::from_utf8(vec![b'0'; hex_digits]).unwrap();
loop {
hasher.update(format!("{}:{:x}", challenge, counter).as_bytes());
hasher.update(format!("{challenge}:{counter:x}").as_bytes());
let result = format!("{:x}", hasher.finalize_reset());
if result[..hex_digits] == zeros {
break format!("{:x}", counter);
break format!("{counter:x}");
};
counter += 1
}

16
packages/common/y-octo/core/src/doc/codec/any.rs
View File

@@ -529,18 +529,18 @@ impl Display for Any {
match self {
Self::True => write!(f, "true"),
Self::False => write!(f, "false"),
Self::String(s) => write!(f, "\"{}\"", s),
Self::Integer(i) => write!(f, "{}", i),
Self::Float32(v) => write!(f, "{}", v),
Self::Float64(v) => write!(f, "{}", v),
Self::BigInt64(v) => write!(f, "{}", v),
Self::String(s) => write!(f, "\"{s}\""),
Self::Integer(i) => write!(f, "{i}"),
Self::Float32(v) => write!(f, "{v}"),
Self::Float64(v) => write!(f, "{v}"),
Self::BigInt64(v) => write!(f, "{v}"),
Self::Object(map) => {
write!(f, "{{")?;
for (i, (key, value)) in map.iter().enumerate() {
if i > 0 {
write!(f, ", ")?;
}
write!(f, "{}: {}", key, value)?;
write!(f, "{key}: {value}")?;
}
write!(f, "}}")
}
@@ -550,11 +550,11 @@ impl Display for Any {
if i > 0 {
write!(f, ", ")?;
}
write!(f, "{}", value)?;
write!(f, "{value}")?;
}
write!(f, "]")
}
Self::Binary(buf) => write!(f, "{:?}", buf),
Self::Binary(buf) => write!(f, "{buf:?}"),
Self::Undefined => write!(f, "undefined"),
Self::Null => write!(f, "null"),
}

2
packages/common/y-octo/core/src/doc/codec/update.rs
View File

@@ -166,7 +166,7 @@ impl Update {
// merge two nodes, mark the index
merged_index.push(index + 1);
} else {
debug!("merge failed: {:?} {:?}", cur, next)
debug!("merge failed: {cur:?} {next:?}")
}
}

6
packages/common/y-octo/core/src/doc/publisher.rs
View File

@@ -96,13 +96,13 @@ impl DocPublisher {
let mut encoder = RawEncoder::default();
if let Err(e) = update.write(&mut encoder) {
warn!("Failed to encode document: {}", e);
warn!("Failed to encode document: {e}");
continue;
}
(encoder.into_inner(), history)
}
Err(e) => {
warn!("Failed to diff document: {}", e);
warn!("Failed to diff document: {e}");
continue;
}
};
@@ -117,7 +117,7 @@ impl DocPublisher {
cb(&binary, &history);
}))
.unwrap_or_else(|e| {
warn!("Failed to call subscriber: {:?}", e);
warn!("Failed to call subscriber: {e:?}");
});
}
} else {

8
packages/common/y-octo/core/src/doc/store.rs
View File

@@ -90,7 +90,7 @@ impl DocStore {
if let Some(last_struct) = structs.back() {
last_struct.clock() + last_struct.len()
} else {
warn!("client {} has no struct info", client);
warn!("client {client} has no struct info");
0
}
} else {
@@ -108,7 +108,7 @@ impl DocStore {
if let Some(last_struct) = structs.back() {
state.insert(*client, last_struct.clock() + last_struct.len());
} else {
warn!("client {} has no struct info", client);
warn!("client {client} has no struct info");
}
}
state
@@ -126,7 +126,7 @@ impl DocStore {
return Err(JwstCodecError::StructClockInvalid { expect, actually });
}
} else {
warn!("client {} has no struct info", client_id);
warn!("client {client_id} has no struct info");
}
structs.push_back(item);
}
@@ -593,7 +593,7 @@ impl DocStore {
self.delete_node(&Node::Item(item_owner_ref.clone()), Some(parent));
} else {
// adjust parent length
if this.parent_sub.is_none() {
if this.parent_sub.is_none() && this.countable() {
parent.len += this.len();
}
}

10
packages/common/y-octo/core/src/doc/types/list/mod.rs
View File

@@ -17,7 +17,7 @@ pub(crate) struct ItemPosition {
impl ItemPosition {
pub fn forward(&mut self) {
if let Some(right) = self.right.get() {
if !right.deleted() {
if right.indexable() {
self.index += right.len();
}
@@ -103,7 +103,7 @@ pub(crate) trait ListType: AsInner<Inner = YTypeRef> {
while remaining > 0 {
if let Some(item) = pos.right.get() {
if !item.deleted() {
if item.indexable() {
let content_len = item.len();
if remaining < content_len {
pos.offset = remaining;
@@ -148,7 +148,9 @@ pub(crate) trait ListType: AsInner<Inner = YTypeRef> {
content: Content,
) -> JwstCodecResult {
if let Some(markers) = &ty.markers {
markers.update_marker_changes(pos.index, content.clock_len() as i64);
if content.countable() {
markers.update_marker_changes(pos.index, content.clock_len() as i64);
}
}
let item = store.create_item(
@@ -214,7 +216,7 @@ pub(crate) trait ListType: AsInner<Inner = YTypeRef> {
while remaining > 0 {
if let Some(item) = pos.right.get() {
if !item.deleted() {
if item.indexable() {
let content_len = item.len();
if remaining < content_len {
store.split_node(item.id, remaining)?;

586
packages/common/y-octo/core/src/doc/types/text.rs
View File

@@ -1,12 +1,43 @@
use std::fmt::Display;
use std::{collections::BTreeMap, fmt::Display};
use super::list::ListType;
use crate::{impl_type, Content, JwstCodecResult};
use super::{list::ListType, AsInner};
use crate::{
doc::{DocStore, ItemRef, Node, Parent, Somr, YType, YTypeRef},
impl_type, Any, Content, JwstCodecError, JwstCodecResult,
};
impl_type!(Text);
impl ListType for Text {}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
#[serde(untagged)]
pub enum TextInsert {
Text(String),
Embed(Vec<Any>),
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
#[serde(untagged)]
pub enum TextDeltaOp {
Insert {
insert: TextInsert,
#[serde(skip_serializing_if = "Option::is_none")]
format: Option<TextAttributes>,
},
Retain {
retain: u64,
#[serde(skip_serializing_if = "Option::is_none")]
format: Option<TextAttributes>,
},
Delete {
delete: u64,
},
}
pub type TextDelta = Vec<TextDeltaOp>;
pub type TextAttributes = BTreeMap<String, Any>;
impl Text {
#[inline]
pub fn len(&self) -> u64 {
@@ -27,13 +58,113 @@ impl Text {
pub fn remove(&mut self, char_index: u64, len: u64) -> JwstCodecResult {
self.remove_at(char_index, len)
}
pub fn to_delta(&self) -> TextDelta {
let mut ops = Vec::new();
let mut attrs = TextAttributes::new();
for item_ref in self.iter_item() {
if let Some(item) = item_ref.get() {
match &item.content {
Content::Format { key, value } => {
if is_nullish(value) {
attrs.remove(key.as_str());
} else {
attrs.insert(key.to_string(), value.clone());
}
}
Content::String(text) => {
push_insert(&mut ops, TextInsert::Text(text.clone()), &attrs);
}
Content::Embed(embed) => {
push_insert(&mut ops, TextInsert::Embed(vec![embed.clone()]), &attrs);
}
Content::Any(any) => {
push_insert(&mut ops, TextInsert::Embed(any.clone()), &attrs);
}
Content::Json(values) => {
let converted = values
.iter()
.map(|value| {
value
.as_ref()
.map(|s| Any::String(s.clone()))
.unwrap_or(Any::Undefined)
})
.collect::<Vec<_>>();
push_insert(&mut ops, TextInsert::Embed(converted), &attrs);
}
Content::Binary(value) => {
push_insert(
&mut ops,
TextInsert::Embed(vec![Any::Binary(value.clone())]),
&attrs,
);
}
_ => {}
}
}
}
ops
}
pub fn apply_delta(&mut self, delta: &[TextDeltaOp]) -> JwstCodecResult {
let (mut store, mut ty) = self.as_inner().write().ok_or(JwstCodecError::DocReleased)?;
let parent = self.as_inner().clone();
let mut pos = TextPosition::new(parent, ty.start.clone());
for op in delta {
match op {
TextDeltaOp::Insert { insert, format } => {
let attrs = format.clone().unwrap_or_default();
match insert {
TextInsert::Text(text) => {
insert_text_content(
&mut store,
&mut ty,
&mut pos,
Content::String(text.clone()),
attrs,
)?;
}
TextInsert::Embed(values) => {
for value in values {
insert_text_content(
&mut store,
&mut ty,
&mut pos,
Content::Embed(value.clone()),
attrs.clone(),
)?;
}
}
}
}
TextDeltaOp::Retain { retain, format } => {
let attrs = format.clone().unwrap_or_default();
if attrs.is_empty() {
advance_text_position(&mut store, &mut pos, *retain)?;
} else {
format_text(&mut store, &mut ty, &mut pos, *retain, attrs)?;
}
}
TextDeltaOp::Delete { delete } => {
delete_text(&mut store, &mut ty, &mut pos, *delete)?;
}
}
}
Ok(())
}
}
impl Display for Text {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.iter_item().try_for_each(|item| {
if let Content::String(str) = &item.get().unwrap().content {
write!(f, "{}", str)
write!(f, "{str}")
} else {
Ok(())
}
@@ -50,15 +181,361 @@ impl serde::Serialize for Text {
}
}
struct TextPosition {
parent: YTypeRef,
left: ItemRef,
right: ItemRef,
index: u64,
attrs: TextAttributes,
}
impl TextPosition {
fn new(parent: YTypeRef, right: ItemRef) -> Self {
Self {
parent,
left: Somr::none(),
right,
index: 0,
attrs: TextAttributes::new(),
}
}
fn forward(&mut self) {
if let Some(right) = self.right.get() {
if !right.deleted() {
if let Content::Format { key, value } = &right.content {
if is_nullish(value) {
self.attrs.remove(key.as_str());
} else {
self.attrs.insert(key.to_string(), value.clone());
}
} else if right.countable() {
self.index += right.len();
}
}
self.left = self.right.clone();
self.right = right.right.clone();
}
}
}
fn is_nullish(value: &Any) -> bool {
matches!(value, Any::Null | Any::Undefined)
}
fn push_insert(ops: &mut Vec<TextDeltaOp>, insert: TextInsert, attrs: &TextAttributes) {
let format = if attrs.is_empty() {
None
} else {
Some(attrs.clone())
};
if let Some(TextDeltaOp::Insert {
insert: TextInsert::Text(prev),
format: prev_format,
}) = ops.last_mut()
{
if let TextInsert::Text(text) = insert {
if prev_format.as_ref() == format.as_ref() {
prev.push_str(&text);
return;
}
ops.push(TextDeltaOp::Insert {
insert: TextInsert::Text(text),
format,
});
return;
}
}
ops.push(TextDeltaOp::Insert { insert, format });
}
fn advance_text_position(
store: &mut DocStore,
pos: &mut TextPosition,
mut remaining: u64,
) -> JwstCodecResult {
while remaining > 0 {
let Some(item) = pos.right.get() else {
return Err(JwstCodecError::IndexOutOfBound(pos.index + remaining));
};
if item.deleted() {
pos.forward();
continue;
}
if matches!(item.content, Content::Format { .. }) {
pos.forward();
continue;
}
let item_len = item.len();
if remaining < item_len {
let (left, right) = store.split_node(item.id, remaining)?;
pos.left = left.as_item();
pos.right = right.as_item();
pos.index += remaining;
break;
}
remaining -= item_len;
pos.forward();
}
Ok(())
}
fn minimize_attribute_changes(pos: &mut TextPosition, attrs: &TextAttributes) {
loop {
let Some(item) = pos.right.get() else {
break;
};
if item.deleted() {
pos.forward();
continue;
}
if let Content::Format { key, value } = &item.content {
let attr = attrs.get(key.as_str()).cloned().unwrap_or(Any::Null);
if attr == *value {
pos.forward();
continue;
}
}
break;
}
}
fn insert_item(
store: &mut DocStore,
ty: &mut YType,
pos: &mut TextPosition,
content: Content,
) -> JwstCodecResult {
if let Some(markers) = &ty.markers {
if content.countable() {
markers.update_marker_changes(pos.index, content.clock_len() as i64);
}
}
let item = store.create_item(
content,
pos.left.clone(),
pos.right.clone(),
Some(Parent::Type(pos.parent.clone())),
None,
);
let item_ref = item.clone();
store.integrate(Node::Item(item), 0, Some(ty))?;
pos.right = item_ref;
pos.forward();
Ok(())
}
fn insert_attributes(
store: &mut DocStore,
ty: &mut YType,
pos: &mut TextPosition,
attrs: &TextAttributes,
) -> JwstCodecResult<TextAttributes> {
let mut negated = TextAttributes::new();
for (key, value) in attrs {
let current = pos.attrs.get(key.as_str()).cloned().unwrap_or(Any::Null);
if current == *value {
continue;
}
negated.insert(key.to_string(), current);
insert_item(
store,
ty,
pos,
Content::Format {
key: key.to_string(),
value: value.clone(),
},
)?;
}
Ok(negated)
}
fn insert_negated_attributes(
store: &mut DocStore,
ty: &mut YType,
pos: &mut TextPosition,
mut negated: TextAttributes,
) -> JwstCodecResult {
loop {
let Some(item) = pos.right.get() else {
break;
};
if item.deleted() {
pos.forward();
continue;
}
if let Content::Format { key, value } = &item.content {
if let Some(negated_value) = negated.get(key.as_str()) {
if negated_value == value {
negated.remove(key.as_str());
pos.forward();
continue;
}
}
}
break;
}
for (key, value) in negated {
insert_item(
store,
ty,
pos,
Content::Format {
key: key.to_string(),
value,
},
)?;
}
Ok(())
}
fn insert_text_content(
store: &mut DocStore,
ty: &mut YType,
pos: &mut TextPosition,
content: Content,
mut attrs: TextAttributes,
) -> JwstCodecResult {
for key in pos.attrs.keys() {
if !attrs.contains_key(key.as_str()) {
attrs.insert(key.to_string(), Any::Null);
}
}
minimize_attribute_changes(pos, &attrs);
let negated = insert_attributes(store, ty, pos, &attrs)?;
insert_item(store, ty, pos, content)?;
insert_negated_attributes(store, ty, pos, negated)?;
Ok(())
}
fn format_text(
store: &mut DocStore,
ty: &mut YType,
pos: &mut TextPosition,
mut remaining: u64,
attrs: TextAttributes,
) -> JwstCodecResult {
if remaining == 0 {
return Ok(());
}
minimize_attribute_changes(pos, &attrs);
let mut negated = insert_attributes(store, ty, pos, &attrs)?;
while remaining > 0 {
let Some(item) = pos.right.get() else {
break;
};
if item.deleted() {
pos.forward();
continue;
}
match &item.content {
Content::Format { key, value } => {
if let Some(attr) = attrs.get(key.as_str()) {
if attr == value {
negated.remove(key.as_str());
} else {
negated.insert(key.to_string(), value.clone());
}
store.delete_item(item, Some(ty));
pos.forward();
} else {
pos.forward();
}
}
_ => {
let item_len = item.len();
if remaining < item_len {
store.split_node(item.id, remaining)?;
remaining = 0;
} else {
remaining -= item_len;
}
pos.forward();
}
}
}
insert_negated_attributes(store, ty, pos, negated)?;
Ok(())
}
fn delete_text(
store: &mut DocStore,
ty: &mut YType,
pos: &mut TextPosition,
mut remaining: u64,
) -> JwstCodecResult {
if remaining == 0 {
return Ok(());
}
let start = remaining;
while remaining > 0 {
let Some(item) = pos.right.get() else {
break;
};
if item.indexable() {
let item_len = item.len();
if remaining < item_len {
store.split_node(item.id, remaining)?;
remaining = 0;
} else {
remaining -= item_len;
}
store.delete_item(item, Some(ty));
}
pos.forward();
}
if let Some(markers) = &ty.markers {
markers.update_marker_changes(pos.index, -((start - remaining) as i64));
}
Ok(())
}
#[cfg(test)]
mod tests {
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use yrs::{Options, Text, Transact};
use super::{TextAttributes, TextDeltaOp, TextInsert};
#[cfg(not(loom))]
use crate::sync::{Arc, AtomicUsize, Ordering};
use crate::{loom_model, sync::thread, Doc};
use crate::{loom_model, sync::thread, Any, Doc};
#[test]
fn test_manipulate_text() {
@@ -290,4 +767,103 @@ mod tests {
assert_eq!(text.to_string(), "hello great world!");
});
}
#[test]
fn test_text_delta_insert_format() {
loom_model!({
let doc = Doc::new();
let mut text = doc.get_or_create_text("text").unwrap();
let mut attrs = TextAttributes::new();
attrs.insert("bold".to_string(), Any::True);
text
.apply_delta(&[TextDeltaOp::Insert {
insert: TextInsert::Text("abc".to_string()),
format: Some(attrs.clone()),
}])
.unwrap();
assert_eq!(text.to_string(), "abc");
assert_eq!(
text.to_delta(),
vec![TextDeltaOp::Insert {
insert: TextInsert::Text("abc".to_string()),
format: Some(attrs),
}]
);
});
}
#[test]
fn test_text_delta_retain_format() {
loom_model!({
let doc = Doc::new();
let mut text = doc.get_or_create_text("text").unwrap();
text
.apply_delta(&[TextDeltaOp::Insert {
insert: TextInsert::Text("abc".to_string()),
format: None,
}])
.unwrap();
let mut attrs = TextAttributes::new();
attrs.insert("bold".to_string(), Any::True);
text
.apply_delta(&[TextDeltaOp::Retain {
retain: 1,
format: Some(attrs.clone()),
}])
.unwrap();
assert_eq!(
text.to_delta(),
vec![
TextDeltaOp::Insert {
insert: TextInsert::Text("a".to_string()),
format: Some(attrs),
},
TextDeltaOp::Insert {
insert: TextInsert::Text("bc".to_string()),
format: None,
}
]
);
});
}
#[test]
fn test_text_delta_utf16_retain() {
loom_model!({
let doc = Doc::new();
let mut text = doc.get_or_create_text("text").unwrap();
text
.apply_delta(&[TextDeltaOp::Insert {
insert: TextInsert::Text("😀".to_string()),
format: None,
}])
.unwrap();
let mut attrs = TextAttributes::new();
attrs.insert("bold".to_string(), Any::True);
text
.apply_delta(&[TextDeltaOp::Retain {
retain: 2,
format: Some(attrs.clone()),
}])
.unwrap();
assert_eq!(
text.to_delta(),
vec![TextDeltaOp::Insert {
insert: TextInsert::Text("😀".to_string()),
format: Some(attrs),
}]
);
});
}
}

4
packages/common/y-octo/core/src/doc/types/value.rs
View File

@@ -131,8 +131,8 @@ impl From<Doc> for Value {
impl Display for Value {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Value::Any(any) => write!(f, "{}", any),
Value::Text(text) => write!(f, "{}", text),
Value::Any(any) => write!(f, "{any}"),
Value::Text(text) => write!(f, "{text}"),
_ => write!(f, ""),
}
}

3
packages/common/y-octo/core/src/lib.rs
View File

@@ -9,7 +9,8 @@ pub use doc::{
encode_awareness_as_message, encode_update_as_message, merge_updates_v1, Any, Array, Awareness,
AwarenessEvent, Client, ClientMap, Clock, CrdtRead, CrdtReader, CrdtWrite, CrdtWriter, Doc,
DocOptions, HashMap as AHashMap, HashMapExt, History, HistoryOptions, Id, Map, RawDecoder,
RawEncoder, StateVector, StoreHistory, Text, Update, Value,
RawEncoder, StateVector, StoreHistory, Text, TextAttributes, TextDelta, TextDeltaOp, TextInsert,
Update, Value,
};
pub(crate) use doc::{Content, Item};
use log::{debug, warn};

2
packages/common/y-octo/core/src/protocol/sync.rs
View File

@@ -62,7 +62,7 @@ pub fn read_sync_message(input: &[u8]) -> IResult<&[u8], SyncMessage> {
let (awareness_tail, awareness) = read_awareness(update)?;
let tail_len = awareness_tail.len();
if tail_len > 0 {
debug!("awareness update has trailing bytes: {}", tail_len);
debug!("awareness update has trailing bytes: {tail_len}");
debug_assert!(tail_len > 0, "awareness update has trailing bytes");
}
awareness