Merge pull request #418 from toeverything/develop

chore: sync changes

.eslintrc.json

@@ -103,7 +103,8 @@
                                     "datasource:db-services",
                                     "components:ui",
                                     "components:icons",
-                                    "library:feature-flags"
+                                    "library:feature-flags",
+                                    "datasource:i18n"
                                 ]
                             },
                             {

.github/deployment/Dockerfile-affine

@@ -1,4 +1,5 @@
 FROM node:16-alpine as builder
+ARG HUBSPOT_API_SECRET
 WORKDIR /app
 RUN apk add git && npm i -g pnpm@7
 COPY . .

.github/workflows/affine-keck.yml

@@ -1,10 +1,7 @@
 name: Build AFFiNE-Local-Keck
 
 on:
-    push:
+    workflow_dispatch:
-        branches: [master]
-    # pull_request:
-    #     branches: [master]
 
 # Cancels all previous workflow runs for pull requests that have not completed.
 # See https://docs.github.com/en/actions/using-jobs/using-concurrency

.github/workflows/languages-download.yml

@@ -13,6 +13,12 @@ concurrency:
     group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
     cancel-in-progress: true
 
+# This action need write permission to create pull requests
+# See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+permissions:
+    contents: write
+    pull-requests: write
+
 jobs:
     main:
         strategy:
@@ -41,7 +47,6 @@ jobs:
               run: pnpm install
 
             - name: Sync Languages
-              if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master'
               working-directory: ./libs/datasource/i18n
               run: pnpm run download-resources
               env:
@@ -61,11 +66,11 @@ jobs:
                 fi
                 git config user.name 'github-actions[bot]'
                 git config user.email 'github-actions[bot]@users.noreply.github.com'
-                git commit --message 'feat(i18n): new translations'
+                git commit --message 'feat(i18n): new translations' --no-verify
                 git remote set-url origin "https://$GITHUB_ACTOR:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY"
                 git push --force origin HEAD:$TARGET_BRANCH
               env:
-                GITHUB_TOKEN: ${{ secrets.github_token }}
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                 TARGET_BRANCH: bot/new-translations
 
             - name: Get current date

.github/workflows/livedemo.yml

@@ -1,4 +1,4 @@
-name: Build AFFiNE-Local
+name: Build AFFiNE-Livedemo
 
 on:
     push:
@@ -64,14 +64,14 @@ jobs:
 
             - name: Build and push Docker image (AFFINE-Local)
               uses: docker/build-push-action@v3
-              env:
-                HUBSPOT_API_SECRET: ${{ secrets.SuperSecret }}
               with:
                   context: .
                   push: true
                   file: ./.github/deployment/Dockerfile-affine
                   tags: ${{ env.LOCAL_CACHE }}
                   target: AFFiNE
+                  build-args: |
+                    HUBSPOT_API_SECRET=${{ secrets.HUBSPOT_API_SECRET }}
 
             - name: Build and push Docker image (AFFINE-Local)
               uses: docker/build-push-action@v3

README.md

@@ -145,6 +145,8 @@ We use the following open source projects to help us build a better development
 
 Thanks a lot to the community for providing such powerful and simple libraries, so that we can focus more on the implementation of the product logic, and we hope that in the future our projects will also provide a more easy-to-use knowledge base for everyone.
 
+You enjoy AFFiNE? You are not alone. [Some amazing companies](./docs/Jobs.md) are looking for developers with some AFFiNE and/or AFFiNE technology experience.
+
 # Contributors
 
 <!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->

apps/keck/src/utils.ts.bak

@@ -0,0 +1,139 @@
+import WebSocket = require('ws');
+import http = require('http');
+import Y = require('yjs');
+import lib0 = require('lib0');
+import syncProtocol = require('y-protocols/sync');
+
+const { encoding, decoding, map } = lib0;
+
+const wsReadyStateConnecting = 0;
+const wsReadyStateOpen = 1;
+
+// disable gc when using snapshots!
+const gcEnabled = process.env.GC !== 'false' && process.env.GC !== '0';
+
+const docs: Map<string, WSSharedDoc> = new Map();
+
+const messageSync = 0;
+
+const updateHandler = (update: Uint8Array, origin: any, doc: WSSharedDoc) => {
+    const encoder = encoding.createEncoder();
+    encoding.writeVarUint(encoder, messageSync);
+    syncProtocol.writeUpdate(encoder, update);
+    const message = encoding.toUint8Array(encoder);
+    doc.conns.forEach((_, conn) => send(doc, conn, message));
+};
+export class WSSharedDoc extends Y.Doc {
+    name: string;
+    conns: Map<any, any>;
+
+    constructor(name: string) {
+        super({ gc: gcEnabled });
+        this.name = name;
+        // Maps from conn to set of controlled user ids. Delete all user ids from awareness when this conn is closed
+        this.conns = new Map();
+
+        this.on('update', updateHandler);
+    }
+}
+
+// Gets a Y.Doc by name, whether in memory or on disk
+const getYDoc = (docname: string, gc = true): WSSharedDoc =>
+    map.setIfUndefined(docs, docname, () => {
+        const doc = new WSSharedDoc(docname);
+        doc.gc = gc;
+        docs.set(docname, doc);
+        return doc;
+    });
+
+const closeConn = (doc: WSSharedDoc, conn: any) => {
+    if (doc.conns.has(conn)) {
+        doc.conns.delete(conn);
+    }
+    conn.close();
+};
+
+const send = (doc: WSSharedDoc, conn: any, m: Uint8Array) => {
+    if (
+        conn.readyState !== wsReadyStateConnecting &&
+        conn.readyState !== wsReadyStateOpen
+    ) {
+        closeConn(doc, conn);
+    }
+    try {
+        conn.send(m, (/** @param {any} err */ err: any) => {
+            err != null && closeConn(doc, conn);
+        });
+    } catch (e) {
+        closeConn(doc, conn);
+    }
+};
+
+export const handleConnection = (
+    socket: WebSocket.WebSocket,
+    request: http.IncomingMessage,
+    docName: string
+) => {
+    const gc = true;
+    socket.binaryType = 'arraybuffer';
+    // get doc, initialize if it does not exist yet
+    const doc = getYDoc(docName, gc);
+    doc.conns.set(socket, new Set());
+    // listen and reply to events
+    socket.on('message', (message: ArrayBuffer) => {
+        try {
+            const encoder = encoding.createEncoder();
+            const decoder = decoding.createDecoder(new Uint8Array(message));
+            const messageType = decoding.readVarUint(decoder);
+            switch (messageType) {
+                case messageSync:
+                    encoding.writeVarUint(encoder, messageSync);
+                    syncProtocol.readSyncMessage(decoder, encoder, doc, null);
+                    if (encoding.length(encoder) > 1) {
+                        send(doc, socket, encoding.toUint8Array(encoder));
+                    }
+                    break;
+            }
+        } catch (err) {
+            console.error(err);
+            doc.emit('error', [err]);
+        }
+    });
+
+    // Check if connection is still alive
+    let pongReceived = true;
+    const pingInterval = setInterval(() => {
+        if (!pongReceived) {
+            if (doc.conns.has(socket)) {
+                closeConn(doc, socket);
+            }
+            clearInterval(pingInterval);
+        } else if (doc.conns.has(socket)) {
+            pongReceived = false;
+            try {
+                socket.ping();
+            } catch (e) {
+                closeConn(doc, socket);
+                clearInterval(pingInterval);
+            }
+        }
+    }, 30 * 1000);
+    socket.on('close', () => {
+        closeConn(doc, socket);
+        clearInterval(pingInterval);
+    });
+    socket.on('pong', () => {
+        pongReceived = true;
+    });
+    // put the following in a variables in a block so the interval handlers don't keep in in
+    // scope
+    {
+        // send sync step 1
+        const encoder = encoding.createEncoder();
+        encoding.writeVarUint(encoder, messageSync);
+        console.log('sync step 0', encoding.toUint8Array(encoder));
+        syncProtocol.writeSyncStep1(encoder, doc);
+        send(doc, socket, encoding.toUint8Array(encoder));
+        console.log('sync step 1 sent', encoding.toUint8Array(encoder));
+    }
+};

apps/ligo-virgo/proxy.conf.json

@@ -1,11 +1,6 @@
 {
-    "/api": {
-        "target": "https://nightly.affine.pro/",
-        "secure": false,
-        "changeOrigin": true
-    },
     "/collaboration": {
-        "target": "https://canary.affine.pro",
+        "target": "http://127.0.0.1:3000/",
         "ws": true,
         "changeOrigin": true,
         "secure": false

apps/venus/src/app/common/Header.tsx

@@ -13,7 +13,7 @@ import AFFiNETextLogo from './affine-text-logo.png';
 import { HoverMenu } from './HoverMenu';
 import { MobileHeader } from './MobileHeader';
 
-import { options } from '../i18n';
+import { LOCALES } from '../i18n';
 
 export const AFFiNEHeader = () => {
     const matches = useMediaQuery('(max-width: 1024px)');
@@ -140,15 +140,15 @@ export const AFFiNEHeader = () => {
                     }}
                 />
                 <Select
-                    defaultValue="en"
+                    defaultValue={i18n.language}
                     sx={{ display: matchesIPAD ? 'none' : 'intial' }}
                     onChange={changeLanguage}
                     size="md"
                     variant="plain"
                 >
-                    {options.map(option => (
+                    {LOCALES.map(lang => (
-                        <Option key={option.value} value={option.value}>
+                        <Option key={lang.tag} value={lang.tag}>
-                            {option.text}
+                            {lang.originalName}
                         </Option>
                     ))}
                 </Select>

apps/venus/src/app/i18n/index.ts

@@ -1,25 +1,63 @@
-import i18next from 'i18next';
+import i18next, { Resource } from 'i18next';
-import { initReactI18next } from 'react-i18next';
+import {
-import en_US from './resources/en.json';
+    I18nextProvider,
-import zh_CN from './resources/zh.json';
+    initReactI18next,
+    useTranslation,
+} from 'react-i18next';
+import { LOCALES } from './resources';
+import type en_US from './resources/en.json';
 
-const resources = {
+// See https://react.i18next.com/latest/typescript
-    en: en_US,
+declare module 'react-i18next' {
-    zh: zh_CN,
+    interface CustomTypeOptions {
-} as const;
+        // custom namespace type if you changed it
+        // defaultNS: 'ns1';
+        // custom resources type
+        resources: {
+            en: typeof en_US;
+        };
+    }
+}
+
+const STORAGE_KEY = 'i18n_lng';
+
+export { i18n, useTranslation, I18nProvider, LOCALES };
+
+const resources = LOCALES.reduce<Resource>(
+    (acc, { tag, res }) => ({ ...acc, [tag]: { translation: res } }),
+    {}
+);
+
+const fallbackLng = LOCALES[0].tag;
+const standardizeLocale = (language: string) => {
+    if (LOCALES.find(locale => locale.tag === language)) return language;
+    if (
+        LOCALES.find(
+            locale => locale.tag === language.slice(0, 2).toLowerCase()
+        )
+    )
+        return language;
+    return fallbackLng;
+};
+
+const language = standardizeLocale(
+    localStorage.getItem(STORAGE_KEY) ?? navigator.language
+);
+
+const i18n = i18next.createInstance();
+i18n.use(initReactI18next).init({
+    lng: language,
+    fallbackLng,
+    debug: process.env['NODE_ENV'] === 'development',
 
-i18next.use(initReactI18next).init({
-    lng: 'en',
-    fallbackLng: 'en',
     resources,
     interpolation: {
         escapeValue: false, // not needed for react as it escapes by default
     },
 });
 
-export const options = [
+i18n.on('languageChanged', lng => {
-    { value: 'en', text: 'English' },
+    localStorage.setItem(STORAGE_KEY, lng);
-    { value: 'zh', text: '简体中文' },
+});
-];
 
-export { i18next };
+const I18nProvider = I18nextProvider;

apps/venus/src/app/i18n/resources/en.json

@@ -1,5 +1,4 @@
 {
-    "translation": {
     "Blog": "Blog",
     "AboutUs": "About Us",
     "Open Source": "Open Source",
@@ -9,6 +8,7 @@
     "Privacy First": "Privacy First",
     "Alternative": "Alternative",
     "Check GitHub": "Check GitHub",
+    "GitHub": "GitHub",
     "Try it Online": "Try it Online",
     "language": "Language",
     "description1": {
@@ -37,4 +37,3 @@
     "KeepUpdated": "Keep Updated on",
     "Join": "Join Our Community"
 }
-}

apps/venus/src/app/i18n/resources/es.json

@@ -0,0 +1,32 @@
+{
+    "Alternative": "Alternativa",
+    "BuildFor": "Construir por un futuro abierto y semántico",
+    "Check GitHub": "Revisar en GitHub",
+    "Join": "Únete a nuestra comunidad",
+    "KeepUpdated": "Mantente actualizado",
+    "Open Source": "Código abierto",
+    "Privacy First": "La privacidad es lo primero",
+    "Try it Online": "Prueba Online",
+    "description1": {
+        "part1": "Affine es un base de conocimiento colaborativa de próxima generación para los profesionales.",
+        "part2": "Es más que una simple colección de Documentos, Pizarra y Tablas.",
+        "part3": "Transforma cualquier bloque como tu quieras. ",
+        "part4": "Dígale adios a la redundancia, Almacene sus datos una vez, y manténgalos como usted quiera."
+    },
+    "description2": {
+        "part1": "Dele forma a sus  páginas.",
+        "part2": "Los Documentos, Kanbans y Base de datos son completamente funcionales en cualquier lugar, en cualquier momento.  Un verdadero entorno de lo que ves y lo que obtienes para tus datos.",
+        "part3": "Todas las páginas tiene una vista de documento y de pizarra. "
+    },
+    "description3": {
+        "part1": "Programe sus tareas.",
+        "part2": "No más caos manejando múltiples vistas.",
+        "part3": "Establezca un TODO con Markdown, y edítelo sin problemas en un Kanban. ",
+        "part4": "La gestión de tablas multidimensionales debería ser así de fácil y ahora lo es."
+    },
+    "description4": {
+        "part1": "Privacidad en primer lugar, y colaboración sin compromiso. ",
+        "part2": "\nNo nos gusta estar encerrados, y a usted tampoco debería. La privacidad está en la base de todo lo que hacemos, pero no debe limitarnos, por eso no hay compromisos.",
+        "part3": "Sus datos son suyos; siempre están almacenados y protegidos de forma local, siempre disponibles para usted. Sin dejar de lado el poder disfrutar de funciones de colaboración como la edición en tiempo real y el uso compartido con otros, sin ninguna configuración en la nube."
+    }
+}

apps/venus/src/app/i18n/resources/index.ts

@@ -0,0 +1,24 @@
+import en from './en.json';
+import es from './es.json';
+import zh_Hans from './zh.json';
+
+export const LOCALES = [
+    {
+        name: 'English',
+        tag: 'en',
+        originalName: 'English',
+        res: en,
+    },
+    {
+        name: 'Simplified Chinese',
+        tag: 'zh-Hans',
+        originalName: '简体中文',
+        res: zh_Hans,
+    },
+    {
+        name: 'Spanish',
+        tag: 'es',
+        originalName: 'español',
+        res: es,
+    },
+] as const;

apps/venus/src/app/i18n/resources/zh.json

@@ -1,5 +1,4 @@
 {
-    "translation": {
     "Blog": "博客",
     "AboutUs": "关于我们",
     "Open Source": "开源",
@@ -37,4 +36,3 @@
     "KeepUpdated": "持续更新在",
     "Join": "加入我们的社区"
 }
-}

docs/Jobs.md

@@ -0,0 +1,5 @@
+# Jobs
+
+-   [<b>Full Stack Platform Engineer</b>](./mysc.app.md) @[mysc.app](https://mysc.app/)
+
+    Rust · GWST · Remote · Shanghai, China

docs/mysc.app.md

@@ -0,0 +1,25 @@
+## Full Stack Platform Engineer
+
+### Your responsibilities will include
+
+-   Build APIs in the Data Platform to support new capabilities within mysc.
+-   Work with backend and client side databases (MongoDB, Redis, SQLite)
+-   Design and implement algorithms that are highly performant, resilient against failures and race conditions and are easy to use by application developers
+-   Build up solid knowledge of our product to understand end to end system behaviour and data flow
+-   Execute performance profiling on existing systems to understand key bottlenecks and improve on their performance characteristics
+
+### What we're looking for
+
+-   You have strong analytical thinking, planning, and problem-solving skills
+-   You have 3-5 years experience in building APIs or Platforms
+-   You have strong computer science fundamentals, including knowledge of data structures, algorithmic complexity, and designing for performance and scalability
+-   You have experience in NodeJS, Typescript and Go
+-   You have experience working with JWST
+-   You have experience with unit / automated testing
+
+### What we offer
+
+-   A fully remote team based on Gather Town
+-   A culture that encourages different opinions, respects different values and advocates work life balance
+-   Real ownership and actual impact
+-   Learning and career opportunities on the long run

libs/components/editor-blocks/src/blocks/group/scene-kanban/RefPage.tsx

@@ -1,6 +1,13 @@
 import { useEditor } from '@toeverything/components/editor-core';
 import { MuiBackdrop, styled, useTheme } from '@toeverything/components/ui';
-import { createContext, ReactNode, useContext, useState } from 'react';
+import {
+    createContext,
+    ReactNode,
+    useContext,
+    useEffect,
+    useRef,
+    useState,
+} from 'react';
 import { createPortal } from 'react-dom';
 
 const Dialog = styled('div')({
@@ -11,7 +18,7 @@ const Dialog = styled('div')({
     boxShadow: '0px 1px 10px rgba(152, 172, 189, 0.6)',
     borderRadius: '10px',
     padding: '72px 120px',
-    overflow: 'scroll',
+    overflowY: 'auto',
 });
 
 const Modal = ({ open, children }: { open: boolean; children?: ReactNode }) => {
@@ -48,13 +55,20 @@ const Modal = ({ open, children }: { open: boolean; children?: ReactNode }) => {
 
 const ModalPage = ({ blockId }: { blockId: string | null }) => {
     const { editor, editorElement } = useEditor();
+    const editorRef = useRef<typeof editor>(null);
 
     const AffineEditor = editorElement as any;
 
+    // Active block after modal open
+    useEffect(() => {
+        editorRef.current?.selectionManager.activeNodeByNodeId(blockId);
+    }, [blockId]);
+
     return (
         <Modal open={!!blockId}>
             {blockId && (
                 <AffineEditor
+                    ref={editorRef}
                     workspace={editor.workspace}
                     rootBlockId={blockId}
                     scrollBlank={false}

libs/components/editor-plugins/src/menu/command-menu/Menu.tsx

@@ -211,6 +211,8 @@ export const CommandMenu = ({ editor, hooks, style }: CommandMenuProps) => {
 
     const handleSelected = async (type: BlockFlavorKeys | string) => {
         const text = await editor.commands.textCommands.getBlockText(blockId);
+        const block = await editor.getBlockById(blockId);
+        let textValue = block.getProperty('text').value;
         editor.blockHelper.removeSearchSlash(blockId, true);
         if (type.startsWith('Virgo')) {
             const handler =
@@ -223,14 +225,18 @@ export const CommandMenu = ({ editor, hooks, style }: CommandMenuProps) => {
             } else {
                 await commonCommandMenuHandler(blockId, type, editor);
             }
-            const block = await editor.getBlockById(blockId);
             const nextBlock = await block.nextSibling();
             setTimeout(() => {
                 editor.selectionManager.activeNodeByNodeId(nextBlock.id);
-            }, 100);
+                if (textValue.length === 1) {
-            if (block.blockProvider.isEmpty()) {
                     block.remove();
+                } else {
+                    block.setProperty('text', {
+                        //@ts-ignore
+                        value: textValue.filter(text => !text.search),
+                    });
                 }
+            }, 100);
         } else {
             if (Protocol.Block.Type[type as BlockFlavorKeys]) {
                 const block = await editor.commands.blockCommands.convertBlock(

libs/components/editor-plugins/src/menu/left-menu/LeftMenu.tsx

@@ -1,14 +1,15 @@
-import { useMemo } from 'react';
-import { Virgo, PluginHooks } from '@toeverything/framework/virgo';
-import { Cascader, CascaderItemProps } from '@toeverything/components/ui';
-import { Protocol } from '@toeverything/datasource/db-service';
-import { TurnIntoMenu } from './TurnIntoMenu';
 import {
     AddViewIcon,
     DeleteCashBinIcon,
     TurnIntoIcon,
     UngroupIcon,
 } from '@toeverything/components/icons';
+import { Cascader, CascaderItemProps } from '@toeverything/components/ui';
+import { Protocol } from '@toeverything/datasource/db-service';
+import { useTranslation } from '@toeverything/datasource/i18n';
+import { PluginHooks, Virgo } from '@toeverything/framework/virgo';
+import { useMemo } from 'react';
+import { TurnIntoMenu } from './TurnIntoMenu';
 
 interface LeftMenuProps {
     anchorEl?: Element;
@@ -21,10 +22,11 @@ interface LeftMenuProps {
 
 export function LeftMenu(props: LeftMenuProps) {
     const { editor, anchorEl, hooks, blockId, onClose } = props;
+    const { t } = useTranslation();
     const menu: CascaderItemProps[] = useMemo(
         () => [
             {
-                title: 'Delete',
+                title: t('Delete'),
                 callback: () => {
                     editor.commands.blockCommands.removeBlock(blockId);
                 },
@@ -32,7 +34,7 @@ export function LeftMenu(props: LeftMenuProps) {
                 icon: <DeleteCashBinIcon />,
             },
             {
-                title: 'Turn into',
+                title: t('Turn into'),
                 subItems: [],
                 children: (
                     <TurnIntoMenu
@@ -48,7 +50,7 @@ export function LeftMenu(props: LeftMenuProps) {
                 icon: <TurnIntoIcon />,
             },
             {
-                title: 'Add A Below Block',
+                title: t('Add A Below Block'),
                 icon: <AddViewIcon />,
                 callback: async () => {
                     const block = await editor.getBlockById(blockId);
@@ -66,14 +68,14 @@ export function LeftMenu(props: LeftMenuProps) {
                 },
             },
             {
-                title: 'Divide Here As A New Group',
+                title: t('Divide Here As A New Group'),
                 icon: <UngroupIcon />,
                 callback: () => {
                     editor.commands.blockCommands.splitGroupFromBlock(blockId);
                 },
             },
         ],
-        [editor, hooks, blockId, onClose]
+        [t, editor, hooks, blockId, onClose]
     );
 
     // const filterItems = (

libs/components/layout/src/settings-sidebar/Layout/LayoutSettings.tsx

@@ -1,9 +1,11 @@
 import { styled } from '@toeverything/components/ui';
+import { useTranslation } from '@toeverything/datasource/i18n';
 
 export const LayoutSettings = () => {
+    const { t } = useTranslation();
     return (
         <StyledText>
-            <p>Layout Settings Coming Soon...</p>
+            <p>{t('ComingSoon')}</p>
         </StyledText>
     );
 };

libs/datasource/db-service/src/services/database/index.ts

@@ -33,13 +33,15 @@ async function _getCurrentToken() {
                 if (user) resolve(user.getIdToken());
             });
         });
+    } else if (process.env['NX_KECK']) {
+        return 'AFFiNE';
     }
     return undefined;
 }
 
 const _enabled = {
     demo: [],
-    AFFiNE: process.env['NX_KECK'] ? ['idb'] : ['idb', 'ws'],
+    AFFiNE: process.env['NX_KECK'] ? ['idb', 'keck'] : ['idb'],
 } as any;
 
 async function _getBlockDatabase(

libs/datasource/i18n/README.md

@@ -2,10 +2,13 @@
 
 ## Usages
 
+-   Update missing translations into the base resources, a.k.a the `src/resources/en.json`
+-   Replace literal text with translation keys
+
 ```tsx
 import { useTranslation } from '@toeverything/datasource/i18n';
 
-// base.json
+// src/resources/en.json
 // {
 //     'Text': 'some text',
 //     'Switch to language': 'Switch to {{language}}', // <- you can interpolation by curly brackets
@@ -33,7 +36,14 @@ const App = () => {
 };
 ```
 
-## How to sync translations
+## How the i18n workflow works?
+
+-   When the `src/resources/en.json`(base language) updated and merged to the develop branch, will trigger the `languages-sync` action.
+-   The `languages-sync` action will check the base language and add missing translations to the Tolgee platform.
+-   This way, partners from the community can update the translations.
+-   Finally, the `languages-download` action will regularly download the latest translation resources from the Tolgee platform.
+
+## How to sync translations manually
 
 -   Set token as environment variable
 

libs/datasource/i18n/project.json

@@ -2,7 +2,7 @@
     "$schema": "../../../node_modules/nx/schemas/project-schema.json",
     "sourceRoot": "libs/datasource/i18n/src",
     "projectType": "library",
-    "tags": ["datasource"],
+    "tags": ["datasource:i18n"],
     "targets": {
         "build": {
             "executor": "@nrwl/web:rollup",

libs/datasource/i18n/src/resources/en.json

@@ -20,5 +20,9 @@
     "Import Workspace": "Import Workspace",
     "Export Workspace": "Export Workspace",
     "Last edited by": "Last edited by {{name}}",
-    "Logout": "Logout"
+    "Logout": "Logout",
+    "Delete": "Delete",
+    "Turn into": "Turn into",
+    "Add A Below Block": "Add A Below Block",
+    "Divide Here As A New Group": "Divide Here As A New Group"
 }

libs/datasource/i18n/src/resources/zh-Hans.json

@@ -21,5 +21,9 @@
     "Import Workspace": "导入 Workspace",
     "Export Workspace": "导出 Workspace",
     "Last edited by": "最后编辑者为 {{name}}",
-    "Logout": "退出登录"
+    "Logout": "退出登录",
+    "Delete": "删除",
+    "Turn into": "转换为",
+    "Add A Below Block": "在下方添加一个新块",
+    "Divide Here As A New Group": "从这里划分一个新组"
 }

libs/datasource/jwt-rpc/src/handler.ts

@@ -4,7 +4,8 @@ import * as authProtocol from 'y-protocols/auth';
 import * as awarenessProtocol from 'y-protocols/awareness';
 import * as syncProtocol from 'y-protocols/sync';
 
-import { WebsocketProvider } from './provider';
+import { KeckProvider } from './keckprovider';
+import { WebsocketProvider } from './wsprovider';
 
 const permissionDeniedHandler = (provider: WebsocketProvider, reason: string) =>
     console.warn(`Permission denied to access ${provider.url}.\n${reason}`);
@@ -19,7 +20,7 @@ export enum Message {
 export type MessageCallback = (
     encoder: encoding.Encoder,
     decoder: decoding.Decoder,
-    provider: WebsocketProvider,
+    provider: WebsocketProvider | KeckProvider,
     emitSynced: boolean,
     messageType: number
 ) => void;
@@ -48,6 +49,7 @@ export const handler: Record<Message, MessageCallback> = {
         emitSynced,
         messageType
     ) => {
+        if (provider instanceof WebsocketProvider) {
             encoding.writeVarUint(encoder, Message.queryAwareness);
             encoding.writeVarUint8Array(
                 encoder,
@@ -56,6 +58,7 @@ export const handler: Record<Message, MessageCallback> = {
                     Array.from(provider.awareness.getStates().keys())
                 )
             );
+        }
     },
 
     [Message.awareness]: (
@@ -65,11 +68,13 @@ export const handler: Record<Message, MessageCallback> = {
         emitSynced,
         messageType
     ) => {
+        if (provider instanceof WebsocketProvider) {
             awarenessProtocol.applyAwarenessUpdate(
                 provider.awareness,
                 decoding.readVarUint8Array(decoder),
                 provider
             );
+        }
     },
 
     [Message.auth]: (encoder, decoder, provider, emitSynced, messageType) => {

libs/datasource/jwt-rpc/src/index.ts

@@ -1,3 +1,4 @@
 export { IndexedDBProvider } from './indexeddb';
-export { WebsocketProvider } from './provider';
+export { KeckProvider } from './keckprovider';
 export { SQLiteProvider } from './sqlite';
+export { WebsocketProvider } from './wsprovider';

libs/datasource/jwt-rpc/src/keckprovider.ts

@@ -0,0 +1,115 @@
+import * as Y from 'yjs';
+
+import { Observable } from 'lib0/observable';
+import * as url from 'lib0/url';
+
+import { handler } from './handler';
+import { registerKeckUpdateHandler } from './processor';
+import { registerWebsocket } from './websocket';
+
+/**
+ * Websocket Provider for Yjs. Creates a websocket connection to sync the shared document.
+ * The document name is attached to the provided url. I.e. the following example
+ * creates a websocket connection to http://localhost:3000/my-document-name
+ *
+ * @example
+ *   import * as Y from 'yjs'
+ *   import { KeckProvider } from 'jwt-rpc'
+ *   const doc = new Y.Doc()
+ *   const provider = new KeckProvider('http://localhost:3000', 'my-document-name', doc)
+ */
+export class KeckProvider extends Observable<string> {
+    maxBackOffTime: number;
+    url: string;
+    roomName: string;
+
+    doc: Y.Doc;
+
+    wsUnsuccessfulReconnects: number;
+    private _synced: boolean;
+
+    broadcastChannel: string;
+    private _broadcast?: {
+        broadcastMessage: (buf: ArrayBuffer) => void;
+        disconnect: () => void;
+    };
+
+    private _websocket?: {
+        broadcastMessage: (buf: ArrayBuffer) => void;
+        disconnect: () => void;
+    };
+
+    private _updateHandlerDestroy: () => void;
+
+    constructor(
+        token: string,
+        serverUrl: string,
+        roomName: string,
+        doc: Y.Doc,
+        { params = {}, resyncInterval = -1, maxBackOffTime = 2500 } = {}
+    ) {
+        super();
+
+        this.roomName = roomName;
+        // ensure that url is always ends with /
+        while (serverUrl[serverUrl.length - 1] === '/') {
+            serverUrl = serverUrl.slice(0, serverUrl.length - 1);
+        }
+        this.broadcastChannel = serverUrl + '/' + roomName + '/';
+        const encodedParams = url.encodeQueryParams(params);
+        this.url =
+            this.broadcastChannel +
+            (encodedParams.length === 0 ? '' : '?' + encodedParams);
+
+        this.doc = doc;
+
+        this.maxBackOffTime = maxBackOffTime;
+        this.wsUnsuccessfulReconnects = 0;
+
+        this._synced = false;
+
+        this._websocket = registerWebsocket(this, token, resyncInterval);
+
+        this._updateHandlerDestroy = registerKeckUpdateHandler(
+            this,
+            doc,
+            buf => {
+                this._websocket?.broadcastMessage(buf);
+                this._broadcast?.broadcastMessage(buf);
+            }
+        );
+    }
+
+    get messageHandlers() {
+        return handler;
+    }
+
+    get synced() {
+        return this._synced;
+    }
+
+    set synced(state) {
+        if (this._synced !== state) {
+            this._synced = state;
+            this.emit('synced', [state]);
+            this.emit('sync', [state]);
+        }
+    }
+
+    override destroy() {
+        if (this._broadcast) {
+            const disconnect = this._broadcast.disconnect;
+            this._broadcast = undefined;
+            disconnect();
+        }
+
+        if (this._websocket) {
+            const disconnect = this._websocket.disconnect;
+            this._websocket = undefined;
+            disconnect();
+        }
+
+        this._updateHandlerDestroy?.();
+        super.destroy();
+    }
+}

libs/datasource/jwt-rpc/src/processor.ts

@@ -5,10 +5,11 @@ import * as syncProtocol from 'y-protocols/sync';
 import * as Y from 'yjs';
 
 import { Message } from './handler';
-import { WebsocketProvider } from './provider';
+import { KeckProvider } from './keckprovider';
+import { WebsocketProvider } from './wsprovider';
 
 export const readMessage = (
-    provider: WebsocketProvider,
+    provider: WebsocketProvider | KeckProvider,
     buf: Uint8Array,
     emitSynced: boolean
 ): encoding.Encoder => {
@@ -24,7 +25,7 @@ export const readMessage = (
     return encoder;
 };
 
-export const registerUpdateHandler = (
+export const registerWsUpdateHandler = (
     provider: WebsocketProvider,
     awareness: awarenessProtocol.Awareness,
     doc: Y.Doc,
@@ -78,3 +79,24 @@ export const registerUpdateHandler = (
         doc.off('update', documentUpdateHandler);
     };
 };
+
+export const registerKeckUpdateHandler = (
+    provider: KeckProvider,
+    doc: Y.Doc,
+    broadcastMessage: (buf: ArrayBuffer) => void
+) => {
+    //  Listens to Yjs updates and sends them to remote peers (ws and broadcastchannel)
+    const documentUpdateHandler = (update: Uint8Array, origin: any) => {
+        if (origin !== provider) {
+            const encoder = encoding.createEncoder();
+            encoding.writeVarUint(encoder, Message.sync);
+            syncProtocol.writeUpdate(encoder, update);
+            broadcastMessage(encoding.toUint8Array(encoder));
+        }
+    };
+
+    doc.on('update', documentUpdateHandler);
+    return () => {
+        doc.off('update', documentUpdateHandler);
+    };
+};

libs/datasource/jwt-rpc/src/websocket.ts

@@ -5,8 +5,9 @@ import * as awarenessProtocol from 'y-protocols/awareness';
 import * as syncProtocol from 'y-protocols/sync';
 
 import { Message } from './handler';
+import { KeckProvider } from './keckprovider';
 import { readMessage } from './processor';
-import { WebsocketProvider } from './provider';
+import { WebsocketProvider } from './wsprovider';
 
 enum WebSocketState {
     disconnected = 0,
@@ -46,14 +47,14 @@ const _getToken = async (
     return resp.json();
 };
 
-const _getTimeout = (provider: WebsocketProvider) =>
+const _getTimeout = (provider: WebsocketProvider | KeckProvider) =>
     math.min(
         math.pow(2, provider.wsUnsuccessfulReconnects) * 100,
         provider.maxBackOffTime
     );
 
 export const registerWebsocket = (
-    provider: WebsocketProvider,
+    provider: WebsocketProvider | KeckProvider,
     token: string,
     resync = -1,
     reconnect = 3,
@@ -105,13 +106,19 @@ export const registerWebsocket = (
                     state = WebSocketState.disconnected;
                     provider.synced = false;
                     // update awareness (all users except local left)
+
+                    const awareness = (provider as any)['awareness'];
+                    if (awareness) {
                         awarenessProtocol.removeAwarenessStates(
-                        provider.awareness,
+                            awareness,
-                        Array.from(
+                            Array.from(awareness.getStates().keys()).filter(
-                            provider.awareness.getStates().keys()
+                                (client): client is number =>
-                        ).filter(client => client !== provider.doc.clientID),
+                                    client !== provider.doc.clientID
+                            ),
                             provider
                         );
+                    }
+
                     provider.emit('status', [{ status: 'disconnected' }]);
                 } else {
                     provider.wsUnsuccessfulReconnects++;
@@ -139,8 +146,10 @@ export const registerWebsocket = (
                 encoding.writeVarUint(encoder, Message.sync);
                 syncProtocol.writeSyncStep1(encoder, provider.doc);
                 websocket?.send(encoding.toUint8Array(encoder));
+
+                const awareness = (provider as any)['awareness'];
                 // broadcast local awareness state
-                if (provider.awareness.getLocalState() !== null) {
+                if (awareness && awareness.getLocalState() !== null) {
                     const encoderAwarenessState = encoding.createEncoder();
                     encoding.writeVarUint(
                         encoderAwarenessState,
@@ -148,10 +157,9 @@ export const registerWebsocket = (
                     );
                     encoding.writeVarUint8Array(
                         encoderAwarenessState,
-                        awarenessProtocol.encodeAwarenessUpdate(
+                        awarenessProtocol.encodeAwarenessUpdate(awareness, [
-                            provider.awareness,
+                            provider.doc.clientID,
-                            [provider.doc.clientID]
+                        ])
-                        )
                     );
                     websocket?.send(
                         encoding.toUint8Array(encoderAwarenessState)

libs/datasource/jwt-rpc/src/wsprovider.ts

@@ -6,7 +6,7 @@ import * as url from 'lib0/url';
 import * as awarenessProtocol from 'y-protocols/awareness';
 
 import { handler } from './handler';
-import { registerUpdateHandler } from './processor';
+import { registerWsUpdateHandler } from './processor';
 import { registerWebsocket } from './websocket';
 
 /**
@@ -85,7 +85,7 @@ export class WebsocketProvider extends Observable<string> {
         //     this.doc
         // );
 
-        this._updateHandlerDestroy = registerUpdateHandler(
+        this._updateHandlerDestroy = registerWsUpdateHandler(
             this,
             awareness,
             doc,

libs/datasource/jwt/src/types/index.ts

@@ -17,7 +17,7 @@ function getCollaborationPoint() {
     const { protocol, host } = getLocation();
     const ws = protocol.startsWith('https') ? 'wss' : 'ws';
     const isOnline = host.endsWith('affine.pro');
-    const site = isOnline ? host : 'localhost:4200';
+    const site = isOnline ? host : 'localhost:3000';
     return `${ws}://${site}/collaboration/`;
 }
 

libs/datasource/jwt/src/yjs/provider.ts

@@ -3,6 +3,7 @@ import { Doc } from 'yjs';
 
 import {
     IndexedDBProvider,
+    KeckProvider,
     SQLiteProvider,
     WebsocketProvider,
 } from '@toeverything/datasource/jwt-rpc';
@@ -22,7 +23,7 @@ export type YjsProvider = (
     instances: YjsDefaultInstances
 ) => Promise<unknown | undefined>;
 
-type ProviderType = 'idb' | 'sqlite' | 'ws';
+type ProviderType = 'idb' | 'sqlite' | 'ws' | 'keck';
 
 export type YjsProviderOptions = {
     enabled: ProviderType[];
@@ -79,6 +80,39 @@ export const getYjsProviders = (
                         }
                     ) as any; // TODO: type is erased after cascading references
 
+                    // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
+                    return new Promise<void>((resolve, reject) => {
+                        // TODO: synced will also be triggered on reconnection after losing sync
+                        // There needs to be an event mechanism to emit the synchronization state to the upper layer
+                        ws.once('synced', () => resolve());
+                        ws.once('lost-connection', () => resolve());
+                        ws.once('connection-error', () => reject());
+                        ws.on('synced', () => instances.emitState('connected'));
+                        ws.on('lost-connection', () =>
+                            instances.emitState('retry')
+                        );
+                        ws.on('connection-error', () =>
+                            instances.emitState('retry')
+                        );
+                    });
+                } else {
+                    return;
+                }
+            }
+        },
+        keck: async (instances: YjsDefaultInstances) => {
+            if (options.enabled.includes('keck')) {
+                if (instances.token) {
+                    const ws = new KeckProvider(
+                        instances.token,
+                        options.backend,
+                        instances.workspace,
+                        instances.doc,
+                        {
+                            params: options.params,
+                        }
+                    ) as any; // TODO: type is erased after cascading references
+
                     // Wait for ws synchronization to complete, otherwise the data will be modified in reverse, which can be optimized later
                     return new Promise<void>((resolve, reject) => {
                         // TODO: synced will also be triggered on reconnection after losing sync

package.json

@@ -9,6 +9,7 @@
         "start": "env-cmd -f .github/env/.env.local-dev nx serve ligo-virgo",
         "start:affine": "nx serve ligo-virgo",
         "start:keck": "nx serve keck",
+        "start:keck-dev": "env-cmd -f .github/env/.env.local-keck nx serve ligo-virgo",
         "start:venus": "nx serve venus",
         "build": "nx build ligo-virgo",
         "build:local": "env-cmd -f .github/env/.env.local-dev nx build ligo-virgo",