Merge pull request #787 from toeverything/test-coverage

chore: add test coverage check

.eslintignore

@@ -1,7 +0,0 @@
-**/webpack.config.js
-**/scripts/*.js
-**/node_modules/**
-.github/**
-**/__tests__/**
-**/tests/**
-

.github/workflows/tests.yml

@@ -5,7 +5,58 @@ on:
   pull_request:
     branches: [master]
 jobs:
-  test:
+  e2e-test:
+    timeout-minutes: 60
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 'latest'
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          registry-url: https://npm.pkg.github.com
+          scope: '@toeverything'
+          cache: 'pnpm'
+
+      - run: node scripts/module-resolve/ci.cjs
+
+      - name: Install dependencies
+        run: pnpm install --no-frozen-lockfile
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_GITHUB_AUTH_TOKEN }}
+
+      - name: Install Playwright browsers
+        run: npx playwright install chromium
+
+      - name: Run E2E tests
+        run: pnpm run test:coverage
+      #   env:
+      #     NEXT_PUBLIC_FIREBASE_API_KEY: ${{ secrets.NEXT_PUBLIC_FIREBASE_API_KEY }}
+      #     NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN: ${{ secrets.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN }}
+      #     NEXT_PUBLIC_FIREBASE_PROJECT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_PROJECT_ID }}
+      #     NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET: ${{ secrets.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET }}
+      #     NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID }}
+      #     NEXT_PUBLIC_FIREBASE_APP_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_APP_ID }}
+      #     NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID }}
+
+      - name: Collect code coverage report
+        run: pnpm exec nyc report -t .nyc_output --report-dir .coverage --reporter=lcov
+
+      - name: Upload e2e test coverage results
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./.coverage/lcov.info
+          flags: e2etest
+          name: affine
+          fail_ci_if_error: true
+
+  unit-test:
     timeout-minutes: 60
     runs-on: ubuntu-latest
 
@@ -33,16 +84,5 @@ jobs:
       - name: Install Playwright browsers
         run: npx playwright install chromium
 
-      # - name: Run E2E tests
-      #   run: pnpm run test:e2e
-      #   env:
-      #     NEXT_PUBLIC_FIREBASE_API_KEY: ${{ secrets.NEXT_PUBLIC_FIREBASE_API_KEY }}
-      #     NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN: ${{ secrets.NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN }}
-      #     NEXT_PUBLIC_FIREBASE_PROJECT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_PROJECT_ID }}
-      #     NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET: ${{ secrets.NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET }}
-      #     NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MESSAGING_SENDER_ID }}
-      #     NEXT_PUBLIC_FIREBASE_APP_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_APP_ID }}
-      #     NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID: ${{ secrets.NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID }}
-
       - name: Run Unit tests
         run: pnpm run test:unit

.gitignore

@@ -4,6 +4,8 @@
 *dist
 /tmp
 /out-tsc
+.nyc_output
+.coverage
 
 # dependencies
 node_modules

.vscode/settings.json

@@ -10,5 +10,11 @@
     "pnpm",
     "selfhosted",
     "testid"
-  ]
+  ],
+  "explorer.fileNesting.patterns": {
+    "*.js": "${capture}.js.map, ${capture}.min.js, ${capture}.d.ts, ${capture}.d.ts.map",
+    "package.json": ".browserslist*, .circleci*, .codecov, .commitlint*, .cz-config.js, .czrc, .dlint.json, .dprint.json, .editorconfig, .eslint*, .firebase*, .flowconfig, .github*, .gitlab*, .gitpod*, .huskyrc*, .jslint*, .lighthouserc.*, .lintstagedrc*, .markdownlint*, .mocha*, .node-version, .nodemon*, .npm*, .nvmrc, .pm2*, .pnp.*, .pnpm*, .prettier*, .releaserc*, .sentry*, .stackblitz*, .styleci*, .stylelint*, .tazerc*, .textlint*, .tool-versions, .travis*, .versionrc*, .vscode*, .watchman*, .xo-config*, .yamllint*, .yarnrc*, Procfile, api-extractor.json, apollo.config.*, appveyor*, ava.config.*, azure-pipelines*, bower.json, build.config.*, commitlint*, crowdin*, cypress.*, dangerfile*, dlint.json, dprint.json, firebase.json, grunt*, gulp*, histoire.config.*, jasmine.*, jenkins*, jest.config.*, jsconfig.*, karma*, lerna*, lighthouserc.*, lint-staged*, nest-cli.*, netlify*, nodemon*, nx.*, package-lock.json, package.nls*.json, phpcs.xml, playwright.config.*, pm2.*, pnpm*, prettier*, pullapprove*, puppeteer.config.*, pyrightconfig.json, release-tasks.sh, renovate*, rollup.config.*, stylelint*, tsconfig.*, tsdoc.*, tslint*, tsup.config.*, turbo*, typedoc*, unlighthouse*, vercel*, vetur.config.*, vitest.config.*, webpack*, workspace.json, xo.config.*, yarn*, babel.*, .babelrc, project.json",
+    "Cargo.toml": "Cargo.lock",
+    "README.md": "LICENSE, CHANGELOG.md, CODE_OF_CONDUCT.md, CONTRIBUTING.md"
+  }
 }

package.json

@@ -8,18 +8,16 @@
   "scripts": {
     "dev": "pnpm --filter=!@affine/app build && pnpm --filter @affine/app dev",
     "dev:ac": "pnpm --filter=!@affine/app build && NODE_API_SERVER=ac pnpm --filter @affine/app dev",
-    "dev:local": "pnpm --filter=!@affine/app build && NODE_API_SERVER=local pnpm --filter @affine/app dev",
+    "dev:local": "pnpm --filter=!@affine/app build && cross-env NODE_API_SERVER=local pnpm --filter @affine/app dev",
     "build": " pnpm --filter=!@affine/app build && pnpm --filter!=@affine/datacenter -r build",
     "export": "pnpm --filter @affine/app export",
     "start": "pnpm --filter @affine/app start",
     "lint": "pnpm --filter @affine/app lint",
     "test": "playwright test",
-    "test:dc": "pnpm --filter @affine/datacenter test",
+    "test:coverage": "cross-env COVERAGE=true pnpm test -- --forbid-only",
-    "test:e2e:codegen": "npx playwright codegen http://localhost:8080",
     "test:unit": "playwright test --config=playwright.config.unit.ts",
     "postinstall": "husky install",
     "notify": "node --experimental-modules scripts/notify.mjs",
-    "check:ci": "pnpm lint & pnpm test",
     "update:blocksuite": "pnpm i --filter @affine/app --filter @affine/datacenter @blocksuite/blocks@nightly @blocksuite/store@nightly && pnpm i --filter @affine/app @blocksuite/editor@nightly"
   },
   "lint-staged": {
@@ -28,12 +26,13 @@
   },
   "devDependencies": {
     "@changesets/cli": "^2.26.0",
-    "@jest/globals": "^29.3.1",
     "@playwright/test": "^1.29.1",
     "@types/eslint": "^8.4.10",
     "@types/node": "^18.11.17",
     "@typescript-eslint/eslint-plugin": "^5.47.0",
     "@typescript-eslint/parser": "^5.47.0",
+    "babel-plugin-istanbul": "^6.1.1",
+    "cross-env": "^7.0.3",
     "eslint": "^8.30.0",
     "eslint-config-next": "12.3.1",
     "eslint-config-prettier": "^8.5.0",
@@ -41,8 +40,8 @@
     "fake-indexeddb": "4.0.1",
     "got": "^12.5.3",
     "husky": "^8.0.2",
-    "jest": "^29.3.1",
     "lint-staged": "^13.1.0",
+    "nyc": "^15.1.0",
     "prettier": "^2.7.1",
     "typescript": "^4.9.3"
   },
@@ -67,9 +66,16 @@
     },
     "reportUnusedDisableDirectives": true,
     "ignorePatterns": [
-      "src/**/*.test.ts",
       "package/**/dist/*",
-      "package/**/sync.js"
+      "package/**/.babelrc.js",
+      "package/**/sync.js",
+      "src/**/*.test.ts",
+      "**/webpack.config.js",
+      "**/scripts/*.js",
+      "**/node_modules/**",
+      ".github/**",
+      "**/__tests__/**",
+      "**/tests/**"
     ]
   }
 }

packages/app/.babelrc.js

@@ -0,0 +1,13 @@
+const plugins = [];
+
+if (process.env.NODE_ENV === 'development' || process.env.COVERAGE === 'true') {
+  console.log(
+    'Detected development environment. Instrumenting code for coverage.'
+  );
+  plugins.push('istanbul');
+}
+
+module.exports = {
+  presets: ['next/babel'],
+  plugins,
+};

packages/app/next.config.js

@@ -12,7 +12,7 @@ const EDITOR_VERSION = enableDebugLocal
 const profileTarget = {
   ac: '100.85.73.88:12001',
   dev: '100.77.180.48:11001',
-  test: '100.85.73.88:12001',
+  test: '100.77.180.48:11001',
   stage: '',
   pro: 'http://pathfinder.affine.pro',
   local: '127.0.0.1:3000',

playwright.config.ts

@@ -35,6 +35,9 @@ const config: PlaywrightTestConfig = {
     port: 8080,
     timeout: 120 * 1000,
     reuseExistingServer: !process.env.CI,
+    env: {
+      COVERAGE: process.env.COVERAGE || 'false',
+    },
   },
 };
 

tests/change-page-mode.spec.ts

@@ -1,10 +1,11 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { clickPageMoreActions } from './libs/page-logic.js';
 loadPage();
 
-test.skip('Change page mode(Paper or Edgeless)', () => {
+test.describe('Change page mode(Paper or Edgeless)', () => {
-  test.skip('Switch to edgeless by switch edgeless item', async ({ page }) => {
+  test('Switch to edgeless by switch edgeless item', async ({ page }) => {
     const switcher = page.locator('[data-testid=editor-mode-switcher]');
     const box = await switcher.boundingBox();
     expect(box?.x).not.toBeUndefined();
@@ -35,7 +36,7 @@ test.skip('Change page mode(Paper or Edgeless)', () => {
     expect(await edgeless.isVisible()).toBe(true);
   });
 
-  test.skip('Convert to edgeless by editor header items', async ({ page }) => {
+  test('Convert to edgeless by editor header items', async ({ page }) => {
     await clickPageMoreActions(page);
     const menusEdgelessItem = page.getByTestId('editor-option-menu-edgeless');
     await menusEdgelessItem.click();

tests/console.spec.ts

@@ -1,4 +1,5 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();

tests/contact-us.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();
 
-test.skip('Open contact us', () => {
+test.describe('Open contact us', () => {
   test.skip('Click about us', async ({ page }) => {
     const currentWorkspace = page.getByTestId('current-workspace');
     await currentWorkspace.click();

tests/exception-page.spec.ts

@@ -1,10 +1,11 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();
 
 // ps aux | grep 8080
-test.skip('exception page', () => {
+test.describe('exception page', () => {
   test('visit 404 page', async ({ page }) => {
     await page.goto('http://localhost:8080/404');
     await page.waitForTimeout(1000);

tests/invite-code-page.spec.ts

@@ -1,10 +1,11 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();
 
 // ps aux | grep 8080
-test.skip('invite code page', () => {
+test.describe('invite code page', () => {
   test('the link has expired', async ({ page }) => {
     await page.goto('http://localhost:8080//invite/abc');
     await page.waitForTimeout(1000);

tests/layout.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();
 
-test.skip('Layout ui', () => {
+test.describe('Layout ui', () => {
   test('Collapse Sidebar', async ({ page }) => {
     await page.getByTestId('sliderBar-arrowButton').click();
     const sliderBarArea = page.getByTestId('sliderBar');

tests/libs/playwright.ts

@@ -0,0 +1,52 @@
+import { test as baseTest } from '@playwright/test';
+import fs from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+import crypto from 'node:crypto';
+
+const istanbulTempDir = process.env.ISTANBUL_TEMP_DIR
+  ? path.resolve(process.env.ISTANBUL_TEMP_DIR)
+  : path.join(process.cwd(), '.nyc_output');
+
+function generateUUID() {
+  return crypto.randomUUID();
+}
+
+const enableCoverage = !!process.env.CI || !!process.env.COVERAGE;
+
+export const test = baseTest.extend({
+  context: async ({ context }, use) => {
+    if (enableCoverage) {
+      await context.addInitScript(() =>
+        window.addEventListener('beforeunload', () =>
+          // @ts-expect-error
+          window.collectIstanbulCoverage(JSON.stringify(window.__coverage__))
+        )
+      );
+
+      await fs.promises.mkdir(istanbulTempDir, { recursive: true });
+      await context.exposeFunction(
+        'collectIstanbulCoverage',
+        (coverageJSON?: string) => {
+          if (coverageJSON)
+            fs.writeFileSync(
+              path.join(
+                istanbulTempDir,
+                `playwright_coverage_${generateUUID()}.json`
+              ),
+              coverageJSON
+            );
+        }
+      );
+    }
+    await use(context);
+    if (enableCoverage) {
+      for (const page of context.pages()) {
+        await page.evaluate(() =>
+          // @ts-expect-error
+          window.collectIstanbulCoverage(JSON.stringify(window.__coverage__))
+        );
+      }
+    }
+  },
+});

tests/local-first-delete-page.spec.ts

@@ -1,10 +1,11 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage } from './libs/page-logic.js';
 
 loadPage();
 
-test.skip('Local first delete page', () => {
+test.describe('Local first delete page', () => {
   test('New a page , then delete it in all pages, permanently delete it', async ({
     page,
   }) => {

tests/local-first-export-page.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage, clickPageMoreActions } from './libs/page-logic.js';
 loadPage();
 
-test.skip('Local first export page', () => {
+test.describe('Local first export page', () => {
   test('New a page ,then open it and export html', async ({ page }) => {
     await newPage(page);
     await page.getByPlaceholder('Title').click();

tests/local-first-favorite-page.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage, clickPageMoreActions } from './libs/page-logic.js';
 loadPage();
 
-test.skip('Local first favorite and cancel favorite  page', () => {
+test.describe('Local first favorite and cancel favorite  page', () => {
   test('New a page and open it ,then favorite it', async ({ page }) => {
     await newPage(page);
     await page.getByPlaceholder('Title').click();

tests/local-first-favorites-items.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage, clickPageMoreActions } from './libs/page-logic.js';
 loadPage();
 
-test.skip('Local first favorite items ui', () => {
+test.describe('Local first favorite items ui', () => {
   test('Show favorite items in sidebar', async ({ page }) => {
     await newPage(page);
     await page.getByPlaceholder('Title').click();

tests/local-first-new-page.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage } from './libs/page-logic.js';
 loadPage();
 
-test.skip('local first new page', () => {
+test.describe('local first new page', () => {
   test('click btn new page', async ({ page }) => {
     const originPageId = page.url().split('/').reverse()[0];
     await newPage(page);

tests/local-first-openpage-newtab.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage } from './libs/page-logic.js';
 loadPage();
 
-test.skip('local first new page', () => {
+test.describe('local first new page', () => {
   test('click btn bew page and open in tab', async ({ page }) => {
     await newPage(page);
     await page.getByPlaceholder('Title').click();

tests/local-first-restore-page.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage } from './libs/page-logic.js';
 loadPage();
 
-test.skip('Local first delete page', () => {
+test.describe('Local first delete page', () => {
   test('New a page , then delete it in all pages, restore it', async ({
     page,
   }) => {

tests/local-first-show-delete-modal.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage, clickPageMoreActions } from './libs/page-logic.js';
 loadPage();
 
-test.skip('Local first delete page', () => {
+test.describe('Local first delete page', () => {
   test('New a page ,then open it and show delete modal', async ({ page }) => {
     await newPage(page);
     await page.getByPlaceholder('Title').click();

tests/local-first-trash-page.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { newPage } from './libs/page-logic.js';
 loadPage();
 
-test.skip('Local first trash page', () => {
+test.describe('Local first trash page', () => {
   test('New a page , then delete it in all pages, finally find it in trash', async ({
     page,
   }) => {

tests/local-first-workspace.spec.ts

@@ -1,4 +1,5 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();

tests/login.spec.ts

@@ -1,9 +1,9 @@
-import { test, expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();
 
-test.skip('Login Flow', () => {
+test.describe('Login Flow', () => {
   test.skip('Open login modal by click current workspace', async ({ page }) => {
     await page.getByTestId('current-workspace').click();
     await page.waitForTimeout(800);
@@ -15,7 +15,7 @@ test.skip('Login Flow', () => {
       .click();
   });
 
-  test('Open login modal by click cloud-unsync-icon', async ({ page }) => {
+  test.skip('Open login modal by click cloud-unsync-icon', async ({ page }) => {
     await page.getByTestId('cloud-unsync-icon').click();
 
     await page.waitForTimeout(800);

tests/quick-search.spec.ts

@@ -1,4 +1,5 @@
-import { test, expect, type Page } from '@playwright/test';
+import { expect, type Page } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 import { withCtrlOrMeta } from './libs/keyboard.js';
 import { newPage } from './libs/page-logic.js';
@@ -21,7 +22,7 @@ async function assertResultList(page: Page, texts: string[]) {
   expect(actual).toEqual(texts);
 }
 
-test.skip('Open quick search', () => {
+test.describe('Open quick search', () => {
   test('Click slider bar button', async ({ page }) => {
     await newPage(page);
     const quickSearchButton = page.locator(
@@ -50,7 +51,7 @@ test.skip('Open quick search', () => {
   });
 });
 
-test.skip('Add new page in quick search', () => {
+test.describe('Add new page in quick search', () => {
   // FIXME: not working
   test('Create a new page without keyword', async ({ page }) => {
     await newPage(page);
@@ -72,7 +73,7 @@ test.skip('Add new page in quick search', () => {
   });
 });
 
-test.skip('Search and select', () => {
+test.describe('Search and select', () => {
   test('Create a new page and search this page', async ({ page }) => {
     await newPage(page);
     await openQuickSearchByShortcut(page);

tests/shortcuts.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();
 
-test.skip('Shortcuts Modal', () => {
+test.describe('Shortcuts Modal', () => {
   test('Open shortcuts modal', async ({ page }) => {
     const faqIcon = page.locator('[data-testid=faq-icon]');
     const box = await faqIcon.boundingBox();

tests/theme.spec.ts

@@ -1,9 +1,10 @@
-import { test, expect } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { test } from './libs/playwright.js';
 import { loadPage } from './libs/load-page.js';
 
 loadPage();
 
-test.skip('Change Theme', () => {
+test.describe('Change Theme', () => {
   test('default white', async ({ page }) => {
     await page.waitForSelector('html');
     const root = page.locator('html');