feat: merge service

.github/actions/deploy/deploy.mjs

@@ -33,24 +33,22 @@ const replicaConfig = {
   stable: {
     front: Number(process.env.PRODUCTION_FRONT_REPLICA) || 2,
     graphql: Number(process.env.PRODUCTION_GRAPHQL_REPLICA) || 2,
-    doc: Number(process.env.PRODUCTION_DOC_REPLICA) || 2,
   },
   beta: {
     front: Number(process.env.BETA_FRONT_REPLICA) || 1,
     graphql: Number(process.env.BETA_GRAPHQL_REPLICA) || 1,
-    doc: Number(process.env.BETA_DOC_REPLICA) || 1,
   },
-  canary: { front: 1, graphql: 1, doc: 1 },
+  canary: { front: 1, graphql: 1 },
 };
 
 const cpuConfig = {
-  beta: { front: '1', graphql: '1', doc: '1' },
+  beta: { front: '1', graphql: '1' },
-  canary: { front: '500m', graphql: '1', doc: '500m' },
+  canary: { front: '500m', graphql: '1' },
 };
 
 const memoryConfig = {
-  beta: { front: '1Gi', graphql: '1Gi', doc: '1Gi' },
+  beta: { front: '2Gi', graphql: '1Gi' },
-  canary: { front: '512Mi', graphql: '512Mi', doc: '512Mi' },
+  canary: { front: '512Mi', graphql: '512Mi' },
 };
 
 const createHelmCommand = ({ isDryRun }) => {
@@ -80,7 +78,6 @@ const createHelmCommand = ({ isDryRun }) => {
   const serviceAnnotations = [
     `--set-json   front.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
     `--set-json   graphql.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
-    `--set-json   doc.serviceAccount.annotations="{ \\"iam.gke.io/gcp-service-account\\": \\"${APP_IAM_ACCOUNT}\\" }"`,
   ].concat(
     isProduction || isBeta || isInternal
       ? [
@@ -98,7 +95,6 @@ const createHelmCommand = ({ isDryRun }) => {
     ? [
         `--set-json   front.nodeSelector="${spotNodeSelector}"`,
         `--set-json   graphql.nodeSelector="${spotNodeSelector}"`,
-        `--set-json   doc.nodeSelector="${spotNodeSelector}"`,
       ]
     : [];
 
@@ -109,14 +105,12 @@ const createHelmCommand = ({ isDryRun }) => {
     resources = resources.concat([
       `--set        front.resources.requests.cpu="${cpu.front}"`,
       `--set        graphql.resources.requests.cpu="${cpu.graphql}"`,
-      `--set        doc.resources.requests.cpu="${cpu.doc}"`,
     ]);
   }
   if (memory) {
     resources = resources.concat([
       `--set        front.resources.requests.memory="${memory.front}"`,
       `--set        graphql.resources.requests.memory="${memory.graphql}"`,
-      `--set        doc.resources.requests.memory="${memory.doc}"`,
     ]);
   }
 
@@ -155,9 +149,6 @@ const createHelmCommand = ({ isDryRun }) => {
     `--set        graphql.replicaCount=${replica.graphql}`,
     `--set-string graphql.image.tag="${imageTag}"`,
     `--set-string graphql.app.host="${primaryHost}"`,
-    `--set-string doc.image.tag="${imageTag}"`,
-    `--set-string doc.app.host="${primaryHost}"`,
-    `--set        doc.replicaCount=${replica.doc}`,
     ...serviceAnnotations,
     ...spotScheduling,
     ...resources,

.github/helm/affine/charts/doc/templates/NOTES.txt

@@ -1,16 +0,0 @@
-1. Get the application URL by running these commands:
-{{- if contains "NodePort" .Values.service.type }}
-  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "doc.fullname" . }})
-  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
-  echo http://$NODE_IP:$NODE_PORT
-{{- else if contains "LoadBalancer" .Values.service.type }}
-     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "doc.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "doc.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
-{{- else if contains "ClusterIP" .Values.service.type }}
-  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "doc.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
-  echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
-{{- end }}

.github/helm/affine/charts/doc/templates/_helpers.tpl

@@ -1,63 +0,0 @@
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "doc.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "doc.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/*
-Create chart name and version as used by the chart label.
-*/}}
-{{- define "doc.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
-
-{{/*
-Common labels
-*/}}
-{{- define "doc.labels" -}}
-helm.sh/chart: {{ include "doc.chart" . }}
-{{ include "doc.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-monitoring: enabled
-{{- end }}
-
-{{/*
-Selector labels
-*/}}
-{{- define "doc.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "doc.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "doc.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "doc.fullname" .) .Values.global.docService.name }}
-{{- else }}
-{{- default "default" .Values.global.docService.name }}
-{{- end }}
-{{- end }}

.github/helm/affine/charts/doc/templates/deployment.yaml

@@ -1,118 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{ include "doc.fullname" . }}
-  labels:
-    {{- include "doc.labels" . | nindent 4 }}
-spec:
-  replicas: {{ .Values.replicaCount }}
-  selector:
-    matchLabels:
-      {{- include "doc.selectorLabels" . | nindent 6 }}
-  template:
-    metadata:
-      {{- with .Values.podAnnotations }}
-      annotations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      labels:
-        {{- include "doc.selectorLabels" . | nindent 8 }}
-    spec:
-      {{- with .Values.imagePullSecrets }}
-      imagePullSecrets:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      serviceAccountName: {{ include "doc.serviceAccountName" . }}
-      containers:
-        - name: {{ .Chart.Name }}
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          env:
-          - name: AFFINE_PRIVATE_KEY
-            valueFrom:
-              secretKeyRef:
-                name: "{{ .Values.global.secret.secretName }}"
-                key: key
-          - name: NODE_ENV
-            value: "{{ .Values.env }}"
-          - name: NODE_OPTIONS
-            value: "--max-old-space-size=4096"
-          - name: NO_COLOR
-            value: "1"
-          - name: DEPLOYMENT_TYPE
-            value: "{{ .Values.global.deployment.type }}"
-          - name: DEPLOYMENT_PLATFORM
-            value: "{{ .Values.global.deployment.platform }}"
-          - name: SERVER_FLAVOR
-            value: "doc"
-          - name: AFFINE_ENV
-            value: "{{ .Release.Namespace }}"
-          - name: DATABASE_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: pg-postgresql
-                key: postgres-password
-          - name: DATABASE_URL
-            value: postgres://{{ .Values.global.database.user }}:$(DATABASE_PASSWORD)@{{ .Values.global.database.host }}:{{ .Values.global.database.port }}/{{ .Values.global.database.name }}
-          - name: REDIS_SERVER_ENABLED
-            value: "true"
-          - name: REDIS_SERVER_HOST
-            value: "{{ .Values.global.redis.host }}"
-          - name: REDIS_SERVER_PORT
-            value: "{{ .Values.global.redis.port }}"
-          - name: REDIS_SERVER_USER
-            value: "{{ .Values.global.redis.username }}"
-          - name: REDIS_SERVER_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: redis
-                key: redis-password
-          - name: REDIS_SERVER_DATABASE
-            value: "{{ .Values.global.redis.database }}"
-          - name: AFFINE_INDEXER_SEARCH_PROVIDER
-            value: "{{ .Values.global.indexer.provider }}"
-          - name: AFFINE_INDEXER_SEARCH_ENDPOINT
-            value: "{{ .Values.global.indexer.endpoint }}"
-          - name: AFFINE_INDEXER_SEARCH_API_KEY
-            valueFrom:
-              secretKeyRef:
-                name: indexer
-                key: indexer-apiKey
-          - name: AFFINE_SERVER_PORT
-            value: "{{ .Values.global.docService.port }}"
-          - name: AFFINE_SERVER_SUB_PATH
-            value: "{{ .Values.app.path }}"
-          - name: AFFINE_SERVER_HOST
-            value: "{{ .Values.app.host }}"
-          - name: AFFINE_SERVER_HTTPS
-            value: "{{ .Values.app.https }}"
-          ports:
-            - name: http
-              containerPort: {{ .Values.global.docService.port }}
-              protocol: TCP
-          livenessProbe:
-            httpGet:
-              path: /info
-              port: http
-            initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }}
-            timeoutSeconds: {{ .Values.probe.timeoutSeconds }}
-          readinessProbe:
-            httpGet:
-              path: /info
-              port: http
-            initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }}
-            timeoutSeconds: {{ .Values.probe.timeoutSeconds }}
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}

.github/helm/affine/charts/doc/templates/serviceaccount.yaml

@@ -1,12 +0,0 @@
-{{- if .Values.serviceAccount.create -}}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: {{ include "doc.serviceAccountName" . }}
-  labels:
-    {{- include "doc.labels" . | nindent 4 }}
-  {{- with .Values.serviceAccount.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-{{- end }}

.github/helm/affine/charts/doc/templates/tests/test-connection.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: "{{ include "doc.fullname" . }}-test-connection"
-  labels:
-    {{- include "doc.labels" . | nindent 4 }}
-  annotations:
-    "helm.sh/hook": test
-spec:
-  containers:
-    - name: wget
-      image: busybox
-      command: ['wget']
-      args: ['{{ include "doc.fullname" . }}:{{ .Values.global.docService.port }}']
-  restartPolicy: Never

.github/helm/affine/charts/front/templates/deployment.yaml

@@ -88,8 +88,6 @@ spec:
               value: "{{ .Values.app.host }}"
             - name: AFFINE_SERVER_HTTPS
               value: "{{ .Values.app.https }}"
-            - name: DOC_SERVICE_ENDPOINT
-              value: "http://{{ .Values.global.docService.name }}:{{ .Values.global.docService.port }}"
           ports:
             - name: http
               containerPort: {{ .Values.app.port }}

.github/helm/affine/charts/front/templates/service-doc.yaml

@@ -1,19 +1,19 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "doc.fullname" . }}
+  name: {{ .Values.global.docService.name }}
   labels:
-    {{- include "doc.labels" . | nindent 4 }}
+    {{- include "front.labels" . | nindent 4 }}
-  {{- with .Values.service.annotations }}
+  {{- with .Values.services.doc.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 spec:
-  type: {{ .Values.service.type }}
+  type: {{ .Values.services.doc.type }}
   ports:
     - port: {{ .Values.global.docService.port }}
       targetPort: http
       protocol: TCP
       name: http
   selector:
-    {{- include "doc.selectorLabels" . | nindent 4 }}
+    {{- include "front.selectorLabels" . | nindent 4 }}

.github/helm/affine/charts/front/values.yaml

@@ -57,6 +57,9 @@ services:
     type: ClusterIP
     port: 8080
     annotations: {}
+  doc:
+    type: ClusterIP
+    annotations: {}
 
 nodeSelector: {}
 tolerations: []

.github/helm/affine/values.yaml

@@ -47,12 +47,6 @@ graphql:
     annotations:
       cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'
 
-doc:
-  service:
-    type: ClusterIP
-    annotations:
-      cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'
-
 front:
   services:
     sync:
@@ -71,3 +65,7 @@ front:
       name: affine-web
       type: ClusterIP
       port: 8080
+    doc:
+      type: ClusterIP
+      annotations:
+        cloud.google.com/backend-config: '{"default": "affine-api-backendconfig"}'

packages/backend/server/src/__tests__/e2e/apps/flavors.spec.ts

@@ -1,5 +1,6 @@
 import { getCurrentUserQuery } from '@affine/graphql';
 
+import { DatabaseDocReader, DocReader } from '../../../core/doc';
 import { createApp } from '../create-app';
 import { e2e } from '../test';
 
@@ -52,4 +53,7 @@ e2e('should init front service', async t => {
 
   const res = await app.GET('/info').expect(200);
   t.is(res.body.flavor, 'front');
+
+  const docReader = app.get(DocReader);
+  t.true(docReader instanceof DatabaseDocReader);
 });

packages/backend/server/src/app.module.ts

@@ -159,8 +159,11 @@ export function buildAppModule(env: Env) {
     // basic
     .use(...FunctionalityModules)
 
-    // enable indexer module on graphql server and doc service
+    // enable indexer module on graphql, doc and front service
-    .useIf(() => env.flavors.graphql || env.flavors.doc, IndexerModule)
+    .useIf(
+      () => env.flavors.graphql || env.flavors.doc || env.flavors.front,
+      IndexerModule
+    )
 
     // auth
     .use(UserModule, AuthModule, PermissionModule)
@@ -202,8 +205,8 @@ export function buildAppModule(env: Env) {
       AccessTokenModule,
       QueueDashboardModule
     )
-    // doc service only
+    // doc service and front service
-    .useIf(() => env.flavors.doc, DocServiceModule)
+    .useIf(() => env.flavors.doc || env.flavors.front, DocServiceModule)
     // worker for and self-hosted API only for self-host and local development only
     .useIf(() => env.dev || env.selfhosted, WorkerModule, SelfhostModule)
     // static frontend routes for front flavor

packages/backend/server/src/base/job/queue/executor.ts

@@ -31,8 +31,8 @@ export class JobExecutor implements OnModuleDestroy {
       ? difference(QUEUES, [Queue.DOC, Queue.INDEXER])
       : [];
 
-    // NOTE(@forehalo): only enable doc queue in doc service
+    // Enable doc/indexer queues in both doc and front service.
-    if (env.flavors.doc) {
+    if (env.flavors.doc || env.flavors.front) {
       queues.push(Queue.DOC);
       // NOTE(@fengmk2): Once the index task cannot be processed in time, it needs to be separated from the doc service and deployed independently.
       queues.push(Queue.INDEXER);

packages/backend/server/src/core/doc/reader.ts

@@ -447,7 +447,7 @@ export class RpcDocReader extends DatabaseDocReader {
 export const DocReaderProvider: FactoryProvider = {
   provide: DocReader,
   useFactory: (ref: ModuleRef) => {
-    if (env.flavors.doc) {
+    if (env.flavors.doc || env.flavors.front) {
       return ref.create(DatabaseDocReader);
     }
     return ref.create(RpcDocReader);