diff --git a/.github/actions/deploy/deploy.mjs b/.github/actions/deploy/deploy.mjs index 126d3b111..204d8471a 100644 --- a/.github/actions/deploy/deploy.mjs +++ b/.github/actions/deploy/deploy.mjs @@ -28,6 +28,7 @@ const { REDIS_PASSWORD, STRIPE_API_KEY, STRIPE_WEBHOOK_KEY, + STATIC_IP_NAME, } = process.env; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -35,17 +36,13 @@ const buildType = BUILD_TYPE || 'canary'; const isProduction = buildType === 'stable'; const isBeta = buildType === 'beta'; +const isInternal = buildType === 'internal'; const createHelmCommand = ({ isDryRun }) => { const flag = isDryRun ? '--dry-run' : '--atomic'; const imageTag = `${buildType}-${GIT_SHORT_HASH}`; - const staticIpName = isProduction - ? 'affine-cluster-production' - : isBeta - ? 'affine-cluster-beta' - : 'affine-cluster-dev'; const redisAndPostgres = - isProduction || isBeta + isProduction || isBeta || isInternal ? [ `--set-string global.database.url=${DATABASE_URL}`, `--set-string global.database.user=${DATABASE_USERNAME}`, @@ -59,26 +56,32 @@ const createHelmCommand = ({ isDryRun }) => { ] : []; const serviceAnnotations = - isProduction || isBeta + isProduction || isBeta || isInternal ? [ `--set-json web.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`, - `--set-json graphql.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_SQL_IAM_ACCOUNT}\\" }\"`, `--set-json graphql.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`, - `--set-json sync.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_SQL_IAM_ACCOUNT}\\" }\"`, `--set-json sync.service.annotations=\"{ \\"cloud.google.com/neg\\": \\"{\\\\\\"ingress\\\\\\": true}\\" }\"`, + `--set-json cloud-sql-proxy.serviceAccount.annotations=\"{ \\"iam.gke.io/gcp-service-account\\": \\"${CLOUD_SQL_IAM_ACCOUNT}\\" }\"`, + `--set-json cloud-sql-proxy.nodeSelector=\"{ \\"iam.gke.io/gke-metadata-server-enabled\\": \\"true\\" }\"`, ] : []; const webReplicaCount = isProduction ? 3 : isBeta ? 2 : 2; const graphqlReplicaCount = isProduction ? 10 : isBeta ? 5 : 2; const syncReplicaCount = isProduction ? 10 : isBeta ? 5 : 2; - const namespace = isProduction ? 'production' : isBeta ? 'beta' : 'dev'; + const namespace = isProduction + ? 'production' + : isBeta + ? 'beta' + : isInternal + ? 'internal' + : 'dev'; // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const host = DEPLOY_HOST || CANARY_DEPLOY_HOST; const deployCommand = [ `helm upgrade --install affine .github/helm/affine`, `--namespace ${namespace}`, `--set global.ingress.enabled=true`, - `--set-json global.ingress.annotations=\"{ \\"kubernetes.io/ingress.class\\": \\"gce\\", \\"kubernetes.io/ingress.allow-http\\": \\"true\\", \\"kubernetes.io/ingress.global-static-ip-name\\": \\"${staticIpName}\\" }\"`, + `--set-json global.ingress.annotations=\"{ \\"kubernetes.io/ingress.class\\": \\"gce\\", \\"kubernetes.io/ingress.allow-http\\": \\"true\\", \\"kubernetes.io/ingress.global-static-ip-name\\": \\"${STATIC_IP_NAME}\\" }\"`, `--set-string global.ingress.host="${host}"`, `--set-string global.version="${APP_VERSION}"`, ...redisAndPostgres, diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/.helmignore b/.github/helm/affine/charts/gcloud-sql-proxy/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/Chart.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/Chart.yaml new file mode 100644 index 000000000..f1038c707 --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: cloud-sql-proxy +description: Google Cloud SQL Proxy +type: application +version: 0.0.0 +appVersion: "2.8.1" diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/NOTES.txt b/.github/helm/affine/charts/gcloud-sql-proxy/templates/NOTES.txt new file mode 100644 index 000000000..b7cb03936 --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/templates/NOTES.txt @@ -0,0 +1,18 @@ +{{- if .Values.global.database.gcloud.enabled -}} +1. Get the application URL by running these commands: +{{- if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "gcloud-sql-proxy.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "gcloud-sql-proxy.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "gcloud-sql-proxy.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "gcloud-sql-proxy.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/_helpers.tpl b/.github/helm/affine/charts/gcloud-sql-proxy/templates/_helpers.tpl new file mode 100644 index 000000000..8d447acbf --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "gcloud-sql-proxy.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "gcloud-sql-proxy.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "gcloud-sql-proxy.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "gcloud-sql-proxy.labels" -}} +helm.sh/chart: {{ include "gcloud-sql-proxy.chart" . }} +{{ include "gcloud-sql-proxy.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "gcloud-sql-proxy.selectorLabels" -}} +app.kubernetes.io/name: {{ include "gcloud-sql-proxy.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "gcloud-sql-proxy.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "gcloud-sql-proxy.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml new file mode 100644 index 000000000..26d783255 --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/templates/deployment.yaml @@ -0,0 +1,132 @@ +{{- if .Values.global.database.gcloud.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "gcloud-sql-proxy.fullname" . }} + labels: + {{- include "gcloud-sql-proxy.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "gcloud-sql-proxy.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "gcloud-sql-proxy.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "gcloud-sql-proxy.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "--address" + - "0.0.0.0" + - "--structured-logs" + - "--auto-iam-authn" + - "{{ .Values.global.database.gcloud.connectionName }}" + env: + # Enable HTTP healthchecks on port 9801. This enables /liveness, + # /readiness and /startup health check endpoints. Allow connections + # listen for connections on any interface (0.0.0.0) so that the + # k8s management components can reach these endpoints. + - name: CSQL_PROXY_HEALTH_CHECK + value: "true" + - name: CSQL_PROXY_HTTP_PORT + value: "9801" + - name: CSQL_PROXY_HTTP_ADDRESS + value: 0.0.0.0 + ports: + - name: cloud-sql-proxy + containerPort: {{ .Values.global.database.gcloud.proxyPort }} + protocol: TCP + - containerPort: 9801 + protocol: TCP + # The /startup probe returns OK when the proxy is ready to receive + # connections from the application. In this example, k8s will check + # once a second for 60 seconds. + startupProbe: + failureThreshold: 60 + httpGet: + path: /startup + port: 9801 + scheme: HTTP + periodSeconds: 1 + successThreshold: 1 + timeoutSeconds: 10 + # The /liveness probe returns OK as soon as the proxy application has + # begun its startup process and continues to return OK until the + # process stops. + livenessProbe: + failureThreshold: 3 + httpGet: + path: /liveness + port: 9801 + scheme: HTTP + # The probe will be checked every 10 seconds. + periodSeconds: 10 + # Number of times the probe is allowed to fail before the transition + # from healthy to failure state. + # + # If periodSeconds = 60, 5 tries will result in five minutes of + # checks. The proxy starts to refresh a certificate five minutes + # before its expiration. If those five minutes lapse without a + # successful refresh, the liveness probe will fail and the pod will be + # restarted. + successThreshold: 1 + # The probe will fail if it does not respond in 10 seconds + timeoutSeconds: 10 + readinessProbe: + # The /readiness probe returns OK when the proxy can establish + # a new connections to its databases. + httpGet: + path: /readiness + port: 9801 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 + # Number of times the probe must report success to transition from failure to healthy state. + # Defaults to 1 for readiness probe. + successThreshold: 1 + failureThreshold: 6 + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/service.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/service.yaml new file mode 100644 index 000000000..b7a0c96cf --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/templates/service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.global.database.gcloud.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "gcloud-sql-proxy.fullname" . }} + labels: + {{- include "gcloud-sql-proxy.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.global.database.port }} + targetPort: cloud-sql-proxy + protocol: TCP + name: cloud-sql-proxy + selector: + {{- include "gcloud-sql-proxy.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/serviceaccount.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/serviceaccount.yaml new file mode 100644 index 000000000..5e9bfa2ad --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +{{- if .Values.global.database.gcloud.enabled -}} +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "gcloud-sql-proxy.serviceAccountName" . }} + labels: + {{- include "gcloud-sql-proxy.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} +{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/templates/tests/test-connection.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/templates/tests/test-connection.yaml new file mode 100644 index 000000000..c6138cc29 --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/templates/tests/test-connection.yaml @@ -0,0 +1,17 @@ +{{- if .Values.global.database.gcloud.enabled -}} +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "gcloud-sql-proxy.fullname" . }}-test-connection" + labels: + {{- include "gcloud-sql-proxy.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "gcloud-sql-proxy.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never +{{- end }} diff --git a/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml b/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml new file mode 100644 index 000000000..947365fe3 --- /dev/null +++ b/.github/helm/affine/charts/gcloud-sql-proxy/values.yaml @@ -0,0 +1,40 @@ +replicaCount: 3 + +image: + # the tag is defined as chart appVersion. + repository: gcr.io/cloud-sql-connectors/cloud-sql-proxy + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + automount: true + annotations: {} + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: + fsGroup: 2000 + +securityContext: + runAsNonRoot: true + +service: + type: ClusterIP + port: 5432 + +resources: + limits: + memory: "4Gi" + cpu: "2" + +volumes: [] +volumeMounts: [] +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/.github/helm/affine/charts/graphql/Chart.yaml b/.github/helm/affine/charts/graphql/Chart.yaml index 8a66ffa1a..04d1c6315 100644 --- a/.github/helm/affine/charts/graphql/Chart.yaml +++ b/.github/helm/affine/charts/graphql/Chart.yaml @@ -4,3 +4,8 @@ description: AFFiNE GraphQL server type: application version: 0.0.0 appVersion: "0.10.3-canary.2" +dependencies: + - name: gcloud-sql-proxy + version: 0.0.0 + repository: "file://../gcloud-sql-proxy" + condition: .global.database.gcloud.enabled diff --git a/.github/helm/affine/charts/graphql/templates/deployment.yaml b/.github/helm/affine/charts/graphql/templates/deployment.yaml index 593824db6..be1473098 100644 --- a/.github/helm/affine/charts/graphql/templates/deployment.yaml +++ b/.github/helm/affine/charts/graphql/templates/deployment.yaml @@ -189,20 +189,6 @@ spec: initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{ if .Values.global.database.gcloud.enabled }} - - name: cloud-sql-proxy - image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.6.0 - args: - - "--structured-logs" - - "--auto-iam-authn" - - "{{ .Values.global.database.gcloud.connectionName }}" - securityContext: - runAsNonRoot: true - resources: - requests: - memory: "2Gi" - cpu: "1" - {{ end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/.github/helm/affine/charts/sync/Chart.yaml b/.github/helm/affine/charts/sync/Chart.yaml index 071318a56..2b0721ad5 100644 --- a/.github/helm/affine/charts/sync/Chart.yaml +++ b/.github/helm/affine/charts/sync/Chart.yaml @@ -1,6 +1,11 @@ apiVersion: v2 name: sync -description: A Helm chart for Kubernetes +description: AFFiNE Sync Server type: application version: 0.0.0 appVersion: "0.10.3-canary.2" +dependencies: + - name: gcloud-sql-proxy + version: 0.0.0 + repository: "file://../gcloud-sql-proxy" + condition: .global.database.gcloud.enabled diff --git a/.github/helm/affine/charts/sync/templates/deployment.yaml b/.github/helm/affine/charts/sync/templates/deployment.yaml index f1b6c4a99..f7e06cce7 100644 --- a/.github/helm/affine/charts/sync/templates/deployment.yaml +++ b/.github/helm/affine/charts/sync/templates/deployment.yaml @@ -82,20 +82,6 @@ spec: initialDelaySeconds: {{ .Values.probe.initialDelaySeconds }} resources: {{- toYaml .Values.resources | nindent 12 }} - {{ if .Values.global.database.gcloud.enabled }} - - name: cloud-sql-proxy - image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.6.0 - args: - - "--structured-logs" - - "--auto-iam-authn" - - "{{ .Values.global.database.gcloud.connectionName }}" - securityContext: - runAsNonRoot: true - resources: - requests: - memory: "2Gi" - cpu: "1" - {{ end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/.github/helm/affine/values.yaml b/.github/helm/affine/values.yaml index b54513091..7326f7029 100644 --- a/.github/helm/affine/values.yaml +++ b/.github/helm/affine/values.yaml @@ -16,6 +16,8 @@ global: cloudSqlInternal: '' connectionName: '' serviceAccount: '' + cloudProxyReplicas: 3 + proxyPort: '5432' redis: enabled: true host: 'redis-master' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 52e03ae9c..d0ec8394a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: flavor: - description: 'Build type (canary, beta, or stable)' + description: 'Build type (canary, beta, or stable or internal)' type: string default: canary @@ -249,3 +249,4 @@ jobs: CLOUD_SQL_IAM_ACCOUNT: ${{ secrets.CLOUD_SQL_IAM_ACCOUNT }} STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }} STRIPE_WEBHOOK_KEY: ${{ secrets.STRIPE_WEBHOOK_KEY }} + STATIC_IP_NAME: ${{ secrets.STATIC_IP_NAME }} diff --git a/.github/workflows/workers.yml b/.github/workflows/workers.yml index b51fc5092..f079df253 100644 --- a/.github/workflows/workers.yml +++ b/.github/workflows/workers.yml @@ -11,7 +11,7 @@ jobs: deploy: runs-on: ubuntu-latest name: Deploy - environment: production + environment: stable steps: - uses: actions/checkout@v4 - name: Publish