From f4d0affd44cb9df250add9c145cd77981b4388b6 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 19 Apr 2020 23:14:17 +0530 Subject: [PATCH] add: Kubernetes Installation README --- installation/kubernetes/README.md | 222 ++++++++++++++++++ .../erpnext/templates/deployment-erpnext.yaml | 6 +- .../templates/deployment-worker-default.yaml | 4 - .../templates/deployment-worker-long.yaml | 4 - .../templates/deployment-worker-short.yaml | 4 - .../erpnext/templates/job-migrate-sites.yaml | 26 ++ .../helm-charts/erpnext/templates/pvc.yaml | 2 +- .../helm-charts/erpnext/values.yaml | 21 +- ....template => backupsitesjob.yaml.template} | 9 +- ...site-job.sh => create-backup-sites-job.sh} | 7 +- ...ite-job.sh => create-migrate-sites-job.sh} | 7 +- .../resources/create-new-site-ingress.sh | 4 +- ...template => migratesitesjob.yaml.template} | 10 +- .../resources/newsiteingress.yaml.template | 4 +- .../resources/newsitejob.yaml.template | 5 +- 15 files changed, 274 insertions(+), 61 deletions(-) create mode 100644 installation/kubernetes/README.md create mode 100644 installation/kubernetes/helm-charts/erpnext/templates/job-migrate-sites.yaml rename installation/kubernetes/resources/{backupsitejob.yaml.template => backupsitesjob.yaml.template} (73%) rename installation/kubernetes/resources/{create-backup-site-job.sh => create-backup-sites-job.sh} (59%) rename installation/kubernetes/resources/{create-migrate-site-job.sh => create-migrate-sites-job.sh} (59%) rename installation/kubernetes/resources/{migratesitejob.yaml.template => migratesitesjob.yaml.template} (69%) diff --git a/installation/kubernetes/README.md b/installation/kubernetes/README.md new file mode 100644 index 00000000..c36a0fc3 --- /dev/null +++ b/installation/kubernetes/README.md @@ -0,0 +1,222 @@ +## Prerequisites + +- Access to Kubernetes cluster. +- [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +- [helm 3](https://helm.sh/) + +## Install Ingress Controller + +You can use Ingress Controller of your choice. +During Creation of new ingress, cert-manager annotations are used. + +```shell +kubectl create namespace nginx-ingress +helm repo add nginx-stable https://helm.nginx.com/stable +helm repo update +helm install --namespace nginx-ingress nginx-controller nginx-stable/nginx-ingress +``` + +Notes: + +- If apps from cluster need to access other apps hosted on same cluster by domain name, set `service.spec.externalTrafficPolicy` to `Cluster`. [Read More](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip) +- Once LoadBalancer Service is up, set Wildcard entry in your DNS Configuration + +## Install Cert Manager + +Cert Manager can be used to automate Letsencrypt certificate management. +During Creation of new ingress, cert-manager annotations are used. + +[Installation](https://cert-manager.io/docs/installation/kubernetes/) +[Configure Issuer](https://cert-manager.io/docs/installation/kubernetes/#configuring-your-first-issuer) + +## Prepare MariaDB + +MariaDB options : +- Host separately (access by Private IP) +- Use managed service (e.g. AWS RDS) +- Install mariadb on kubernetes cluster + +### Install MariaDB Helm chart + +Download and edit values.yaml for frappe related mariadb config. + +``` +wget -c https://raw.githubusercontent.com/bitnami/charts/master/bitnami/mariadb/values-production.yaml + +# Use editor of choice +code values-production.yaml +``` + +Set `rootUser.password` and `replication.password`. + +```yaml +rootUser: + password: super_secret_password + +replication: + password: super_secret_password +``` + +Change `master.config` as follows: + +```yaml + config: |- + [mysqld] + character-set-client-handshake=FALSE + skip-name-resolve + explicit_defaults_for_timestamp + basedir=/opt/bitnami/mariadb + plugin_dir=/opt/bitnami/mariadb/plugin + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + tmpdir=/opt/bitnami/mariadb/tmp + max_allowed_packet=16M + bind-address=0.0.0.0 + pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid + log-error=/opt/bitnami/mariadb/logs/mysqld.log + character-set-server=utf8mb4 + collation-server=utf8mb4_unicode_ci + + [client] + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + default-character-set=utf8mb4 + plugin_dir=/opt/bitnami/mariadb/plugin + + [manager] + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid +``` + +Change `slave.config` as follows: + +```yaml + config: |- + [mysqld] + character-set-client-handshake=FALSE + skip-name-resolve + explicit_defaults_for_timestamp + basedir=/opt/bitnami/mariadb + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + tmpdir=/opt/bitnami/mariadb/tmp + max_allowed_packet=16M + bind-address=0.0.0.0 + pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid + log-error=/opt/bitnami/mariadb/logs/mysqld.log + character-set-server=utf8mb4 + collation-server=utf8mb4_unicode_ci + + [client] + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + default-character-set=utf8mb4 + + [manager] + port=3306 + socket=/opt/bitnami/mariadb/tmp/mysql.sock + pid-file=/opt/bitnami/mariadb/tmp/mysqld.pid +``` + +Create namespace and Install Helm Chart + +```shell +kubectl create namespace mariadb + +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo update +helm install -n mariadb bitnami/mariadb -f values-production.yaml +``` + +## Prepare Shared Filesystem + +Options are: + +- [NFS](https://github.com/helm/charts/tree/master/stable/nfs-server-provisioner), recommended for small cluster +- Rook/Ceph, [Hyper-converged infrastructure](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure) + - [Quickstart](https://rook.io/docs/rook/v1.3/ceph-quickstart.html) + - [Shared Filesystem](https://rook.io/docs/rook/v1.3/ceph-filesystem.html) + +Note: After preparing storage, we get a `storageClass` which has `ReadWriteMany` `accessMode` available. e.g. `nfs` or `rook-cephfs` + +## Install Frappe/ERPNext Helm Chart + +```shell +kubectl create namespace erpnext +helm repo add erpnext https://helm.erpnext.com/repo +helm repo update + +helm install frappe-bench-0001 --namespace erpnext-v12 erpnext \ + --set mariadbHost=mariadb.mariadb.svc.cluster.local \ + --set persistence.storageClass=rook-cephfs +``` + +## Site Operations + +Following scripts take environment variables and generate a YAML file. +Generated YAML file can be modified as per need. + +### Create MariaDB Root Password Secret + +Generate Root Password. Export environment variable `BASE64_PASSWORD` and set it to base64 encoded mariadb root password. + +```shell +# In case mariadb helm chart is installed +export BASE64_PASSWORD=$(kubectl get secret --namespace mariadb mariadb -o jsonpath="{.data.mariadb-root-password}") + +./create-mariadb-root-password-secret.sh + +kubectl -n erpnext apply -f mariadbrootpasswordsecret.yaml +``` + +### Create New Site + +``` +export SITE_NAME=mysite.example.com +export DB_ROOT_USER=root +export ADMIN_PASSWORD=$(cat /tmp/site_admin_password) +export SITES_PVC=erpnext-v12 +export VERSION=v12 + +./create-new-site-job.sh + +kubectl -n erpnext apply -f newsitejob-mysite.example.com-1587301207.yaml +``` + +Note: Site admin password is set in `/tmp/site_admin_password` file. + +### Create New Ingress + +```shell +export SITE_NAME=mysite.example.com +export INGRESS_NAME=$SITE_NAME +export FRAPPE_SERVICE=erpnext-v12 +export TLS_SECRET_NAME=mysite-example-com-tls + +./create-new-site-ingress.sh + +kubectl -n erpnext apply -f newsiteingress_mysite.example.com.yaml +``` + +### Backup New Site + +```shell +export SITES_PVC=erpnext-v12 +export VERSION=v12 + +./create-backup-sites-job.sh + +kubectl -n erpnext apply -f backupsitesjob-1587303964.yaml +``` + +### Migrate Sites + +```shell +export SITES_PVC=erpnext-v12 +export VERSION=v12 + +./create-migrate-sites-job.sh + +kubectl -n erpnext apply -f migratesitesjob-1587306818.yaml +``` diff --git a/installation/kubernetes/helm-charts/erpnext/templates/deployment-erpnext.yaml b/installation/kubernetes/helm-charts/erpnext/templates/deployment-erpnext.yaml index d9c94d32..9ab4aad3 100644 --- a/installation/kubernetes/helm-charts/erpnext/templates/deployment-erpnext.yaml +++ b/installation/kubernetes/helm-charts/erpnext/templates/deployment-erpnext.yaml @@ -60,12 +60,8 @@ spec: mountPath: /home/frappe/frappe-bench/sites imagePullPolicy: {{ .Values.pythonImage.pullPolicy }} env: - {{ if .Values.runAsRoot }} - - name: "RUN_AS_ROOT" - value: "1" - {{ end }} - name: "MARIADB_HOST" - value: {{ .Values.mariadbHost }} + value: {{ required "A valid .Values.mariadbHost entry required!" .Values.mariadbHost }} - name: "REDIS_QUEUE" value: {{ include "erpnext.fullname" . }}-redis-queue:{{ .Values.redisQueueService.port }} - name: "REDIS_CACHE" diff --git a/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-default.yaml b/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-default.yaml index 1fd2b5cb..7c89b1d4 100644 --- a/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-default.yaml +++ b/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-default.yaml @@ -35,10 +35,6 @@ spec: command: ["docker-entrypoint.sh"] args: ["worker"] env: - {{ if .Values.runAsRoot }} - - name: "RUN_AS_ROOT" - value: "1" - {{ end }} - name: "WORKER_TYPE" value: "default" livenessProbe: diff --git a/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-long.yaml b/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-long.yaml index ae74eebb..de178c82 100644 --- a/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-long.yaml +++ b/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-long.yaml @@ -37,10 +37,6 @@ spec: env: - name: "WORKER_TYPE" value: "long" - {{ if .Values.runAsRoot }} - - name: "RUN_AS_ROOT" - value: "1" - {{ end }} livenessProbe: exec: command: diff --git a/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-short.yaml b/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-short.yaml index d315631e..ebe02fc2 100644 --- a/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-short.yaml +++ b/installation/kubernetes/helm-charts/erpnext/templates/deployment-worker-short.yaml @@ -37,10 +37,6 @@ spec: env: - name: "WORKER_TYPE" value: "short" - {{ if .Values.runAsRoot }} - - name: "RUN_AS_ROOT" - value: "1" - {{ end }} livenessProbe: exec: command: diff --git a/installation/kubernetes/helm-charts/erpnext/templates/job-migrate-sites.yaml b/installation/kubernetes/helm-charts/erpnext/templates/job-migrate-sites.yaml new file mode 100644 index 00000000..acbcfe84 --- /dev/null +++ b/installation/kubernetes/helm-charts/erpnext/templates/job-migrate-sites.yaml @@ -0,0 +1,26 @@ +{{ if .Values.migrateJob }} +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-sites-1587307712 +spec: + backoffLimit: 1 + template: + spec: + securityContext: + supplementalGroups: [1000] + containers: + - name: erpnext + image: frappe/erpnext-worker:v12 + command: ["migrate"] + imagePullPolicy: IfNotPresent + volumeMounts: + - name: sites-dir + mountPath: /home/frappe/frappe-bench/sites + restartPolicy: Never + volumes: + - name: sites-dir + persistentVolumeClaim: + claimName: erpnext-v12 + readOnly: false +{{ end }} diff --git a/installation/kubernetes/helm-charts/erpnext/templates/pvc.yaml b/installation/kubernetes/helm-charts/erpnext/templates/pvc.yaml index 2cbfadf1..0deadd25 100644 --- a/installation/kubernetes/helm-charts/erpnext/templates/pvc.yaml +++ b/installation/kubernetes/helm-charts/erpnext/templates/pvc.yaml @@ -14,5 +14,5 @@ spec: resources: requests: storage: {{ .Values.persistence.size | quote }} - storageClassName: {{ .Values.persistence.storageClass }} + storageClassName: {{ required "A valid .Values.persistence.storageClass entry required!" .Values.persistence.storageClass }} {{- end }} diff --git a/installation/kubernetes/helm-charts/erpnext/values.yaml b/installation/kubernetes/helm-charts/erpnext/values.yaml index 230e1f02..86c506ab 100644 --- a/installation/kubernetes/helm-charts/erpnext/values.yaml +++ b/installation/kubernetes/helm-charts/erpnext/values.yaml @@ -7,24 +7,27 @@ replicaCount: 1 nginxImage: repository: frappe/erpnext-nginx tag: edge - pullPolicy: Always + pullPolicy: IfNotPresent pythonImage: repository: frappe/erpnext-worker tag: edge - pullPolicy: Always + pullPolicy: IfNotPresent socketIOImage: repository: frappe/frappe-socketio tag: edge - pullPolicy: Always + pullPolicy: IfNotPresent # Asset Image Env Variables frappePyPort: "8000" socketIOPort: "9000" # Python Image Env Variables -mariadbHost: "mariadb.mariadb.svc.cluster.local" +# mariadbHost: "mariadb.mariadb.svc.cluster.local" + +# Set this to true to run migrate as part of helm install/upgrade +migrateJob: false imagePullSecrets: [] nameOverride: "" @@ -75,15 +78,11 @@ redisCacheService: persistence: enabled: true # existingClaim: "" - size: 10Gi - # storageClass: "csi-s3" - # storageClass: "standard" - storageClass: "nfs" + size: 2Gi + # storageClass: "nfs" resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following + # If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: # cpu: 100m diff --git a/installation/kubernetes/resources/backupsitejob.yaml.template b/installation/kubernetes/resources/backupsitesjob.yaml.template similarity index 73% rename from installation/kubernetes/resources/backupsitejob.yaml.template rename to installation/kubernetes/resources/backupsitesjob.yaml.template index 7f5a714f..9ec2293f 100644 --- a/installation/kubernetes/resources/backupsitejob.yaml.template +++ b/installation/kubernetes/resources/backupsitesjob.yaml.template @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: backup-site-${SITE_NAME}-${TIMESTAMP} + name: backup-sites-${TIMESTAMP} spec: backoffLimit: 1 template: @@ -11,14 +11,11 @@ spec: containers: - name: erpnext image: frappe/erpnext-worker:${VERSION} - command: ["docker-entrypoint.sh"] - args: ["backup"] - imagePullPolicy: Always + command: "backup" + imagePullPolicy: IfNotPresent env: - name: "WITH_FILES" value: "1" - - name: "SITES" - value: "${SITE_NAME}" volumeMounts: - name: sites-dir mountPath: /home/frappe/frappe-bench/sites diff --git a/installation/kubernetes/resources/create-backup-site-job.sh b/installation/kubernetes/resources/create-backup-sites-job.sh similarity index 59% rename from installation/kubernetes/resources/create-backup-site-job.sh rename to installation/kubernetes/resources/create-backup-sites-job.sh index e8e581e2..d472010e 100755 --- a/installation/kubernetes/resources/create-backup-site-job.sh +++ b/installation/kubernetes/resources/create-backup-sites-job.sh @@ -1,10 +1,6 @@ #!/bin/bash set -e -if [[ -z "$SITE_NAME" ]]; then - echo "SITE_NAME is not set" - exit 1 -fi if [[ -z "$SITES_PVC" ]]; then echo "SITES_PVC is not set" exit 1 @@ -17,7 +13,6 @@ fi export TIMESTAMP=$(date +%s) envsubst '${TIMESTAMP} - ${SITE_NAME} ${VERSION} ${SITES_PVC}' \ - < ./backupsitejob.yaml.template > backupsitejob-$SITE_NAME-$TIMESTAMP.yaml + < ./backupsitesjob.yaml.template > backupsitesjob-$TIMESTAMP.yaml diff --git a/installation/kubernetes/resources/create-migrate-site-job.sh b/installation/kubernetes/resources/create-migrate-sites-job.sh similarity index 59% rename from installation/kubernetes/resources/create-migrate-site-job.sh rename to installation/kubernetes/resources/create-migrate-sites-job.sh index 21c65f41..314b1300 100755 --- a/installation/kubernetes/resources/create-migrate-site-job.sh +++ b/installation/kubernetes/resources/create-migrate-sites-job.sh @@ -1,10 +1,6 @@ #!/bin/bash set -e -if [[ -z "$SITE_NAME" ]]; then - echo "SITE_NAME is not set" - exit 1 -fi if [[ -z "$SITES_PVC" ]]; then echo "SITES_PVC is not set" exit 1 @@ -17,7 +13,6 @@ fi export TIMESTAMP=$(date +%s) envsubst '${TIMESTAMP} - ${SITE_NAME} ${VERSION} ${SITES_PVC}' \ - < ./migratesitejob.yaml.template > migratesitejob-$SITE_NAME-$TIMESTAMP.yaml + < ./migratesitesjob.yaml.template > migratesitesjob-$TIMESTAMP.yaml diff --git a/installation/kubernetes/resources/create-new-site-ingress.sh b/installation/kubernetes/resources/create-new-site-ingress.sh index 75c0eaa8..f5e31c59 100755 --- a/installation/kubernetes/resources/create-new-site-ingress.sh +++ b/installation/kubernetes/resources/create-new-site-ingress.sh @@ -5,8 +5,8 @@ if [[ -z "$INGRESS_NAME" ]]; then echo "INGRESS_NAME is not set" exit 1 fi -if [[ -z "$ERPNEXT_SERVICE" ]]; then - echo "ERPNEXT_SERVICE is not set" +if [[ -z "$FRAPPE_SERVICE" ]]; then + echo "FRAPPE_SERVICE is not set" exit 1 fi if [[ -z "$SITE_NAME" ]]; then diff --git a/installation/kubernetes/resources/migratesitejob.yaml.template b/installation/kubernetes/resources/migratesitesjob.yaml.template similarity index 69% rename from installation/kubernetes/resources/migratesitejob.yaml.template rename to installation/kubernetes/resources/migratesitesjob.yaml.template index 1809df36..a77b67fe 100644 --- a/installation/kubernetes/resources/migratesitejob.yaml.template +++ b/installation/kubernetes/resources/migratesitesjob.yaml.template @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: migrate-site-${SITE_NAME}-${TIMESTAMP} + name: migrate-sites-${TIMESTAMP} spec: backoffLimit: 1 template: @@ -11,12 +11,8 @@ spec: containers: - name: erpnext image: frappe/erpnext-worker:${VERSION} - command: ["docker-entrypoint.sh"] - args: ["migrate"] - imagePullPolicy: Always - env: - - name: "SITES" - value: "${SITE_NAME}" + command: "migrate" + imagePullPolicy: IfNotPresent volumeMounts: - name: sites-dir mountPath: /home/frappe/frappe-bench/sites diff --git a/installation/kubernetes/resources/newsiteingress.yaml.template b/installation/kubernetes/resources/newsiteingress.yaml.template index 916488d9..5f216efc 100644 --- a/installation/kubernetes/resources/newsiteingress.yaml.template +++ b/installation/kubernetes/resources/newsiteingress.yaml.template @@ -4,7 +4,7 @@ metadata: name: ${INGRESS_NAME} # Optional Labels labels: - app.kubernetes.io/instance: ${ERPNEXT_SERVICE} + app.kubernetes.io/instance: ${FRAPPE_SERVICE} annotations: # required for cert-manager letsencrypt cert-manager.io/cluster-issuer: letsencrypt-prod @@ -17,7 +17,7 @@ spec: http: paths: - backend: - serviceName: ${ERPNEXT_SERVICE} + serviceName: ${FRAPPE_SERVICE} servicePort: 80 path: / tls: diff --git a/installation/kubernetes/resources/newsitejob.yaml.template b/installation/kubernetes/resources/newsitejob.yaml.template index 1ecb5eaa..78a4499e 100644 --- a/installation/kubernetes/resources/newsitejob.yaml.template +++ b/installation/kubernetes/resources/newsitejob.yaml.template @@ -11,9 +11,8 @@ spec: containers: - name: erpnext image: frappe/erpnext-worker:${VERSION} - command: ["docker-entrypoint.sh"] - args: ["new"] - imagePullPolicy: Always + command: "new" + imagePullPolicy: IfNotPresent volumeMounts: - name: sites-dir mountPath: /home/frappe/frappe-bench/sites