diff --git a/.github/workflows/ecr_build_push.yml b/.github/workflows/ecr_build_push.yml new file mode 100644 index 00000000..b411cd8f --- /dev/null +++ b/.github/workflows/ecr_build_push.yml @@ -0,0 +1,55 @@ +name: Deploy to production + +on: + push: + branches: [ main ] + +jobs: + + deploy: + name: Build image + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Install kubectl + uses: azure/setup-kubectl@v1 + with: + version: 'v1.21.3' + id: install + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ap-southeast-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Load secrets and save to app.env + run: aws secretsmanager get-secret-value --secret-id simple_bank --query SecretString --output text | jq -r 'to_entries|map("\(.key)=\(.value)")|.[]' > app.env + + - name: Build, tag, and push image to Amazon ECR + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: selen_frappe + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:latest . + docker push -a $ECR_REGISTRY/$ECR_REPOSITORY + + - name: Update kube config + run: aws eks update-kubeconfig --name seleneks --region ap-southeast-1 + + - name: Deploy image to Amazon EKS + run: | + kubectl apply -f eks/aws-auth.yaml + kubectl apply -f eks/deployment.yaml + kubectl apply -f eks/service.yaml + kubectl apply -f eks/issuer.yaml + kubectl apply -f eks/ingress.yaml diff --git a/build_push_image.sh b/build_push_image.sh new file mode 100755 index 00000000..d7160681 --- /dev/null +++ b/build_push_image.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +source ./ci/build.env + +export APPS_JSON_BASE64=$(base64 -w 0 ./ci/apps.json) + +TAG="${1:-selen_frappe:latest}" + +aws ecr get-login-password --region ap-southeast-1 | docker login --username AWS --password-stdin 119186933498.dkr.ecr.ap-southeast-1.amazonaws.com + +docker build \ + --build-arg FRAPPE_PATH="$FRAPPE_PATH" \ + --build-arg FRAPPE_BRANCH="$FRAPPE_BRANCH" \ + --build-arg PYTHON_VERSION="$PYTHON_VERSION" \ + --build-arg NODE_VERSION="$NODE_VERSION" \ + --build-arg APPS_JSON_BASE64="$APPS_JSON_BASE64" \ + --tag "$TAG"\ + ./ci + +docker tag ${TAG} 119186933498.dkr.ecr.ap-southeast-1.amazonaws.com/selen_frappe:latest + +docker push 119186933498.dkr.ecr.ap-southeast-1.amazonaws.com/selen_frappe:latest diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 00000000..e9200833 --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,149 @@ +ARG PYTHON_VERSION=3.11.4 +ARG DEBIAN_BASE=bookworm +FROM python:${PYTHON_VERSION}-slim-${DEBIAN_BASE} AS base + +COPY resources/nginx-template.conf /templates/nginx/frappe.conf.template +COPY resources/nginx-entrypoint.sh /usr/local/bin/nginx-entrypoint.sh + +ARG WKHTMLTOPDF_VERSION=0.12.6.1-3 +ARG WKHTMLTOPDF_DISTRO=bookworm +ARG NODE_VERSION=18.16.1 +ENV NVM_DIR=/home/frappe/.nvm +ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} + +RUN useradd -ms /bin/bash frappe \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + curl \ + git \ + vim \ + nginx \ + gettext-base \ + # weasyprint dependencies + libpango-1.0-0 \ + libharfbuzz0b \ + libpangoft2-1.0-0 \ + libpangocairo-1.0-0 \ + # For backups + restic \ + # MariaDB + mariadb-client \ + # Postgres + libpq-dev \ + postgresql-client \ + # For healthcheck + wait-for-it \ + jq \ + # NodeJS + && mkdir -p ${NVM_DIR} \ + && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.2/install.sh | bash \ + && . ${NVM_DIR}/nvm.sh \ + && nvm install ${NODE_VERSION} \ + && nvm use v${NODE_VERSION} \ + && npm install -g yarn \ + && nvm alias default v${NODE_VERSION} \ + && rm -rf ${NVM_DIR}/.cache \ + && echo 'export NVM_DIR="/home/frappe/.nvm"' >>/home/frappe/.bashrc \ + && echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >>/home/frappe/.bashrc \ + && echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >>/home/frappe/.bashrc \ + # Install wkhtmltopdf with patched qt + && if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ + && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ + && downloaded_file=wkhtmltox_${WKHTMLTOPDF_VERSION}.${WKHTMLTOPDF_DISTRO}_${ARCH}.deb \ + && curl -sLO https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ + && apt-get install -y ./$downloaded_file \ + && rm $downloaded_file \ + # Clean up + && rm -rf /var/lib/apt/lists/* \ + && rm -fr /etc/nginx/sites-enabled/default \ + && pip3 install frappe-bench \ + # Fixes for non-root nginx and logs to stdout + && sed -i '/user www-data/d' /etc/nginx/nginx.conf \ + && ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log \ + && touch /run/nginx.pid \ + && chown -R frappe:frappe /etc/nginx/conf.d \ + && chown -R frappe:frappe /etc/nginx/nginx.conf \ + && chown -R frappe:frappe /var/log/nginx \ + && chown -R frappe:frappe /var/lib/nginx \ + && chown -R frappe:frappe /run/nginx.pid \ + && chmod 755 /usr/local/bin/nginx-entrypoint.sh \ + && chmod 644 /templates/nginx/frappe.conf.template + +FROM base AS builder + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + # For frappe framework + wget \ + # For psycopg2 + libpq-dev \ + # Other + libffi-dev \ + liblcms2-dev \ + libldap2-dev \ + libmariadb-dev \ + libsasl2-dev \ + libtiff5-dev \ + libwebp-dev \ + redis-tools \ + rlwrap \ + tk8.6-dev \ + cron \ + # For pandas + gcc \ + build-essential \ + libbz2-dev \ + && rm -rf /var/lib/apt/lists/* + +# apps.json includes +ARG APPS_JSON_BASE64 +RUN if [ -n "${APPS_JSON_BASE64}" ]; then \ + mkdir /opt/frappe && echo "${APPS_JSON_BASE64}" | base64 -d > /opt/frappe/apps.json; \ + fi + +USER frappe + +ARG FRAPPE_BRANCH=version-14 +ARG FRAPPE_PATH=https://github.com/frappe/frappe +RUN export APP_INSTALL_ARGS="" && \ + if [ -n "${APPS_JSON_BASE64}" ]; then \ + export APP_INSTALL_ARGS="--apps_path=/opt/frappe/apps.json"; \ + fi && \ + bench init ${APP_INSTALL_ARGS}\ + --frappe-branch=${FRAPPE_BRANCH} \ + --frappe-path=${FRAPPE_PATH} \ + --no-procfile \ + --no-backups \ + --skip-redis-config-generation \ + --verbose \ + /home/frappe/frappe-bench && \ + cd /home/frappe/frappe-bench && \ + echo "{}" > sites/common_site_config.json && \ + find apps -mindepth 1 -path "*/.git" | xargs rm -fr + +FROM base as backend + +USER frappe + +COPY --from=builder --chown=frappe:frappe /home/frappe/frappe-bench /home/frappe/frappe-bench + +WORKDIR /home/frappe/frappe-bench + +VOLUME [ \ + "/home/frappe/frappe-bench/sites", \ + "/home/frappe/frappe-bench/sites/assets", \ + "/home/frappe/frappe-bench/logs" \ +] + +CMD [ \ + "/home/frappe/frappe-bench/env/bin/gunicorn", \ + "--chdir=/home/frappe/frappe-bench/sites", \ + "--bind=0.0.0.0:8000", \ + "--threads=4", \ + "--workers=2", \ + "--worker-class=gthread", \ + "--worker-tmp-dir=/dev/shm", \ + "--timeout=120", \ + "--preload", \ + "frappe.app:application" \ +] diff --git a/ci/Readme.md b/ci/Readme.md new file mode 100644 index 00000000..abb88fd8 --- /dev/null +++ b/ci/Readme.md @@ -0,0 +1,23 @@ +# Steps + +- git clone frappe_docker +- cd frappe_docker +- export APPS_JSON_BASE64=$(base64 -w 0 ./ci/apps.json) +- cp ./images/custom/Containerfile ./ci/Dockerfile +- aws / ecr docker login (https://ap-southeast-1.console.aws.amazon.com/ecr/repositories?region=ap-southeast-1) +- run ./build_push_image.sh {TAG} or CI --> set workflow (use github_ci aws iam user) + +## AWS Links & Resources + +### Users + +- root +- selen_user +- github_ci / deployment group (AmazonEC2ContainerRegistryFullAccess) + +## Github + +### Secrets + +- AWS_ACCESS_KEY_ID +- AWS_SECRET_ACCESS_KEY diff --git a/ci/apps.json b/ci/apps.json new file mode 100644 index 00000000..9609a68f --- /dev/null +++ b/ci/apps.json @@ -0,0 +1,18 @@ +[ + { + "url": "https://github.com/frappe/erpnext.git", + "branch": "version-14" + }, + { + "url": "https://github.com/frappe/payments.git", + "branch": "version-14" + }, + { + "url": "https://github.com/frappe/helpdesk.git", + "branch": "main" + }, + { + "url": "https://github.com/frappe/hrms.git", + "branch": "version-14" + } +] diff --git a/ci/build.env b/ci/build.env new file mode 100644 index 00000000..4cc1b922 --- /dev/null +++ b/ci/build.env @@ -0,0 +1,8 @@ +PYTHON_VERSION=3.11.4 +DEBIAN_BASE=bookworm +WKHTMLTOPDF_VERSION=0.12.6.1-3 +WKHTMLTOPDF_DISTRO=bookworm +NODE_VERSION=16.20.1 +FRAPPE_BRANCH=v14.40.1 +FRAPPE_PATH=https://github.com/frappe/frappe +DOCKERFILE=Containerfile diff --git a/ci/resources/nginx-entrypoint.sh b/ci/resources/nginx-entrypoint.sh new file mode 100755 index 00000000..50408e56 --- /dev/null +++ b/ci/resources/nginx-entrypoint.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# Set variables that do not exist +if [[ -z "$BACKEND" ]]; then + echo "BACKEND defaulting to 0.0.0.0:8000" + export BACKEND=0.0.0.0:8000 +fi +if [[ -z "$SOCKETIO" ]]; then + echo "SOCKETIO defaulting to 0.0.0.0:9000" + export SOCKETIO=0.0.0.0:9000 +fi +if [[ -z "$UPSTREAM_REAL_IP_ADDRESS" ]]; then + echo "UPSTREAM_REAL_IP_ADDRESS defaulting to 127.0.0.1" + export UPSTREAM_REAL_IP_ADDRESS=127.0.0.1 +fi +if [[ -z "$UPSTREAM_REAL_IP_HEADER" ]]; then + echo "UPSTREAM_REAL_IP_HEADER defaulting to X-Forwarded-For" + export UPSTREAM_REAL_IP_HEADER=X-Forwarded-For +fi +if [[ -z "$UPSTREAM_REAL_IP_RECURSIVE" ]]; then + echo "UPSTREAM_REAL_IP_RECURSIVE defaulting to off" + export UPSTREAM_REAL_IP_RECURSIVE=off +fi +if [[ -z "$FRAPPE_SITE_NAME_HEADER" ]]; then + # shellcheck disable=SC2016 + echo 'FRAPPE_SITE_NAME_HEADER defaulting to $host' + # shellcheck disable=SC2016 + export FRAPPE_SITE_NAME_HEADER='$host' +fi + +if [[ -z "$PROXY_READ_TIMEOUT" ]]; then + echo "PROXY_READ_TIMEOUT defaulting to 120" + export PROXY_READ_TIMEOUT=120 +fi + +if [[ -z "$CLIENT_MAX_BODY_SIZE" ]]; then + echo "CLIENT_MAX_BODY_SIZE defaulting to 50m" + export CLIENT_MAX_BODY_SIZE=50m +fi + +# shellcheck disable=SC2016 +envsubst '${BACKEND} + ${SOCKETIO} + ${UPSTREAM_REAL_IP_ADDRESS} + ${UPSTREAM_REAL_IP_HEADER} + ${UPSTREAM_REAL_IP_RECURSIVE} + ${FRAPPE_SITE_NAME_HEADER} + ${PROXY_READ_TIMEOUT} + ${CLIENT_MAX_BODY_SIZE}' \ + /etc/nginx/conf.d/frappe.conf + +nginx -g 'daemon off;' diff --git a/ci/resources/nginx-template.conf b/ci/resources/nginx-template.conf new file mode 100644 index 00000000..944bce3c --- /dev/null +++ b/ci/resources/nginx-template.conf @@ -0,0 +1,114 @@ +upstream backend-server { + server ${BACKEND} fail_timeout=0; +} + +upstream socketio-server { + server ${SOCKETIO} fail_timeout=0; +} + +# Parse the X-Forwarded-Proto header - if set - defaulting to $scheme. +map $http_x_forwarded_proto $proxy_x_forwarded_proto { + default $scheme; + https https; +} + +server { + listen 8080; + server_name ${FRAPPE_SITE_NAME_HEADER}; + root /home/frappe/frappe-bench/sites; + + proxy_buffer_size 128k; + proxy_buffers 4 256k; + proxy_busy_buffers_size 256k; + + add_header X-Frame-Options "SAMEORIGIN"; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + add_header Referrer-Policy "same-origin, strict-origin-when-cross-origin"; + + set_real_ip_from ${UPSTREAM_REAL_IP_ADDRESS}; + real_ip_header ${UPSTREAM_REAL_IP_HEADER}; + real_ip_recursive ${UPSTREAM_REAL_IP_RECURSIVE}; + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /${FRAPPE_SITE_NAME_HEADER}/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER}; + proxy_set_header Origin $scheme://${FRAPPE_SITE_NAME_HEADER}; + proxy_set_header Host $host; + + proxy_pass http://socketio-server; + } + + location / { + rewrite ^(.+)/$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent; + rewrite ^(.+)/index\.html$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent; + rewrite ^(.+)\.html$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent; + + location ~ ^/files/.*.(htm|html|svg|xml) { + add_header Content-disposition "attachment"; + try_files /${FRAPPE_SITE_NAME_HEADER}/public/$uri @webserver; + } + + try_files /${FRAPPE_SITE_NAME_HEADER}/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER}; + proxy_set_header Host $host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout ${PROXY_READ_TIMEOUT}; + proxy_redirect off; + + proxy_pass http://backend-server; + } + + # optimizations + sendfile on; + keepalive_timeout 15; + client_max_body_size ${CLIENT_MAX_BODY_SIZE}; + client_body_buffer_size 16K; + client_header_buffer_size 1k; + + # enable gzip compression + # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge + gzip on; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component; + # text/html is always compressed by HttpGzipModule +} diff --git a/ci/version.txt b/ci/version.txt new file mode 100644 index 00000000..addd9e6e --- /dev/null +++ b/ci/version.txt @@ -0,0 +1 @@ +14.40.1