From 6204941a5d0d6963369752f81c56a469a32d7c3b Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 4 Jan 2021 18:30:24 +0530 Subject: [PATCH 01/13] feat(frappe-nginx): configurable nginx http timeout --- build/common/nginx-default.conf.template | 2 +- build/frappe-nginx/docker-entrypoint.sh | 11 +++++++---- docs/environment-variables.md | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) create mode 100644 docs/environment-variables.md diff --git a/build/common/nginx-default.conf.template b/build/common/nginx-default.conf.template index 09575391..2008e655 100644 --- a/build/common/nginx-default.conf.template +++ b/build/common/nginx-default.conf.template @@ -55,7 +55,7 @@ server { proxy_set_header X-Frappe-Site-Name $host; proxy_set_header Host $http_host; proxy_set_header X-Use-X-Accel-Redirect True; - proxy_read_timeout 120; + proxy_read_timeout ${HTTP_TIMEOUT}; proxy_redirect off; proxy_pass http://frappe-server; diff --git a/build/frappe-nginx/docker-entrypoint.sh b/build/frappe-nginx/docker-entrypoint.sh index 69431038..5d905bdf 100755 --- a/build/frappe-nginx/docker-entrypoint.sh +++ b/build/frappe-nginx/docker-entrypoint.sh @@ -31,12 +31,15 @@ if [[ -z "$SOCKETIO_PORT" ]]; then export SOCKETIO_PORT=9000 fi -envsubst '${API_HOST} - ${API_PORT} - ${FRAPPE_PY} +if [[ -z "$HTTP_TIMEOUT" ]]; then + export HTTP_TIMEOUT=120 +fi + +envsubst '${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} - ${SOCKETIO_PORT}' \ + ${SOCKETIO_PORT} + ${HTTP_TIMEOUT}' \ < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf echo "Waiting for frappe-python to be available on $FRAPPE_PY port $FRAPPE_PY_PORT" diff --git a/docs/environment-variables.md b/docs/environment-variables.md new file mode 100644 index 00000000..943db4fc --- /dev/null +++ b/docs/environment-variables.md @@ -0,0 +1,22 @@ +List of environment variables for containers + +### frappe-worker and erpnext-worker + +Following environment variables are set into sites volume as `common_site_config.json`. It means once the file is created in volume, the variables won't have any effect, you need to edit the common_site_config.json to update the configuration + +- `DB_HOST`: MariaDB host, domain name or ip address. +- `DB_PORT`: MariaDB port. +- `REDIS_CACHE`: Redis cache host, domain name or ip address. +- `REDIS_QUEUE`: Redis queue host, domain name or ip address. +- `REDIS_SOCKETIO`: Redis queue host, domain name or ip address. +- `SOCKETIO_PORT: `: Port on which the SocketIO should start. + +### frappe-nginx and erpnext-nginx + +These variables are set on every container start. Change in these variables will reflect on every container start. + +- `FRAPPE_PY`: Gunicorn host to reverse proxy. Default: 0.0.0.0 +- `FRAPPE_PY_PORT`: Gunicorn port to reverse proxy. Default: 8000 +- `FRAPPE_SOCKETIO`: SocketIO host to reverse proxy. Default: 0.0.0.0 +- `SOCKETIO_PORT`: SocketIO port to reverse proxy. Default: 9000 +- `HTTP_TIMEOUT`: Nginx http timeout. Default: 120 From 092348d7489c86207e16851d9cd9afdceebb1b2d Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 4 Jan 2021 18:45:42 +0530 Subject: [PATCH 02/13] docs(frappe-nginx): configurable nginx http timeout [skip travis] --- README.md | 5 +++-- docs/environment-variables.md | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6b980d81..3a47895e 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,9 @@ It takes care of the following: 1. [Docker Swarm](docs/docker-swarm.md) 2. [Kubernetes](https://helm.erpnext.com) 3. [Site Operations](docs/site-operations.md) -4. [Custom apps for production](docs/custom-apps-for-production.md) -5. [Tips for moving deployments](docs/tips-for-moving-deployments.md) +4. [Environment Variables](docs/environment-variables.md) +5. [Custom apps for production](docs/custom-apps-for-production.md) +6. [Tips for moving deployments](docs/tips-for-moving-deployments.md) ## Development Setup diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 943db4fc..3aa2e7cc 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -20,3 +20,7 @@ These variables are set on every container start. Change in these variables will - `FRAPPE_SOCKETIO`: SocketIO host to reverse proxy. Default: 0.0.0.0 - `SOCKETIO_PORT`: SocketIO port to reverse proxy. Default: 9000 - `HTTP_TIMEOUT`: Nginx http timeout. Default: 120 + +### frappe-socketio + +This container takes configuration from `common_site_config.json` which is already created by erpnext gunicorn container. It doesn't use any environment variables. From 35dd53cc6f9c210083dce4cee0e1e71675899b2e Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 4 Jan 2021 18:52:52 +0530 Subject: [PATCH 03/13] fix(frappe-worker): allow non-root password to not expire old image tags seems to stop working if password is set to expire fixes https://github.com/frappe/helm/issues/58 --- build/frappe-worker/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 948836cf..6eaa9c95 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -1,6 +1,9 @@ FROM bitnami/python:3.7-prod -RUN useradd -ms /bin/bash frappe +# Add non root user without password +RUN sed -i 's/^PASS_MAX_DAYS.*/PASS_MAX_DAYS -1/' /etc/login.defs \ + && sed -i 's/^PASS_MIN_DAYS.*/PASS_MIN_DAYS -1/' /etc/login.defs \ + && useradd -ms /bin/bash frappe ARG GIT_BRANCH=develop ENV PYTHONUNBUFFERED 1 From 062eea917a55259d8cb8ea1d89b0069550d14143 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 4 Jan 2021 18:54:11 +0530 Subject: [PATCH 04/13] update: nodejs lts to latest minor and patch version --- build/bench/Dockerfile | 4 ++-- build/frappe-nginx/Dockerfile | 2 +- build/frappe-worker/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index 920f9cc2..98fb9cde 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -85,8 +85,8 @@ RUN bash -c "bench --version" # https://nodejs.org/download/release/latest-v10.x/ # https://nodejs.org/download/release/latest-v12.x/ # https://nodejs.org/download/release/latest-v13.x/ -ENV NODE_VERSION=12.18.2 -ENV NODE_VERSION_FRAPPEV11=10.21.0 +ENV NODE_VERSION=12.20.0 +ENV NODE_VERSION_FRAPPEV11=10.23.0 # Install nvm with node RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index 191d72bf..9e23fa29 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -5,7 +5,7 @@ FROM bitnami/python:3.7-prod ENV NVM_DIR=/root/.nvm -ENV NODE_VERSION=12.18.3 +ENV NODE_VERSION=12.20.0 ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" RUN install_packages wget \ && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 6eaa9c95..77777d58 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -8,7 +8,7 @@ RUN sed -i 's/^PASS_MAX_DAYS.*/PASS_MAX_DAYS -1/' /etc/login.defs \ ARG GIT_BRANCH=develop ENV PYTHONUNBUFFERED 1 ENV NVM_DIR=/home/frappe/.nvm -ENV NODE_VERSION=12.18.3 +ENV NODE_VERSION=12.20.0 ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" # Install dependencies From 4f904fe3b925a6e1c8006085e70b0e81fcb6d58f Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 5 Jan 2021 14:47:01 +0530 Subject: [PATCH 05/13] feat: make images arm64 compatible --- build/bench/Dockerfile | 10 ++++++---- build/erpnext-nginx/Dockerfile | 11 ++++++++--- build/erpnext-nginx/install_app.sh | 2 -- build/frappe-nginx/Dockerfile | 11 ++++++----- build/frappe-worker/Dockerfile | 20 ++++++++++++-------- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index 98fb9cde..f4c1f705 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -1,8 +1,8 @@ # Frappe Bench Dockerfile -FROM bitnami/minideb:latest +FROM debian:stable-slim LABEL author=frappé -RUN install_packages \ +RUN apt-get update -y && apt-get install \ git \ wkhtmltopdf \ mariadb-client \ @@ -49,7 +49,9 @@ RUN install_packages \ python3-setuptools \ python3-tk \ python-virtualenv \ - less + less -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ && dpkg-reconfigure --frontend=noninteractive locales @@ -89,7 +91,7 @@ ENV NODE_VERSION=12.20.0 ENV NODE_VERSION_FRAPPEV11=10.23.0 # Install nvm with node -RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh +RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh RUN chmod +x install.sh RUN ./install.sh ENV NVM_DIR=/home/frappe/.nvm diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx/Dockerfile index 72488ab7..6394acaa 100644 --- a/build/erpnext-nginx/Dockerfile +++ b/build/erpnext-nginx/Dockerfile @@ -1,9 +1,14 @@ -ARG NODE_IMAGE_TAG=12-prod +ARG NODE_IMAGE_TAG=12-buster-slim ARG GIT_BRANCH=develop -FROM bitnami/node:${NODE_IMAGE_TAG} +FROM node:${NODE_IMAGE_TAG} ARG GIT_BRANCH -COPY build/erpnext-nginx/install_app.sh /install_app +COPY install_app.sh /install_app +RUN chmod +x /install_app && \ + apt-get update -y && \ + apt-get install build-essential git python2 -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* RUN /install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh index 7af233d2..f4928b2d 100755 --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx/install_app.sh @@ -10,8 +10,6 @@ mkdir -p /home/frappe/frappe-bench/sites/assets cd /home/frappe/frappe-bench echo -e "frappe\n${APP_NAME}" > /home/frappe/frappe-bench/sites/apps.txt -install_packages git python2 - mkdir -p apps cd apps git clone --depth 1 https://github.com/frappe/frappe ${BRANCH} diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index 9e23fa29..161a5357 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -2,13 +2,16 @@ # This is done to ensures that node-sass binary remains common. # node-sass is required to enable website theme feature used # by Website Manager role in Frappe Framework -FROM bitnami/python:3.7-prod +FROM python:3.7-slim-buster ENV NVM_DIR=/root/.nvm ENV NODE_VERSION=12.20.0 ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" -RUN install_packages wget \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ +RUN apt-get update -y \ + && apt-get install wget python2 git build-essential -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ && chmod +x install.sh \ && ./install.sh \ && . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} \ @@ -18,8 +21,6 @@ WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt -RUN install_packages git - ARG GIT_BRANCH=develop RUN mkdir -p apps sites/assets/css \ && cd apps \ diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 77777d58..56c4e522 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -1,4 +1,4 @@ -FROM bitnami/python:3.7-prod +FROM python:3.7-slim-buster # Add non root user without password RUN sed -i 's/^PASS_MAX_DAYS.*/PASS_MAX_DAYS -1/' /etc/login.defs \ @@ -13,28 +13,32 @@ ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" # Install dependencies WORKDIR /home/frappe/frappe-bench -RUN install_packages \ +RUN apt-get update -y && apt-get install \ git \ mariadb-client \ postgresql-client \ gettext-base \ wget \ # for PDF + fonts-cantarell \ + libffi-dev \ + libjpeg-dev \ libjpeg62-turbo \ - libx11-6 \ libxcb1 \ libxext6 \ + libx11-6 \ + libxml2 \ libxrender1 \ libssl-dev \ - fonts-cantarell \ xfonts-75dpi \ xfonts-base \ + zlib1g-dev \ # For psycopg2 libpq-dev \ - wait-for-it \ - && wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb \ - && dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ + wait-for-it -y \ + && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_arm64.deb \ + && dpkg -i wkhtmltox_0.12.6-1.buster_arm64.deb && rm wkhtmltox_0.12.6-1.buster_arm64.deb \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ && apt-get purge -y wget && apt-get autoremove -y \ && chown -R frappe:frappe /home/frappe From ed60f76e7844d87fe098cd86e8399c38f0f91f44 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 5 Jan 2021 17:47:06 +0530 Subject: [PATCH 06/13] fix: travis.py for node image tag --- travis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis.py b/travis.py index f8224053..0602c156 100755 --- a/travis.py +++ b/travis.py @@ -41,7 +41,7 @@ def git_version(service, version, branch): def build(service, tag, image, branch): build_args = f'--build-arg GIT_BRANCH={branch}' if service == 'nginx' and branch == 'version-11': - build_args += f' --build-arg NODE_IMAGE_TAG=10-prod' + build_args += f' --build-arg NODE_IMAGE_TAG=10-buster-slim' print(f'Building {service} {image} image') subprocess.run(f'docker build {build_args} -t {service}-{image} -f build/{service}-{image}/Dockerfile .', shell=True) From 117de9d4b8d4ce16361cc0a10df00a5c7f121232 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 5 Jan 2021 17:48:11 +0530 Subject: [PATCH 07/13] fix(erpnext-nginx): COPY path for install_app.sh --- build/erpnext-nginx/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx/Dockerfile index 6394acaa..13f1c2de 100644 --- a/build/erpnext-nginx/Dockerfile +++ b/build/erpnext-nginx/Dockerfile @@ -3,7 +3,7 @@ ARG GIT_BRANCH=develop FROM node:${NODE_IMAGE_TAG} ARG GIT_BRANCH -COPY install_app.sh /install_app +COPY build/erpnext-nginx/install_app.sh /install_app RUN chmod +x /install_app && \ apt-get update -y && \ apt-get install build-essential git python2 -y && \ From e55d7a8c6a16b29e56060414fa78a13923c8947f Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 5 Jan 2021 17:48:54 +0530 Subject: [PATCH 08/13] fix(frappe-worker): default to amd64 as arch --- build/frappe-worker/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 56c4e522..14d6a266 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -6,6 +6,7 @@ RUN sed -i 's/^PASS_MAX_DAYS.*/PASS_MAX_DAYS -1/' /etc/login.defs \ && useradd -ms /bin/bash frappe ARG GIT_BRANCH=develop +ARG ARCH=amd64 ENV PYTHONUNBUFFERED 1 ENV NVM_DIR=/home/frappe/.nvm ENV NODE_VERSION=12.20.0 @@ -19,6 +20,7 @@ RUN apt-get update -y && apt-get install \ postgresql-client \ gettext-base \ wget \ + curl \ # for PDF fonts-cantarell \ libffi-dev \ @@ -36,8 +38,8 @@ RUN apt-get update -y && apt-get install \ # For psycopg2 libpq-dev \ wait-for-it -y \ - && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_arm64.deb \ - && dpkg -i wkhtmltox_0.12.6-1.buster_arm64.deb && rm wkhtmltox_0.12.6-1.buster_arm64.deb \ + && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ + && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ && apt-get purge -y wget && apt-get autoremove -y \ && chown -R frappe:frappe /home/frappe From 94f667caebac5cf75ed4cff2a27a712fb7b72894 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 5 Jan 2021 22:02:00 +0530 Subject: [PATCH 09/13] fix(frappe-worker): add gcc required for arm64 env --- build/frappe-worker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 14d6a266..0294364d 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -37,7 +37,9 @@ RUN apt-get update -y && apt-get install \ zlib1g-dev \ # For psycopg2 libpq-dev \ - wait-for-it -y \ + wait-for-it \ + # For arm64 + gcc -y \ && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ From 948e1bf00af0f88b3e58ed83d764a9ca4f341344 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 6 Jan 2021 00:14:25 +0530 Subject: [PATCH 10/13] ci: build multi arch images --- .travis.yml | 15 ++++++++++++++- build/frappe-worker/Dockerfile | 2 ++ travis.py | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc5b100b..404ad47a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,12 +11,25 @@ services: - docker before_install: + # Install latest docker (https://www.docker.com/blog/multi-arch-build-what-about-travis) + - sudo rm -rf /var/lib/apt/lists/* + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge" + - sudo apt-get update + - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + - mkdir -vp ~/.docker/cli-plugins/ + - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64" > ~/.docker/cli-plugins/docker-buildx + - chmod a+x ~/.docker/cli-plugins/docker-buildx + - docker buildx create --use + - if [[ "$BUILD" != "development" ]]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin; sudo apt-get update && sudo apt-get -y install git; fi - if [[ $BUILD == "development" ]];then - sudo apt-get update && sudo apt-get -y install docker-compose; + sudo apt-get update && sudo apt-get -y install curl; + sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose; + sudo chmod +x /usr/local/bin/docker-compose; fi - chmod u+x ./travis.py diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 0294364d..1aae32d0 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -40,6 +40,8 @@ RUN apt-get update -y && apt-get install \ wait-for-it \ # For arm64 gcc -y \ + && if [ `uname -m` = 'aarch64' ]; then export ARCH=arm64; fi \ + && if [ `uname -m` = 'x86_64' ]; then export ARCH=amd64; fi \ && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ diff --git a/travis.py b/travis.py index 0602c156..f2c1c784 100755 --- a/travis.py +++ b/travis.py @@ -44,7 +44,7 @@ def build(service, tag, image, branch): build_args += f' --build-arg NODE_IMAGE_TAG=10-buster-slim' print(f'Building {service} {image} image') - subprocess.run(f'docker build {build_args} -t {service}-{image} -f build/{service}-{image}/Dockerfile .', shell=True) + subprocess.run(f'docker buildx build --platform linux/amd64,linux/arm64 --load {build_args} -t {service}-{image} -f build/{service}-{image}/Dockerfile .', shell=True) tag_and_push(f'{service}-{image}', tag) From 8d2c139fc62bdcf4edde06cc9a21b931e82b567f Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 7 Jan 2021 19:37:46 +0530 Subject: [PATCH 11/13] ci: use docker buildx in travis --- .travis.yml | 283 ++++++++++++++++++++--------- build/bench/Dockerfile | 10 +- build/erpnext-nginx/Dockerfile | 3 +- build/erpnext-nginx/install_app.sh | 1 - build/erpnext-worker/Dockerfile | 3 +- build/frappe-nginx/Dockerfile | 1 - build/frappe-socketio/Dockerfile | 2 +- build/frappe-worker/Dockerfile | 10 +- travis.py | 72 -------- 9 files changed, 215 insertions(+), 170 deletions(-) delete mode 100755 travis.py diff --git a/.travis.yml b/.travis.yml index 404ad47a..8a9f2558 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ before_install: - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64" > ~/.docker/cli-plugins/docker-buildx - chmod a+x ~/.docker/cli-plugins/docker-buildx - docker buildx create --use + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - docker buildx inspect --bootstrap - if [[ "$BUILD" != "development" ]]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin; @@ -31,7 +33,6 @@ before_install: sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose; sudo chmod +x /usr/local/bin/docker-compose; fi - - chmod u+x ./travis.py after_success: - docker --version @@ -41,128 +42,244 @@ jobs: - stage: "Build Frappe bench development environment (latest)" if: branch = develop AND type != pull_request script: - - docker build -t frappe/bench:latest -f build/bench/Dockerfile . - - docker push frappe/bench:latest - - stage: "Frappe (edge)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/bench:latest + -f build/bench/Dockerfile . + - stage: "Frappe Worker (edge)" if: branch = develop AND type != pull_request script: - - ./travis.py frappe --worker --tag latest - - ./travis.py frappe --worker --tag edge --tag-only - - ./travis.py frappe --worker --tag develop --tag-only - - stage: "Frappe (edge)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:edge + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:develop + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:latest + -f build/frappe-worker/Dockerfile . + - stage: "Frappe Nginx (edge)" if: branch = develop AND type != pull_request script: - - ./travis.py frappe --nginx --tag latest - - ./travis.py frappe --nginx --tag edge --tag-only - - ./travis.py frappe --nginx --tag develop --tag-only - - stage: "Frappe (edge)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:edge + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:develop + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:latest + -f build/frappe-nginx/Dockerfile . + - stage: "Frappe SocketIO (edge)" if: branch = develop AND type != pull_request script: - - ./travis.py frappe --socketio --tag latest - - ./travis.py frappe --socketio --tag edge --tag-only - - ./travis.py frappe --socketio --tag develop --tag-only - - stage: "ERPNext (edge)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:edge + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:develop + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:latest + -f build/frappe-socketio/Dockerfile . + - stage: "ERPNext Worker (edge)" if: branch = develop AND type != pull_request script: - - ./travis.py erpnext --worker --tag latest - - ./travis.py erpnext --worker --tag edge --tag-only - - ./travis.py erpnext --worker --tag develop --tag-only - - stage: "ERPNext (edge)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:edge + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:develop + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:latest + -f build/erpnext-worker/Dockerfile . + - stage: "ERPNext Nginx (edge)" if: branch = develop AND type != pull_request script: - - ./travis.py erpnext --nginx --tag latest - - ./travis.py erpnext --nginx --tag edge --tag-only - - ./travis.py erpnext --nginx --tag develop --tag-only - - stage: "Frappe (v13-beta)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:edge + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:develop + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:latest + -f build/erpnext-nginx/Dockerfile . + - stage: "Frappe Worker (v13-beta)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --worker --git-version 13-beta - - ./travis.py frappe --worker --tag v13-beta --tag-only - - ./travis.py frappe --worker --tag version-13-beta --tag-only - - stage: "Frappe (v13-beta)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-13-beta + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:v13 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:${VERSION} + -f build/frappe-worker/Dockerfile . + - stage: "Frappe Nginx (v13-beta)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --nginx --git-version 13-beta - - ./travis.py frappe --nginx --tag v13-beta --tag-only - - ./travis.py frappe --nginx --tag version-13-beta --tag-only - - stage: "Frappe (v13-beta)" + - > + docker buildx build + --no-cache + --build-arg GIT_BRANCH=version-13-beta + --platform linux/amd64,linux/arm64 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:v13 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:${VERSION} + -f build/frappe-nginx/Dockerfile . + - stage: "Frappe SocketIO (v13-beta)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --socketio --git-version 13-beta - - ./travis.py frappe --socketio --tag v13-beta --tag-only - - ./travis.py frappe --socketio --tag version-13-beta --tag-only - - stage: "ERPNext (v13-beta)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-13-beta + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:v13 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:${VERSION} + -f build/frappe-socketio/Dockerfile . + - stage: "ERPNext Worker (v13-beta)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --worker --git-version 13-beta - - ./travis.py erpnext --worker --tag v13-beta --tag-only - - ./travis.py erpnext --worker --tag version-13-beta --tag-only - - stage: "ERPNext (v13-beta)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-13-beta + --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:v13 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:${VERSION} + -f build/erpnext-worker/Dockerfile . + - stage: "ERPNext Nginx (v13-beta)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --nginx --git-version 13-beta - - ./travis.py erpnext --nginx --tag v13-beta --tag-only - - ./travis.py erpnext --nginx --tag version-13-beta --tag-only - - stage: "Frappe (v12)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-13-beta + --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:v13 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:${VERSION} + -f build/erpnext-nginx/Dockerfile . + - stage: "Frappe Worker (v12)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --worker --git-version 12 - - ./travis.py frappe --worker --tag v12 --tag-only - - ./travis.py frappe --worker --tag version-12 --tag-only - - stage: "Frappe (v12)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-12 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:v12 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:${VERSION} + -f build/frappe-worker/Dockerfile . + - stage: "Frappe Nginx (v12)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --nginx --git-version 12 - - ./travis.py frappe --nginx --tag v12 --tag-only - - ./travis.py frappe --nginx --tag version-12 --tag-only - - stage: "Frappe (v12)" + - > + docker buildx build + --no-cache + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-12 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:v12 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:${VERSION} + -f build/frappe-nginx/Dockerfile . + - stage: "Frappe SocketIO (v12)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --socketio --git-version 12 - - ./travis.py frappe --socketio --tag v12 --tag-only - - ./travis.py frappe --socketio --tag version-12 --tag-only - - stage: "ERPNext (v12)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-12 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:v12 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:${VERSION} + -f build/frappe-socketio/Dockerfile . + - stage: "ERPNext Worker (v12)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --worker --git-version 12 - - ./travis.py erpnext --worker --tag v12 --tag-only - - ./travis.py erpnext --worker --tag version-12 --tag-only - - stage: "ERPNext (v12)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-12 + --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:v12 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:${VERSION} + -f build/erpnext-worker/Dockerfile . + - stage: "ERPNext Nginx (v12)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --nginx --git-version 12 - - ./travis.py erpnext --nginx --tag v12 --tag-only - - ./travis.py erpnext --nginx --tag version-12 --tag-only - - stage: "Frappe (v11)" + - docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-12 + --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:v12 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:${VERSION} + -f build/erpnext-nginx/Dockerfile . + - stage: "Frappe Worker (v11)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --worker --git-version 11 - - ./travis.py frappe --worker --tag v11 --tag-only - - ./travis.py frappe --worker --tag version-11 --tag-only - - stage: "Frappe (v11)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-11 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:v11 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:${VERSION} + -f build/frappe-worker/Dockerfile . + - stage: "Frappe Nginx (v11)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --nginx --git-version 11 - - ./travis.py frappe --nginx --tag v11 --tag-only - - ./travis.py frappe --nginx --tag version-11 --tag-only - - stage: "Frappe (v11)" + - > + docker buildx build + --no-cache + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-11 + --build-arg NODE_IMAGE_TAG=10-buster-slim + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:v11 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:${VERSION} + -f build/frappe-nginx/Dockerfile . + - stage: "Frappe SocketIO (v11)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --socketio --git-version 11 - - ./travis.py frappe --socketio --tag v11 --tag-only - - ./travis.py frappe --socketio --tag version-11 --tag-only - - stage: "ERPNext (v11)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-11 + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:v11 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:${VERSION} + -f build/frappe-socketio/Dockerfile . + - stage: "ERPNext Worker (v11)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --worker --git-version 11 - - ./travis.py erpnext --worker --tag v11 --tag-only - - ./travis.py erpnext --worker --tag version-11 --tag-only - - stage: "ERPNext (v11)" + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-11 + --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:v11 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:${VERSION} + -f build/erpnext-worker/Dockerfile . + - stage: "ERPNext Nginx (v11)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --nginx --git-version 11 - - ./travis.py erpnext --nginx --tag v11 --tag-only - - ./travis.py erpnext --nginx --tag version-11 --tag-only + - > + docker buildx build + --platform linux/amd64,linux/arm64 + --build-arg GIT_BRANCH=version-11 + --build-arg NODE_IMAGE_TAG=10-buster-slim + --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} + --push + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:v11 + -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:${VERSION} + -f build/erpnext-nginx/Dockerfile . - stage: "Build and test edge images" if: type = pull_request before_install: diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index f4c1f705..d185fadb 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -1,5 +1,5 @@ # Frappe Bench Dockerfile -FROM debian:stable-slim +FROM debian:buster-slim LABEL author=frappé RUN apt-get update -y && apt-get install \ @@ -56,9 +56,11 @@ RUN apt-get update -y && apt-get install \ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ && dpkg-reconfigure --frontend=noninteractive locales -# Install wkhtmltox correctly -RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb -RUN dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb +# Detect arch, download and install wkhtmltox +RUN if [ `uname -m` = 'aarch64' ]; then export ARCH=arm64; fi \ + && if [ `uname -m` = 'x86_64' ]; then export ARCH=amd64; fi \ + && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ + && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb # Create new user with home directory, improve docker compatibility with UID/GID 1000, add user to sudo group, allow passwordless sudo, switch to that user and change directory to user home directory RUN groupadd -g 1000 frappe diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx/Dockerfile index 13f1c2de..61bfa49d 100644 --- a/build/erpnext-nginx/Dockerfile +++ b/build/erpnext-nginx/Dockerfile @@ -1,5 +1,6 @@ ARG NODE_IMAGE_TAG=12-buster-slim ARG GIT_BRANCH=develop +ARG DOCKER_REGISTRY_PREFIX=frappe FROM node:${NODE_IMAGE_TAG} ARG GIT_BRANCH @@ -12,7 +13,7 @@ RUN chmod +x /install_app && \ RUN /install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} -FROM frappe/frappe-nginx:${GIT_BRANCH} +FROM ${DOCKER_REGISTRY_PREFIX}/frappe-nginx:${GIT_BRANCH} COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ COPY --from=0 /rsync /rsync diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh index f4928b2d..f886ced1 100755 --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx/install_app.sh @@ -18,7 +18,6 @@ git clone --depth 1 ${APP_REPO} ${BRANCH} ${APP_NAME} cd /home/frappe/frappe-bench/apps/frappe yarn yarn production --app ${APP_NAME} -rm -fr node_modules yarn install --production=true yarn add node-sass diff --git a/build/erpnext-worker/Dockerfile b/build/erpnext-worker/Dockerfile index 90aad213..ef8718ca 100644 --- a/build/erpnext-worker/Dockerfile +++ b/build/erpnext-worker/Dockerfile @@ -1,5 +1,6 @@ ARG GIT_BRANCH=develop -FROM frappe/frappe-worker:${GIT_BRANCH} +ARG DOCKER_REGISTRY_PREFIX=frappe +FROM ${DOCKER_REGISTRY_PREFIX}/frappe-worker:${GIT_BRANCH} ARG GIT_BRANCH RUN install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index 161a5357..11b02b02 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -29,7 +29,6 @@ RUN mkdir -p apps sites/assets/css \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && yarn \ && yarn run production \ - && rm -fr node_modules \ && yarn install --production=true RUN node --version \ diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile index 9455dbd7..1157a469 100644 --- a/build/frappe-socketio/Dockerfile +++ b/build/frappe-socketio/Dockerfile @@ -1,4 +1,4 @@ -FROM node:slim +FROM node:buster-slim # Add frappe user RUN useradd -ms /bin/bash frappe diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 1aae32d0..7a8131d6 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -1,9 +1,7 @@ FROM python:3.7-slim-buster # Add non root user without password -RUN sed -i 's/^PASS_MAX_DAYS.*/PASS_MAX_DAYS -1/' /etc/login.defs \ - && sed -i 's/^PASS_MIN_DAYS.*/PASS_MIN_DAYS -1/' /etc/login.defs \ - && useradd -ms /bin/bash frappe +RUN useradd -ms /bin/bash frappe ARG GIT_BRANCH=develop ARG ARCH=amd64 @@ -20,7 +18,6 @@ RUN apt-get update -y && apt-get install \ postgresql-client \ gettext-base \ wget \ - curl \ # for PDF fonts-cantarell \ libffi-dev \ @@ -39,13 +36,14 @@ RUN apt-get update -y && apt-get install \ libpq-dev \ wait-for-it \ # For arm64 - gcc -y \ + gcc \ + g++ -y \ + # Detect arch, download and install wkhtmltox && if [ `uname -m` = 'aarch64' ]; then export ARCH=arm64; fi \ && if [ `uname -m` = 'x86_64' ]; then export ARCH=amd64; fi \ && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ - && apt-get purge -y wget && apt-get autoremove -y \ && chown -R frappe:frappe /home/frappe USER frappe diff --git a/travis.py b/travis.py deleted file mode 100755 index f2c1c784..00000000 --- a/travis.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import subprocess -import os - -def parse_args(): - parser = argparse.ArgumentParser(description='frappe_docker common CI elements', add_help=True) - - parser.add_argument('service', action='store', type=str, help='Name of the service to build: "erpnext" or "frappe"') - parser.add_argument('-o', '--tag-only', required=False, action='store_true', dest='tag_only', help='Only tag an image and push it.') - - image_type = parser.add_mutually_exclusive_group(required=True) - image_type.add_argument('-a', '--nginx', action='store_const', dest='image_type', const='nginx', help='Build the nginx + static assets image') - image_type.add_argument('-s', '--socketio', action='store_const', dest='image_type', const='socketio', help='Build the frappe-socketio image') - image_type.add_argument('-w', '--worker', action='store_const', dest='image_type', const='worker', help='Build the python environment image') - - tag_type = parser.add_mutually_exclusive_group(required=True) - tag_type.add_argument('-g', '--git-version', action='store', type=str, dest='version', help='The version number of service (i.e. "11", "12", etc.)') - tag_type.add_argument('-t', '--tag', action='store', type=str, dest='tag', help='The image tag (i.e. erpnext-worker:$TAG )') - - args = parser.parse_args() - return args - - -def git_version(service, version, branch): - print(f'Pulling {service} v{version}') - subprocess.run(f'git clone https://github.com/frappe/{service} --branch {branch}', shell=True) - cd = os.getcwd() - os.chdir(os.getcwd() + f'/{service}') - subprocess.run('git fetch --tags', shell=True) - - # XX-beta becomes XX for tags search - version = version.split('-')[0] - - version_tag = subprocess.check_output(f'git tag --list --sort=-version:refname "v{version}*" | sed -n 1p | sed -e \'s#.*@\(\)#\\1#\'', shell=True).strip().decode('ascii') - os.chdir(cd) - return version_tag - - -def build(service, tag, image, branch): - build_args = f'--build-arg GIT_BRANCH={branch}' - if service == 'nginx' and branch == 'version-11': - build_args += f' --build-arg NODE_IMAGE_TAG=10-buster-slim' - - print(f'Building {service} {image} image') - subprocess.run(f'docker buildx build --platform linux/amd64,linux/arm64 --load {build_args} -t {service}-{image} -f build/{service}-{image}/Dockerfile .', shell=True) - tag_and_push(f'{service}-{image}', tag) - - -def tag_and_push(image_name, tag): - print(f'Tagging {image_name} as "{tag}" and pushing') - subprocess.run(f'docker tag {image_name} frappe/{image_name}:{tag}', shell=True) - subprocess.run(f'docker push frappe/{image_name}:{tag}', shell=True) - - -def main(): - args = parse_args() - tag = args.tag - branch = 'develop' - - if args.version: - branch = 'version-' + args.version - tag = git_version(args.service, args.version, branch) - - if args.tag_only: - tag_and_push(f'{args.service}-{args.image_type}', tag) - else: - build(args.service, tag, args.image_type, branch) - -if __name__ == "__main__": - main() From 77b9eda56707c764c891a40e7abc8b141b3a2734 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Thu, 7 Jan 2021 19:44:40 +0530 Subject: [PATCH 12/13] Revert "feat: make images arm64 compatible" --- .travis.yml | 298 ++++++++--------------------- build/bench/Dockerfile | 18 +- build/erpnext-nginx/Dockerfile | 12 +- build/erpnext-nginx/install_app.sh | 3 + build/erpnext-worker/Dockerfile | 3 +- build/frappe-nginx/Dockerfile | 12 +- build/frappe-socketio/Dockerfile | 2 +- build/frappe-worker/Dockerfile | 30 ++- travis.py | 72 +++++++ 9 files changed, 188 insertions(+), 262 deletions(-) create mode 100755 travis.py diff --git a/.travis.yml b/.travis.yml index 8a9f2558..fc5b100b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,28 +11,14 @@ services: - docker before_install: - # Install latest docker (https://www.docker.com/blog/multi-arch-build-what-about-travis) - - sudo rm -rf /var/lib/apt/lists/* - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) edge" - - sudo apt-get update - - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - - mkdir -vp ~/.docker/cli-plugins/ - - curl --silent -L "https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-amd64" > ~/.docker/cli-plugins/docker-buildx - - chmod a+x ~/.docker/cli-plugins/docker-buildx - - docker buildx create --use - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - docker buildx inspect --bootstrap - - if [[ "$BUILD" != "development" ]]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin; sudo apt-get update && sudo apt-get -y install git; fi - if [[ $BUILD == "development" ]];then - sudo apt-get update && sudo apt-get -y install curl; - sudo curl -L "https://github.com/docker/compose/releases/download/1.27.4/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose; - sudo chmod +x /usr/local/bin/docker-compose; + sudo apt-get update && sudo apt-get -y install docker-compose; fi + - chmod u+x ./travis.py after_success: - docker --version @@ -42,244 +28,128 @@ jobs: - stage: "Build Frappe bench development environment (latest)" if: branch = develop AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/bench:latest - -f build/bench/Dockerfile . - - stage: "Frappe Worker (edge)" + - docker build -t frappe/bench:latest -f build/bench/Dockerfile . + - docker push frappe/bench:latest + - stage: "Frappe (edge)" if: branch = develop AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:edge - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:develop - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:latest - -f build/frappe-worker/Dockerfile . - - stage: "Frappe Nginx (edge)" + - ./travis.py frappe --worker --tag latest + - ./travis.py frappe --worker --tag edge --tag-only + - ./travis.py frappe --worker --tag develop --tag-only + - stage: "Frappe (edge)" if: branch = develop AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:edge - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:develop - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:latest - -f build/frappe-nginx/Dockerfile . - - stage: "Frappe SocketIO (edge)" + - ./travis.py frappe --nginx --tag latest + - ./travis.py frappe --nginx --tag edge --tag-only + - ./travis.py frappe --nginx --tag develop --tag-only + - stage: "Frappe (edge)" if: branch = develop AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:edge - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:develop - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:latest - -f build/frappe-socketio/Dockerfile . - - stage: "ERPNext Worker (edge)" + - ./travis.py frappe --socketio --tag latest + - ./travis.py frappe --socketio --tag edge --tag-only + - ./travis.py frappe --socketio --tag develop --tag-only + - stage: "ERPNext (edge)" if: branch = develop AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:edge - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:develop - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:latest - -f build/erpnext-worker/Dockerfile . - - stage: "ERPNext Nginx (edge)" + - ./travis.py erpnext --worker --tag latest + - ./travis.py erpnext --worker --tag edge --tag-only + - ./travis.py erpnext --worker --tag develop --tag-only + - stage: "ERPNext (edge)" if: branch = develop AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:edge - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:develop - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:latest - -f build/erpnext-nginx/Dockerfile . - - stage: "Frappe Worker (v13-beta)" + - ./travis.py erpnext --nginx --tag latest + - ./travis.py erpnext --nginx --tag edge --tag-only + - ./travis.py erpnext --nginx --tag develop --tag-only + - stage: "Frappe (v13-beta)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-13-beta - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:v13 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:${VERSION} - -f build/frappe-worker/Dockerfile . - - stage: "Frappe Nginx (v13-beta)" + - ./travis.py frappe --worker --git-version 13-beta + - ./travis.py frappe --worker --tag v13-beta --tag-only + - ./travis.py frappe --worker --tag version-13-beta --tag-only + - stage: "Frappe (v13-beta)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --no-cache - --build-arg GIT_BRANCH=version-13-beta - --platform linux/amd64,linux/arm64 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:v13 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:${VERSION} - -f build/frappe-nginx/Dockerfile . - - stage: "Frappe SocketIO (v13-beta)" + - ./travis.py frappe --nginx --git-version 13-beta + - ./travis.py frappe --nginx --tag v13-beta --tag-only + - ./travis.py frappe --nginx --tag version-13-beta --tag-only + - stage: "Frappe (v13-beta)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-13-beta - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:v13 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:${VERSION} - -f build/frappe-socketio/Dockerfile . - - stage: "ERPNext Worker (v13-beta)" + - ./travis.py frappe --socketio --git-version 13-beta + - ./travis.py frappe --socketio --tag v13-beta --tag-only + - ./travis.py frappe --socketio --tag version-13-beta --tag-only + - stage: "ERPNext (v13-beta)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-13-beta - --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:v13 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:${VERSION} - -f build/erpnext-worker/Dockerfile . - - stage: "ERPNext Nginx (v13-beta)" + - ./travis.py erpnext --worker --git-version 13-beta + - ./travis.py erpnext --worker --tag v13-beta --tag-only + - ./travis.py erpnext --worker --tag version-13-beta --tag-only + - stage: "ERPNext (v13-beta)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-13-beta - --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:v13 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:${VERSION} - -f build/erpnext-nginx/Dockerfile . - - stage: "Frappe Worker (v12)" + - ./travis.py erpnext --nginx --git-version 13-beta + - ./travis.py erpnext --nginx --tag v13-beta --tag-only + - ./travis.py erpnext --nginx --tag version-13-beta --tag-only + - stage: "Frappe (v12)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-12 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:v12 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:${VERSION} - -f build/frappe-worker/Dockerfile . - - stage: "Frappe Nginx (v12)" + - ./travis.py frappe --worker --git-version 12 + - ./travis.py frappe --worker --tag v12 --tag-only + - ./travis.py frappe --worker --tag version-12 --tag-only + - stage: "Frappe (v12)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --no-cache - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-12 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:v12 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:${VERSION} - -f build/frappe-nginx/Dockerfile . - - stage: "Frappe SocketIO (v12)" + - ./travis.py frappe --nginx --git-version 12 + - ./travis.py frappe --nginx --tag v12 --tag-only + - ./travis.py frappe --nginx --tag version-12 --tag-only + - stage: "Frappe (v12)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-12 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:v12 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:${VERSION} - -f build/frappe-socketio/Dockerfile . - - stage: "ERPNext Worker (v12)" + - ./travis.py frappe --socketio --git-version 12 + - ./travis.py frappe --socketio --tag v12 --tag-only + - ./travis.py frappe --socketio --tag version-12 --tag-only + - stage: "ERPNext (v12)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-12 - --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:v12 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:${VERSION} - -f build/erpnext-worker/Dockerfile . - - stage: "ERPNext Nginx (v12)" + - ./travis.py erpnext --worker --git-version 12 + - ./travis.py erpnext --worker --tag v12 --tag-only + - ./travis.py erpnext --worker --tag version-12 --tag-only + - stage: "ERPNext (v12)" if: branch = master AND type != pull_request script: - - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-12 - --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:v12 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:${VERSION} - -f build/erpnext-nginx/Dockerfile . - - stage: "Frappe Worker (v11)" + - ./travis.py erpnext --nginx --git-version 12 + - ./travis.py erpnext --nginx --tag v12 --tag-only + - ./travis.py erpnext --nginx --tag version-12 --tag-only + - stage: "Frappe (v11)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-11 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:v11 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-worker:${VERSION} - -f build/frappe-worker/Dockerfile . - - stage: "Frappe Nginx (v11)" + - ./travis.py frappe --worker --git-version 11 + - ./travis.py frappe --worker --tag v11 --tag-only + - ./travis.py frappe --worker --tag version-11 --tag-only + - stage: "Frappe (v11)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --no-cache - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-11 - --build-arg NODE_IMAGE_TAG=10-buster-slim - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:v11 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-nginx:${VERSION} - -f build/frappe-nginx/Dockerfile . - - stage: "Frappe SocketIO (v11)" + - ./travis.py frappe --nginx --git-version 11 + - ./travis.py frappe --nginx --tag v11 --tag-only + - ./travis.py frappe --nginx --tag version-11 --tag-only + - stage: "Frappe (v11)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-11 - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:v11 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/frappe-socketio:${VERSION} - -f build/frappe-socketio/Dockerfile . - - stage: "ERPNext Worker (v11)" + - ./travis.py frappe --socketio --git-version 11 + - ./travis.py frappe --socketio --tag v11 --tag-only + - ./travis.py frappe --socketio --tag version-11 --tag-only + - stage: "ERPNext (v11)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-11 - --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:v11 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-worker:${VERSION} - -f build/erpnext-worker/Dockerfile . - - stage: "ERPNext Nginx (v11)" + - ./travis.py erpnext --worker --git-version 11 + - ./travis.py erpnext --worker --tag v11 --tag-only + - ./travis.py erpnext --worker --tag version-11 --tag-only + - stage: "ERPNext (v11)" if: branch = master AND type != pull_request script: - - > - docker buildx build - --platform linux/amd64,linux/arm64 - --build-arg GIT_BRANCH=version-11 - --build-arg NODE_IMAGE_TAG=10-buster-slim - --build-arg DOCKER_REGISTRY_PREFIX:=frappe=${DOCKER_REGISTRY_PREFIX:=frappe} - --push - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:v11 - -t ${DOCKER_REGISTRY_PREFIX:=frappe}/erpnext-nginx:${VERSION} - -f build/erpnext-nginx/Dockerfile . + - ./travis.py erpnext --nginx --git-version 11 + - ./travis.py erpnext --nginx --tag v11 --tag-only + - ./travis.py erpnext --nginx --tag version-11 --tag-only - stage: "Build and test edge images" if: type = pull_request before_install: diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index d185fadb..98fb9cde 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -1,8 +1,8 @@ # Frappe Bench Dockerfile -FROM debian:buster-slim +FROM bitnami/minideb:latest LABEL author=frappé -RUN apt-get update -y && apt-get install \ +RUN install_packages \ git \ wkhtmltopdf \ mariadb-client \ @@ -49,18 +49,14 @@ RUN apt-get update -y && apt-get install \ python3-setuptools \ python3-tk \ python-virtualenv \ - less -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* + less RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ && dpkg-reconfigure --frontend=noninteractive locales -# Detect arch, download and install wkhtmltox -RUN if [ `uname -m` = 'aarch64' ]; then export ARCH=arm64; fi \ - && if [ `uname -m` = 'x86_64' ]; then export ARCH=amd64; fi \ - && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ - && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb +# Install wkhtmltox correctly +RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb +RUN dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb # Create new user with home directory, improve docker compatibility with UID/GID 1000, add user to sudo group, allow passwordless sudo, switch to that user and change directory to user home directory RUN groupadd -g 1000 frappe @@ -93,7 +89,7 @@ ENV NODE_VERSION=12.20.0 ENV NODE_VERSION_FRAPPEV11=10.23.0 # Install nvm with node -RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh +RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh RUN chmod +x install.sh RUN ./install.sh ENV NVM_DIR=/home/frappe/.nvm diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx/Dockerfile index 61bfa49d..72488ab7 100644 --- a/build/erpnext-nginx/Dockerfile +++ b/build/erpnext-nginx/Dockerfile @@ -1,19 +1,13 @@ -ARG NODE_IMAGE_TAG=12-buster-slim +ARG NODE_IMAGE_TAG=12-prod ARG GIT_BRANCH=develop -ARG DOCKER_REGISTRY_PREFIX=frappe -FROM node:${NODE_IMAGE_TAG} +FROM bitnami/node:${NODE_IMAGE_TAG} ARG GIT_BRANCH COPY build/erpnext-nginx/install_app.sh /install_app -RUN chmod +x /install_app && \ - apt-get update -y && \ - apt-get install build-essential git python2 -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* RUN /install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} -FROM ${DOCKER_REGISTRY_PREFIX}/frappe-nginx:${GIT_BRANCH} +FROM frappe/frappe-nginx:${GIT_BRANCH} COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ COPY --from=0 /rsync /rsync diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh index f886ced1..7af233d2 100755 --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx/install_app.sh @@ -10,6 +10,8 @@ mkdir -p /home/frappe/frappe-bench/sites/assets cd /home/frappe/frappe-bench echo -e "frappe\n${APP_NAME}" > /home/frappe/frappe-bench/sites/apps.txt +install_packages git python2 + mkdir -p apps cd apps git clone --depth 1 https://github.com/frappe/frappe ${BRANCH} @@ -18,6 +20,7 @@ git clone --depth 1 ${APP_REPO} ${BRANCH} ${APP_NAME} cd /home/frappe/frappe-bench/apps/frappe yarn yarn production --app ${APP_NAME} +rm -fr node_modules yarn install --production=true yarn add node-sass diff --git a/build/erpnext-worker/Dockerfile b/build/erpnext-worker/Dockerfile index ef8718ca..90aad213 100644 --- a/build/erpnext-worker/Dockerfile +++ b/build/erpnext-worker/Dockerfile @@ -1,6 +1,5 @@ ARG GIT_BRANCH=develop -ARG DOCKER_REGISTRY_PREFIX=frappe -FROM ${DOCKER_REGISTRY_PREFIX}/frappe-worker:${GIT_BRANCH} +FROM frappe/frappe-worker:${GIT_BRANCH} ARG GIT_BRANCH RUN install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index 11b02b02..9e23fa29 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -2,16 +2,13 @@ # This is done to ensures that node-sass binary remains common. # node-sass is required to enable website theme feature used # by Website Manager role in Frappe Framework -FROM python:3.7-slim-buster +FROM bitnami/python:3.7-prod ENV NVM_DIR=/root/.nvm ENV NODE_VERSION=12.20.0 ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" -RUN apt-get update -y \ - && apt-get install wget python2 git build-essential -y \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ +RUN install_packages wget \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ && chmod +x install.sh \ && ./install.sh \ && . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} \ @@ -21,6 +18,8 @@ WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt +RUN install_packages git + ARG GIT_BRANCH=develop RUN mkdir -p apps sites/assets/css \ && cd apps \ @@ -29,6 +28,7 @@ RUN mkdir -p apps sites/assets/css \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && yarn \ && yarn run production \ + && rm -fr node_modules \ && yarn install --production=true RUN node --version \ diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile index 1157a469..9455dbd7 100644 --- a/build/frappe-socketio/Dockerfile +++ b/build/frappe-socketio/Dockerfile @@ -1,4 +1,4 @@ -FROM node:buster-slim +FROM node:slim # Add frappe user RUN useradd -ms /bin/bash frappe diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 7a8131d6..77777d58 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -1,10 +1,11 @@ -FROM python:3.7-slim-buster +FROM bitnami/python:3.7-prod # Add non root user without password -RUN useradd -ms /bin/bash frappe +RUN sed -i 's/^PASS_MAX_DAYS.*/PASS_MAX_DAYS -1/' /etc/login.defs \ + && sed -i 's/^PASS_MIN_DAYS.*/PASS_MIN_DAYS -1/' /etc/login.defs \ + && useradd -ms /bin/bash frappe ARG GIT_BRANCH=develop -ARG ARCH=amd64 ENV PYTHONUNBUFFERED 1 ENV NVM_DIR=/home/frappe/.nvm ENV NODE_VERSION=12.20.0 @@ -12,38 +13,29 @@ ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" # Install dependencies WORKDIR /home/frappe/frappe-bench -RUN apt-get update -y && apt-get install \ +RUN install_packages \ git \ mariadb-client \ postgresql-client \ gettext-base \ wget \ # for PDF - fonts-cantarell \ - libffi-dev \ - libjpeg-dev \ libjpeg62-turbo \ + libx11-6 \ libxcb1 \ libxext6 \ - libx11-6 \ - libxml2 \ libxrender1 \ libssl-dev \ + fonts-cantarell \ xfonts-75dpi \ xfonts-base \ - zlib1g-dev \ # For psycopg2 libpq-dev \ wait-for-it \ - # For arm64 - gcc \ - g++ -y \ - # Detect arch, download and install wkhtmltox - && if [ `uname -m` = 'aarch64' ]; then export ARCH=arm64; fi \ - && if [ `uname -m` = 'x86_64' ]; then export ARCH=amd64; fi \ - && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ - && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ + && wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb \ + && dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ + && apt-get purge -y wget && apt-get autoremove -y \ && chown -R frappe:frappe /home/frappe USER frappe diff --git a/travis.py b/travis.py new file mode 100755 index 00000000..f8224053 --- /dev/null +++ b/travis.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import argparse +import subprocess +import os + +def parse_args(): + parser = argparse.ArgumentParser(description='frappe_docker common CI elements', add_help=True) + + parser.add_argument('service', action='store', type=str, help='Name of the service to build: "erpnext" or "frappe"') + parser.add_argument('-o', '--tag-only', required=False, action='store_true', dest='tag_only', help='Only tag an image and push it.') + + image_type = parser.add_mutually_exclusive_group(required=True) + image_type.add_argument('-a', '--nginx', action='store_const', dest='image_type', const='nginx', help='Build the nginx + static assets image') + image_type.add_argument('-s', '--socketio', action='store_const', dest='image_type', const='socketio', help='Build the frappe-socketio image') + image_type.add_argument('-w', '--worker', action='store_const', dest='image_type', const='worker', help='Build the python environment image') + + tag_type = parser.add_mutually_exclusive_group(required=True) + tag_type.add_argument('-g', '--git-version', action='store', type=str, dest='version', help='The version number of service (i.e. "11", "12", etc.)') + tag_type.add_argument('-t', '--tag', action='store', type=str, dest='tag', help='The image tag (i.e. erpnext-worker:$TAG )') + + args = parser.parse_args() + return args + + +def git_version(service, version, branch): + print(f'Pulling {service} v{version}') + subprocess.run(f'git clone https://github.com/frappe/{service} --branch {branch}', shell=True) + cd = os.getcwd() + os.chdir(os.getcwd() + f'/{service}') + subprocess.run('git fetch --tags', shell=True) + + # XX-beta becomes XX for tags search + version = version.split('-')[0] + + version_tag = subprocess.check_output(f'git tag --list --sort=-version:refname "v{version}*" | sed -n 1p | sed -e \'s#.*@\(\)#\\1#\'', shell=True).strip().decode('ascii') + os.chdir(cd) + return version_tag + + +def build(service, tag, image, branch): + build_args = f'--build-arg GIT_BRANCH={branch}' + if service == 'nginx' and branch == 'version-11': + build_args += f' --build-arg NODE_IMAGE_TAG=10-prod' + + print(f'Building {service} {image} image') + subprocess.run(f'docker build {build_args} -t {service}-{image} -f build/{service}-{image}/Dockerfile .', shell=True) + tag_and_push(f'{service}-{image}', tag) + + +def tag_and_push(image_name, tag): + print(f'Tagging {image_name} as "{tag}" and pushing') + subprocess.run(f'docker tag {image_name} frappe/{image_name}:{tag}', shell=True) + subprocess.run(f'docker push frappe/{image_name}:{tag}', shell=True) + + +def main(): + args = parse_args() + tag = args.tag + branch = 'develop' + + if args.version: + branch = 'version-' + args.version + tag = git_version(args.service, args.version, branch) + + if args.tag_only: + tag_and_push(f'{args.service}-{args.image_type}', tag) + else: + build(args.service, tag, args.image_type, branch) + +if __name__ == "__main__": + main() From c90783d6be73cfa924781fd5220a83094e5ea932 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 8 Jan 2021 17:45:22 +0530 Subject: [PATCH 13/13] feat: make images arm64 compatible --- build/bench/Dockerfile | 18 +++++++++++------- build/erpnext-nginx/Dockerfile | 12 +++++++++--- build/erpnext-nginx/install_app.sh | 4 ---- build/erpnext-worker/Dockerfile | 3 ++- build/frappe-nginx/Dockerfile | 12 ++++++------ build/frappe-socketio/Dockerfile | 2 +- build/frappe-worker/Dockerfile | 29 +++++++++++++++++++---------- 7 files changed, 48 insertions(+), 32 deletions(-) diff --git a/build/bench/Dockerfile b/build/bench/Dockerfile index 98fb9cde..d185fadb 100644 --- a/build/bench/Dockerfile +++ b/build/bench/Dockerfile @@ -1,8 +1,8 @@ # Frappe Bench Dockerfile -FROM bitnami/minideb:latest +FROM debian:buster-slim LABEL author=frappé -RUN install_packages \ +RUN apt-get update -y && apt-get install \ git \ wkhtmltopdf \ mariadb-client \ @@ -49,14 +49,18 @@ RUN install_packages \ python3-setuptools \ python3-tk \ python-virtualenv \ - less + less -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ && dpkg-reconfigure --frontend=noninteractive locales -# Install wkhtmltox correctly -RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb -RUN dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb +# Detect arch, download and install wkhtmltox +RUN if [ `uname -m` = 'aarch64' ]; then export ARCH=arm64; fi \ + && if [ `uname -m` = 'x86_64' ]; then export ARCH=amd64; fi \ + && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ + && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb # Create new user with home directory, improve docker compatibility with UID/GID 1000, add user to sudo group, allow passwordless sudo, switch to that user and change directory to user home directory RUN groupadd -g 1000 frappe @@ -89,7 +93,7 @@ ENV NODE_VERSION=12.20.0 ENV NODE_VERSION_FRAPPEV11=10.23.0 # Install nvm with node -RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh +RUN wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh RUN chmod +x install.sh RUN ./install.sh ENV NVM_DIR=/home/frappe/.nvm diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx/Dockerfile index 72488ab7..61bfa49d 100644 --- a/build/erpnext-nginx/Dockerfile +++ b/build/erpnext-nginx/Dockerfile @@ -1,13 +1,19 @@ -ARG NODE_IMAGE_TAG=12-prod +ARG NODE_IMAGE_TAG=12-buster-slim ARG GIT_BRANCH=develop -FROM bitnami/node:${NODE_IMAGE_TAG} +ARG DOCKER_REGISTRY_PREFIX=frappe +FROM node:${NODE_IMAGE_TAG} ARG GIT_BRANCH COPY build/erpnext-nginx/install_app.sh /install_app +RUN chmod +x /install_app && \ + apt-get update -y && \ + apt-get install build-essential git python2 -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* RUN /install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} -FROM frappe/frappe-nginx:${GIT_BRANCH} +FROM ${DOCKER_REGISTRY_PREFIX}/frappe-nginx:${GIT_BRANCH} COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ COPY --from=0 /rsync /rsync diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh index 7af233d2..f1c71860 100755 --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx/install_app.sh @@ -10,8 +10,6 @@ mkdir -p /home/frappe/frappe-bench/sites/assets cd /home/frappe/frappe-bench echo -e "frappe\n${APP_NAME}" > /home/frappe/frappe-bench/sites/apps.txt -install_packages git python2 - mkdir -p apps cd apps git clone --depth 1 https://github.com/frappe/frappe ${BRANCH} @@ -20,9 +18,7 @@ git clone --depth 1 ${APP_REPO} ${BRANCH} ${APP_NAME} cd /home/frappe/frappe-bench/apps/frappe yarn yarn production --app ${APP_NAME} -rm -fr node_modules yarn install --production=true -yarn add node-sass mkdir -p /home/frappe/frappe-bench/sites/assets/${APP_NAME} cp -R /home/frappe/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public/* /home/frappe/frappe-bench/sites/assets/${APP_NAME} diff --git a/build/erpnext-worker/Dockerfile b/build/erpnext-worker/Dockerfile index 90aad213..ef8718ca 100644 --- a/build/erpnext-worker/Dockerfile +++ b/build/erpnext-worker/Dockerfile @@ -1,5 +1,6 @@ ARG GIT_BRANCH=develop -FROM frappe/frappe-worker:${GIT_BRANCH} +ARG DOCKER_REGISTRY_PREFIX=frappe +FROM ${DOCKER_REGISTRY_PREFIX}/frappe-worker:${GIT_BRANCH} ARG GIT_BRANCH RUN install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index 9e23fa29..11b02b02 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -2,13 +2,16 @@ # This is done to ensures that node-sass binary remains common. # node-sass is required to enable website theme feature used # by Website Manager role in Frappe Framework -FROM bitnami/python:3.7-prod +FROM python:3.7-slim-buster ENV NVM_DIR=/root/.nvm ENV NODE_VERSION=12.20.0 ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" -RUN install_packages wget \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ +RUN apt-get update -y \ + && apt-get install wget python2 git build-essential -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ && chmod +x install.sh \ && ./install.sh \ && . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} \ @@ -18,8 +21,6 @@ WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt -RUN install_packages git - ARG GIT_BRANCH=develop RUN mkdir -p apps sites/assets/css \ && cd apps \ @@ -28,7 +29,6 @@ RUN mkdir -p apps sites/assets/css \ RUN cd /home/frappe/frappe-bench/apps/frappe \ && yarn \ && yarn run production \ - && rm -fr node_modules \ && yarn install --production=true RUN node --version \ diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio/Dockerfile index 9455dbd7..1157a469 100644 --- a/build/frappe-socketio/Dockerfile +++ b/build/frappe-socketio/Dockerfile @@ -1,4 +1,4 @@ -FROM node:slim +FROM node:buster-slim # Add frappe user RUN useradd -ms /bin/bash frappe diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 77777d58..3d3167b1 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -1,11 +1,10 @@ -FROM bitnami/python:3.7-prod +FROM python:3.7-slim-buster # Add non root user without password -RUN sed -i 's/^PASS_MAX_DAYS.*/PASS_MAX_DAYS -1/' /etc/login.defs \ - && sed -i 's/^PASS_MIN_DAYS.*/PASS_MIN_DAYS -1/' /etc/login.defs \ - && useradd -ms /bin/bash frappe +RUN useradd -ms /bin/bash frappe ARG GIT_BRANCH=develop +ARG ARCH=amd64 ENV PYTHONUNBUFFERED 1 ENV NVM_DIR=/home/frappe/.nvm ENV NODE_VERSION=12.20.0 @@ -13,12 +12,14 @@ ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" # Install dependencies WORKDIR /home/frappe/frappe-bench -RUN install_packages \ +RUN apt-get update -y && apt-get install \ + # for frappe framework git \ mariadb-client \ postgresql-client \ gettext-base \ wget \ + wait-for-it \ # for PDF libjpeg62-turbo \ libx11-6 \ @@ -29,13 +30,21 @@ RUN install_packages \ fonts-cantarell \ xfonts-75dpi \ xfonts-base \ + libxml2 \ + libffi-dev \ + libjpeg-dev \ + zlib1g-dev \ # For psycopg2 libpq-dev \ - wait-for-it \ - && wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb \ - && dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ - && apt-get purge -y wget && apt-get autoremove -y \ + # For arm64 python wheel builds + gcc \ + g++ -y \ + # Detect arch, download and install wkhtmltox + && if [ `uname -m` = 'aarch64' ]; then export ARCH=arm64; fi \ + && if [ `uname -m` = 'x86_64' ]; then export ARCH=amd64; fi \ + && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ + && dpkg -i wkhtmltox_0.12.6-1.buster_${ARCH}.deb && rm wkhtmltox_0.12.6-1.buster_${ARCH}.deb \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.37.2/install.sh \ && chown -R frappe:frappe /home/frappe USER frappe