Merge branch 'frappe:main' into pytest

.github/workflows/lint.yml

@@ -25,7 +25,7 @@ jobs:
           go-version: "^1.14"
 
       - name: Cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             ~/.cache/pre-commit

.github/workflows/pre-commit-autoupdate.yml

@@ -16,7 +16,7 @@ jobs:
         uses: vrslev/pre-commit-autoupdate@v1.0.0
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@v4
         with:
           branch: pre-commit-autoupdate
           title: "chore(deps): Update pre-commit hooks"

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       - id: isort
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.5.1
+    rev: v2.6.0
     hooks:
       - id: prettier
 

custom_app/README.md

@@ -18,15 +18,17 @@ To get started, install Docker and [Buildx](https://github.com/docker/buildx#ins
 Before the next step—to build images—replace "custom_app" with your app's name in `docker-bake.hcl`. After that, let's try to build:
 
 ```bash
-FRAPPE_VERSION=<Frappe version you need> docker buildx bake
+FRAPPE_VERSION=... ERPNEXT_VERSION=... docker buildx bake
 ```
 
+> 💡 We assume that majority of our users use ERPNext, that's why images in this tutorial are based on ERPNext images. If don't want ERPNext, change base image in Dockerfiles and remove ERPNEXT_VERSION from bake file.
+
 If something goes wrong feel free to leave an issue.
 
 To test if site works, setup `.env` file (check [example](<(https://github.com/frappe/frappe_docker/blob/main/example.env)>)) and run:
 
 ```bash
-docker-compose up -d
+docker-compose -f compose.yaml -f overrides/compose.noproxy.yaml -f overrides/compose.mariadb.yaml -f overrides/compose.redis.yaml -f custom_app/compose.override.yaml up -d
 docker-compose exec backend \
   bench new-site 127.0.0.1 \
     --mariadb-root-password 123 \
@@ -36,3 +38,7 @@ docker-compose restart backend
 ```
 
 Cool! You just containerized your app!
+
+## Installing multiple apps
+
+Both backend and frontend builds contain `install-app` script that places app where it should be. Each call to script installs given app. Usage: `install-app [APP_NAME] [BRANCH?] [GIT_URL?]`.

custom_app/backend.Dockerfile

@@ -1,8 +1,20 @@
-ARG FRAPPE_VERSION
-FROM frappe/frappe-worker:${FRAPPE_VERSION}
+# syntax=docker/dockerfile:1.3
+
+ARG ERPNEXT_VERSION
+FROM frappe/erpnext-worker:${ERPNEXT_VERSION}
+
+USER root
 
 ARG APP_NAME
-COPY --chown=frappe . ../apps/${APP_NAME}
+COPY . ../apps/${APP_NAME}
 
-RUN echo "frappe\n${APP_NAME}" >/home/frappe/frappe-bench/sites/apps.txt \
-    && ../env/bin/pip install --no-cache-dir -e ../apps/${APP_NAME}
+RUN --mount=type=cache,target=/root/.cache/pip \
+    install-app ${APP_NAME}
+
+# or with git:
+# ARG APP_NAME
+# ARG BRANCH
+# ARG GIT_URL
+# RUN install-assets ${APP_NAME} ${BRANCH} ${GIT_URL}
+
+USER frappe

custom_app/docker-bake.hcl

@@ -1,6 +1,7 @@
 APP_NAME="custom_app"
 
 variable "FRAPPE_VERSION" {}
+variable "ERPNEXT_VERSION" {}
 
 group "default" {
     targets = ["backend", "frontend"]
@@ -10,7 +11,7 @@ target "backend" {
     dockerfile = "backend.Dockerfile"
     tags = ["custom_app/worker:latest"]
     args = {
-      "FRAPPE_VERSION" = FRAPPE_VERSION
+      "ERPNEXT_VERSION" = ERPNEXT_VERSION
       "APP_NAME" = APP_NAME
     }
 }
@@ -20,6 +21,7 @@ target "frontend" {
     tags = ["custom_app/nginx:latest"]
     args = {
       "FRAPPE_VERSION" = FRAPPE_VERSION
+      "ERPNEXT_VERSION" = ERPNEXT_VERSION
       "APP_NAME" = APP_NAME
     }
 }

custom_app/frontend.Dockerfile

@@ -1,52 +1,19 @@
 ARG FRAPPE_VERSION
-FROM node:14-bullseye-slim as prod_node_modules
-
-RUN apt-get update \
-    && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-    git \
-    build-essential \
-    python \
-    ca-certificates \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /root/frappe-bench
-RUN mkdir -p sites/assets
-
-ARG FRAPPE_VERSION
-RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe
-
-RUN yarn --cwd apps/frappe
+ARG ERPNEXT_VERSION
 
+FROM frappe/assets-builder:${FRAPPE_VERSION} as assets
 
 ARG APP_NAME
 COPY . apps/${APP_NAME}
+RUN install-app ${APP_NAME}
 
-# Install production node modules
-RUN yarn --cwd apps/${APP_NAME} --prod
+# or with git:
+# ARG APP_NAME
+# ARG BRANCH
+# ARG GIT_URL
+# RUN install-app ${APP_NAME} ${BRANCH} ${GIT_URL}
 
 
+FROM frappe/erpnext-nginx:${ERPNEXT_VERSION}
 
-FROM prod_node_modules as assets
-
-ARG APP_NAME
-
-# Install development node modules
-RUN yarn --cwd apps/${APP_NAME}
-
-# Build assets
-RUN echo "frappe\n${APP_NAME}" >sites/apps.txt \
-    && yarn --cwd apps/frappe production --app ${APP_NAME} \
-    && rm sites/apps.txt
-
-
-
-FROM frappe/frappe-nginx:${FRAPPE_VERSION}
-
-ARG APP_NAME
-
-# Copy all not built assets
-COPY --from=prod_node_modules /root/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public /usr/share/nginx/html/assets/${APP_NAME}
-# Copy production node modules
-COPY --from=prod_node_modules /root/frappe-bench/apps/${APP_NAME}/node_modules /usr/share/nginx/html/assets/${APP_NAME}/node_modules
-# Copy built assets
-COPY --from=assets /root/frappe-bench/sites /usr/share/nginx/html
+COPY --from=assets /out /usr/share/nginx/html

docker-bake.hcl

@@ -30,7 +30,7 @@ target "bench-test" {
 # Base for all other targets
 
 group "frappe" {
-    targets = ["frappe-worker", "frappe-nginx", "frappe-socketio"]
+    targets = ["frappe-worker", "frappe-nginx", "frappe-socketio", "assets-builder"]
 }
 
 group "erpnext" {
@@ -81,6 +81,13 @@ target "frappe-nginx" {
     tags =  tag("frappe-nginx", "${FRAPPE_VERSION}")
 }
 
+target "assets-builder" {
+  inherits = ["default-args"]
+  context = "images/nginx"
+  target = "assets_builder"
+  tags = tag("assets-builder", "${FRAPPE_VERSION}")
+}
+
 target "erpnext-nginx" {
     inherits = ["default-args"]
     context = "images/nginx"

docs/docker-swarm.md

@@ -52,4 +52,4 @@ Notes:
 
 #### Site Operations
 
-Refer [site operations documentation](./site-operations) to create new site, migrate site, drop site and perform other site operations.
+Refer [site operations documentation](./site-operations.md) to create new site, migrate site, drop site and perform other site operations.

docs/images-and-compose-files.md

@@ -51,6 +51,8 @@ It is quite simple to run overrides. All we need to do is to specify compose fil
 docker-compose -f compose.yaml -f overrides/compose.erpnext.yaml config
 ```
 
+⚠ Make sure to use docker-compose v2 (run `docker-compose -v` to check). If you want to use v1 make sure the correct `$`-signs as they get duplicated by the `config` command!
+
 That's it! Of course, we also have to setup `.env` before all of that, but that's not the point.
 
 ## Configuration

example.env

@@ -1,9 +1,9 @@
 # Reference: https://github.com/frappe/frappe_docker/blob/main/docs/images-and-compose-files.md
 
-FRAPPE_VERSION=v13.22.1
+FRAPPE_VERSION=v13.23.0
 
 # Only with ERPNext override
-ERPNEXT_VERSION=v13.22.1
+ERPNEXT_VERSION=v13.23.1
 
 DB_PASSWORD=123
 

images/nginx/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:14-bullseye-slim as base
+FROM node:14-bullseye-slim as assets_builder
 
 RUN apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
@@ -8,81 +8,55 @@ RUN apt-get update \
     ca-certificates \
     && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /root/frappe-bench
-RUN mkdir -p sites/assets
+WORKDIR /frappe-bench
+
+RUN mkdir -p sites/assets /out/assets \
+    && echo frappe >sites/apps.txt
 
 ARG FRAPPE_VERSION
 RUN git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe
 
-
-
-FROM base as frappe_prod_node_modules
-
-# Install production node modules
-RUN yarn --cwd apps/frappe --prod
-
-
-
-FROM frappe_prod_node_modules as frappe_assets
-
 # Install development node modules
-RUN yarn --cwd apps/frappe
-
-# Build assets they're stored in frappe-bench/sites/assets
-RUN echo "frappe" >sites/apps.txt \
-    && yarn --cwd apps/frappe run production \
-    && rm sites/apps.txt \
+RUN yarn --cwd apps/frappe \
     # TODO: Currently `yarn run production` doesn't create .build on develop branch: https://github.com/frappe/frappe/issues/15396
-    && if [ ! -f sites/.build ]; then touch sites/.build; fi
+    && if [ ! -f sites/.build ]; then touch sites/.build; fi \
+    && cp sites/.build /out
+
+COPY install-app.sh /usr/local/bin/install-app
 
 
+FROM assets_builder as frappe_assets
 
-FROM base as erpnext_prod_node_modules
+RUN install-app frappe
+
+
+FROM assets_builder as erpnext_assets
 
 ARG ERPNEXT_VERSION
-RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext
-
-RUN yarn --cwd apps/erpnext --prod
+RUN install-app erpnext ${ERPNEXT_VERSION} https://github.com/frappe/erpnext
 
 
+FROM alpine/git as bench
 
-FROM erpnext_prod_node_modules as erpnext_assets
-
-RUN yarn --cwd apps/erpnext
-
-COPY --from=frappe_assets /root/frappe-bench/apps/frappe/node_modules /root/frappe-bench/apps/frappe/node_modules
-COPY --from=frappe_assets /root/frappe-bench/apps/frappe/package.json /root/frappe-bench/apps/frappe/yarn.lock /root/frappe-bench/apps/frappe/
-
-RUN echo "frappe\nerpnext" >sites/apps.txt \
-    && yarn --cwd apps/frappe run production --app erpnext \
-    && rm sites/apps.txt
-
-
-
-FROM base as error_pages
-
-RUN git clone --depth 1 https://github.com/frappe/bench /root/bench
-
+# Error pages
+RUN git clone --depth 1 https://github.com/frappe/bench /tmp/bench \
+    && mkdir /out \
+    && mv /tmp/bench/bench/config/templates/502.html /out/
 
 
 FROM nginxinc/nginx-unprivileged:1.21.6-alpine as frappe
 
-COPY --from=error_pages /root/bench/bench/config/templates/502.html /usr/share/nginx/html
-COPY --from=base /root/frappe-bench/apps/frappe/frappe/public /usr/share/nginx/html/assets/frappe
-COPY --from=frappe_prod_node_modules /root/frappe-bench/apps/frappe/node_modules /usr/share/nginx/html/assets/frappe/node_modules
-COPY --from=frappe_assets /root/frappe-bench/sites /usr/share/nginx/html
-
 # https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/20-envsubst-on-templates.sh
 COPY nginx-template.conf /etc/nginx/templates/default.conf.template
 # https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/docker-entrypoint.sh
 COPY entrypoint.sh /docker-entrypoint.d/frappe-entrypoint.sh
 
-USER 1000
+COPY --from=bench /out /usr/share/nginx/html/
+COPY --from=frappe_assets /out /usr/share/nginx/html
 
+USER 1000
 
 
 FROM frappe as erpnext
 
-COPY --from=erpnext_prod_node_modules /root/frappe-bench/apps/erpnext/erpnext/public /usr/share/nginx/html/assets/erpnext
-COPY --from=erpnext_prod_node_modules /root/frappe-bench/apps/erpnext/node_modules /usr/share/nginx/html/assets/erpnext/node_modules
-COPY --from=erpnext_assets /root/frappe-bench/sites /usr/share/nginx/html
+COPY --from=erpnext_assets /out /usr/share/nginx/html

images/nginx/install-app.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+set -x
+
+APP=$1 BRANCH=$2 GIT_URL=$3
+
+cd /frappe-bench
+
+if test "$BRANCH" && test "$GIT_URL"; then
+  # Clone in case not copied manually
+  git clone --depth 1 -b "$BRANCH" "$GIT_URL" "apps/$APP"
+fi
+
+# Add all not built assets
+cp -r "apps/$APP/$APP/public" "/out/assets/$APP"
+
+# Add production node modules
+yarn --cwd "apps/$APP" --prod
+cp -r "apps/$APP/node_modules" "/out/assets/$APP/node_modules"
+
+# Add built assets
+yarn --cwd "apps/$APP"
+echo "$APP" >>sites/apps.txt
+yarn --cwd apps/frappe run production --app "$APP"
+cp -r sites/assets /out
+
+# Cleanup
+rm -rf "apps/$APP"
+rm -rf sites/assets

images/worker/Dockerfile

@@ -14,12 +14,10 @@ USER frappe
 RUN mkdir -p /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/logs /home/frappe/frappe-bench/sites
 WORKDIR /home/frappe/frappe-bench
 
-RUN --mount=type=cache,target=/home/frappe/.cache/pip \
-    pip install -U pip wheel \
+USER root
+RUN pip install -U pip wheel \
     && python -m venv env \
     && env/bin/pip install -U pip wheel
-USER root
-
 
 
 FROM base as build_deps
@@ -37,51 +35,44 @@ RUN apt-get update \
     # Make is required to build wheels of ERPNext deps in develop branch for linux/arm64
     make \
     && rm -rf /var/lib/apt/lists/*
-USER frappe
 
+COPY install-app.sh /usr/local/bin/install-app
 
 
 FROM build_deps as frappe_builder
 
 ARG FRAPPE_VERSION
-RUN --mount=type=cache,target=/home/frappe/.cache/pip \
-    git clone --depth 1 -b ${FRAPPE_VERSION} https://github.com/frappe/frappe apps/frappe \
-    && env/bin/pip install -e apps/frappe \
+RUN --mount=type=cache,target=/root/.cache/pip \
+    install-app frappe ${FRAPPE_VERSION} https://github.com/frappe/frappe \
     && env/bin/pip install -U gevent \
-    && rm -r apps/frappe/.git \
     # Link Frappe's node_modules/ to make Website Theme work
     && mkdir -p /home/frappe/frappe-bench/sites/assets/frappe/node_modules \
     && ln -s /home/frappe/frappe-bench/sites/assets/frappe/node_modules /home/frappe/frappe-bench/apps/frappe/node_modules
 
 
-
-# We split ERPNext wheels build in separate stage to achieve concurrency with Frappe build
-FROM build_deps as erpnext_wheels
-
-ARG ERPNEXT_VERSION
-RUN git clone --depth 1 -b ${ERPNEXT_VERSION} https://github.com/frappe/erpnext apps/erpnext \
-    && rm -r apps/erpnext/.git
-
-RUN --mount=type=cache,target=/home/frappe/.cache/pip \
-    pip wheel --wheel-dir /home/frappe/erpnext-wheels -r apps/erpnext/requirements.txt
-
-
-
 FROM frappe_builder as erpnext_builder
 
-COPY --from=erpnext_wheels --chown=frappe /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext
-RUN --mount=type=bind,target=/home/frappe/erpnext-wheels,source=/home/frappe/erpnext-wheels,from=erpnext_wheels \
-    --mount=type=cache,target=/home/frappe/.cache/pip \
-    --mount=type=cache,target=/home/frappe/.cache/pip,source=/home/frappe/.cache/pip,from=erpnext_wheels \
-    env/bin/pip install --find-links=/home/frappe/erpnext-wheels -e apps/erpnext
-
+ARG ERPNEXT_VERSION
+RUN --mount=type=cache,target=/root/.cache/pip \
+    install-app erpnext ${ERPNEXT_VERSION} https://github.com/frappe/erpnext
 
 
 FROM base as configured_base
 
+ARG WKHTMLTOPDF_VERSION=0.12.6-1
+
 RUN apt-get update \
+    # Setup Node lists
     && apt-get install --no-install-recommends -y curl \
     && curl -sL https://deb.nodesource.com/setup_14.x | bash - \
+    # Install wkhtmltopdf with patched qt
+    && if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \
+    && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \
+    && downloaded_file=wkhtmltox_$WKHTMLTOPDF_VERSION.buster_${ARCH}.deb \
+    && curl -sLO https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \
+    && apt-get install -y ./$downloaded_file \
+    && rm $downloaded_file \
+    # Cleanup
     && apt-get purge -y --auto-remove curl \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
@@ -89,17 +80,12 @@ RUN apt-get update \
     mariadb-client \
     # Postgres
     postgresql-client \
-    # wkhtmltopdf
-    xvfb \
-    libfontconfig \
-    wkhtmltopdf \
     # For healthcheck
     wait-for-it \
     jq \
     # other
     nodejs \
     && rm -rf /var/lib/apt/lists/*
-USER frappe
 
 COPY pretend-bench.sh /usr/local/bin/bench
 COPY push_backup.py /usr/local/bin/push-backup
@@ -113,14 +99,19 @@ CMD [ "/home/frappe/frappe-bench/env/bin/gunicorn", "-b", "0.0.0.0:8000", "frapp
 
 FROM configured_base as frappe
 
-RUN echo "frappe" >/home/frappe/frappe-bench/sites/apps.txt
 COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe
 COPY --from=frappe_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env
+COPY --from=frappe_builder /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/
+
+USER frappe
 
 
+# Split frappe and erpnext to reduce image size (because of frappe-bench/env/ directory)
 FROM configured_base as erpnext
 
-RUN echo "frappe\nerpnext" >/home/frappe/frappe-bench/sites/apps.txt
 COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe
 COPY --from=erpnext_builder /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext
 COPY --from=erpnext_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env
+COPY --from=erpnext_builder /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/
+
+USER frappe

images/worker/install-app.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+set -x
+
+APP=$1 BRANCH=$2 GIT_URL=$3
+
+cd /home/frappe/frappe-bench
+
+if test "$BRANCH" && test "$GIT_URL"; then
+  # Clone in case not copied manually
+  git clone --depth 1 -b "$BRANCH" "$GIT_URL" "apps/$APP"
+  rm -r "apps/$APP/.git"
+fi
+
+env/bin/pip install -e "apps/$APP"
+
+echo "$APP" >>sites/apps.txt