armv64 support

build/bench-arm/Dockerfile

@@ -1,8 +1,8 @@
 # Frappe Bench Dockerfile
-FROM bitnami/minideb:latest
+FROM debian:stable-slim
 LABEL author=frappé
 
-RUN install_packages \
+RUN  apt-get update -y && apt-get install  \
     git \
     wkhtmltopdf \
     mariadb-client \
@@ -49,7 +49,7 @@ RUN install_packages \
     python3-setuptools \
     python3-tk \
     python-virtualenv \
-    less
+    less -y && apt-get clean
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \
   && dpkg-reconfigure --frontend=noninteractive locales
@@ -85,7 +85,7 @@ RUN bash -c "bench --version"
 # https://nodejs.org/download/release/latest-v10.x/
 # https://nodejs.org/download/release/latest-v12.x/
 # https://nodejs.org/download/release/latest-v13.x/
-ENV NODE_VERSION=12.18.2
+ENV NODE_VERSION=12.19.1
 ENV NODE_VERSION_FRAPPEV11=10.21.0
 
 # Install nvm with node

build/erpnext-nginx-arm/Dockerfile

@@ -1,13 +1,14 @@
-ARG NODE_IMAGE_TAG=12-prod
+ARG NODE_IMAGE_TAG=12.19.1-buster-slim
 ARG GIT_BRANCH=develop
-FROM bitnami/node:${NODE_IMAGE_TAG}
+FROM node:${NODE_IMAGE_TAG}
 
 ARG GIT_BRANCH
-COPY build/erpnext-nginx/install_app.sh /install_app
+COPY install_app.sh /install_app 
+RUN chmod +x /install_app && apt-get update -y && apt-get install build-essential git python2 -y
 
 RUN /install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH}
 
-FROM frappe/frappe-nginx:${GIT_BRANCH}
+FROM frappe/frappe-nginx-arm:${GIT_BRANCH}
 
 COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/
 COPY --from=0 /rsync /rsync

build/erpnext-nginx-arm/install_app.sh

@@ -10,7 +10,7 @@ mkdir -p /home/frappe/frappe-bench/sites/assets
 cd /home/frappe/frappe-bench
 echo -e "frappe\n${APP_NAME}" > /home/frappe/frappe-bench/sites/apps.txt
 
-install_packages git python2
+# install_packages git python2
 
 mkdir -p apps
 cd apps

build/erpnext-worker-arm/Dockerfile

@@ -1,5 +1,6 @@
 ARG GIT_BRANCH=develop
-FROM frappe/frappe-worker:${GIT_BRANCH}
+FROM frappe/frappe-worker-arm:${GIT_BRANCH}
 
 ARG GIT_BRANCH
+RUN pip install bootstrapped --no-deps
 RUN install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH}

build/frappe-nginx-arm/Dockerfile

@@ -2,12 +2,12 @@
 # This is done to ensures that node-sass binary remains common.
 # node-sass is required to enable website theme feature used
 # by Website Manager role in Frappe Framework
-FROM bitnami/python:3.7-prod
+FROM python:3.7.9-slim-buster
 
 ENV NVM_DIR=/root/.nvm
-ENV NODE_VERSION=12.18.3
+ENV NODE_VERSION=12.19.1
 ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
-RUN install_packages wget \
+RUN apt-get update -y && apt-get install wget python2 git -y build-essential \
     && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \
     && chmod +x install.sh \
     && ./install.sh \
@@ -18,7 +18,7 @@ WORKDIR /home/frappe/frappe-bench
 RUN mkdir -p /home/frappe/frappe-bench/sites \
     && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt
 
-RUN install_packages git
+# RUN install_packages git
 
 ARG GIT_BRANCH=develop
 RUN mkdir -p apps sites/assets/css  \
@@ -45,8 +45,8 @@ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/fra
 FROM nginx:latest
 COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/
 COPY --from=0 /var/www/error_pages /var/www/
-COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template
-COPY build/frappe-nginx/docker-entrypoint.sh /
+COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template
+COPY docker-entrypoint.sh /
 
 RUN apt-get update && apt-get install -y rsync && apt-get clean \
     && echo "#!/bin/bash" > /rsync \

build/frappe-nginx-arm/nginx-default.conf.template

@@ -0,0 +1,104 @@
+upstream frappe-server {
+    server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0;
+}
+
+upstream socketio-server {
+    server ${FRAPPE_SOCKETIO}:${SOCKETIO_PORT} fail_timeout=0;
+}
+
+server {
+    listen 80;
+    server_name $http_host;
+    root /var/www/html;
+
+    add_header X-Frame-Options "SAMEORIGIN";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
+    add_header X-Content-Type-Options nosniff;
+    add_header X-XSS-Protection "1; mode=block";
+
+    location /assets {
+        try_files $uri =404;
+    }
+
+    location ~ ^/protected/(.*) {
+        internal;
+        try_files /sites/$http_host/$1 =404;
+    }
+
+    location /socket.io {
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header X-Frappe-Site-Name $host;
+        proxy_set_header Origin $scheme://$http_host;
+        proxy_set_header Host $http_host;
+
+        proxy_pass http://socketio-server;
+    }
+
+    location / {
+        rewrite ^(.+)/$ $1 permanent;
+        rewrite ^(.+)/index\.html$ $1 permanent;
+        rewrite ^(.+)\.html$ $1 permanent;
+
+        location ~ ^/files/.*.(htm|html|svg|xml) {
+                add_header Content-disposition "attachment";
+                try_files /sites/$http_host/public/$uri @webserver;
+        }
+
+        try_files /sites/$http_host/public/$uri @webserver;
+    }
+
+    location @webserver {
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Frappe-Site-Name $host;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Use-X-Accel-Redirect True;
+        proxy_read_timeout 120;
+        proxy_redirect off;
+
+        proxy_pass  http://frappe-server;
+    }
+
+    # error pages
+    error_page 502 /502.html;
+    location /502.html {
+        root /var/www/templates;
+        internal;
+    }
+
+    # optimizations
+    sendfile on;
+    keepalive_timeout 15;
+    client_max_body_size 50m;
+    client_body_buffer_size 16K;
+    client_header_buffer_size 1k;
+
+    # enable gzip compresion
+    # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge
+    gzip on;
+    gzip_http_version 1.1;
+    gzip_comp_level 5;
+    gzip_min_length 256;
+    gzip_proxied any;
+    gzip_vary on;
+    gzip_types
+        application/atom+xml
+        application/javascript
+        application/json
+        application/rss+xml
+        application/vnd.ms-fontobject
+        application/x-font-ttf
+        application/font-woff
+        application/x-web-app-manifest+json
+        application/xhtml+xml
+        application/xml
+        font/opentype
+        image/svg+xml
+        image/x-icon
+        text/css
+        text/plain
+        text/x-component;
+        # text/html is always compressed by HttpGzipModule
+}

build/frappe-socketio-arm/Dockerfile

@@ -8,7 +8,7 @@ RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frap
     && chown -R frappe:frappe /home/frappe
 
 # Download socketio and purge curl package
-COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe
+COPY package.json /home/frappe/frappe-bench/apps/frappe
 ARG GIT_BRANCH=develop
 RUN apt-get update && apt-get install -y curl \
     && cd /home/frappe/frappe-bench/apps/frappe \
@@ -25,7 +25,7 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \
     && npm --version
 
 # Setup docker-entrypoint
-COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
 
 WORKDIR /home/frappe/frappe-bench/sites

build/frappe-worker-arm/Dockerfile

@@ -1,39 +1,42 @@
-FROM bitnami/python:3.7-prod
+FROM python:3.7.9-stretch
 
 RUN useradd -ms /bin/bash frappe
 
 ARG GIT_BRANCH=develop
 ENV PYTHONUNBUFFERED 1
 ENV NVM_DIR=/home/frappe/.nvm
-ENV NODE_VERSION=12.18.3
+ENV NODE_VERSION=12.19.1
 ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
 
 # Install dependencies
 WORKDIR /home/frappe/frappe-bench
-RUN install_packages \
+RUN apt-get update -y && apt-get install build-essential \
     git \
     mariadb-client \
     postgresql-client \
     gettext-base \
     wget \
+    curl \
     # for PDF
     libjpeg62-turbo \
     libx11-6 \
     libxcb1 \
     libxext6 \
     libxrender1 \
+    libffi-dev libjpeg-dev libxml2  \
     libssl-dev \
     fonts-cantarell \
     xfonts-75dpi \
     xfonts-base \
+    zlib1g-dev \
     # For psycopg2
     libpq-dev \
-    wait-for-it \
-    && wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb \
-    && dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb \
-    && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \
+    wait-for-it -y \
+    && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_arm64.deb \
+    && dpkg -i wkhtmltox_0.12.6-1.buster_arm64.deb && rm wkhtmltox_0.12.6-1.buster_arm64.deb \
+    && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh && chmod +x install.sh \
     && apt-get purge -y wget && apt-get autoremove -y \
-    && chown -R frappe:frappe /home/frappe
+    && chown -R frappe:frappe /home/frappe && pip install bootstrapped --no-deps
 
 USER frappe
 # Install nvm with node
@@ -55,14 +58,18 @@ RUN python -m venv env \
 
 USER root
 # Copy scripts and templates
-COPY build/common/commands/* /home/frappe/frappe-bench/commands/
-COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template
-COPY build/common/worker/install_app.sh /usr/local/bin/install_app
-COPY build/common/worker/bench /usr/local/bin/bench
-COPY build/common/worker/healthcheck.sh /usr/local/bin/healthcheck.sh
+COPY common/commands/* /home/frappe/frappe-bench/commands/
+RUN chmod +x -R /home/frappe/frappe-bench/commands/
+COPY common/common_site_config.json.template /opt/frappe/common_site_config.json.template
+COPY common/worker/install_app.sh /usr/local/bin/install_app
+RUN chmod +x /usr/local/bin/install_app
+COPY common/worker/bench /usr/local/bin/bench 
+RUN chmod +x /usr/local/bin/bench
+COPY common/worker/healthcheck.sh /usr/local/bin/healthcheck.sh
+RUN chmod +x /usr/local/bin/healthcheck.sh
 
 # Setup docker-entrypoint
-COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+COPY common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat
 
 # Use sites volume as working directory

build/frappe-worker-arm/common/commands/auto_migrate.py

@@ -0,0 +1,64 @@
+import os
+import semantic_version
+import git
+
+from migrate import migrate_sites
+from utils import (
+    save_version_file,
+    get_apps,
+    get_container_versions,
+    get_version_file,
+    get_config
+)
+
+
+def main():
+    is_ready = False
+    apps = get_apps()
+
+    container_versions = get_container_versions(apps)
+
+    version_file = get_version_file()
+
+    if not version_file:
+        version_file = container_versions
+        save_version_file(version_file)
+
+    for app in apps:
+        container_version = None
+        file_version = None
+        version_file_hash = None
+        container_hash = None
+
+        repo = git.Repo(os.path.join('..', 'apps', app))
+        branch = repo.active_branch.name
+
+        if branch == 'develop':
+            version_file_hash = version_file.get(app+'_git_hash')
+            container_hash = container_versions.get(app+'_git_hash')
+            if container_hash and version_file_hash:
+                if container_hash != version_file_hash:
+                    is_ready = True
+                    break
+
+        if version_file.get(app):
+            file_version = semantic_version.Version(version_file.get(app))
+
+        if container_versions.get(app):
+            container_version = semantic_version.Version(container_versions.get(app))
+
+        if file_version and container_version:
+            if container_version > file_version:
+                is_ready = True
+                break
+
+    config = get_config()
+
+    if is_ready and config.get('maintenance_mode') != 1:
+        migrate_sites(maintenance_mode=True)
+        version_file = container_versions
+        save_version_file(version_file)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/background.py

@@ -0,0 +1,11 @@
+from frappe.utils.scheduler import start_scheduler
+
+
+def main():
+    print("Starting background scheduler . . .")
+    start_scheduler()
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/backup.py

@@ -0,0 +1,39 @@
+import os
+import frappe
+from frappe.utils.backups import scheduled_backup
+from frappe.utils import cint, get_sites, now
+
+
+def backup(sites, with_files=False):
+    for site in sites:
+        frappe.init(site)
+        frappe.connect()
+        odb = scheduled_backup(
+            ignore_files=not with_files,
+            backup_path_db=None,
+            backup_path_files=None,
+            backup_path_private_files=None,
+            force=True
+        )
+        print("database backup taken -", odb.backup_path_db, "- on", now())
+        if with_files:
+            print("files backup taken -", odb.backup_path_files, "- on", now())
+            print("private files backup taken -", odb.backup_path_private_files, "- on", now())
+        frappe.destroy()
+
+
+def main():
+    installed_sites = ":".join(get_sites())
+    sites = os.environ.get("SITES", installed_sites).split(":")
+    with_files = cint(os.environ.get("WITH_FILES"))
+
+    backup(sites, with_files)
+
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/check_connection.py

@@ -0,0 +1,130 @@
+import socket
+import time
+from six.moves.urllib.parse import urlparse
+from utils import get_config
+from constants import (
+    REDIS_QUEUE_KEY,
+    REDIS_CACHE_KEY,
+    REDIS_SOCKETIO_KEY,
+    DB_HOST_KEY,
+    DB_PORT_KEY,
+    DB_PORT
+)
+
+
+def is_open(ip, port, timeout=30):
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    s.settimeout(timeout)
+    try:
+        s.connect((ip, int(port)))
+        s.shutdown(socket.SHUT_RDWR)
+        return True
+    except Exception:
+        return False
+    finally:
+        s.close()
+
+
+def check_host(ip, port, retry=10, delay=3, print_attempt=True):
+    ipup = False
+    for i in range(retry):
+        if print_attempt:
+            print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1))
+        if is_open(ip, port):
+            ipup = True
+            break
+        else:
+            time.sleep(delay)
+    return ipup
+
+
+# Check service
+def check_service(
+    retry=10,
+    delay=3,
+    print_attempt=True,
+    service_name=None,
+    service_port=None):
+
+    config = get_config()
+    if not service_name:
+        service_name = config.get(DB_HOST_KEY, 'mariadb')
+    if not service_port:
+        service_port = config.get(DB_PORT_KEY, DB_PORT)
+
+    is_db_connected = False
+    is_db_connected = check_host(
+        service_name,
+        service_port,
+        retry,
+        delay,
+        print_attempt)
+    if not is_db_connected:
+        print("Connection to {service_name}:{service_port} timed out".format(
+            service_name=service_name,
+            service_port=service_port,
+        ))
+        exit(1)
+
+
+# Check redis queue
+def check_redis_queue(retry=10, delay=3, print_attempt=True):
+    check_redis_queue = False
+    config = get_config()
+    redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc
+    redis_queue, redis_queue_port = redis_queue_url.split(":")
+    check_redis_queue = check_host(
+        redis_queue,
+        redis_queue_port,
+        retry,
+        delay,
+        print_attempt)
+    if not check_redis_queue:
+        print("Connection to redis queue timed out")
+        exit(1)
+
+
+# Check redis cache
+def check_redis_cache(retry=10, delay=3, print_attempt=True):
+    check_redis_cache = False
+    config = get_config()
+    redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc
+    redis_cache, redis_cache_port = redis_cache_url.split(":")
+    check_redis_cache = check_host(
+        redis_cache,
+        redis_cache_port,
+        retry,
+        delay,
+        print_attempt)
+    if not check_redis_cache:
+        print("Connection to redis cache timed out")
+        exit(1)
+
+
+# Check redis socketio
+def check_redis_socketio(retry=10, delay=3, print_attempt=True):
+    check_redis_socketio = False
+    config = get_config()
+    redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc
+    redis_socketio, redis_socketio_port = redis_socketio_url.split(":")
+    check_redis_socketio = check_host(
+        redis_socketio,
+        redis_socketio_port,
+        retry,
+        delay,
+        print_attempt)
+    if not check_redis_socketio:
+        print("Connection to redis socketio timed out")
+        exit(1)
+
+
+def main():
+    check_service()
+    check_redis_queue()
+    check_redis_cache()
+    check_redis_socketio()
+    print('Connections OK')
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/console.py

@@ -0,0 +1,32 @@
+import sys
+import frappe
+import IPython
+
+from frappe.utils import get_sites
+
+
+def console(site):
+    "Start ipython console for a site"
+    if site not in get_sites():
+        print("Site {0} does not exist on the current bench".format(site))
+        return
+
+    frappe.init(site=site)
+    frappe.connect()
+    frappe.local.lang = frappe.db.get_default("lang")
+    all_apps = frappe.get_installed_apps()
+    for app in all_apps:
+        locals()[app] = __import__(app)
+    print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
+    IPython.embed(display_banner="", header="")
+
+
+def main():
+    site = sys.argv[-1]
+    console(site)
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/constants.py

@@ -0,0 +1,13 @@
+REDIS_QUEUE_KEY = 'redis_queue'
+REDIS_CACHE_KEY = 'redis_cache'
+REDIS_SOCKETIO_KEY = 'redis_socketio'
+DB_HOST_KEY = 'db_host'
+DB_PORT_KEY = 'db_port'
+DB_PORT = 3306
+APP_VERSIONS_JSON_FILE = 'app_versions.json'
+APPS_TXT_FILE = 'apps.txt'
+COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
+DATE_FORMAT = "%Y%m%d_%H%M%S"
+RDS_DB = 'rds_db'
+RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES"
+ARCHIVE_SITES_PATH = '/home/frappe/frappe-bench/sites/archive_sites'

build/frappe-worker-arm/common/commands/doctor.py

@@ -0,0 +1,61 @@
+import argparse
+
+from check_connection import (
+    check_service,
+    check_redis_cache,
+    check_redis_queue,
+    check_redis_socketio,
+)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-p',
+        '--ping-service',
+        dest='ping_services',
+        action='append',
+        type=str,
+        help='list of services to ping, e.g. doctor -p "postgres:5432" --ping-service "mariadb:3306"',
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+    check_service(retry=1, delay=0, print_attempt=False)
+    print("Bench database Connected")
+    check_redis_cache(retry=1, delay=0, print_attempt=False)
+    print("Redis Cache Connected")
+    check_redis_queue(retry=1, delay=0, print_attempt=False)
+    print("Redis Queue Connected")
+    check_redis_socketio(retry=1, delay=0, print_attempt=False)
+    print("Redis SocketIO Connected")
+
+    if(args.ping_services):
+        for service in args.ping_services:
+            service_name = None
+            service_port = None
+
+            try:
+                service_name, service_port = service.split(':')
+            except ValueError:
+                print('Service should be in format host:port, e.g postgres:5432')
+                exit(1)
+
+            check_service(
+                retry=1,
+                delay=0,
+                print_attempt=False,
+                service_name=service_name,
+                service_port=service_port,
+            )
+            print("{0}:{1} Connected".format(service_name, service_port))
+
+    print("Health check successful")
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/drop.py

@@ -0,0 +1,39 @@
+import os
+import frappe
+
+from frappe.commands.site import _drop_site
+from constants import ARCHIVE_SITES_PATH
+from utils import get_password
+
+
+def main():
+    site_name = os.environ.get("SITE_NAME", 'site1.localhost')
+    db_root_username = os.environ.get("DB_ROOT_USER", 'root')
+    mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin')
+    postgres_root_password = get_password("POSTGRES_PASSWORD")
+    db_root_password = mariadb_root_password
+
+    if postgres_root_password:
+        db_root_password = postgres_root_password
+
+    force = True if os.environ.get("FORCE", None) else False
+    no_backup = True if os.environ.get("NO_BACKUP", None) else False
+    frappe.init(site_name, new_site=True)
+
+    _drop_site(
+        site=site_name,
+        root_login=db_root_username,
+        root_password=db_root_password,
+        archived_sites_path=ARCHIVE_SITES_PATH,
+        force=force,
+        no_backup=no_backup
+    )
+
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/migrate.py

@@ -0,0 +1,51 @@
+import os
+import frappe
+
+from frappe.utils import cint, get_sites
+from utils import get_config, save_config
+
+
+def set_maintenance_mode(enable=True):
+    conf = get_config()
+
+    if enable:
+        conf.update({"maintenance_mode": 1, "pause_scheduler": 1})
+        save_config(conf)
+
+    if not enable:
+        conf.update({"maintenance_mode": 0, "pause_scheduler": 0})
+        save_config(conf)
+
+
+def migrate_sites(maintenance_mode=False):
+    installed_sites = ":".join(get_sites())
+    sites = os.environ.get("SITES", installed_sites).split(":")
+    if not maintenance_mode:
+        maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE"))
+
+    if maintenance_mode:
+        set_maintenance_mode(True)
+
+    for site in sites:
+        print('Migrating', site)
+        frappe.init(site=site)
+        frappe.connect()
+        try:
+            from frappe.migrate import migrate
+            migrate()
+        finally:
+            frappe.destroy()
+
+    # Disable maintenance mode after migration
+    set_maintenance_mode(False)
+
+
+def main():
+    migrate_sites()
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/new.py

@@ -0,0 +1,109 @@
+import os
+import frappe
+import semantic_version
+
+from frappe.commands.site import _new_site
+from frappe.installer import update_site_config
+from constants import COMMON_SITE_CONFIG_FILE, RDS_DB, RDS_PRIVILEGES
+from utils import (
+    run_command,
+    get_config,
+    get_site_config,
+    get_password,
+)
+
+
+def main():
+    config = get_config()
+    db_type = 'mariadb'
+    db_port = config.get('db_port', 3306)
+    db_host = config.get('db_host')
+    site_name = os.environ.get("SITE_NAME", 'site1.localhost')
+    db_root_username = os.environ.get("DB_ROOT_USER", 'root')
+    mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin')
+    postgres_root_password = get_password("POSTGRES_PASSWORD")
+    db_root_password = mariadb_root_password
+
+    if postgres_root_password:
+        db_type = 'postgres'
+        db_host = os.environ.get("POSTGRES_HOST")
+        db_port = 5432
+        db_root_password = postgres_root_password
+        if not db_host:
+            db_host = config.get('db_host')
+            print('Environment variable POSTGRES_HOST not found.')
+            print('Using db_host from common_site_config.json')
+
+        sites_path = os.getcwd()
+        common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE)
+        update_site_config("root_login", db_root_username, validate = False, site_config_path = common_site_config_path)
+        update_site_config("root_password", db_root_password, validate = False, site_config_path = common_site_config_path)
+
+    force = True if os.environ.get("FORCE", None) else False
+    install_apps = os.environ.get("INSTALL_APPS", None)
+    install_apps = install_apps.split(',') if install_apps else []
+    frappe.init(site_name, new_site=True)
+
+    if semantic_version.Version(frappe.__version__).major > 11:
+        _new_site(
+            None,
+            site_name,
+            mariadb_root_username=db_root_username,
+            mariadb_root_password=db_root_password,
+            admin_password=get_password("ADMIN_PASSWORD", 'admin'),
+            verbose=True,
+            install_apps=install_apps,
+            source_sql=None,
+            force=force,
+            db_type=db_type,
+            reinstall=False,
+            db_host=db_host,
+            db_port=db_port,
+        )
+    else:
+        _new_site(
+            None,
+            site_name,
+            mariadb_root_username=db_root_username,
+            mariadb_root_password=db_root_password,
+            admin_password=get_password("ADMIN_PASSWORD", 'admin'),
+            verbose=True,
+            install_apps=install_apps,
+            source_sql=None,
+            force=force,
+            reinstall=False,
+        )
+
+
+    if db_type == "mariadb":
+        site_config = get_site_config(site_name)
+        db_name = site_config.get('db_name')
+        db_password = site_config.get('db_password')
+
+        mysql_command = ["mysql", f"-h{db_host}", f"-u{db_root_username}", f"-p{mariadb_root_password}", "-e"]
+
+        # Drop User if exists
+        command = mysql_command + [f"DROP USER IF EXISTS '{db_name}'; FLUSH PRIVILEGES;"]
+        run_command(command)
+
+        # Grant permission to database and set password
+        grant_privileges = "ALL PRIVILEGES"
+
+        # for Amazon RDS
+        if config.get(RDS_DB) or site_config.get(RDS_DB):
+            grant_privileges = RDS_PRIVILEGES
+
+        command = mysql_command + [f"\
+            CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; \
+            GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%'; \
+            FLUSH PRIVILEGES;"]
+        run_command(command)
+
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/push_backup.py

@@ -0,0 +1,156 @@
+import os
+import time
+import boto3
+
+import datetime
+from glob import glob
+from frappe.utils import get_sites
+from constants import DATE_FORMAT
+from utils import (
+    get_s3_config,
+    upload_file_to_s3,
+    check_s3_environment_variables,
+)
+
+
+def get_file_ext():
+    return {
+        "database": "-database.sql.gz",
+        "private_files": "-private-files.tar",
+        "public_files": "-files.tar",
+        "site_config": "-site_config_backup.json"
+    }
+
+
+def get_backup_details(sitename):
+    backup_details = dict()
+    file_ext = get_file_ext()
+
+    # add trailing slash https://stackoverflow.com/a/15010678
+    site_backup_path = os.path.join(os.getcwd(), sitename, "private", "backups", "")
+
+    if os.path.exists(site_backup_path):
+        for filetype, ext in file_ext.items():
+            site_slug = sitename.replace('.', '_')
+            pattern = site_backup_path + '*-' + site_slug + ext
+            backup_files = list(filter(os.path.isfile, glob(pattern)))
+
+            if len(backup_files) > 0:
+                backup_files.sort(key=lambda file: os.stat(os.path.join(site_backup_path, file)).st_ctime)
+                backup_date = datetime.datetime.strptime(time.ctime(os.path.getmtime(backup_files[0])), "%a %b %d %H:%M:%S %Y")
+                backup_details[filetype] = {
+                    "sitename": sitename,
+                    "file_size_in_bytes": os.stat(backup_files[-1]).st_size,
+                    "file_path": os.path.abspath(backup_files[-1]),
+                    "filename": os.path.basename(backup_files[-1]),
+                    "backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S")
+                }
+
+    return backup_details
+
+
+def delete_old_backups(limit, bucket, site_name):
+    all_backups = list()
+    all_backup_dates = list()
+    backup_limit = int(limit)
+    check_s3_environment_variables()
+    bucket_dir = os.environ.get('BUCKET_DIR')
+    oldest_backup_date = None
+
+    s3 = boto3.resource(
+        's3',
+        region_name=os.environ.get('REGION'),
+        aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
+        aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
+        endpoint_url=os.environ.get('ENDPOINT_URL')
+    )
+
+    bucket = s3.Bucket(bucket)
+    objects = bucket.meta.client.list_objects_v2(
+        Bucket=bucket.name,
+        Delimiter='/')
+
+    if objects:
+        for obj in objects.get('CommonPrefixes'):
+            if obj.get('Prefix') == bucket_dir + '/':
+                for backup_obj in bucket.objects.filter(Prefix=obj.get('Prefix')):
+                    if backup_obj.get()["ContentType"] == "application/x-directory":
+                        continue
+                    try:
+                        # backup_obj.key is bucket_dir/site/date_time/backupfile.extension
+                        bucket_dir, site_slug, date_time, backupfile = backup_obj.key.split('/')
+                        date_time_object = datetime.datetime.strptime(
+                            date_time, DATE_FORMAT
+                        )
+
+                        if site_name in backup_obj.key:
+                            all_backup_dates.append(date_time_object)
+                            all_backups.append(backup_obj.key)
+                    except IndexError as error:
+                        print(error)
+                        exit(1)
+
+    if len(all_backup_dates) > 0:
+        oldest_backup_date = min(all_backup_dates)
+
+    if len(all_backups) / 3 > backup_limit:
+        oldest_backup = None
+        for backup in all_backups:
+            try:
+                # backup is bucket_dir/site/date_time/backupfile.extension
+                backup_dir, site_slug, backup_dt_string, filename = backup.split('/')
+                backup_datetime = datetime.datetime.strptime(
+                    backup_dt_string, DATE_FORMAT
+                )
+                if backup_datetime == oldest_backup_date:
+                    oldest_backup = backup
+
+            except IndexError as error:
+                print(error)
+                exit(1)
+
+            if oldest_backup:
+                for obj in bucket.objects.filter(Prefix=oldest_backup):
+                    # delete all keys that are inside the oldest_backup
+                    if bucket_dir in obj.key:
+                        print('Deleteing ' + obj.key)
+                        s3.Object(bucket.name, obj.key).delete()
+
+
+def main():
+    details = dict()
+    sites = get_sites()
+    conn, bucket = get_s3_config()
+
+    for site in sites:
+        details = get_backup_details(site)
+        db_file = details.get('database', {}).get('file_path')
+        folder = os.environ.get('BUCKET_DIR') + '/' + site + '/'
+        if db_file:
+            folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(db_file)[:15] + '/'
+            upload_file_to_s3(db_file, folder, conn, bucket)
+
+            # Archive site_config.json
+            site_config_file = details.get('site_config', {}).get('file_path')
+            if not site_config_file:
+                site_config_file = os.path.join(os.getcwd(), site, 'site_config.json')
+            upload_file_to_s3(site_config_file, folder, conn, bucket)
+
+        public_files = details.get('public_files', {}).get('file_path')
+        if public_files:
+            folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(public_files)[:15] + '/'
+            upload_file_to_s3(public_files, folder, conn, bucket)
+
+        private_files = details.get('private_files', {}).get('file_path')
+        if private_files:
+            folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(private_files)[:15] + '/'
+            upload_file_to_s3(private_files, folder, conn, bucket)
+
+        delete_old_backups(os.environ.get('BACKUP_LIMIT', '3'), bucket, site)
+
+    print('push-backup complete')
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/restore_backup.py

@@ -0,0 +1,294 @@
+import os
+import datetime
+import tarfile
+import hashlib
+import frappe
+import boto3
+
+from frappe.utils import get_sites, random_string
+from frappe.installer import (
+    make_conf,
+    get_conf_params,
+    make_site_dirs,
+    update_site_config
+)
+from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES
+from utils import (
+    run_command,
+    list_directories,
+    set_key_in_site_config,
+    get_site_config,
+    get_config,
+    get_password,
+    check_s3_environment_variables,
+)
+
+
+def get_backup_dir():
+    return os.path.join(
+        os.path.expanduser('~'),
+        'backups'
+    )
+
+
+def decompress_db(database_file, site):
+    command = ["gunzip", "-c", database_file]
+    with open(database_file.replace(".gz", ""), "w") as db_file:
+        print('Extract Database GZip for site {}'.format(site))
+        run_command(command, stdout=db_file)
+
+
+def restore_database(files_base, site_config_path, site):
+    # restore database
+    database_file = files_base + '-database.sql.gz'
+    decompress_db(database_file, site)
+    config = get_config()
+
+    # Set db_type if it exists in backup site_config.json
+    set_key_in_site_config('db_type', site, site_config_path)
+    # Set db_host if it exists in backup site_config.json
+    set_key_in_site_config('db_host', site, site_config_path)
+    # Set db_port if it exists in backup site_config.json
+    set_key_in_site_config('db_port', site, site_config_path)
+
+    # get updated site_config
+    site_config = get_site_config(site)
+
+    # if no db_type exists, default to mariadb
+    db_type = site_config.get('db_type', 'mariadb')
+    is_database_restored = False
+
+    if db_type == 'mariadb':
+        restore_mariadb(
+            config=config,
+            site_config=site_config,
+            database_file=database_file)
+        is_database_restored = True
+    elif db_type == 'postgres':
+        restore_postgres(
+            config=config,
+            site_config=site_config,
+            database_file=database_file)
+        is_database_restored = True
+
+    if is_database_restored:
+        # Set encryption_key if it exists in backup site_config.json
+        set_key_in_site_config('encryption_key', site, site_config_path)
+
+
+def restore_files(files_base):
+    public_files = files_base + '-files.tar'
+    # extract tar
+    public_tar = tarfile.open(public_files)
+    print('Extracting {}'.format(public_files))
+    public_tar.extractall()
+
+
+def restore_private_files(files_base):
+    private_files = files_base + '-private-files.tar'
+    private_tar = tarfile.open(private_files)
+    print('Extracting {}'.format(private_files))
+    private_tar.extractall()
+
+
+def pull_backup_from_s3():
+    check_s3_environment_variables()
+
+    # https://stackoverflow.com/a/54672690
+    s3 = boto3.resource(
+        's3',
+        region_name=os.environ.get('REGION'),
+        aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
+        aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
+        endpoint_url=os.environ.get('ENDPOINT_URL')
+    )
+
+    bucket_dir = os.environ.get('BUCKET_DIR')
+    bucket_name = os.environ.get('BUCKET_NAME')
+    bucket = s3.Bucket(bucket_name)
+
+    # Change directory to /home/frappe/backups
+    os.chdir(get_backup_dir())
+
+    backup_files = []
+    sites = set()
+    site_timestamps = set()
+    download_backups = []
+
+    for obj in bucket.objects.filter(Prefix=bucket_dir):
+        if obj.get()["ContentType"] == "application/x-directory":
+            continue
+        backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '')
+        backup_files.append(backup_file)
+        site_name, timestamp, backup_type = backup_file.split('/')
+        site_timestamp = site_name + '/' + timestamp
+        sites.add(site_name)
+        site_timestamps.add(site_timestamp)
+
+    # sort sites for latest backups
+    for site in sites:
+        backup_timestamps = []
+        for site_timestamp in site_timestamps:
+            site_name, timestamp = site_timestamp.split('/')
+            if site == site_name:
+                timestamp_datetime = datetime.datetime.strptime(
+                    timestamp, DATE_FORMAT
+                )
+                backup_timestamps.append(timestamp)
+        download_backups.append(site + '/' + max(backup_timestamps))
+
+    # Only download latest backups
+    for backup_file in backup_files:
+        for backup in download_backups:
+            if backup in backup_file:
+                if not os.path.exists(os.path.dirname(backup_file)):
+                    os.makedirs(os.path.dirname(backup_file))
+                print('Downloading {}'.format(backup_file))
+                bucket.download_file(bucket_dir + '/' + backup_file, backup_file)
+
+    os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites'))
+
+
+def restore_postgres(config, site_config, database_file):
+    # common config
+    common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE)
+
+    db_root_user = config.get('root_login')
+    if not db_root_user:
+        postgres_user = os.environ.get('DB_ROOT_USER')
+        if not postgres_user:
+            print('Variable DB_ROOT_USER not set')
+            exit(1)
+
+        db_root_user = postgres_user
+        update_site_config(
+            "root_login",
+            db_root_user,
+            validate=False,
+            site_config_path=common_site_config_path)
+
+    db_root_password = config.get('root_password')
+    if not db_root_password:
+        root_password = get_password('POSTGRES_PASSWORD')
+        if not root_password:
+            print('Variable POSTGRES_PASSWORD not set')
+            exit(1)
+
+        db_root_password = root_password
+        update_site_config(
+            "root_password",
+            db_root_password,
+            validate=False,
+            site_config_path=common_site_config_path)
+
+    # site config
+    db_host = site_config.get('db_host')
+    db_port = site_config.get('db_port', 5432)
+    db_name = site_config.get('db_name')
+    db_password = site_config.get('db_password')
+
+    psql_command = ["psql"]
+    psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}"
+
+    print('Restoring PostgreSQL')
+    run_command(psql_command + [psql_uri, "-c", f"DROP DATABASE IF EXISTS \"{db_name}\""])
+    run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"])
+    run_command(psql_command + [psql_uri, "-c", f"CREATE DATABASE \"{db_name}\""])
+    run_command(psql_command + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"])
+    run_command(psql_command + [psql_uri, "-c", f"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"])
+    with open(database_file.replace('.gz', ''), 'r') as db_file:
+        run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file)
+
+
+def restore_mariadb(config, site_config, database_file):
+    db_root_password = get_password('MYSQL_ROOT_PASSWORD')
+    if not db_root_password:
+        print('Variable MYSQL_ROOT_PASSWORD not set')
+        exit(1)
+
+    db_root_user = os.environ.get("DB_ROOT_USER", 'root')
+
+    db_host = site_config.get('db_host', config.get('db_host'))
+    db_port = site_config.get('db_port', config.get('db_port', 3306))
+    db_name = site_config.get('db_name')
+    db_password = site_config.get('db_password')
+
+    # mysql command prefix
+    mysql_command = ["mysql", f"-u{db_root_user}", f"-h{db_host}", f"-p{db_root_password}", f"-P{db_port}"]
+
+    # drop db if exists for clean restore
+    drop_database = mysql_command + ["-e",  f"DROP DATABASE IF EXISTS `{db_name}`;"]
+    run_command(drop_database)
+
+    # create db
+    create_database = mysql_command + ["-e", f"CREATE DATABASE IF NOT EXISTS `{db_name}`;"]
+    run_command(create_database)
+
+    # create user
+    create_user = mysql_command + ["-e", f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"]
+    run_command(create_user)
+
+    # grant db privileges to user
+
+    grant_privileges = "ALL PRIVILEGES"
+
+    # for Amazon RDS
+    if config.get(RDS_DB) or site_config.get(RDS_DB):
+        grant_privileges = RDS_PRIVILEGES
+
+    grant_privileges_command = mysql_command + ["-e", f"GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"]
+    run_command(grant_privileges_command)
+
+    print('Restoring MariaDB')
+    with open(database_file.replace('.gz', ''), 'r') as db_file:
+        run_command(mysql_command + [f"{db_name}"], stdin=db_file)
+
+
+def main():
+    backup_dir = get_backup_dir()
+
+    if len(list_directories(backup_dir)) == 0:
+        pull_backup_from_s3()
+
+    for site in list_directories(backup_dir):
+        site_slug = site.replace('.', '_')
+        backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))]
+        latest_backup = max(backups).strftime(DATE_FORMAT)
+        files_base = os.path.join(backup_dir, site, latest_backup, '')
+        files_base += latest_backup + '-' + site_slug
+        site_config_path = files_base + '-site_config_backup.json'
+        if not os.path.exists(site_config_path):
+            site_config_path = os.path.join(backup_dir, site, 'site_config.json')
+        if site in get_sites():
+            print('Overwrite site {}'.format(site))
+            restore_database(files_base, site_config_path, site)
+            restore_private_files(files_base)
+            restore_files(files_base)
+        else:
+            site_config = get_conf_params(
+                db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16],
+                db_password=random_string(16)
+            )
+
+            frappe.local.site = site
+            frappe.local.sites_path = os.getcwd()
+            frappe.local.site_path = os.getcwd() + '/' + site
+            make_conf(
+                db_name=site_config.get('db_name'),
+                db_password=site_config.get('db_password'),
+            )
+            make_site_dirs()
+
+            print('Create site {}'.format(site))
+            restore_database(files_base, site_config_path, site)
+            restore_private_files(files_base)
+            restore_files(files_base)
+
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/commands/utils.py

@@ -0,0 +1,204 @@
+import json
+import os
+import subprocess
+import boto3
+import git
+
+from frappe.installer import update_site_config
+from constants import (
+    APP_VERSIONS_JSON_FILE,
+    APPS_TXT_FILE,
+    COMMON_SITE_CONFIG_FILE
+)
+
+def run_command(command, stdout=None, stdin=None, stderr=None):
+    stdout = stdout or subprocess.PIPE
+    stderr = stderr or subprocess.PIPE
+    stdin = stdin or subprocess.PIPE
+    process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr)
+    out, error = process.communicate()
+    if process.returncode:
+        print("Something went wrong:")
+        print(f"return code: {process.returncode}")
+        print(f"stdout:\n{out}")
+        print(f"\nstderr:\n{error}")
+        exit(process.returncode)
+
+
+def save_version_file(versions):
+    with open(APP_VERSIONS_JSON_FILE, 'w') as f:
+        return json.dump(versions, f, indent=1, sort_keys=True)
+
+
+def get_apps():
+    apps = []
+    try:
+        with open(APPS_TXT_FILE) as apps_file:
+            for app in apps_file.readlines():
+                if app.strip():
+                    apps.append(app.strip())
+
+    except FileNotFoundError as exception:
+        print(exception)
+        exit(1)
+    except Exception:
+        print(APPS_TXT_FILE + " is not valid")
+        exit(1)
+
+    return apps
+
+
+def get_container_versions(apps):
+    versions = {}
+    for app in apps:
+        try:
+            version = __import__(app).__version__
+            versions.update({app: version})
+        except Exception:
+            pass
+
+        try:
+            path = os.path.join('..', 'apps', app)
+            repo = git.Repo(path)
+            commit_hash = repo.head.object.hexsha
+            versions.update({app+'_git_hash': commit_hash})
+        except Exception:
+            pass
+
+    return versions
+
+
+def get_version_file():
+    versions = None
+    try:
+        with open(APP_VERSIONS_JSON_FILE) as versions_file:
+            versions = json.load(versions_file)
+    except Exception:
+        pass
+    return versions
+
+
+def get_config():
+    config = None
+    try:
+        with open(COMMON_SITE_CONFIG_FILE) as config_file:
+            config = json.load(config_file)
+    except FileNotFoundError as exception:
+        print(exception)
+        exit(1)
+    except Exception:
+        print(COMMON_SITE_CONFIG_FILE + " is not valid")
+        exit(1)
+    return config
+
+
+def get_site_config(site_name):
+    site_config = None
+    with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file:
+        site_config = json.load(site_config_file)
+    return site_config
+
+
+def save_config(config):
+    with open(COMMON_SITE_CONFIG_FILE, 'w') as f:
+        return json.dump(config, f, indent=1, sort_keys=True)
+
+
+def get_password(env_var, default=None):
+    return os.environ.get(env_var) or get_password_from_secret(f"{env_var}_FILE") or default
+
+
+def get_password_from_secret(env_var):
+    """Fetches the secret value from the docker secret file
+    usually located inside /run/secrets/
+    Arguments:
+        env_var {str} -- Name of the environment variable
+        containing the path to the secret file.
+    Returns:
+        [str] -- Secret value
+    """
+    passwd = None
+    secret_file_path = os.environ.get(env_var)
+    if secret_file_path:
+        with open(secret_file_path) as secret_file:
+            passwd = secret_file.read().strip()
+
+    return passwd
+
+
+def get_s3_config():
+    check_s3_environment_variables()
+    bucket = os.environ.get('BUCKET_NAME')
+
+    conn = boto3.client(
+        's3',
+        region_name=os.environ.get('REGION'),
+        aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
+        aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
+        endpoint_url=os.environ.get('ENDPOINT_URL')
+    )
+
+    return conn, bucket
+
+
+def upload_file_to_s3(filename, folder, conn, bucket):
+
+    destpath = os.path.join(folder, os.path.basename(filename))
+    try:
+        print("Uploading file:", filename)
+        conn.upload_file(filename, bucket, destpath)
+
+    except Exception as e:
+        print("Error uploading: %s" % (e))
+        exit(1)
+
+
+def list_directories(path):
+    directories = []
+    for name in os.listdir(path):
+        if os.path.isdir(os.path.join(path, name)):
+            directories.append(name)
+    return directories
+
+
+def get_site_config_from_path(site_config_path):
+    site_config = dict()
+    if os.path.exists(site_config_path):
+        with open(site_config_path, 'r') as sc:
+            site_config = json.load(sc)
+    return site_config
+
+
+def set_key_in_site_config(key, site, site_config_path):
+    site_config = get_site_config_from_path(site_config_path)
+    value = site_config.get(key)
+    if value:
+        print('Set {key} in site config for site: {site}'.format(key=key, site=site))
+        update_site_config(key, value,
+                            site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
+
+
+def check_s3_environment_variables():
+    if 'BUCKET_NAME' not in os.environ:
+        print('Variable BUCKET_NAME not set')
+        exit(1)
+
+    if 'ACCESS_KEY_ID' not in os.environ:
+        print('Variable ACCESS_KEY_ID not set')
+        exit(1)
+
+    if 'SECRET_ACCESS_KEY' not in os.environ:
+        print('Variable SECRET_ACCESS_KEY not set')
+        exit(1)
+
+    if 'ENDPOINT_URL' not in os.environ:
+        print('Variable ENDPOINT_URL not set')
+        exit(1)
+
+    if 'BUCKET_DIR' not in os.environ:
+        print('Variable BUCKET_DIR not set')
+        exit(1)
+
+    if 'REGION' not in os.environ:
+        print('Variable REGION not set')
+        exit(1)

build/frappe-worker-arm/common/commands/worker.py

@@ -0,0 +1,12 @@
+import os
+from frappe.utils.background_jobs import start_worker
+
+
+def main():
+    queue = os.environ.get("WORKER_TYPE", "default")
+    start_worker(queue, False)
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/frappe-worker-arm/common/common_site_config.json.template

@@ -0,0 +1,8 @@
+{
+    "db_host": "${DB_HOST}",
+    "db_port": ${DB_PORT},
+    "redis_cache": "redis://${REDIS_CACHE}",
+    "redis_queue": "redis://${REDIS_QUEUE}",
+    "redis_socketio": "redis://${REDIS_SOCKETIO}",
+    "socketio_port": ${SOCKETIO_PORT}
+}

build/frappe-worker-arm/common/nginx-default.conf.template

@@ -0,0 +1,104 @@
+upstream frappe-server {
+    server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0;
+}
+
+upstream socketio-server {
+    server ${FRAPPE_SOCKETIO}:${SOCKETIO_PORT} fail_timeout=0;
+}
+
+server {
+    listen 80;
+    server_name $http_host;
+    root /var/www/html;
+
+    add_header X-Frame-Options "SAMEORIGIN";
+    add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
+    add_header X-Content-Type-Options nosniff;
+    add_header X-XSS-Protection "1; mode=block";
+
+    location /assets {
+        try_files $uri =404;
+    }
+
+    location ~ ^/protected/(.*) {
+        internal;
+        try_files /sites/$http_host/$1 =404;
+    }
+
+    location /socket.io {
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header X-Frappe-Site-Name $host;
+        proxy_set_header Origin $scheme://$http_host;
+        proxy_set_header Host $http_host;
+
+        proxy_pass http://socketio-server;
+    }
+
+    location / {
+        rewrite ^(.+)/$ $1 permanent;
+        rewrite ^(.+)/index\.html$ $1 permanent;
+        rewrite ^(.+)\.html$ $1 permanent;
+
+        location ~ ^/files/.*.(htm|html|svg|xml) {
+                add_header Content-disposition "attachment";
+                try_files /sites/$http_host/public/$uri @webserver;
+        }
+
+        try_files /sites/$http_host/public/$uri @webserver;
+    }
+
+    location @webserver {
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Frappe-Site-Name $host;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Use-X-Accel-Redirect True;
+        proxy_read_timeout 120;
+        proxy_redirect off;
+
+        proxy_pass  http://frappe-server;
+    }
+
+    # error pages
+    error_page 502 /502.html;
+    location /502.html {
+        root /var/www/templates;
+        internal;
+    }
+
+    # optimizations
+    sendfile on;
+    keepalive_timeout 15;
+    client_max_body_size 50m;
+    client_body_buffer_size 16K;
+    client_header_buffer_size 1k;
+
+    # enable gzip compresion
+    # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge
+    gzip on;
+    gzip_http_version 1.1;
+    gzip_comp_level 5;
+    gzip_min_length 256;
+    gzip_proxied any;
+    gzip_vary on;
+    gzip_types
+        application/atom+xml
+        application/javascript
+        application/json
+        application/rss+xml
+        application/vnd.ms-fontobject
+        application/x-font-ttf
+        application/font-woff
+        application/x-web-app-manifest+json
+        application/xhtml+xml
+        application/xml
+        font/opentype
+        image/svg+xml
+        image/x-icon
+        text/css
+        text/plain
+        text/x-component;
+        # text/html is always compressed by HttpGzipModule
+}

build/frappe-worker-arm/common/worker/bench

@@ -0,0 +1,19 @@
+#!/home/frappe/frappe-bench/env/bin/python
+
+import subprocess
+import sys
+import os
+
+
+if __name__ == "__main__":
+    bench_dir = os.path.join(os.sep, 'home', 'frappe', 'frappe-bench')
+    sites_dir = os.path.join(bench_dir, 'sites')
+    bench_helper = os.path.join(
+        bench_dir, 'apps', 'frappe',
+        'frappe', 'utils', 'bench_helper.py',
+    )
+    cwd = os.getcwd()
+    os.chdir(sites_dir)
+    subprocess.check_call(
+        [sys.executable, bench_helper, 'frappe'] + sys.argv[1:],
+    )

build/frappe-worker-arm/common/worker/docker-entrypoint.sh

@@ -0,0 +1,220 @@
+#!/bin/bash
+
+function configureEnv() {
+  if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then
+
+    if [[ -z "$MARIADB_HOST" ]]; then
+      if [[ -z "$POSTGRES_HOST" ]]; then
+        echo "MARIADB_HOST or POSTGRES_HOST is not set"
+        exit 1
+      fi
+    fi
+
+    if [[ -z "$REDIS_CACHE" ]]; then
+      echo "REDIS_CACHE is not set"
+      exit 1
+    fi
+
+    if [[ -z "$REDIS_QUEUE" ]]; then
+      echo "REDIS_QUEUE is not set"
+      exit 1
+    fi
+
+    if [[ -z "$REDIS_SOCKETIO" ]]; then
+      echo "REDIS_SOCKETIO is not set"
+      exit 1
+    fi
+
+    if [[ -z "$SOCKETIO_PORT" ]]; then
+      echo "SOCKETIO_PORT is not set"
+      exit 1
+    fi
+
+    if [[ -z "$DB_PORT" ]]; then
+      export DB_PORT=3306
+    fi
+
+    export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}"
+
+    envsubst '${DB_HOST}
+      ${DB_PORT}
+      ${REDIS_CACHE}
+      ${REDIS_QUEUE}
+      ${REDIS_SOCKETIO}
+      ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json
+  fi
+}
+
+function checkConnection() {
+  su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+    && python /home/frappe/frappe-bench/commands/check_connection.py"
+}
+
+function checkConfigExists() {
+  COUNTER=0
+  while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do
+      sleep 1
+      (( COUNTER=COUNTER+1 ))
+      echo "config file not created, retry $COUNTER"
+  done
+
+  if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then
+    echo "timeout: config file not created"
+    exit 1
+  fi
+}
+
+if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then
+  find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -r > /home/frappe/frappe-bench/sites/apps.txt
+fi
+
+# Allow user process to create files in logs directory
+chown -R frappe:frappe /home/frappe/frappe-bench/logs
+
+# symlink node_modules
+ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \
+  /home/frappe/frappe-bench/apps/frappe/node_modules
+
+if [ "$1" = 'start' ]; then
+  configureEnv
+  checkConnection
+
+  chown frappe:frappe /home/frappe/frappe-bench/sites/common_site_config.json
+
+  if [[ -z "$WORKERS" ]]; then
+    export WORKERS=2
+  fi
+
+  if [[ -z "$FRAPPE_PORT" ]]; then
+    export FRAPPE_PORT=8000
+  fi
+
+  if [[ ! -z "$AUTO_MIGRATE" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/auto_migrate.py"
+  fi
+
+  if [[ -z "$RUN_AS_ROOT" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && gunicorn -b 0.0.0.0:$FRAPPE_PORT \
+      --worker-tmp-dir /dev/shm \
+      --threads=4 \
+      --workers $WORKERS \
+      --worker-class=gthread \
+      --log-file=- \
+      -t 120 frappe.app:application --preload"
+  else
+    . /home/frappe/frappe-bench/env/bin/activate
+    gunicorn -b 0.0.0.0:$FRAPPE_PORT \
+      --worker-tmp-dir /dev/shm \
+      --threads=4 \
+      --workers $WORKERS \
+      --worker-class=gthread \
+      --log-file=- \
+      -t 120 frappe.app:application --preload
+  fi
+
+elif [ "$1" = 'worker' ]; then
+  checkConfigExists
+  checkConnection
+  # default WORKER_TYPE=default
+  if [[ -z "$RUN_AS_ROOT" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/worker.py"
+  else
+    . /home/frappe/frappe-bench/env/bin/activate
+    python /home/frappe/frappe-bench/commands/worker.py
+  fi
+
+elif [ "$1" = 'schedule' ]; then
+  checkConfigExists
+  checkConnection
+  if [[ -z "$RUN_AS_ROOT" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/background.py"
+  else
+    . /home/frappe/frappe-bench/env/bin/activate
+    python /home/frappe/frappe-bench/commands/background.py
+  fi
+
+elif [ "$1" = 'new' ]; then
+  checkConfigExists
+  checkConnection
+  if [[ -z "$RUN_AS_ROOT" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/new.py"
+    exit
+  else
+    . /home/frappe/frappe-bench/env/bin/activate
+    python /home/frappe/frappe-bench/commands/new.py
+  fi
+
+elif [ "$1" = 'drop' ]; then
+  checkConfigExists
+  checkConnection
+  if [[ -z "$RUN_AS_ROOT" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/drop.py"
+    exit
+  else
+    . /home/frappe/frappe-bench/env/bin/activate
+    python /home/frappe/frappe-bench/commands/drop.py
+  fi
+
+elif [ "$1" = 'migrate' ]; then
+
+  su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+    && python /home/frappe/frappe-bench/commands/migrate.py"
+  exit
+
+elif [ "$1" = 'doctor' ]; then
+
+  su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+    && python /home/frappe/frappe-bench/commands/doctor.py ${@:2}"
+  exit
+
+elif [ "$1" = 'backup' ]; then
+
+  if [[ -z "$RUN_AS_ROOT" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/backup.py"
+    exit
+  else
+    . /home/frappe/frappe-bench/env/bin/activate
+    python /home/frappe/frappe-bench/commands/backup.py
+  fi
+
+elif [ "$1" = 'console' ]; then
+
+  if [[ -z "$2" ]]; then
+    echo "Need to specify a sitename with the command:"
+    echo "console <sitename>"
+    exit 1
+  fi
+
+  if [[ -z "$RUN_AS_ROOT" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/console.py $2"
+    exit
+  else
+    . /home/frappe/frappe-bench/env/bin/activate
+    python /home/frappe/frappe-bench/commands/console.py "$2"
+  fi
+
+elif [ "$1" = 'push-backup' ]; then
+
+  su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+    && python /home/frappe/frappe-bench/commands/push_backup.py"
+  exit
+
+elif [ "$1" = 'restore-backup' ]; then
+
+  su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+    && python /home/frappe/frappe-bench/commands/restore_backup.py"
+  exit
+
+else
+
+  exec $@
+
+fi

build/frappe-worker-arm/common/worker/healthcheck.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+export COMMON_SITE_CONFIG_JSON='/home/frappe/frappe-bench/sites/common_site_config.json'
+
+# Set DB Host and port
+export DB_HOST=`cat $COMMON_SITE_CONFIG_JSON | awk '/db_host/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'`
+export DB_PORT=`cat $COMMON_SITE_CONFIG_JSON | awk '/db_port/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'`
+if [[ -z "$DB_PORT" ]]; then
+    export DB_PORT=3306
+fi
+
+# Set REDIS host:port
+export REDIS_CACHE=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_cache/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'`
+if [[ "$REDIS_CACHE" == *"/"* ]]; then
+  export REDIS_CACHE=`echo $REDIS_CACHE | cut -f1 -d"/"`
+fi
+
+export REDIS_QUEUE=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_queue/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'`
+if [[ "$REDIS_QUEUE" == *"/"* ]]; then
+  export REDIS_QUEUE=`echo $REDIS_QUEUE | cut -f1 -d"/"`
+fi
+
+export REDIS_SOCKETIO=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_socketio/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'`
+if [[ "$REDIS_SOCKETIO" == *"/"* ]]; then
+  export REDIS_SOCKETIO=`echo $REDIS_SOCKETIO | cut -f1 -d"/"`
+fi
+
+echo "Check $DB_HOST:$DB_PORT"
+wait-for-it $DB_HOST:$DB_PORT -t 1
+echo "Check $REDIS_CACHE"
+wait-for-it $REDIS_CACHE -t 1
+echo "Check $REDIS_QUEUE"
+wait-for-it $REDIS_QUEUE -t 1
+echo "Check $REDIS_SOCKETIO"
+wait-for-it $REDIS_SOCKETIO -t 1
+
+if [[ "$1" = "-p" ]] || [[ "$1" = "--ping-service" ]]; then
+    echo "Check $2"
+    wait-for-it $2 -t 1
+fi

build/frappe-worker-arm/common/worker/install_app.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+APP_NAME=${1}
+APP_REPO=${2}
+APP_BRANCH=${3}
+
+cd /home/frappe/frappe-bench/
+
+. env/bin/activate
+
+cd ./apps
+
+[ "${APP_BRANCH}" ] && BRANCH="-b ${APP_BRANCH}"
+
+git clone --depth 1 -o upstream ${APP_REPO} ${BRANCH} ${APP_NAME}
+pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/${APP_NAME}