From 3332996dfa08d608bcee6f80ac6dac8784d9a377 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 1 Dec 2020 08:05:05 +0000 Subject: [PATCH] armv64 support --- build/{bench => bench-arm}/Dockerfile | 8 +- build/common/common_site_config.json.template | 0 build/common/worker/bench | 0 build/common/worker/docker-entrypoint.sh | 0 build/common/worker/healthcheck.sh | 0 build/common/worker/install_app.sh | 0 .../Dockerfile | 9 +- .../install_app.sh | 2 +- .../Dockerfile | 3 +- .../Dockerfile | 12 +- .../docker-entrypoint.sh | 0 .../nginx-default.conf.template | 104 +++++++ .../Dockerfile | 4 +- .../docker-entrypoint.sh | 0 .../package.json | 0 .../Dockerfile | 35 ++- .../common/commands/auto_migrate.py | 64 ++++ .../common/commands/background.py | 11 + .../common/commands/backup.py | 39 +++ .../common/commands/check_connection.py | 130 ++++++++ .../common/commands/console.py | 32 ++ .../common/commands/constants.py | 13 + .../common/commands/doctor.py | 61 ++++ .../frappe-worker-arm/common/commands/drop.py | 39 +++ .../common/commands/migrate.py | 51 +++ .../frappe-worker-arm/common/commands/new.py | 109 +++++++ .../common/commands/push_backup.py | 156 ++++++++++ .../common/commands/restore_backup.py | 294 ++++++++++++++++++ .../common/commands/utils.py | 204 ++++++++++++ .../common/commands/worker.py | 12 + .../common/common_site_config.json.template | 8 + .../common/nginx-default.conf.template | 104 +++++++ build/frappe-worker-arm/common/worker/bench | 19 ++ .../common/worker/docker-entrypoint.sh | 220 +++++++++++++ .../common/worker/healthcheck.sh | 40 +++ .../common/worker/install_app.sh | 16 + frappe-installer | 0 tests/docker-test.sh | 0 travis.py | 0 39 files changed, 1767 insertions(+), 32 deletions(-) rename build/{bench => bench-arm}/Dockerfile (96%) mode change 100755 => 100644 build/common/common_site_config.json.template mode change 100755 => 100644 build/common/worker/bench mode change 100755 => 100644 build/common/worker/docker-entrypoint.sh mode change 100755 => 100644 build/common/worker/healthcheck.sh mode change 100755 => 100644 build/common/worker/install_app.sh rename build/{erpnext-nginx => erpnext-nginx-arm}/Dockerfile (58%) rename build/{erpnext-nginx => erpnext-nginx-arm}/install_app.sh (97%) mode change 100755 => 100644 rename build/{erpnext-worker => erpnext-worker-arm}/Dockerfile (57%) rename build/{frappe-nginx => frappe-nginx-arm}/Dockerfile (87%) rename build/{frappe-nginx => frappe-nginx-arm}/docker-entrypoint.sh (100%) mode change 100755 => 100644 create mode 100644 build/frappe-nginx-arm/nginx-default.conf.template rename build/{frappe-socketio => frappe-socketio-arm}/Dockerfile (87%) rename build/{frappe-socketio => frappe-socketio-arm}/docker-entrypoint.sh (100%) mode change 100755 => 100644 rename build/{frappe-socketio => frappe-socketio-arm}/package.json (100%) rename build/{frappe-worker => frappe-worker-arm}/Dockerfile (60%) create mode 100644 build/frappe-worker-arm/common/commands/auto_migrate.py create mode 100644 build/frappe-worker-arm/common/commands/background.py create mode 100644 build/frappe-worker-arm/common/commands/backup.py create mode 100644 build/frappe-worker-arm/common/commands/check_connection.py create mode 100644 build/frappe-worker-arm/common/commands/console.py create mode 100644 build/frappe-worker-arm/common/commands/constants.py create mode 100644 build/frappe-worker-arm/common/commands/doctor.py create mode 100644 build/frappe-worker-arm/common/commands/drop.py create mode 100644 build/frappe-worker-arm/common/commands/migrate.py create mode 100644 build/frappe-worker-arm/common/commands/new.py create mode 100644 build/frappe-worker-arm/common/commands/push_backup.py create mode 100644 build/frappe-worker-arm/common/commands/restore_backup.py create mode 100644 build/frappe-worker-arm/common/commands/utils.py create mode 100644 build/frappe-worker-arm/common/commands/worker.py create mode 100644 build/frappe-worker-arm/common/common_site_config.json.template create mode 100644 build/frappe-worker-arm/common/nginx-default.conf.template create mode 100644 build/frappe-worker-arm/common/worker/bench create mode 100644 build/frappe-worker-arm/common/worker/docker-entrypoint.sh create mode 100644 build/frappe-worker-arm/common/worker/healthcheck.sh create mode 100644 build/frappe-worker-arm/common/worker/install_app.sh mode change 100755 => 100644 frappe-installer mode change 100755 => 100644 tests/docker-test.sh mode change 100755 => 100644 travis.py diff --git a/build/bench/Dockerfile b/build/bench-arm/Dockerfile similarity index 96% rename from build/bench/Dockerfile rename to build/bench-arm/Dockerfile index 920f9cc2..e1920eb3 100644 --- a/build/bench/Dockerfile +++ b/build/bench-arm/Dockerfile @@ -1,8 +1,8 @@ # Frappe Bench Dockerfile -FROM bitnami/minideb:latest +FROM debian:stable-slim LABEL author=frappé -RUN install_packages \ +RUN apt-get update -y && apt-get install \ git \ wkhtmltopdf \ mariadb-client \ @@ -49,7 +49,7 @@ RUN install_packages \ python3-setuptools \ python3-tk \ python-virtualenv \ - less + less -y && apt-get clean RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ && dpkg-reconfigure --frontend=noninteractive locales @@ -85,7 +85,7 @@ RUN bash -c "bench --version" # https://nodejs.org/download/release/latest-v10.x/ # https://nodejs.org/download/release/latest-v12.x/ # https://nodejs.org/download/release/latest-v13.x/ -ENV NODE_VERSION=12.18.2 +ENV NODE_VERSION=12.19.1 ENV NODE_VERSION_FRAPPEV11=10.21.0 # Install nvm with node diff --git a/build/common/common_site_config.json.template b/build/common/common_site_config.json.template old mode 100755 new mode 100644 diff --git a/build/common/worker/bench b/build/common/worker/bench old mode 100755 new mode 100644 diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh old mode 100755 new mode 100644 diff --git a/build/common/worker/healthcheck.sh b/build/common/worker/healthcheck.sh old mode 100755 new mode 100644 diff --git a/build/common/worker/install_app.sh b/build/common/worker/install_app.sh old mode 100755 new mode 100644 diff --git a/build/erpnext-nginx/Dockerfile b/build/erpnext-nginx-arm/Dockerfile similarity index 58% rename from build/erpnext-nginx/Dockerfile rename to build/erpnext-nginx-arm/Dockerfile index 72488ab7..cf3bcaa9 100644 --- a/build/erpnext-nginx/Dockerfile +++ b/build/erpnext-nginx-arm/Dockerfile @@ -1,13 +1,14 @@ -ARG NODE_IMAGE_TAG=12-prod +ARG NODE_IMAGE_TAG=12.19.1-buster-slim ARG GIT_BRANCH=develop -FROM bitnami/node:${NODE_IMAGE_TAG} +FROM node:${NODE_IMAGE_TAG} ARG GIT_BRANCH -COPY build/erpnext-nginx/install_app.sh /install_app +COPY install_app.sh /install_app +RUN chmod +x /install_app && apt-get update -y && apt-get install build-essential git python2 -y RUN /install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} -FROM frappe/frappe-nginx:${GIT_BRANCH} +FROM frappe/frappe-nginx-arm:${GIT_BRANCH} COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ COPY --from=0 /rsync /rsync diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx-arm/install_app.sh old mode 100755 new mode 100644 similarity index 97% rename from build/erpnext-nginx/install_app.sh rename to build/erpnext-nginx-arm/install_app.sh index 7af233d2..0725f13d --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx-arm/install_app.sh @@ -10,7 +10,7 @@ mkdir -p /home/frappe/frappe-bench/sites/assets cd /home/frappe/frappe-bench echo -e "frappe\n${APP_NAME}" > /home/frappe/frappe-bench/sites/apps.txt -install_packages git python2 +# install_packages git python2 mkdir -p apps cd apps diff --git a/build/erpnext-worker/Dockerfile b/build/erpnext-worker-arm/Dockerfile similarity index 57% rename from build/erpnext-worker/Dockerfile rename to build/erpnext-worker-arm/Dockerfile index 90aad213..715ece7d 100644 --- a/build/erpnext-worker/Dockerfile +++ b/build/erpnext-worker-arm/Dockerfile @@ -1,5 +1,6 @@ ARG GIT_BRANCH=develop -FROM frappe/frappe-worker:${GIT_BRANCH} +FROM frappe/frappe-worker-arm:${GIT_BRANCH} ARG GIT_BRANCH +RUN pip install bootstrapped --no-deps RUN install_app erpnext https://github.com/frappe/erpnext ${GIT_BRANCH} diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx-arm/Dockerfile similarity index 87% rename from build/frappe-nginx/Dockerfile rename to build/frappe-nginx-arm/Dockerfile index 191d72bf..58413b7c 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx-arm/Dockerfile @@ -2,12 +2,12 @@ # This is done to ensures that node-sass binary remains common. # node-sass is required to enable website theme feature used # by Website Manager role in Frappe Framework -FROM bitnami/python:3.7-prod +FROM python:3.7.9-slim-buster ENV NVM_DIR=/root/.nvm -ENV NODE_VERSION=12.18.3 +ENV NODE_VERSION=12.19.1 ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" -RUN install_packages wget \ +RUN apt-get update -y && apt-get install wget python2 git -y build-essential \ && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ && chmod +x install.sh \ && ./install.sh \ @@ -18,7 +18,7 @@ WORKDIR /home/frappe/frappe-bench RUN mkdir -p /home/frappe/frappe-bench/sites \ && echo "frappe" > /home/frappe/frappe-bench/sites/apps.txt -RUN install_packages git +# RUN install_packages git ARG GIT_BRANCH=develop RUN mkdir -p apps sites/assets/css \ @@ -45,8 +45,8 @@ RUN cp -R /home/frappe/frappe-bench/apps/frappe/frappe/public/* /home/frappe/fra FROM nginx:latest COPY --from=0 /home/frappe/frappe-bench/sites /var/www/html/ COPY --from=0 /var/www/error_pages /var/www/ -COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template -COPY build/frappe-nginx/docker-entrypoint.sh / +COPY nginx-default.conf.template /etc/nginx/conf.d/default.conf.template +COPY docker-entrypoint.sh / RUN apt-get update && apt-get install -y rsync && apt-get clean \ && echo "#!/bin/bash" > /rsync \ diff --git a/build/frappe-nginx/docker-entrypoint.sh b/build/frappe-nginx-arm/docker-entrypoint.sh old mode 100755 new mode 100644 similarity index 100% rename from build/frappe-nginx/docker-entrypoint.sh rename to build/frappe-nginx-arm/docker-entrypoint.sh diff --git a/build/frappe-nginx-arm/nginx-default.conf.template b/build/frappe-nginx-arm/nginx-default.conf.template new file mode 100644 index 00000000..09575391 --- /dev/null +++ b/build/frappe-nginx-arm/nginx-default.conf.template @@ -0,0 +1,104 @@ +upstream frappe-server { + server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0; +} + +upstream socketio-server { + server ${FRAPPE_SOCKETIO}:${SOCKETIO_PORT} fail_timeout=0; +} + +server { + listen 80; + server_name $http_host; + root /var/www/html; + + add_header X-Frame-Options "SAMEORIGIN"; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /sites/$http_host/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name $host; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $http_host; + + proxy_pass http://socketio-server; + } + + location / { + rewrite ^(.+)/$ $1 permanent; + rewrite ^(.+)/index\.html$ $1 permanent; + rewrite ^(.+)\.html$ $1 permanent; + + location ~ ^/files/.*.(htm|html|svg|xml) { + add_header Content-disposition "attachment"; + try_files /sites/$http_host/public/$uri @webserver; + } + + try_files /sites/$http_host/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frappe-Site-Name $host; + proxy_set_header Host $http_host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout 120; + proxy_redirect off; + + proxy_pass http://frappe-server; + } + + # error pages + error_page 502 /502.html; + location /502.html { + root /var/www/templates; + internal; + } + + # optimizations + sendfile on; + keepalive_timeout 15; + client_max_body_size 50m; + client_body_buffer_size 16K; + client_header_buffer_size 1k; + + # enable gzip compresion + # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge + gzip on; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component; + # text/html is always compressed by HttpGzipModule +} diff --git a/build/frappe-socketio/Dockerfile b/build/frappe-socketio-arm/Dockerfile similarity index 87% rename from build/frappe-socketio/Dockerfile rename to build/frappe-socketio-arm/Dockerfile index 9455dbd7..a648b297 100644 --- a/build/frappe-socketio/Dockerfile +++ b/build/frappe-socketio-arm/Dockerfile @@ -8,7 +8,7 @@ RUN mkdir -p /home/frappe/frappe-bench/sites /home/frappe/frappe-bench/apps/frap && chown -R frappe:frappe /home/frappe # Download socketio and purge curl package -COPY build/frappe-socketio/package.json /home/frappe/frappe-bench/apps/frappe +COPY package.json /home/frappe/frappe-bench/apps/frappe ARG GIT_BRANCH=develop RUN apt-get update && apt-get install -y curl \ && cd /home/frappe/frappe-bench/apps/frappe \ @@ -25,7 +25,7 @@ RUN cd /home/frappe/frappe-bench/apps/frappe \ && npm --version # Setup docker-entrypoint -COPY build/frappe-socketio/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat WORKDIR /home/frappe/frappe-bench/sites diff --git a/build/frappe-socketio/docker-entrypoint.sh b/build/frappe-socketio-arm/docker-entrypoint.sh old mode 100755 new mode 100644 similarity index 100% rename from build/frappe-socketio/docker-entrypoint.sh rename to build/frappe-socketio-arm/docker-entrypoint.sh diff --git a/build/frappe-socketio/package.json b/build/frappe-socketio-arm/package.json similarity index 100% rename from build/frappe-socketio/package.json rename to build/frappe-socketio-arm/package.json diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker-arm/Dockerfile similarity index 60% rename from build/frappe-worker/Dockerfile rename to build/frappe-worker-arm/Dockerfile index 948836cf..99e8483b 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker-arm/Dockerfile @@ -1,39 +1,42 @@ -FROM bitnami/python:3.7-prod +FROM python:3.7.9-stretch RUN useradd -ms /bin/bash frappe ARG GIT_BRANCH=develop ENV PYTHONUNBUFFERED 1 ENV NVM_DIR=/home/frappe/.nvm -ENV NODE_VERSION=12.18.3 +ENV NODE_VERSION=12.19.1 ENV PATH="/home/frappe/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" # Install dependencies WORKDIR /home/frappe/frappe-bench -RUN install_packages \ +RUN apt-get update -y && apt-get install build-essential \ git \ mariadb-client \ postgresql-client \ gettext-base \ wget \ + curl \ # for PDF libjpeg62-turbo \ libx11-6 \ libxcb1 \ libxext6 \ libxrender1 \ + libffi-dev libjpeg-dev libxml2 \ libssl-dev \ fonts-cantarell \ xfonts-75dpi \ xfonts-base \ + zlib1g-dev \ # For psycopg2 libpq-dev \ - wait-for-it \ - && wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb \ - && dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb \ - && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ + wait-for-it -y \ + && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_arm64.deb \ + && dpkg -i wkhtmltox_0.12.6-1.buster_arm64.deb && rm wkhtmltox_0.12.6-1.buster_arm64.deb \ + && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh && chmod +x install.sh \ && apt-get purge -y wget && apt-get autoremove -y \ - && chown -R frappe:frappe /home/frappe + && chown -R frappe:frappe /home/frappe && pip install bootstrapped --no-deps USER frappe # Install nvm with node @@ -55,14 +58,18 @@ RUN python -m venv env \ USER root # Copy scripts and templates -COPY build/common/commands/* /home/frappe/frappe-bench/commands/ -COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template -COPY build/common/worker/install_app.sh /usr/local/bin/install_app -COPY build/common/worker/bench /usr/local/bin/bench -COPY build/common/worker/healthcheck.sh /usr/local/bin/healthcheck.sh +COPY common/commands/* /home/frappe/frappe-bench/commands/ +RUN chmod +x -R /home/frappe/frappe-bench/commands/ +COPY common/common_site_config.json.template /opt/frappe/common_site_config.json.template +COPY common/worker/install_app.sh /usr/local/bin/install_app +RUN chmod +x /usr/local/bin/install_app +COPY common/worker/bench /usr/local/bin/bench +RUN chmod +x /usr/local/bin/bench +COPY common/worker/healthcheck.sh /usr/local/bin/healthcheck.sh +RUN chmod +x /usr/local/bin/healthcheck.sh # Setup docker-entrypoint -COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN ln -s /usr/local/bin/docker-entrypoint.sh / # backwards compat # Use sites volume as working directory diff --git a/build/frappe-worker-arm/common/commands/auto_migrate.py b/build/frappe-worker-arm/common/commands/auto_migrate.py new file mode 100644 index 00000000..ce5f3c2f --- /dev/null +++ b/build/frappe-worker-arm/common/commands/auto_migrate.py @@ -0,0 +1,64 @@ +import os +import semantic_version +import git + +from migrate import migrate_sites +from utils import ( + save_version_file, + get_apps, + get_container_versions, + get_version_file, + get_config +) + + +def main(): + is_ready = False + apps = get_apps() + + container_versions = get_container_versions(apps) + + version_file = get_version_file() + + if not version_file: + version_file = container_versions + save_version_file(version_file) + + for app in apps: + container_version = None + file_version = None + version_file_hash = None + container_hash = None + + repo = git.Repo(os.path.join('..', 'apps', app)) + branch = repo.active_branch.name + + if branch == 'develop': + version_file_hash = version_file.get(app+'_git_hash') + container_hash = container_versions.get(app+'_git_hash') + if container_hash and version_file_hash: + if container_hash != version_file_hash: + is_ready = True + break + + if version_file.get(app): + file_version = semantic_version.Version(version_file.get(app)) + + if container_versions.get(app): + container_version = semantic_version.Version(container_versions.get(app)) + + if file_version and container_version: + if container_version > file_version: + is_ready = True + break + + config = get_config() + + if is_ready and config.get('maintenance_mode') != 1: + migrate_sites(maintenance_mode=True) + version_file = container_versions + save_version_file(version_file) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/background.py b/build/frappe-worker-arm/common/commands/background.py new file mode 100644 index 00000000..a26b7934 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/background.py @@ -0,0 +1,11 @@ +from frappe.utils.scheduler import start_scheduler + + +def main(): + print("Starting background scheduler . . .") + start_scheduler() + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/backup.py b/build/frappe-worker-arm/common/commands/backup.py new file mode 100644 index 00000000..10ed6a92 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/backup.py @@ -0,0 +1,39 @@ +import os +import frappe +from frappe.utils.backups import scheduled_backup +from frappe.utils import cint, get_sites, now + + +def backup(sites, with_files=False): + for site in sites: + frappe.init(site) + frappe.connect() + odb = scheduled_backup( + ignore_files=not with_files, + backup_path_db=None, + backup_path_files=None, + backup_path_private_files=None, + force=True + ) + print("database backup taken -", odb.backup_path_db, "- on", now()) + if with_files: + print("files backup taken -", odb.backup_path_files, "- on", now()) + print("private files backup taken -", odb.backup_path_private_files, "- on", now()) + frappe.destroy() + + +def main(): + installed_sites = ":".join(get_sites()) + sites = os.environ.get("SITES", installed_sites).split(":") + with_files = cint(os.environ.get("WITH_FILES")) + + backup(sites, with_files) + + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/check_connection.py b/build/frappe-worker-arm/common/commands/check_connection.py new file mode 100644 index 00000000..359cc58f --- /dev/null +++ b/build/frappe-worker-arm/common/commands/check_connection.py @@ -0,0 +1,130 @@ +import socket +import time +from six.moves.urllib.parse import urlparse +from utils import get_config +from constants import ( + REDIS_QUEUE_KEY, + REDIS_CACHE_KEY, + REDIS_SOCKETIO_KEY, + DB_HOST_KEY, + DB_PORT_KEY, + DB_PORT +) + + +def is_open(ip, port, timeout=30): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(timeout) + try: + s.connect((ip, int(port))) + s.shutdown(socket.SHUT_RDWR) + return True + except Exception: + return False + finally: + s.close() + + +def check_host(ip, port, retry=10, delay=3, print_attempt=True): + ipup = False + for i in range(retry): + if print_attempt: + print("Attempt {i} to connect to {ip}:{port}".format(ip=ip, port=port, i=i+1)) + if is_open(ip, port): + ipup = True + break + else: + time.sleep(delay) + return ipup + + +# Check service +def check_service( + retry=10, + delay=3, + print_attempt=True, + service_name=None, + service_port=None): + + config = get_config() + if not service_name: + service_name = config.get(DB_HOST_KEY, 'mariadb') + if not service_port: + service_port = config.get(DB_PORT_KEY, DB_PORT) + + is_db_connected = False + is_db_connected = check_host( + service_name, + service_port, + retry, + delay, + print_attempt) + if not is_db_connected: + print("Connection to {service_name}:{service_port} timed out".format( + service_name=service_name, + service_port=service_port, + )) + exit(1) + + +# Check redis queue +def check_redis_queue(retry=10, delay=3, print_attempt=True): + check_redis_queue = False + config = get_config() + redis_queue_url = urlparse(config.get(REDIS_QUEUE_KEY, "redis://redis-queue:6379")).netloc + redis_queue, redis_queue_port = redis_queue_url.split(":") + check_redis_queue = check_host( + redis_queue, + redis_queue_port, + retry, + delay, + print_attempt) + if not check_redis_queue: + print("Connection to redis queue timed out") + exit(1) + + +# Check redis cache +def check_redis_cache(retry=10, delay=3, print_attempt=True): + check_redis_cache = False + config = get_config() + redis_cache_url = urlparse(config.get(REDIS_CACHE_KEY, "redis://redis-cache:6379")).netloc + redis_cache, redis_cache_port = redis_cache_url.split(":") + check_redis_cache = check_host( + redis_cache, + redis_cache_port, + retry, + delay, + print_attempt) + if not check_redis_cache: + print("Connection to redis cache timed out") + exit(1) + + +# Check redis socketio +def check_redis_socketio(retry=10, delay=3, print_attempt=True): + check_redis_socketio = False + config = get_config() + redis_socketio_url = urlparse(config.get(REDIS_SOCKETIO_KEY, "redis://redis-socketio:6379")).netloc + redis_socketio, redis_socketio_port = redis_socketio_url.split(":") + check_redis_socketio = check_host( + redis_socketio, + redis_socketio_port, + retry, + delay, + print_attempt) + if not check_redis_socketio: + print("Connection to redis socketio timed out") + exit(1) + + +def main(): + check_service() + check_redis_queue() + check_redis_cache() + check_redis_socketio() + print('Connections OK') + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/console.py b/build/frappe-worker-arm/common/commands/console.py new file mode 100644 index 00000000..b5c06b5e --- /dev/null +++ b/build/frappe-worker-arm/common/commands/console.py @@ -0,0 +1,32 @@ +import sys +import frappe +import IPython + +from frappe.utils import get_sites + + +def console(site): + "Start ipython console for a site" + if site not in get_sites(): + print("Site {0} does not exist on the current bench".format(site)) + return + + frappe.init(site=site) + frappe.connect() + frappe.local.lang = frappe.db.get_default("lang") + all_apps = frappe.get_installed_apps() + for app in all_apps: + locals()[app] = __import__(app) + print("Apps in this namespace:\n{}".format(", ".join(all_apps))) + IPython.embed(display_banner="", header="") + + +def main(): + site = sys.argv[-1] + console(site) + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/constants.py b/build/frappe-worker-arm/common/commands/constants.py new file mode 100644 index 00000000..9f888099 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/constants.py @@ -0,0 +1,13 @@ +REDIS_QUEUE_KEY = 'redis_queue' +REDIS_CACHE_KEY = 'redis_cache' +REDIS_SOCKETIO_KEY = 'redis_socketio' +DB_HOST_KEY = 'db_host' +DB_PORT_KEY = 'db_port' +DB_PORT = 3306 +APP_VERSIONS_JSON_FILE = 'app_versions.json' +APPS_TXT_FILE = 'apps.txt' +COMMON_SITE_CONFIG_FILE = 'common_site_config.json' +DATE_FORMAT = "%Y%m%d_%H%M%S" +RDS_DB = 'rds_db' +RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES" +ARCHIVE_SITES_PATH = '/home/frappe/frappe-bench/sites/archive_sites' \ No newline at end of file diff --git a/build/frappe-worker-arm/common/commands/doctor.py b/build/frappe-worker-arm/common/commands/doctor.py new file mode 100644 index 00000000..1cb9b200 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/doctor.py @@ -0,0 +1,61 @@ +import argparse + +from check_connection import ( + check_service, + check_redis_cache, + check_redis_queue, + check_redis_socketio, +) + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + '-p', + '--ping-service', + dest='ping_services', + action='append', + type=str, + help='list of services to ping, e.g. doctor -p "postgres:5432" --ping-service "mariadb:3306"', + ) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + check_service(retry=1, delay=0, print_attempt=False) + print("Bench database Connected") + check_redis_cache(retry=1, delay=0, print_attempt=False) + print("Redis Cache Connected") + check_redis_queue(retry=1, delay=0, print_attempt=False) + print("Redis Queue Connected") + check_redis_socketio(retry=1, delay=0, print_attempt=False) + print("Redis SocketIO Connected") + + if(args.ping_services): + for service in args.ping_services: + service_name = None + service_port = None + + try: + service_name, service_port = service.split(':') + except ValueError: + print('Service should be in format host:port, e.g postgres:5432') + exit(1) + + check_service( + retry=1, + delay=0, + print_attempt=False, + service_name=service_name, + service_port=service_port, + ) + print("{0}:{1} Connected".format(service_name, service_port)) + + print("Health check successful") + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/drop.py b/build/frappe-worker-arm/common/commands/drop.py new file mode 100644 index 00000000..da759524 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/drop.py @@ -0,0 +1,39 @@ +import os +import frappe + +from frappe.commands.site import _drop_site +from constants import ARCHIVE_SITES_PATH +from utils import get_password + + +def main(): + site_name = os.environ.get("SITE_NAME", 'site1.localhost') + db_root_username = os.environ.get("DB_ROOT_USER", 'root') + mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin') + postgres_root_password = get_password("POSTGRES_PASSWORD") + db_root_password = mariadb_root_password + + if postgres_root_password: + db_root_password = postgres_root_password + + force = True if os.environ.get("FORCE", None) else False + no_backup = True if os.environ.get("NO_BACKUP", None) else False + frappe.init(site_name, new_site=True) + + _drop_site( + site=site_name, + root_login=db_root_username, + root_password=db_root_password, + archived_sites_path=ARCHIVE_SITES_PATH, + force=force, + no_backup=no_backup + ) + + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/migrate.py b/build/frappe-worker-arm/common/commands/migrate.py new file mode 100644 index 00000000..6e2846b0 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/migrate.py @@ -0,0 +1,51 @@ +import os +import frappe + +from frappe.utils import cint, get_sites +from utils import get_config, save_config + + +def set_maintenance_mode(enable=True): + conf = get_config() + + if enable: + conf.update({"maintenance_mode": 1, "pause_scheduler": 1}) + save_config(conf) + + if not enable: + conf.update({"maintenance_mode": 0, "pause_scheduler": 0}) + save_config(conf) + + +def migrate_sites(maintenance_mode=False): + installed_sites = ":".join(get_sites()) + sites = os.environ.get("SITES", installed_sites).split(":") + if not maintenance_mode: + maintenance_mode = cint(os.environ.get("MAINTENANCE_MODE")) + + if maintenance_mode: + set_maintenance_mode(True) + + for site in sites: + print('Migrating', site) + frappe.init(site=site) + frappe.connect() + try: + from frappe.migrate import migrate + migrate() + finally: + frappe.destroy() + + # Disable maintenance mode after migration + set_maintenance_mode(False) + + +def main(): + migrate_sites() + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/new.py b/build/frappe-worker-arm/common/commands/new.py new file mode 100644 index 00000000..ade7e476 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/new.py @@ -0,0 +1,109 @@ +import os +import frappe +import semantic_version + +from frappe.commands.site import _new_site +from frappe.installer import update_site_config +from constants import COMMON_SITE_CONFIG_FILE, RDS_DB, RDS_PRIVILEGES +from utils import ( + run_command, + get_config, + get_site_config, + get_password, +) + + +def main(): + config = get_config() + db_type = 'mariadb' + db_port = config.get('db_port', 3306) + db_host = config.get('db_host') + site_name = os.environ.get("SITE_NAME", 'site1.localhost') + db_root_username = os.environ.get("DB_ROOT_USER", 'root') + mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin') + postgres_root_password = get_password("POSTGRES_PASSWORD") + db_root_password = mariadb_root_password + + if postgres_root_password: + db_type = 'postgres' + db_host = os.environ.get("POSTGRES_HOST") + db_port = 5432 + db_root_password = postgres_root_password + if not db_host: + db_host = config.get('db_host') + print('Environment variable POSTGRES_HOST not found.') + print('Using db_host from common_site_config.json') + + sites_path = os.getcwd() + common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE) + update_site_config("root_login", db_root_username, validate = False, site_config_path = common_site_config_path) + update_site_config("root_password", db_root_password, validate = False, site_config_path = common_site_config_path) + + force = True if os.environ.get("FORCE", None) else False + install_apps = os.environ.get("INSTALL_APPS", None) + install_apps = install_apps.split(',') if install_apps else [] + frappe.init(site_name, new_site=True) + + if semantic_version.Version(frappe.__version__).major > 11: + _new_site( + None, + site_name, + mariadb_root_username=db_root_username, + mariadb_root_password=db_root_password, + admin_password=get_password("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=install_apps, + source_sql=None, + force=force, + db_type=db_type, + reinstall=False, + db_host=db_host, + db_port=db_port, + ) + else: + _new_site( + None, + site_name, + mariadb_root_username=db_root_username, + mariadb_root_password=db_root_password, + admin_password=get_password("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=install_apps, + source_sql=None, + force=force, + reinstall=False, + ) + + + if db_type == "mariadb": + site_config = get_site_config(site_name) + db_name = site_config.get('db_name') + db_password = site_config.get('db_password') + + mysql_command = ["mysql", f"-h{db_host}", f"-u{db_root_username}", f"-p{mariadb_root_password}", "-e"] + + # Drop User if exists + command = mysql_command + [f"DROP USER IF EXISTS '{db_name}'; FLUSH PRIVILEGES;"] + run_command(command) + + # Grant permission to database and set password + grant_privileges = "ALL PRIVILEGES" + + # for Amazon RDS + if config.get(RDS_DB) or site_config.get(RDS_DB): + grant_privileges = RDS_PRIVILEGES + + command = mysql_command + [f"\ + CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; \ + GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%'; \ + FLUSH PRIVILEGES;"] + run_command(command) + + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/push_backup.py b/build/frappe-worker-arm/common/commands/push_backup.py new file mode 100644 index 00000000..246eb362 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/push_backup.py @@ -0,0 +1,156 @@ +import os +import time +import boto3 + +import datetime +from glob import glob +from frappe.utils import get_sites +from constants import DATE_FORMAT +from utils import ( + get_s3_config, + upload_file_to_s3, + check_s3_environment_variables, +) + + +def get_file_ext(): + return { + "database": "-database.sql.gz", + "private_files": "-private-files.tar", + "public_files": "-files.tar", + "site_config": "-site_config_backup.json" + } + + +def get_backup_details(sitename): + backup_details = dict() + file_ext = get_file_ext() + + # add trailing slash https://stackoverflow.com/a/15010678 + site_backup_path = os.path.join(os.getcwd(), sitename, "private", "backups", "") + + if os.path.exists(site_backup_path): + for filetype, ext in file_ext.items(): + site_slug = sitename.replace('.', '_') + pattern = site_backup_path + '*-' + site_slug + ext + backup_files = list(filter(os.path.isfile, glob(pattern))) + + if len(backup_files) > 0: + backup_files.sort(key=lambda file: os.stat(os.path.join(site_backup_path, file)).st_ctime) + backup_date = datetime.datetime.strptime(time.ctime(os.path.getmtime(backup_files[0])), "%a %b %d %H:%M:%S %Y") + backup_details[filetype] = { + "sitename": sitename, + "file_size_in_bytes": os.stat(backup_files[-1]).st_size, + "file_path": os.path.abspath(backup_files[-1]), + "filename": os.path.basename(backup_files[-1]), + "backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S") + } + + return backup_details + + +def delete_old_backups(limit, bucket, site_name): + all_backups = list() + all_backup_dates = list() + backup_limit = int(limit) + check_s3_environment_variables() + bucket_dir = os.environ.get('BUCKET_DIR') + oldest_backup_date = None + + s3 = boto3.resource( + 's3', + region_name=os.environ.get('REGION'), + aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), + endpoint_url=os.environ.get('ENDPOINT_URL') + ) + + bucket = s3.Bucket(bucket) + objects = bucket.meta.client.list_objects_v2( + Bucket=bucket.name, + Delimiter='/') + + if objects: + for obj in objects.get('CommonPrefixes'): + if obj.get('Prefix') == bucket_dir + '/': + for backup_obj in bucket.objects.filter(Prefix=obj.get('Prefix')): + if backup_obj.get()["ContentType"] == "application/x-directory": + continue + try: + # backup_obj.key is bucket_dir/site/date_time/backupfile.extension + bucket_dir, site_slug, date_time, backupfile = backup_obj.key.split('/') + date_time_object = datetime.datetime.strptime( + date_time, DATE_FORMAT + ) + + if site_name in backup_obj.key: + all_backup_dates.append(date_time_object) + all_backups.append(backup_obj.key) + except IndexError as error: + print(error) + exit(1) + + if len(all_backup_dates) > 0: + oldest_backup_date = min(all_backup_dates) + + if len(all_backups) / 3 > backup_limit: + oldest_backup = None + for backup in all_backups: + try: + # backup is bucket_dir/site/date_time/backupfile.extension + backup_dir, site_slug, backup_dt_string, filename = backup.split('/') + backup_datetime = datetime.datetime.strptime( + backup_dt_string, DATE_FORMAT + ) + if backup_datetime == oldest_backup_date: + oldest_backup = backup + + except IndexError as error: + print(error) + exit(1) + + if oldest_backup: + for obj in bucket.objects.filter(Prefix=oldest_backup): + # delete all keys that are inside the oldest_backup + if bucket_dir in obj.key: + print('Deleteing ' + obj.key) + s3.Object(bucket.name, obj.key).delete() + + +def main(): + details = dict() + sites = get_sites() + conn, bucket = get_s3_config() + + for site in sites: + details = get_backup_details(site) + db_file = details.get('database', {}).get('file_path') + folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + if db_file: + folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(db_file)[:15] + '/' + upload_file_to_s3(db_file, folder, conn, bucket) + + # Archive site_config.json + site_config_file = details.get('site_config', {}).get('file_path') + if not site_config_file: + site_config_file = os.path.join(os.getcwd(), site, 'site_config.json') + upload_file_to_s3(site_config_file, folder, conn, bucket) + + public_files = details.get('public_files', {}).get('file_path') + if public_files: + folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(public_files)[:15] + '/' + upload_file_to_s3(public_files, folder, conn, bucket) + + private_files = details.get('private_files', {}).get('file_path') + if private_files: + folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(private_files)[:15] + '/' + upload_file_to_s3(private_files, folder, conn, bucket) + + delete_old_backups(os.environ.get('BACKUP_LIMIT', '3'), bucket, site) + + print('push-backup complete') + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/restore_backup.py b/build/frappe-worker-arm/common/commands/restore_backup.py new file mode 100644 index 00000000..9989d18d --- /dev/null +++ b/build/frappe-worker-arm/common/commands/restore_backup.py @@ -0,0 +1,294 @@ +import os +import datetime +import tarfile +import hashlib +import frappe +import boto3 + +from frappe.utils import get_sites, random_string +from frappe.installer import ( + make_conf, + get_conf_params, + make_site_dirs, + update_site_config +) +from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT, RDS_DB, RDS_PRIVILEGES +from utils import ( + run_command, + list_directories, + set_key_in_site_config, + get_site_config, + get_config, + get_password, + check_s3_environment_variables, +) + + +def get_backup_dir(): + return os.path.join( + os.path.expanduser('~'), + 'backups' + ) + + +def decompress_db(database_file, site): + command = ["gunzip", "-c", database_file] + with open(database_file.replace(".gz", ""), "w") as db_file: + print('Extract Database GZip for site {}'.format(site)) + run_command(command, stdout=db_file) + + +def restore_database(files_base, site_config_path, site): + # restore database + database_file = files_base + '-database.sql.gz' + decompress_db(database_file, site) + config = get_config() + + # Set db_type if it exists in backup site_config.json + set_key_in_site_config('db_type', site, site_config_path) + # Set db_host if it exists in backup site_config.json + set_key_in_site_config('db_host', site, site_config_path) + # Set db_port if it exists in backup site_config.json + set_key_in_site_config('db_port', site, site_config_path) + + # get updated site_config + site_config = get_site_config(site) + + # if no db_type exists, default to mariadb + db_type = site_config.get('db_type', 'mariadb') + is_database_restored = False + + if db_type == 'mariadb': + restore_mariadb( + config=config, + site_config=site_config, + database_file=database_file) + is_database_restored = True + elif db_type == 'postgres': + restore_postgres( + config=config, + site_config=site_config, + database_file=database_file) + is_database_restored = True + + if is_database_restored: + # Set encryption_key if it exists in backup site_config.json + set_key_in_site_config('encryption_key', site, site_config_path) + + +def restore_files(files_base): + public_files = files_base + '-files.tar' + # extract tar + public_tar = tarfile.open(public_files) + print('Extracting {}'.format(public_files)) + public_tar.extractall() + + +def restore_private_files(files_base): + private_files = files_base + '-private-files.tar' + private_tar = tarfile.open(private_files) + print('Extracting {}'.format(private_files)) + private_tar.extractall() + + +def pull_backup_from_s3(): + check_s3_environment_variables() + + # https://stackoverflow.com/a/54672690 + s3 = boto3.resource( + 's3', + region_name=os.environ.get('REGION'), + aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), + endpoint_url=os.environ.get('ENDPOINT_URL') + ) + + bucket_dir = os.environ.get('BUCKET_DIR') + bucket_name = os.environ.get('BUCKET_NAME') + bucket = s3.Bucket(bucket_name) + + # Change directory to /home/frappe/backups + os.chdir(get_backup_dir()) + + backup_files = [] + sites = set() + site_timestamps = set() + download_backups = [] + + for obj in bucket.objects.filter(Prefix=bucket_dir): + if obj.get()["ContentType"] == "application/x-directory": + continue + backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '') + backup_files.append(backup_file) + site_name, timestamp, backup_type = backup_file.split('/') + site_timestamp = site_name + '/' + timestamp + sites.add(site_name) + site_timestamps.add(site_timestamp) + + # sort sites for latest backups + for site in sites: + backup_timestamps = [] + for site_timestamp in site_timestamps: + site_name, timestamp = site_timestamp.split('/') + if site == site_name: + timestamp_datetime = datetime.datetime.strptime( + timestamp, DATE_FORMAT + ) + backup_timestamps.append(timestamp) + download_backups.append(site + '/' + max(backup_timestamps)) + + # Only download latest backups + for backup_file in backup_files: + for backup in download_backups: + if backup in backup_file: + if not os.path.exists(os.path.dirname(backup_file)): + os.makedirs(os.path.dirname(backup_file)) + print('Downloading {}'.format(backup_file)) + bucket.download_file(bucket_dir + '/' + backup_file, backup_file) + + os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites')) + + +def restore_postgres(config, site_config, database_file): + # common config + common_site_config_path = os.path.join(os.getcwd(), COMMON_SITE_CONFIG_FILE) + + db_root_user = config.get('root_login') + if not db_root_user: + postgres_user = os.environ.get('DB_ROOT_USER') + if not postgres_user: + print('Variable DB_ROOT_USER not set') + exit(1) + + db_root_user = postgres_user + update_site_config( + "root_login", + db_root_user, + validate=False, + site_config_path=common_site_config_path) + + db_root_password = config.get('root_password') + if not db_root_password: + root_password = get_password('POSTGRES_PASSWORD') + if not root_password: + print('Variable POSTGRES_PASSWORD not set') + exit(1) + + db_root_password = root_password + update_site_config( + "root_password", + db_root_password, + validate=False, + site_config_path=common_site_config_path) + + # site config + db_host = site_config.get('db_host') + db_port = site_config.get('db_port', 5432) + db_name = site_config.get('db_name') + db_password = site_config.get('db_password') + + psql_command = ["psql"] + psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}" + + print('Restoring PostgreSQL') + run_command(psql_command + [psql_uri, "-c", f"DROP DATABASE IF EXISTS \"{db_name}\""]) + run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"]) + run_command(psql_command + [psql_uri, "-c", f"CREATE DATABASE \"{db_name}\""]) + run_command(psql_command + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"]) + run_command(psql_command + [psql_uri, "-c", f"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"]) + with open(database_file.replace('.gz', ''), 'r') as db_file: + run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file) + + +def restore_mariadb(config, site_config, database_file): + db_root_password = get_password('MYSQL_ROOT_PASSWORD') + if not db_root_password: + print('Variable MYSQL_ROOT_PASSWORD not set') + exit(1) + + db_root_user = os.environ.get("DB_ROOT_USER", 'root') + + db_host = site_config.get('db_host', config.get('db_host')) + db_port = site_config.get('db_port', config.get('db_port', 3306)) + db_name = site_config.get('db_name') + db_password = site_config.get('db_password') + + # mysql command prefix + mysql_command = ["mysql", f"-u{db_root_user}", f"-h{db_host}", f"-p{db_root_password}", f"-P{db_port}"] + + # drop db if exists for clean restore + drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"] + run_command(drop_database) + + # create db + create_database = mysql_command + ["-e", f"CREATE DATABASE IF NOT EXISTS `{db_name}`;"] + run_command(create_database) + + # create user + create_user = mysql_command + ["-e", f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] + run_command(create_user) + + # grant db privileges to user + + grant_privileges = "ALL PRIVILEGES" + + # for Amazon RDS + if config.get(RDS_DB) or site_config.get(RDS_DB): + grant_privileges = RDS_PRIVILEGES + + grant_privileges_command = mysql_command + ["-e", f"GRANT {grant_privileges} ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] + run_command(grant_privileges_command) + + print('Restoring MariaDB') + with open(database_file.replace('.gz', ''), 'r') as db_file: + run_command(mysql_command + [f"{db_name}"], stdin=db_file) + + +def main(): + backup_dir = get_backup_dir() + + if len(list_directories(backup_dir)) == 0: + pull_backup_from_s3() + + for site in list_directories(backup_dir): + site_slug = site.replace('.', '_') + backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir, site))] + latest_backup = max(backups).strftime(DATE_FORMAT) + files_base = os.path.join(backup_dir, site, latest_backup, '') + files_base += latest_backup + '-' + site_slug + site_config_path = files_base + '-site_config_backup.json' + if not os.path.exists(site_config_path): + site_config_path = os.path.join(backup_dir, site, 'site_config.json') + if site in get_sites(): + print('Overwrite site {}'.format(site)) + restore_database(files_base, site_config_path, site) + restore_private_files(files_base) + restore_files(files_base) + else: + site_config = get_conf_params( + db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16], + db_password=random_string(16) + ) + + frappe.local.site = site + frappe.local.sites_path = os.getcwd() + frappe.local.site_path = os.getcwd() + '/' + site + make_conf( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password'), + ) + make_site_dirs() + + print('Create site {}'.format(site)) + restore_database(files_base, site_config_path, site) + restore_private_files(files_base) + restore_files(files_base) + + if frappe.redis_server: + frappe.redis_server.connection_pool.disconnect() + + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/commands/utils.py b/build/frappe-worker-arm/common/commands/utils.py new file mode 100644 index 00000000..eaa9a2cf --- /dev/null +++ b/build/frappe-worker-arm/common/commands/utils.py @@ -0,0 +1,204 @@ +import json +import os +import subprocess +import boto3 +import git + +from frappe.installer import update_site_config +from constants import ( + APP_VERSIONS_JSON_FILE, + APPS_TXT_FILE, + COMMON_SITE_CONFIG_FILE +) + +def run_command(command, stdout=None, stdin=None, stderr=None): + stdout = stdout or subprocess.PIPE + stderr = stderr or subprocess.PIPE + stdin = stdin or subprocess.PIPE + process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr) + out, error = process.communicate() + if process.returncode: + print("Something went wrong:") + print(f"return code: {process.returncode}") + print(f"stdout:\n{out}") + print(f"\nstderr:\n{error}") + exit(process.returncode) + + +def save_version_file(versions): + with open(APP_VERSIONS_JSON_FILE, 'w') as f: + return json.dump(versions, f, indent=1, sort_keys=True) + + +def get_apps(): + apps = [] + try: + with open(APPS_TXT_FILE) as apps_file: + for app in apps_file.readlines(): + if app.strip(): + apps.append(app.strip()) + + except FileNotFoundError as exception: + print(exception) + exit(1) + except Exception: + print(APPS_TXT_FILE + " is not valid") + exit(1) + + return apps + + +def get_container_versions(apps): + versions = {} + for app in apps: + try: + version = __import__(app).__version__ + versions.update({app: version}) + except Exception: + pass + + try: + path = os.path.join('..', 'apps', app) + repo = git.Repo(path) + commit_hash = repo.head.object.hexsha + versions.update({app+'_git_hash': commit_hash}) + except Exception: + pass + + return versions + + +def get_version_file(): + versions = None + try: + with open(APP_VERSIONS_JSON_FILE) as versions_file: + versions = json.load(versions_file) + except Exception: + pass + return versions + + +def get_config(): + config = None + try: + with open(COMMON_SITE_CONFIG_FILE) as config_file: + config = json.load(config_file) + except FileNotFoundError as exception: + print(exception) + exit(1) + except Exception: + print(COMMON_SITE_CONFIG_FILE + " is not valid") + exit(1) + return config + + +def get_site_config(site_name): + site_config = None + with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + site_config = json.load(site_config_file) + return site_config + + +def save_config(config): + with open(COMMON_SITE_CONFIG_FILE, 'w') as f: + return json.dump(config, f, indent=1, sort_keys=True) + + +def get_password(env_var, default=None): + return os.environ.get(env_var) or get_password_from_secret(f"{env_var}_FILE") or default + + +def get_password_from_secret(env_var): + """Fetches the secret value from the docker secret file + usually located inside /run/secrets/ + Arguments: + env_var {str} -- Name of the environment variable + containing the path to the secret file. + Returns: + [str] -- Secret value + """ + passwd = None + secret_file_path = os.environ.get(env_var) + if secret_file_path: + with open(secret_file_path) as secret_file: + passwd = secret_file.read().strip() + + return passwd + + +def get_s3_config(): + check_s3_environment_variables() + bucket = os.environ.get('BUCKET_NAME') + + conn = boto3.client( + 's3', + region_name=os.environ.get('REGION'), + aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), + endpoint_url=os.environ.get('ENDPOINT_URL') + ) + + return conn, bucket + + +def upload_file_to_s3(filename, folder, conn, bucket): + + destpath = os.path.join(folder, os.path.basename(filename)) + try: + print("Uploading file:", filename) + conn.upload_file(filename, bucket, destpath) + + except Exception as e: + print("Error uploading: %s" % (e)) + exit(1) + + +def list_directories(path): + directories = [] + for name in os.listdir(path): + if os.path.isdir(os.path.join(path, name)): + directories.append(name) + return directories + + +def get_site_config_from_path(site_config_path): + site_config = dict() + if os.path.exists(site_config_path): + with open(site_config_path, 'r') as sc: + site_config = json.load(sc) + return site_config + + +def set_key_in_site_config(key, site, site_config_path): + site_config = get_site_config_from_path(site_config_path) + value = site_config.get(key) + if value: + print('Set {key} in site config for site: {site}'.format(key=key, site=site)) + update_site_config(key, value, + site_config_path=os.path.join(os.getcwd(), site, "site_config.json")) + + +def check_s3_environment_variables(): + if 'BUCKET_NAME' not in os.environ: + print('Variable BUCKET_NAME not set') + exit(1) + + if 'ACCESS_KEY_ID' not in os.environ: + print('Variable ACCESS_KEY_ID not set') + exit(1) + + if 'SECRET_ACCESS_KEY' not in os.environ: + print('Variable SECRET_ACCESS_KEY not set') + exit(1) + + if 'ENDPOINT_URL' not in os.environ: + print('Variable ENDPOINT_URL not set') + exit(1) + + if 'BUCKET_DIR' not in os.environ: + print('Variable BUCKET_DIR not set') + exit(1) + + if 'REGION' not in os.environ: + print('Variable REGION not set') + exit(1) diff --git a/build/frappe-worker-arm/common/commands/worker.py b/build/frappe-worker-arm/common/commands/worker.py new file mode 100644 index 00000000..ba107220 --- /dev/null +++ b/build/frappe-worker-arm/common/commands/worker.py @@ -0,0 +1,12 @@ +import os +from frappe.utils.background_jobs import start_worker + + +def main(): + queue = os.environ.get("WORKER_TYPE", "default") + start_worker(queue, False) + exit(0) + + +if __name__ == "__main__": + main() diff --git a/build/frappe-worker-arm/common/common_site_config.json.template b/build/frappe-worker-arm/common/common_site_config.json.template new file mode 100644 index 00000000..f7272fd6 --- /dev/null +++ b/build/frappe-worker-arm/common/common_site_config.json.template @@ -0,0 +1,8 @@ +{ + "db_host": "${DB_HOST}", + "db_port": ${DB_PORT}, + "redis_cache": "redis://${REDIS_CACHE}", + "redis_queue": "redis://${REDIS_QUEUE}", + "redis_socketio": "redis://${REDIS_SOCKETIO}", + "socketio_port": ${SOCKETIO_PORT} +} diff --git a/build/frappe-worker-arm/common/nginx-default.conf.template b/build/frappe-worker-arm/common/nginx-default.conf.template new file mode 100644 index 00000000..09575391 --- /dev/null +++ b/build/frappe-worker-arm/common/nginx-default.conf.template @@ -0,0 +1,104 @@ +upstream frappe-server { + server ${FRAPPE_PY}:${FRAPPE_PY_PORT} fail_timeout=0; +} + +upstream socketio-server { + server ${FRAPPE_SOCKETIO}:${SOCKETIO_PORT} fail_timeout=0; +} + +server { + listen 80; + server_name $http_host; + root /var/www/html; + + add_header X-Frame-Options "SAMEORIGIN"; + add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload"; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + + location /assets { + try_files $uri =404; + } + + location ~ ^/protected/(.*) { + internal; + try_files /sites/$http_host/$1 =404; + } + + location /socket.io { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name $host; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $http_host; + + proxy_pass http://socketio-server; + } + + location / { + rewrite ^(.+)/$ $1 permanent; + rewrite ^(.+)/index\.html$ $1 permanent; + rewrite ^(.+)\.html$ $1 permanent; + + location ~ ^/files/.*.(htm|html|svg|xml) { + add_header Content-disposition "attachment"; + try_files /sites/$http_host/public/$uri @webserver; + } + + try_files /sites/$http_host/public/$uri @webserver; + } + + location @webserver { + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frappe-Site-Name $host; + proxy_set_header Host $http_host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout 120; + proxy_redirect off; + + proxy_pass http://frappe-server; + } + + # error pages + error_page 502 /502.html; + location /502.html { + root /var/www/templates; + internal; + } + + # optimizations + sendfile on; + keepalive_timeout 15; + client_max_body_size 50m; + client_body_buffer_size 16K; + client_header_buffer_size 1k; + + # enable gzip compresion + # based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge + gzip on; + gzip_http_version 1.1; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types + application/atom+xml + application/javascript + application/json + application/rss+xml + application/vnd.ms-fontobject + application/x-font-ttf + application/font-woff + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/svg+xml + image/x-icon + text/css + text/plain + text/x-component; + # text/html is always compressed by HttpGzipModule +} diff --git a/build/frappe-worker-arm/common/worker/bench b/build/frappe-worker-arm/common/worker/bench new file mode 100644 index 00000000..e9c931c2 --- /dev/null +++ b/build/frappe-worker-arm/common/worker/bench @@ -0,0 +1,19 @@ +#!/home/frappe/frappe-bench/env/bin/python + +import subprocess +import sys +import os + + +if __name__ == "__main__": + bench_dir = os.path.join(os.sep, 'home', 'frappe', 'frappe-bench') + sites_dir = os.path.join(bench_dir, 'sites') + bench_helper = os.path.join( + bench_dir, 'apps', 'frappe', + 'frappe', 'utils', 'bench_helper.py', + ) + cwd = os.getcwd() + os.chdir(sites_dir) + subprocess.check_call( + [sys.executable, bench_helper, 'frappe'] + sys.argv[1:], + ) diff --git a/build/frappe-worker-arm/common/worker/docker-entrypoint.sh b/build/frappe-worker-arm/common/worker/docker-entrypoint.sh new file mode 100644 index 00000000..6b617e56 --- /dev/null +++ b/build/frappe-worker-arm/common/worker/docker-entrypoint.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +function configureEnv() { + if [ ! -f /home/frappe/frappe-bench/sites/common_site_config.json ]; then + + if [[ -z "$MARIADB_HOST" ]]; then + if [[ -z "$POSTGRES_HOST" ]]; then + echo "MARIADB_HOST or POSTGRES_HOST is not set" + exit 1 + fi + fi + + if [[ -z "$REDIS_CACHE" ]]; then + echo "REDIS_CACHE is not set" + exit 1 + fi + + if [[ -z "$REDIS_QUEUE" ]]; then + echo "REDIS_QUEUE is not set" + exit 1 + fi + + if [[ -z "$REDIS_SOCKETIO" ]]; then + echo "REDIS_SOCKETIO is not set" + exit 1 + fi + + if [[ -z "$SOCKETIO_PORT" ]]; then + echo "SOCKETIO_PORT is not set" + exit 1 + fi + + if [[ -z "$DB_PORT" ]]; then + export DB_PORT=3306 + fi + + export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}" + + envsubst '${DB_HOST} + ${DB_PORT} + ${REDIS_CACHE} + ${REDIS_QUEUE} + ${REDIS_SOCKETIO} + ${SOCKETIO_PORT}' < /opt/frappe/common_site_config.json.template > /home/frappe/frappe-bench/sites/common_site_config.json + fi +} + +function checkConnection() { + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/check_connection.py" +} + +function checkConfigExists() { + COUNTER=0 + while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]] && [[ $COUNTER -le 30 ]] ; do + sleep 1 + (( COUNTER=COUNTER+1 )) + echo "config file not created, retry $COUNTER" + done + + if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then + echo "timeout: config file not created" + exit 1 + fi +} + +if [[ ! -e /home/frappe/frappe-bench/sites/apps.txt ]]; then + find /home/frappe/frappe-bench/apps -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort -r > /home/frappe/frappe-bench/sites/apps.txt +fi + +# Allow user process to create files in logs directory +chown -R frappe:frappe /home/frappe/frappe-bench/logs + +# symlink node_modules +ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \ + /home/frappe/frappe-bench/apps/frappe/node_modules + +if [ "$1" = 'start' ]; then + configureEnv + checkConnection + + chown frappe:frappe /home/frappe/frappe-bench/sites/common_site_config.json + + if [[ -z "$WORKERS" ]]; then + export WORKERS=2 + fi + + if [[ -z "$FRAPPE_PORT" ]]; then + export FRAPPE_PORT=8000 + fi + + if [[ ! -z "$AUTO_MIGRATE" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/auto_migrate.py" + fi + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload" + else + . /home/frappe/frappe-bench/env/bin/activate + gunicorn -b 0.0.0.0:$FRAPPE_PORT \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers $WORKERS \ + --worker-class=gthread \ + --log-file=- \ + -t 120 frappe.app:application --preload + fi + +elif [ "$1" = 'worker' ]; then + checkConfigExists + checkConnection + # default WORKER_TYPE=default + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/worker.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/worker.py + fi + +elif [ "$1" = 'schedule' ]; then + checkConfigExists + checkConnection + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/background.py" + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/background.py + fi + +elif [ "$1" = 'new' ]; then + checkConfigExists + checkConnection + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/new.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/new.py + fi + +elif [ "$1" = 'drop' ]; then + checkConfigExists + checkConnection + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/drop.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/drop.py + fi + +elif [ "$1" = 'migrate' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/migrate.py" + exit + +elif [ "$1" = 'doctor' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/doctor.py ${@:2}" + exit + +elif [ "$1" = 'backup' ]; then + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/backup.py" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/backup.py + fi + +elif [ "$1" = 'console' ]; then + + if [[ -z "$2" ]]; then + echo "Need to specify a sitename with the command:" + echo "console " + exit 1 + fi + + if [[ -z "$RUN_AS_ROOT" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/console.py $2" + exit + else + . /home/frappe/frappe-bench/env/bin/activate + python /home/frappe/frappe-bench/commands/console.py "$2" + fi + +elif [ "$1" = 'push-backup' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/push_backup.py" + exit + +elif [ "$1" = 'restore-backup' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/restore_backup.py" + exit + +else + + exec $@ + +fi diff --git a/build/frappe-worker-arm/common/worker/healthcheck.sh b/build/frappe-worker-arm/common/worker/healthcheck.sh new file mode 100644 index 00000000..6fac34b7 --- /dev/null +++ b/build/frappe-worker-arm/common/worker/healthcheck.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +export COMMON_SITE_CONFIG_JSON='/home/frappe/frappe-bench/sites/common_site_config.json' + +# Set DB Host and port +export DB_HOST=`cat $COMMON_SITE_CONFIG_JSON | awk '/db_host/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'` +export DB_PORT=`cat $COMMON_SITE_CONFIG_JSON | awk '/db_port/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'` +if [[ -z "$DB_PORT" ]]; then + export DB_PORT=3306 +fi + +# Set REDIS host:port +export REDIS_CACHE=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_cache/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` +if [[ "$REDIS_CACHE" == *"/"* ]]; then + export REDIS_CACHE=`echo $REDIS_CACHE | cut -f1 -d"/"` +fi + +export REDIS_QUEUE=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_queue/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` +if [[ "$REDIS_QUEUE" == *"/"* ]]; then + export REDIS_QUEUE=`echo $REDIS_QUEUE | cut -f1 -d"/"` +fi + +export REDIS_SOCKETIO=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_socketio/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` +if [[ "$REDIS_SOCKETIO" == *"/"* ]]; then + export REDIS_SOCKETIO=`echo $REDIS_SOCKETIO | cut -f1 -d"/"` +fi + +echo "Check $DB_HOST:$DB_PORT" +wait-for-it $DB_HOST:$DB_PORT -t 1 +echo "Check $REDIS_CACHE" +wait-for-it $REDIS_CACHE -t 1 +echo "Check $REDIS_QUEUE" +wait-for-it $REDIS_QUEUE -t 1 +echo "Check $REDIS_SOCKETIO" +wait-for-it $REDIS_SOCKETIO -t 1 + +if [[ "$1" = "-p" ]] || [[ "$1" = "--ping-service" ]]; then + echo "Check $2" + wait-for-it $2 -t 1 +fi diff --git a/build/frappe-worker-arm/common/worker/install_app.sh b/build/frappe-worker-arm/common/worker/install_app.sh new file mode 100644 index 00000000..ceb8d3bd --- /dev/null +++ b/build/frappe-worker-arm/common/worker/install_app.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +APP_NAME=${1} +APP_REPO=${2} +APP_BRANCH=${3} + +cd /home/frappe/frappe-bench/ + +. env/bin/activate + +cd ./apps + +[ "${APP_BRANCH}" ] && BRANCH="-b ${APP_BRANCH}" + +git clone --depth 1 -o upstream ${APP_REPO} ${BRANCH} ${APP_NAME} +pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/${APP_NAME} \ No newline at end of file diff --git a/frappe-installer b/frappe-installer old mode 100755 new mode 100644 diff --git a/tests/docker-test.sh b/tests/docker-test.sh old mode 100755 new mode 100644 diff --git a/travis.py b/travis.py old mode 100755 new mode 100644