Merge pull request #287 from frappe/develop

chore: merge updates to master

.travis.yml

@@ -153,7 +153,10 @@ jobs:
     - stage: "Build and test edge images"
       if: type = pull_request
       before_install:
-        - sudo apt-get update && sudo apt-get -y install docker-compose
+        - sudo curl -L "https://github.com/docker/compose/releases/download/1.26.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        - sudo chmod +x /usr/local/bin/docker-compose
+        - sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
+        - sudo apt-get update && sudo apt-get -y install w3m
       script:
         - docker build -t frappe/frappe-socketio:edge -f build/frappe-socketio/Dockerfile .
         - docker build -t frappe/frappe-worker:develop -f build/frappe-worker/Dockerfile .
@@ -164,7 +167,7 @@ jobs:
     - stage: "Pull and test edge images"
       if: branch = develop AND type != pull_request
       before_install:
-        - sudo apt-get update && sudo apt-get -y install docker-compose
+        - sudo apt-get update && sudo apt-get -y install docker-compose w3m
       script:
         - ./tests/docker-test.sh
     - stage: "Helm Chart Release"

build/common/commands/push_backup.py

@@ -13,7 +13,8 @@ def get_file_ext():
     return {
         "database": "-database.sql.gz",
         "private_files": "-private-files.tar",
-        "public_files": "-files.tar"
+        "public_files": "-files.tar",
+        "site_config": "-site_config_backup.json"
     }
 
 
@@ -177,7 +178,9 @@ def main():
             upload_file_to_s3(db_file, folder, conn, bucket)
 
             # Archive site_config.json
-            site_config_file = os.path.join(os.getcwd(), site, 'site_config.json')
+            site_config_file = details.get('site_config', {}).get('file_path')
+            if not site_config_file:
+                site_config_file = os.path.join(os.getcwd(), site, 'site_config.json')
             upload_file_to_s3(site_config_file, folder, conn, bucket)
 
         public_files = details.get('public_files', {}).get('file_path')

build/common/commands/restore_backup.py

@@ -1,5 +1,6 @@
 import os
 import datetime
+import json
 import tarfile
 import hashlib
 import frappe
@@ -8,7 +9,7 @@ import boto3
 from new import get_password
 from push_backup import DATE_FORMAT, check_environment_variables
 from frappe.utils import get_sites, random_string
-from frappe.installer import make_conf, get_conf_params, make_site_dirs
+from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config
 from check_connection import get_site_config, get_config
 
 
@@ -38,7 +39,7 @@ def decompress_db(files_base, site):
     os.system(command)
 
 
-def restore_database(files_base, site):
+def restore_database(files_base, site_config_path, site):
     db_root_password = get_password('MYSQL_ROOT_PASSWORD')
     if not db_root_password:
         print('Variable MYSQL_ROOT_PASSWORD not set')
@@ -96,6 +97,15 @@ def restore_database(files_base, site):
     print('Restoring database for site: {}'.format(site))
     os.system(command)
 
+    if os.path.exists(site_config_path):
+        with open(site_config_path, 'r') as sc:
+            site_config = json.load(sc)
+        encryption_key = site_config.get("encryption_key")
+        if encryption_key:
+            print('Restoring site config for site: {}'.format(site))
+            update_site_config('encryption_key', encryption_key,
+                               site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
+
 
 def restore_files(files_base):
     public_files = files_base + '-files.tar'
@@ -180,8 +190,11 @@ def main():
         latest_backup = max(backups).strftime(DATE_FORMAT)
         files_base = os.path.join(backup_dir, site, latest_backup, '')
         files_base += latest_backup + '-' + site_slug
+        site_config_path = files_base + '-site_config_backup.json'
+        if not os.path.exists(site_config_path):
+            site_config_path = os.path.join(backup_dir, site, 'site_config.json')
         if site in get_sites():
-            restore_database(files_base, site)
+            restore_database(files_base, site_config_path, site)
             restore_private_files(files_base)
             restore_files(files_base)
         else:
@@ -203,7 +216,7 @@ def main():
                 db_password=site_config.get('db_password'),
             )
             make_site_dirs()
-            restore_database(files_base, site)
+            restore_database(files_base, site_config_path, site)
             restore_private_files(files_base)
             restore_files(files_base)
 

tests/docker-test.sh

@@ -41,13 +41,21 @@ function loopHealthCheck() {
     echo "Health check successful"
 }
 
-echo "Copy env-example file"
+echo -e "\e[1m\e[4mCopy env-example file\e[0m"
 cp env-example .env
+echo -e "\n"
 
-echo "Set version to v12"
+echo -e "\e[1m\e[4mSet version to v12\e[0m"
 sed -i -e "s/edge/v12/g" .env
+echo -e "\n"
 
-echo "Start Services"
+echo -e "\e[1m\e[4mStart Services\e[0m"
+docker-compose \
+    --project-name frappebench00 \
+    -f installation/docker-compose-common.yml \
+    -f installation/docker-compose-erpnext.yml \
+    -f installation/erpnext-publish.yml \
+    pull
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
@@ -56,23 +64,30 @@ docker-compose \
     up -d
 
 loopHealthCheck
+echo -e "\n"
 
-echo "Create new site (v12)"
+echo -e "\e[1m\e[4mCreate new site (v12)\e[0m"
 docker run -it \
     -e "SITE_NAME=test.localhost" \
     -e "INSTALL_APPS=erpnext" \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:v12 new
+echo -e "\n"
 
-echo "Ping created site"
-curl -S http://test.localhost/api/method/version
-echo ""
+echo -e "\e[1m\e[4mPing created site\e[0m"
+curl -sS http://test.localhost/api/method/version
+echo -e "\n"
 
-echo "Set version to edge"
+echo -e "\e[1m\e[4mCheck Created Site Index Page\e[0m"
+curl -s http://test.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mSet version to edge\e[0m"
 sed -i -e "s/v12/edge/g" .env
+echo -e "\n"
 
-echo "Restart containers with edge image"
+echo -e "\e[1m\e[4mRestart containers with edge image\e[0m"
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
@@ -87,31 +102,37 @@ docker-compose \
     up -d
 
 checkMigrationComplete
+echo -e "\n"
 
-echo "Ping migrated site"
+echo -e "\e[1m\e[4mPing migrated site\e[0m"
 sleep 3
-curl -S http://test.localhost/api/method/version
-echo ""
+curl -sS http://test.localhost/api/method/version
+echo -e "\n"
 
-echo "Backup site"
+echo -e "\e[1m\e[4mCheck Migrated Site Index Page\e[0m"
+curl -s http://test.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mBackup site\e[0m"
 docker run -it \
     -e "WITH_FILES=1" \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:edge backup
+echo -e "\n"
 
 export MINIO_ACCESS_KEY="AKIAIOSFODNN7EXAMPLE"
 export MINIO_SECRET_KEY="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
 
-echo "Start MinIO container for s3 compatible storage"
+echo -e "\e[1m\e[4mStart MinIO container for s3 compatible storage\e[0m"
 docker run -d --name minio \
   -e "MINIO_ACCESS_KEY=$MINIO_ACCESS_KEY" \
   -e "MINIO_SECRET_KEY=$MINIO_SECRET_KEY" \
   --network frappebench00_default \
   minio/minio server /data
+echo -e "\n"
 
-echo "Create bucket named erpnext after 3 sec."
-sleep 3
+echo -e "\e[1m\e[4mCreate bucket named erpnext\e[0m"
 docker run \
     --network frappebench00_default \
     vltgroup/s3cmd:latest s3cmd --access_key=$MINIO_ACCESS_KEY \
@@ -121,8 +142,9 @@ docker run \
     --host=minio:9000 \
     --host-bucket=minio:9000 \
     mb s3://erpnext
+echo -e "\n"
 
-echo "Push backup to MinIO s3"
+echo -e "\e[1m\e[4mPush backup to MinIO s3\e[0m"
 docker run \
     -e BUCKET_NAME=erpnext \
     -e REGION=us-east-1 \
@@ -133,19 +155,22 @@ docker run \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:edge push-backup
+echo -e "\n"
 
-echo "Stop Services"
+echo -e "\e[1m\e[4mStop Services\e[0m"
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
     -f installation/docker-compose-erpnext.yml \
     -f installation/erpnext-publish.yml \
     stop
+echo -e "\n"
 
-echo "Prune Containers"
+echo -e "\e[1m\e[4mPrune Containers\e[0m"
 docker container prune -f && docker volume prune -f
+echo -e "\n"
 
-echo "Start Services"
+echo -e "\e[1m\e[4mStart Services\e[0m"
 docker-compose \
     --project-name frappebench00 \
     -f installation/docker-compose-common.yml \
@@ -154,8 +179,9 @@ docker-compose \
     up -d
 
 loopHealthCheck
+echo -e "\n"
 
-echo "Restore backup from MinIO / S3"
+echo -e "\e[1m\e[4mRestore backup from MinIO / S3\e[0m"
 docker run \
     -e MYSQL_ROOT_PASSWORD=admin \
     -e BUCKET_NAME=erpnext \
@@ -167,15 +193,16 @@ docker run \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
     frappe/erpnext-worker:edge restore-backup
+echo -e "\n"
 
-echo "Check Restored Site"
+echo -e "\e[1m\e[4mCheck Restored Site\e[0m"
 sleep 3
-RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
+RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
 INCREMENT=0
 while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
     sleep 1
     echo "Wait for restoration to complete ..."
-    RESTORE_STATUS=$(curl -S http://test.localhost/api/method/version || echo "")
+    RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
     ((INCREMENT=INCREMENT+1))
     if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
         CONTAINER_ID=$(docker-compose \
@@ -189,10 +216,54 @@ while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
     fi
 done
 
-echo "Ping restored site"
+echo -e "\e[1m\e[4mPing restored site\e[0m"
 echo $RESTORE_STATUS
+echo -e "\n"
 
-echo "Migrate command in edge container"
+echo -e "\e[1m\e[4mCheck Restored Site Index Page\e[0m"
+curl -s http://test.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCreate new site (edge)\e[0m"
+docker run -it \
+    -e "SITE_NAME=edge.localhost" \
+    -e "INSTALL_APPS=erpnext" \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge new
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck New Edge Site\e[0m"
+sleep 3
+RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "")
+INCREMENT=0
+while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
+    sleep 1
+    echo -e "\e[1m\e[4mWait for restoration to complete ..."
+    RESTORE_STATUS=$(curl -sS http://edge.localhost/api/method/version || echo "")
+    ((INCREMENT=INCREMENT+1))
+    if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
+        CONTAINER_ID=$(docker-compose \
+            --project-name frappebench00 \
+            -f installation/docker-compose-common.yml \
+            -f installation/docker-compose-erpnext.yml \
+            -f installation/erpnext-publish.yml \
+            ps -q erpnext-python)
+        docker logs $CONTAINER_ID
+        exit 1
+    fi
+done
+echo -e "\n"
+
+echo -e "\e[1m\e[4mPing new edge site\e[0m"
+echo $RESTORE_STATUS
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck New Edge Index Page\e[0m"
+curl -s http://edge.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mMigrate command in edge container\e[0m"
 docker run -it \
     -e "MAINTENANCE_MODE=1" \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
@@ -200,16 +271,53 @@ docker run -it \
     frappe/erpnext-worker:edge migrate
 
 checkMigrationComplete
+echo -e "\n"
 
-echo "Create new site (edge)"
-docker run -it \
-    -e "SITE_NAME=edge.localhost" \
-    -e "INSTALL_APPS=erpnext" \
+echo -e "\e[1m\e[4mRestore backup from MinIO / S3 (Overwrite)\e[0m"
+docker run \
+    -e MYSQL_ROOT_PASSWORD=admin \
+    -e BUCKET_NAME=erpnext \
+    -e BUCKET_DIR=local \
+    -e ACCESS_KEY_ID=$MINIO_ACCESS_KEY \
+    -e SECRET_ACCESS_KEY=$MINIO_SECRET_KEY \
+    -e ENDPOINT_URL=http://minio:9000 \
+    -e REGION=us-east-1 \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \
-    frappe/erpnext-worker:edge new
+    frappe/erpnext-worker:edge restore-backup
+echo -e "\n"
 
-echo "Check console command for site test.localhost"
+echo -e "\e[1m\e[4mCheck Overwritten Site\e[0m"
+sleep 3
+RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
+INCREMENT=0
+while [[ -z "$RESTORE_STATUS" && $INCREMENT -lt 60 ]]; do
+    sleep 1
+    echo -e "\e[1m\e[4mWait for restoration to complete ..."
+    RESTORE_STATUS=$(curl -sS http://test.localhost/api/method/version || echo "")
+    ((INCREMENT=INCREMENT+1))
+    if [[ -z "$RESTORE_STATUS" && $INCREMENT -eq 60 ]]; then
+        CONTAINER_ID=$(docker-compose \
+            --project-name frappebench00 \
+            -f installation/docker-compose-common.yml \
+            -f installation/docker-compose-erpnext.yml \
+            -f installation/erpnext-publish.yml \
+            ps -q erpnext-python)
+        docker logs $CONTAINER_ID
+        exit 1
+    fi
+done
+echo -e "\n"
+
+echo -e "\e[1m\e[4mPing overwritten site\e[0m"
+echo $RESTORE_STATUS
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck Overwritten Index Page\e[0m"
+curl -s http://test.localhost | w3m -T text/html -dump
+echo -e "\n"
+
+echo -e "\e[1m\e[4mCheck console command for site test.localhost\e[0m"
 docker run \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
     --network frappebench00_default \

tests/pwd.yml

@@ -0,0 +1,181 @@
+version: "3"
+
+services:
+  traefik:
+    image: "traefik:v2.2"
+    command:
+      - "--log.level=DEBUG"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.web.address=:80"
+    labels:
+      # enable traefik
+      - "traefik.enable=true"
+    ports:
+      - "80:80"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+
+  erpnext-nginx:
+    image: frappe/erpnext-nginx:edge
+    restart: on-failure
+    environment:
+      - FRAPPE_PY=erpnext-python
+      - FRAPPE_PY_PORT=8000
+      - FRAPPE_SOCKETIO=frappe-socketio
+      - SOCKETIO_PORT=9000
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.erpnext-nginx.rule=HostRegexp(`{catchall:.*}`)"
+      - "traefik.http.middlewares.erpnext-nginx.headers.customrequestheaders.Host=mysite.localhost"
+      - "traefik.http.routers.erpnext-nginx.middlewares=erpnext-nginx"
+      - "traefik.http.routers.erpnext-nginx.entrypoints=web"
+      - "traefik.http.services.erpnext-nginx.loadbalancer.server.port=80"
+    volumes:
+      - sites-vol:/var/www/html/sites:rw
+      - assets-vol:/assets:rw
+
+  erpnext-python:
+    image: frappe/erpnext-worker:edge
+    restart: on-failure
+    environment:
+      - MARIADB_HOST=mariadb
+      - REDIS_CACHE=redis-cache:6379
+      - REDIS_QUEUE=redis-queue:6379
+      - REDIS_SOCKETIO=redis-socketio:6379
+      - SOCKETIO_PORT=9000
+      - AUTO_MIGRATE=1
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+      - assets-vol:/home/frappe/frappe-bench/sites/assets:rw
+
+  frappe-socketio:
+    image: frappe/frappe-socketio:edge
+    restart: on-failure
+    depends_on:
+      - redis-socketio
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  erpnext-worker-default:
+    image: frappe/erpnext-worker:edge
+    restart: on-failure
+    command: worker
+    depends_on:
+      - redis-queue
+      - redis-cache
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  erpnext-worker-short:
+    image: frappe/erpnext-worker:edge
+    restart: on-failure
+    command: worker
+    environment:
+      - WORKER_TYPE=short
+    depends_on:
+      - redis-queue
+      - redis-cache
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  erpnext-worker-long:
+    image: frappe/erpnext-worker:edge
+    restart: on-failure
+    command: worker
+    environment:
+      - WORKER_TYPE=long
+    depends_on:
+      - redis-queue
+      - redis-cache
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  erpnext-schedule:
+    image: frappe/erpnext-worker:edge
+    restart: on-failure
+    command: schedule
+    depends_on:
+      - redis-queue
+      - redis-cache
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+  redis-cache:
+    image: redis:latest
+    restart: on-failure
+    volumes:
+      - redis-cache-vol:/data
+
+  redis-queue:
+    image: redis:latest
+    restart: on-failure
+    volumes:
+      - redis-queue-vol:/data
+
+  redis-socketio:
+    image: redis:latest
+    restart: on-failure
+    volumes:
+      - redis-socketio-vol:/data
+
+  site-configurator:
+    image: frappe/erpnext-worker:edge
+    command: ["bash", "-c", "echo mysite.localhost > /sites/currentsite.txt"]
+    volumes:
+      - sites-vol:/sites:rw
+
+  mariadb-configurator:
+    image: mariadb:10.3
+    command:
+      - "bash"
+      - "-c"
+      - >
+        echo -e "[mysqld]\n
+        skip-host-cache\n
+        skip-name-resolve\n
+        character-set-client-handshake = FALSE\n
+        character-set-server = utf8mb4\n
+        collation-server = utf8mb4_unicode_ci\n
+        [mysql]\n
+        default-character-set = utf8mb4\n
+        [mysqld_safe]\n
+        skip_log_error\n
+        syslog\n" > /data/frappe.cnf
+    volumes:
+      - mariadb-conf-vol:/data:rw
+
+  mariadb:
+    image: mariadb:10.3
+    restart: on-failure
+    depends_on:
+      - mariadb-configurator
+    environment:
+      - MYSQL_ROOT_PASSWORD=admin
+    volumes:
+      - mariadb-conf-vol:/etc/mysql/conf.d
+      - mariadb-vol:/var/lib/mysql
+
+  site-creator:
+    image: frappe/erpnext-worker:edge
+    restart: "no"
+    command: new
+    depends_on:
+      - erpnext-python
+    environment:
+      - SITE_NAME=mysite.localhost
+      - DB_ROOT_USER=root
+      - MYSQL_ROOT_PASSWORD=admin
+      - ADMIN_PASSWORD=admin
+      - INSTALL_APPS=erpnext
+    volumes:
+      - sites-vol:/home/frappe/frappe-bench/sites:rw
+
+volumes:
+  mariadb-vol:
+  mariadb-conf-vol:
+  redis-cache-vol:
+  redis-queue-vol:
+  redis-socketio-vol:
+  assets-vol:
+  sites-vol: