Merge branch 'main' into tests-on-pr-push

.github/dependabot.yml

@@ -4,3 +4,13 @@ updates:
     directory: /
     schedule:
       interval: daily
+
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: daily
+
+  - package-ecosystem: npm
+    directory: build/frappe-socketio
+    schedule:
+      interval: daily

.github/scripts/get-latest-tags.sh

@@ -4,7 +4,7 @@ set -e
 set -x
 
 get_tag() {
-    tags=$(git ls-remote --refs --tags --sort='v:refname' https://github.com/$1 "v$2.*")
+    tags=$(git ls-remote --refs --tags --sort='v:refname' "https://github.com/$1" "v$2.*")
     tag=$(echo "$tags" | tail -n1 | sed 's/.*\///')
     echo "$tag"
 }
@@ -12,11 +12,9 @@ get_tag() {
 FRAPPE_VERSION=$(get_tag frappe/frappe "$VERSION")
 ERPNEXT_VERSION=$(get_tag frappe/erpnext "$VERSION")
 
-# shellcheck disable=SC2086
+cat <<EOL >>"$GITHUB_ENV"
-echo "FRAPPE_VERSION=$FRAPPE_VERSION" >>$GITHUB_ENV
+FRAPPE_VERSION=$FRAPPE_VERSION
-# shellcheck disable=SC2086
+ERPNEXT_VERSION=$ERPNEXT_VERSION
-echo "ERPNEXT_VERSION=$ERPNEXT_VERSION" >>$GITHUB_ENV
+GIT_BRANCH=version-$VERSION
-# shellcheck disable=SC2086
+VERSION=$VERSION
-echo "GIT_BRANCH=version-$VERSION" >>$GITHUB_ENV
+EOL
-# shellcheck disable=SC2086
-echo "VERSION=$VERSION" >>$GITHUB_ENV

.github/workflows/lint.yml

@@ -14,3 +14,5 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: ludeeus/action-shellcheck@master
+        env:
+          SHELLCHECK_OPTS: -x

build/erpnext-nginx/install_app.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 APP_NAME=${1}
 APP_REPO=${2}
 APP_BRANCH=${3}
@@ -13,26 +15,29 @@ echo -ne "frappe\n${APP_NAME}" >/home/frappe/frappe-bench/sites/apps.txt
 
 mkdir -p apps
 cd apps
-git clone --depth 1 https://github.com/frappe/frappe -b ${FRAPPE_BRANCH}
+git clone --depth 1 https://github.com/frappe/frappe -b "${FRAPPE_BRANCH}"
-git clone --depth 1 ${APP_REPO} ${BRANCH} ${APP_NAME}
+# shellcheck disable=SC2086
+git clone --depth 1 "${APP_REPO}" ${BRANCH} "${APP_NAME}"
 
 echo "Install frappe NodeJS dependencies . . ."
 cd /home/frappe/frappe-bench/apps/frappe
 yarn
 echo "Install ${APP_NAME} NodeJS dependencies . . ."
-cd /home/frappe/frappe-bench/apps/${APP_NAME}
+cd "/home/frappe/frappe-bench/apps/${APP_NAME}"
 yarn
 echo "Build browser assets . . ."
 cd /home/frappe/frappe-bench/apps/frappe
-yarn production --app ${APP_NAME}
+yarn production --app "${APP_NAME}"
 echo "Install frappe NodeJS production dependencies . . ."
 cd /home/frappe/frappe-bench/apps/frappe
 yarn install --production=true
 echo "Install ${APP_NAME} NodeJS production dependencies . . ."
-cd /home/frappe/frappe-bench/apps/${APP_NAME}
+cd "/home/frappe/frappe-bench/apps/${APP_NAME}"
 yarn install --production=true
 
+# shellcheck disable=SC2086
 mkdir -p /home/frappe/frappe-bench/sites/assets/${APP_NAME}
+# shellcheck disable=SC2086
 cp -R /home/frappe/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public/* /home/frappe/frappe-bench/sites/assets/${APP_NAME}
 
 # Add frappe and all the apps available under in frappe-bench here

build/frappe-nginx/docker-entrypoint.sh

@@ -8,7 +8,7 @@ rsync -a --delete /var/www/html/assets/* /assets
 
 /rsync
 
-touch /var/www/html/sites/.build -r "$(ls -td /assets/* | head -n 1)"
+touch /var/www/html/sites/.build -r "$(find /assets -maxdepth 1 -name "*.*" | head -n 1)"
 
 [[ -z "${FRAPPE_PY}" ]] && FRAPPE_PY='0.0.0.0'
 
@@ -36,6 +36,7 @@ if [[ ${SKIP_NGINX_TEMPLATE_GENERATION} == 1 ]]; then
   echo "Skipping default NGINX template generation. Please mount your own NGINX config file inside /etc/nginx/conf.d"
 else
   echo "Generating default template"
+  # shellcheck disable=SC2016
   envsubst '${FRAPPE_PY}
         ${FRAPPE_PY_PORT}
         ${FRAPPE_SOCKETIO}
@@ -50,9 +51,11 @@ else
 fi
 
 echo "Waiting for frappe-python to be available on ${FRAPPE_PY} port ${FRAPPE_PY_PORT}"
+# shellcheck disable=SC2016
 timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' ${FRAPPE_PY} ${FRAPPE_PY_PORT}
 echo "Frappe-python available on ${FRAPPE_PY} port ${FRAPPE_PY_PORT}"
 echo "Waiting for frappe-socketio to be available on ${FRAPPE_SOCKETIO} port ${SOCKETIO_PORT}"
+# shellcheck disable=SC2016
 timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' ${FRAPPE_SOCKETIO} ${SOCKETIO_PORT}
 echo "Frappe-socketio available on ${FRAPPE_SOCKETIO} port ${SOCKETIO_PORT}"
 

build/frappe-socketio/docker-entrypoint.sh

@@ -1,15 +1,17 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e
 
 function checkConfigExists() {
   COUNTER=0
   while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json && ${COUNTER} -le 30 ]]; do
-    ((COUNTER = COUNTER + 1))
-    echo "config file not created, retry ${COUNTER}"
     sleep 1
+    ((COUNTER = COUNTER + 1))
+    echo "config file not created, retry ${COUNTER}" >&2
   done
 
   if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then
-    echo "timeout: config file not created"
+    echo "timeout: config file not created" >&2
     exit 1
   fi
 }
@@ -17,11 +19,6 @@ function checkConfigExists() {
 if [[ "$1" == 'start' ]]; then
   checkConfigExists
   node /home/frappe/frappe-bench/apps/frappe/socketio.js
-
-elif [[ "$1" == 'doctor' ]]; then
-  node /home/frappe/frappe-bench/apps/frappe/health.js
-
 else
   exec -c "$@"
-
 fi

build/frappe-worker/commands/background.py

@@ -1,11 +0,0 @@
-from frappe.utils.scheduler import start_scheduler
-
-
-def main():
-    print("Starting background scheduler . . .")
-    start_scheduler()
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/backup.py

@@ -1,39 +0,0 @@
-import os
-import frappe
-from frappe.utils.backups import scheduled_backup
-from frappe.utils import cint, get_sites, now
-
-
-def backup(sites, with_files=False):
-    for site in sites:
-        frappe.init(site)
-        frappe.connect()
-        odb = scheduled_backup(
-            ignore_files=not with_files,
-            backup_path_db=None,
-            backup_path_files=None,
-            backup_path_private_files=None,
-            force=True
-        )
-        print("database backup taken -", odb.backup_path_db, "- on", now())
-        if with_files:
-            print("files backup taken -", odb.backup_path_files, "- on", now())
-            print("private files backup taken -", odb.backup_path_private_files, "- on", now())
-        frappe.destroy()
-
-
-def main():
-    installed_sites = ":".join(get_sites())
-    sites = os.environ.get("SITES", installed_sites).split(":")
-    with_files = cint(os.environ.get("WITH_FILES"))
-
-    backup(sites, with_files)
-
-    if frappe.redis_server:
-        frappe.redis_server.connection_pool.disconnect()
-
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/console.py

@@ -1,32 +0,0 @@
-import sys
-import frappe
-import IPython
-
-from frappe.utils import get_sites
-
-
-def console(site):
-    "Start ipython console for a site"
-    if site not in get_sites():
-        print("Site {0} does not exist on the current bench".format(site))
-        return
-
-    frappe.init(site=site)
-    frappe.connect()
-    frappe.local.lang = frappe.db.get_default("lang")
-    all_apps = frappe.get_installed_apps()
-    for app in all_apps:
-        locals()[app] = __import__(app)
-    print("Apps in this namespace:\n{}".format(", ".join(all_apps)))
-    IPython.embed(display_banner="", header="")
-
-
-def main():
-    site = sys.argv[-1]
-    console(site)
-    if frappe.redis_server:
-        frappe.redis_server.connection_pool.disconnect()
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/drop.py

@@ -1,39 +0,0 @@
-import os
-import frappe
-
-from frappe.commands.site import _drop_site
-from constants import ARCHIVE_SITES_PATH
-from utils import get_password
-
-
-def main():
-    site_name = os.environ.get("SITE_NAME", 'site1.localhost')
-    db_root_username = os.environ.get("DB_ROOT_USER", 'root')
-    mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin')
-    postgres_root_password = get_password("POSTGRES_PASSWORD")
-    db_root_password = mariadb_root_password
-
-    if postgres_root_password:
-        db_root_password = postgres_root_password
-
-    force = True if os.environ.get("FORCE", None) else False
-    no_backup = True if os.environ.get("NO_BACKUP", None) else False
-    frappe.init(site_name, new_site=True)
-
-    _drop_site(
-        site=site_name,
-        root_login=db_root_username,
-        root_password=db_root_password,
-        archived_sites_path=ARCHIVE_SITES_PATH,
-        force=force,
-        no_backup=no_backup
-    )
-
-    if frappe.redis_server:
-        frappe.redis_server.connection_pool.disconnect()
-
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/commands/worker.py

@@ -1,12 +0,0 @@
-import os
-from frappe.utils.background_jobs import start_worker
-
-
-def main():
-    queue = os.environ.get("WORKER_TYPE", "default")
-    start_worker(queue, False)
-    exit(0)
-
-
-if __name__ == "__main__":
-    main()

build/frappe-worker/docker-entrypoint.sh

@@ -34,6 +34,7 @@ function configureEnv() {
 
     export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}"
 
+    # shellcheck disable=SC2016
     envsubst '${DB_HOST}
       ${DB_PORT}
       ${REDIS_CACHE}
@@ -72,7 +73,7 @@ ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \
 
 case "$1" in
 
-  start)
+start)
   configureEnv
   checkConnection
 
@@ -90,6 +91,7 @@ case "$1" in
     /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/auto_migrate.py
   fi
 
+  # shellcheck disable=SC2086
   /home/frappe/frappe-bench/env/bin/gunicorn ${LOAD_CONFIG_FILE} -b 0.0.0.0:${FRAPPE_PORT} \
     --worker-tmp-dir /dev/shm \
     --threads=4 \
@@ -99,23 +101,22 @@ case "$1" in
     -t 120 frappe.app:application --preload
   ;;
 
-  worker)
+worker)
-    checkConfigExists
-    checkConnection
-    # default WORKER_TYPE=default
-
-    /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/worker.py
-    ;;
-
-  schedule)
   checkConfigExists
   checkConnection
 
-    /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/background.py
+  : "${WORKER_TYPE:=default}"
-
+  bench worker --queue $WORKER_TYPE
   ;;
 
-  new)
+schedule)
+  checkConfigExists
+  checkConnection
+
+  bench schedule
+  ;;
+
+new)
   checkConfigExists
   checkConnection
 
@@ -123,51 +124,70 @@ case "$1" in
   exit
   ;;
 
-  drop)
+drop)
   checkConfigExists
   checkConnection
 
-    /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/drop.py
+  : "${SITE_NAME:=site1.localhost}"
-    exit
+  : "${DB_ROOT_USER:=root}"
+  : "${DB_ROOT_PASSWORD:=$POSTGRES_PASSWORD}"
+  : "${DB_ROOT_PASSWORD:=$MYSQL_ROOT_PASSWORD}"
+  : "${DB_ROOT_PASSWORD:=admin}"
+  if [[ -n $NO_BACKUP ]]; then
+    NO_BACKUP=--no-backup
+  fi
+  if [[ -n $FORCE ]]; then
+    FORCE=--force
+  fi
+
+  bench drop-site \
+    $SITE_NAME \
+    --root-login $DB_ROOT_USER \
+    --root-password $DB_ROOT_PASSWORD \
+    --archived-sites-path /home/frappe/frappe-bench/sites/archive_sites \
+    $NO_BACKUP $FORCE
   ;;
 
-  migrate)
+migrate)
   /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/migrate.py
   exit
   ;;
 
-  doctor)
+doctor)
   /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/doctor.py "${@:2}"
   exit
   ;;
 
-  backup)
+backup)
+  if [[ -n $WITH_FILES ]]; then
+    WITH_FILES=--with-files
+  fi
 
-    /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py
+  for site in ${SITES//:/ }; do
-    exit
+    bench --site "$site" backup $WITH_FILES
+  done
   ;;
 
-  console)
+console)
   if [[ -z "$2" ]]; then
     echo "Need to specify a sitename with the command:" >&2
     echo "console <sitename>" >&2
     exit 1
   fi
 
-    /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/console.py "$2"
+  bench --site "$2" console
-    exit
   ;;
 
-  push-backup)
+push-backup)
   /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/push_backup.py
   exit
   ;;
 
-  restore-backup)
+restore-backup)
   /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/restore_backup.py
   exit
   ;;
-  *)
+*)
   exec "$@"
   ;;
 esac

build/frappe-worker/healthcheck.sh

@@ -17,17 +17,17 @@ fi
 # Set REDIS host:port
 REDIS_CACHE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_cache" | sed 's|redis://||g')
 if [[ "${REDIS_CACHE}" == *"/"* ]]; then
-  REDIS_CACHE=$(echo ${REDIS_CACHE} | cut -f1 -d"/")
+  REDIS_CACHE=$(echo "${REDIS_CACHE}" | cut -f1 -d"/")
 fi
 
 REDIS_QUEUE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_queue" | sed 's|redis://||g')
 if [[ "${REDIS_QUEUE}" == *"/"* ]]; then
-  REDIS_QUEUE=$(echo ${REDIS_QUEUE} | cut -f1 -d"/")
+  REDIS_QUEUE=$(echo "${REDIS_QUEUE}" | cut -f1 -d"/")
 fi
 
 REDIS_SOCKETIO=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_socketio" | sed 's|redis://||g')
 if [[ "${REDIS_SOCKETIO}" == *"/"* ]]; then
-  REDIS_SOCKETIO=$(echo ${REDIS_SOCKETIO} | cut -f1 -d"/")
+  REDIS_SOCKETIO=$(echo "${REDIS_SOCKETIO}" | cut -f1 -d"/")
 fi
 
 echo "Check ${DB_HOST}:${DB_PORT}"

build/frappe-worker/install_app.sh

@@ -6,5 +6,6 @@ APP_BRANCH=${3}
 
 [[ -n "${APP_BRANCH}" ]] && BRANCH="-b ${APP_BRANCH}"
 
-git clone --depth 1 -o upstream ${APP_REPO} ${BRANCH} /home/frappe/frappe-bench/apps/${APP_NAME}
+# shellcheck disable=SC2086
-/home/frappe/frappe-bench/env/bin/pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/${APP_NAME}
+git clone --depth 1 -o upstream "${APP_REPO}" ${BRANCH} "/home/frappe/frappe-bench/apps/${APP_NAME}"
+/home/frappe/frappe-bench/env/bin/pip3 install --no-cache-dir -e "/home/frappe/frappe-bench/apps/${APP_NAME}"

tests/integration-test.sh

@@ -2,7 +2,6 @@
 
 set -e
 
-# shellcheck source=functions.sh
 source tests/functions.sh
 
 project_name=frappe_bench_00

tests/test-erpnext.sh

@@ -2,7 +2,6 @@
 
 set -e
 
-# shellcheck source=functions.sh
 source tests/functions.sh
 
 project_name="test_erpnext"

tests/test-frappe.sh

@@ -2,7 +2,6 @@
 
 set -e
 
-# shellcheck source=functions.sh
 source tests/functions.sh
 
 project_name="test_frappe"