diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 253bcb76..5753cf9c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,3 +4,13 @@ updates: directory: / schedule: interval: daily + + - package-ecosystem: docker + directory: / + schedule: + interval: daily + + - package-ecosystem: npm + directory: build/frappe-socketio + schedule: + interval: daily diff --git a/.github/scripts/get-latest-tags.sh b/.github/scripts/get-latest-tags.sh index 0ca74acb..cfcf85e9 100755 --- a/.github/scripts/get-latest-tags.sh +++ b/.github/scripts/get-latest-tags.sh @@ -4,7 +4,7 @@ set -e set -x get_tag() { - tags=$(git ls-remote --refs --tags --sort='v:refname' https://github.com/$1 "v$2.*") + tags=$(git ls-remote --refs --tags --sort='v:refname' "https://github.com/$1" "v$2.*") tag=$(echo "$tags" | tail -n1 | sed 's/.*\///') echo "$tag" } @@ -12,11 +12,9 @@ get_tag() { FRAPPE_VERSION=$(get_tag frappe/frappe "$VERSION") ERPNEXT_VERSION=$(get_tag frappe/erpnext "$VERSION") -# shellcheck disable=SC2086 -echo "FRAPPE_VERSION=$FRAPPE_VERSION" >>$GITHUB_ENV -# shellcheck disable=SC2086 -echo "ERPNEXT_VERSION=$ERPNEXT_VERSION" >>$GITHUB_ENV -# shellcheck disable=SC2086 -echo "GIT_BRANCH=version-$VERSION" >>$GITHUB_ENV -# shellcheck disable=SC2086 -echo "VERSION=$VERSION" >>$GITHUB_ENV +cat <>"$GITHUB_ENV" +FRAPPE_VERSION=$FRAPPE_VERSION +ERPNEXT_VERSION=$ERPNEXT_VERSION +GIT_BRANCH=version-$VERSION +VERSION=$VERSION +EOL diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d7320dff..1dbd9007 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -14,3 +14,5 @@ jobs: steps: - uses: actions/checkout@v2 - uses: ludeeus/action-shellcheck@master + env: + SHELLCHECK_OPTS: -x diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh index d3e30769..49f40e78 100755 --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx/install_app.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + APP_NAME=${1} APP_REPO=${2} APP_BRANCH=${3} @@ -13,26 +15,29 @@ echo -ne "frappe\n${APP_NAME}" >/home/frappe/frappe-bench/sites/apps.txt mkdir -p apps cd apps -git clone --depth 1 https://github.com/frappe/frappe -b ${FRAPPE_BRANCH} -git clone --depth 1 ${APP_REPO} ${BRANCH} ${APP_NAME} +git clone --depth 1 https://github.com/frappe/frappe -b "${FRAPPE_BRANCH}" +# shellcheck disable=SC2086 +git clone --depth 1 "${APP_REPO}" ${BRANCH} "${APP_NAME}" echo "Install frappe NodeJS dependencies . . ." cd /home/frappe/frappe-bench/apps/frappe yarn echo "Install ${APP_NAME} NodeJS dependencies . . ." -cd /home/frappe/frappe-bench/apps/${APP_NAME} +cd "/home/frappe/frappe-bench/apps/${APP_NAME}" yarn echo "Build browser assets . . ." cd /home/frappe/frappe-bench/apps/frappe -yarn production --app ${APP_NAME} +yarn production --app "${APP_NAME}" echo "Install frappe NodeJS production dependencies . . ." cd /home/frappe/frappe-bench/apps/frappe yarn install --production=true echo "Install ${APP_NAME} NodeJS production dependencies . . ." -cd /home/frappe/frappe-bench/apps/${APP_NAME} +cd "/home/frappe/frappe-bench/apps/${APP_NAME}" yarn install --production=true +# shellcheck disable=SC2086 mkdir -p /home/frappe/frappe-bench/sites/assets/${APP_NAME} +# shellcheck disable=SC2086 cp -R /home/frappe/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public/* /home/frappe/frappe-bench/sites/assets/${APP_NAME} # Add frappe and all the apps available under in frappe-bench here diff --git a/build/frappe-nginx/docker-entrypoint.sh b/build/frappe-nginx/docker-entrypoint.sh index e1a29277..94277fc3 100755 --- a/build/frappe-nginx/docker-entrypoint.sh +++ b/build/frappe-nginx/docker-entrypoint.sh @@ -8,7 +8,7 @@ rsync -a --delete /var/www/html/assets/* /assets /rsync -touch /var/www/html/sites/.build -r "$(ls -td /assets/* | head -n 1)" +touch /var/www/html/sites/.build -r "$(find /assets -maxdepth 1 -name "*.*" | head -n 1)" [[ -z "${FRAPPE_PY}" ]] && FRAPPE_PY='0.0.0.0' @@ -36,6 +36,7 @@ if [[ ${SKIP_NGINX_TEMPLATE_GENERATION} == 1 ]]; then echo "Skipping default NGINX template generation. Please mount your own NGINX config file inside /etc/nginx/conf.d" else echo "Generating default template" + # shellcheck disable=SC2016 envsubst '${FRAPPE_PY} ${FRAPPE_PY_PORT} ${FRAPPE_SOCKETIO} @@ -50,9 +51,11 @@ else fi echo "Waiting for frappe-python to be available on ${FRAPPE_PY} port ${FRAPPE_PY_PORT}" +# shellcheck disable=SC2016 timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' ${FRAPPE_PY} ${FRAPPE_PY_PORT} echo "Frappe-python available on ${FRAPPE_PY} port ${FRAPPE_PY_PORT}" echo "Waiting for frappe-socketio to be available on ${FRAPPE_SOCKETIO} port ${SOCKETIO_PORT}" +# shellcheck disable=SC2016 timeout 10 bash -c 'until printf "" 2>>/dev/null >>/dev/tcp/$0/$1; do sleep 1; done' ${FRAPPE_SOCKETIO} ${SOCKETIO_PORT} echo "Frappe-socketio available on ${FRAPPE_SOCKETIO} port ${SOCKETIO_PORT}" diff --git a/build/frappe-socketio/docker-entrypoint.sh b/build/frappe-socketio/docker-entrypoint.sh index a7ac66b2..ec347813 100755 --- a/build/frappe-socketio/docker-entrypoint.sh +++ b/build/frappe-socketio/docker-entrypoint.sh @@ -1,15 +1,17 @@ -#!/bin/bash -e +#!/bin/bash + +set -e function checkConfigExists() { COUNTER=0 while [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json && ${COUNTER} -le 30 ]]; do - ((COUNTER = COUNTER + 1)) - echo "config file not created, retry ${COUNTER}" sleep 1 + ((COUNTER = COUNTER + 1)) + echo "config file not created, retry ${COUNTER}" >&2 done if [[ ! -e /home/frappe/frappe-bench/sites/common_site_config.json ]]; then - echo "timeout: config file not created" + echo "timeout: config file not created" >&2 exit 1 fi } @@ -17,11 +19,6 @@ function checkConfigExists() { if [[ "$1" == 'start' ]]; then checkConfigExists node /home/frappe/frappe-bench/apps/frappe/socketio.js - -elif [[ "$1" == 'doctor' ]]; then - node /home/frappe/frappe-bench/apps/frappe/health.js - else exec -c "$@" - fi diff --git a/build/frappe-worker/commands/background.py b/build/frappe-worker/commands/background.py deleted file mode 100644 index a26b7934..00000000 --- a/build/frappe-worker/commands/background.py +++ /dev/null @@ -1,11 +0,0 @@ -from frappe.utils.scheduler import start_scheduler - - -def main(): - print("Starting background scheduler . . .") - start_scheduler() - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/backup.py b/build/frappe-worker/commands/backup.py deleted file mode 100644 index 10ed6a92..00000000 --- a/build/frappe-worker/commands/backup.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import frappe -from frappe.utils.backups import scheduled_backup -from frappe.utils import cint, get_sites, now - - -def backup(sites, with_files=False): - for site in sites: - frappe.init(site) - frappe.connect() - odb = scheduled_backup( - ignore_files=not with_files, - backup_path_db=None, - backup_path_files=None, - backup_path_private_files=None, - force=True - ) - print("database backup taken -", odb.backup_path_db, "- on", now()) - if with_files: - print("files backup taken -", odb.backup_path_files, "- on", now()) - print("private files backup taken -", odb.backup_path_private_files, "- on", now()) - frappe.destroy() - - -def main(): - installed_sites = ":".join(get_sites()) - sites = os.environ.get("SITES", installed_sites).split(":") - with_files = cint(os.environ.get("WITH_FILES")) - - backup(sites, with_files) - - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/console.py b/build/frappe-worker/commands/console.py deleted file mode 100644 index b5c06b5e..00000000 --- a/build/frappe-worker/commands/console.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys -import frappe -import IPython - -from frappe.utils import get_sites - - -def console(site): - "Start ipython console for a site" - if site not in get_sites(): - print("Site {0} does not exist on the current bench".format(site)) - return - - frappe.init(site=site) - frappe.connect() - frappe.local.lang = frappe.db.get_default("lang") - all_apps = frappe.get_installed_apps() - for app in all_apps: - locals()[app] = __import__(app) - print("Apps in this namespace:\n{}".format(", ".join(all_apps))) - IPython.embed(display_banner="", header="") - - -def main(): - site = sys.argv[-1] - console(site) - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/drop.py b/build/frappe-worker/commands/drop.py deleted file mode 100644 index da759524..00000000 --- a/build/frappe-worker/commands/drop.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import frappe - -from frappe.commands.site import _drop_site -from constants import ARCHIVE_SITES_PATH -from utils import get_password - - -def main(): - site_name = os.environ.get("SITE_NAME", 'site1.localhost') - db_root_username = os.environ.get("DB_ROOT_USER", 'root') - mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin') - postgres_root_password = get_password("POSTGRES_PASSWORD") - db_root_password = mariadb_root_password - - if postgres_root_password: - db_root_password = postgres_root_password - - force = True if os.environ.get("FORCE", None) else False - no_backup = True if os.environ.get("NO_BACKUP", None) else False - frappe.init(site_name, new_site=True) - - _drop_site( - site=site_name, - root_login=db_root_username, - root_password=db_root_password, - archived_sites_path=ARCHIVE_SITES_PATH, - force=force, - no_backup=no_backup - ) - - if frappe.redis_server: - frappe.redis_server.connection_pool.disconnect() - - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/commands/worker.py b/build/frappe-worker/commands/worker.py deleted file mode 100644 index ba107220..00000000 --- a/build/frappe-worker/commands/worker.py +++ /dev/null @@ -1,12 +0,0 @@ -import os -from frappe.utils.background_jobs import start_worker - - -def main(): - queue = os.environ.get("WORKER_TYPE", "default") - start_worker(queue, False) - exit(0) - - -if __name__ == "__main__": - main() diff --git a/build/frappe-worker/docker-entrypoint.sh b/build/frappe-worker/docker-entrypoint.sh index 56ec3a9e..5065d2a9 100755 --- a/build/frappe-worker/docker-entrypoint.sh +++ b/build/frappe-worker/docker-entrypoint.sh @@ -34,6 +34,7 @@ function configureEnv() { export DB_HOST="${MARIADB_HOST:-$POSTGRES_HOST}" + # shellcheck disable=SC2016 envsubst '${DB_HOST} ${DB_PORT} ${REDIS_CACHE} @@ -72,102 +73,121 @@ ln -sfn /home/frappe/frappe-bench/sites/assets/frappe/node_modules \ case "$1" in - start) - configureEnv - checkConnection +start) + configureEnv + checkConnection - [[ -z "${WORKERS}" ]] && WORKERS='2' + [[ -z "${WORKERS}" ]] && WORKERS='2' - [[ -z "${FRAPPE_PORT}" ]] && FRAPPE_PORT='8000' + [[ -z "${FRAPPE_PORT}" ]] && FRAPPE_PORT='8000' - [[ -z "${WORKER_CLASS}" ]] && WORKER_CLASS='gthread' + [[ -z "${WORKER_CLASS}" ]] && WORKER_CLASS='gthread' - LOAD_CONFIG_FILE="" - [[ "${WORKER_CLASS}" == "gevent" ]] && - LOAD_CONFIG_FILE="-c /home/frappe/frappe-bench/commands/gevent_patch.py" + LOAD_CONFIG_FILE="" + [[ "${WORKER_CLASS}" == "gevent" ]] && + LOAD_CONFIG_FILE="-c /home/frappe/frappe-bench/commands/gevent_patch.py" - if [[ -n "${AUTO_MIGRATE}" ]]; then - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/auto_migrate.py - fi + if [[ -n "${AUTO_MIGRATE}" ]]; then + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/auto_migrate.py + fi - /home/frappe/frappe-bench/env/bin/gunicorn ${LOAD_CONFIG_FILE} -b 0.0.0.0:${FRAPPE_PORT} \ - --worker-tmp-dir /dev/shm \ - --threads=4 \ - --workers ${WORKERS} \ - --worker-class=${WORKER_CLASS} \ - --log-file=- \ - -t 120 frappe.app:application --preload - ;; + # shellcheck disable=SC2086 + /home/frappe/frappe-bench/env/bin/gunicorn ${LOAD_CONFIG_FILE} -b 0.0.0.0:${FRAPPE_PORT} \ + --worker-tmp-dir /dev/shm \ + --threads=4 \ + --workers ${WORKERS} \ + --worker-class=${WORKER_CLASS} \ + --log-file=- \ + -t 120 frappe.app:application --preload + ;; - worker) - checkConfigExists - checkConnection - # default WORKER_TYPE=default +worker) + checkConfigExists + checkConnection - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/worker.py - ;; + : "${WORKER_TYPE:=default}" + bench worker --queue $WORKER_TYPE + ;; - schedule) - checkConfigExists - checkConnection +schedule) + checkConfigExists + checkConnection - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/background.py + bench schedule + ;; - ;; +new) + checkConfigExists + checkConnection - new) - checkConfigExists - checkConnection + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/new.py + exit + ;; - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/new.py - exit - ;; +drop) + checkConfigExists + checkConnection - drop) - checkConfigExists - checkConnection + : "${SITE_NAME:=site1.localhost}" + : "${DB_ROOT_USER:=root}" + : "${DB_ROOT_PASSWORD:=$POSTGRES_PASSWORD}" + : "${DB_ROOT_PASSWORD:=$MYSQL_ROOT_PASSWORD}" + : "${DB_ROOT_PASSWORD:=admin}" + if [[ -n $NO_BACKUP ]]; then + NO_BACKUP=--no-backup + fi + if [[ -n $FORCE ]]; then + FORCE=--force + fi - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/drop.py - exit - ;; + bench drop-site \ + $SITE_NAME \ + --root-login $DB_ROOT_USER \ + --root-password $DB_ROOT_PASSWORD \ + --archived-sites-path /home/frappe/frappe-bench/sites/archive_sites \ + $NO_BACKUP $FORCE + ;; - migrate) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/migrate.py - exit - ;; +migrate) + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/migrate.py + exit + ;; - doctor) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/doctor.py "${@:2}" - exit - ;; +doctor) + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/doctor.py "${@:2}" + exit + ;; - backup) +backup) + if [[ -n $WITH_FILES ]]; then + WITH_FILES=--with-files + fi - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/backup.py - exit - ;; + for site in ${SITES//:/ }; do + bench --site "$site" backup $WITH_FILES + done + ;; - console) - if [[ -z "$2" ]]; then - echo "Need to specify a sitename with the command:" >&2 - echo "console " >&2 - exit 1 - fi +console) + if [[ -z "$2" ]]; then + echo "Need to specify a sitename with the command:" >&2 + echo "console " >&2 + exit 1 + fi - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/console.py "$2" - exit - ;; + bench --site "$2" console + ;; - push-backup) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/push_backup.py - exit - ;; +push-backup) + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/push_backup.py + exit + ;; - restore-backup) - /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/restore_backup.py - exit - ;; - *) - exec "$@" - ;; +restore-backup) + /home/frappe/frappe-bench/env/bin/python /home/frappe/frappe-bench/commands/restore_backup.py + exit + ;; +*) + exec "$@" + ;; esac diff --git a/build/frappe-worker/healthcheck.sh b/build/frappe-worker/healthcheck.sh index 6d378ded..3c4266ee 100755 --- a/build/frappe-worker/healthcheck.sh +++ b/build/frappe-worker/healthcheck.sh @@ -17,17 +17,17 @@ fi # Set REDIS host:port REDIS_CACHE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_cache" | sed 's|redis://||g') if [[ "${REDIS_CACHE}" == *"/"* ]]; then - REDIS_CACHE=$(echo ${REDIS_CACHE} | cut -f1 -d"/") + REDIS_CACHE=$(echo "${REDIS_CACHE}" | cut -f1 -d"/") fi REDIS_QUEUE=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_queue" | sed 's|redis://||g') if [[ "${REDIS_QUEUE}" == *"/"* ]]; then - REDIS_QUEUE=$(echo ${REDIS_QUEUE} | cut -f1 -d"/") + REDIS_QUEUE=$(echo "${REDIS_QUEUE}" | cut -f1 -d"/") fi REDIS_SOCKETIO=$(getUrl "${COMMON_SITE_CONFIG_JSON}" "redis_socketio" | sed 's|redis://||g') if [[ "${REDIS_SOCKETIO}" == *"/"* ]]; then - REDIS_SOCKETIO=$(echo ${REDIS_SOCKETIO} | cut -f1 -d"/") + REDIS_SOCKETIO=$(echo "${REDIS_SOCKETIO}" | cut -f1 -d"/") fi echo "Check ${DB_HOST}:${DB_PORT}" diff --git a/build/frappe-worker/install_app.sh b/build/frappe-worker/install_app.sh index d7de0842..6e55ec74 100755 --- a/build/frappe-worker/install_app.sh +++ b/build/frappe-worker/install_app.sh @@ -6,5 +6,6 @@ APP_BRANCH=${3} [[ -n "${APP_BRANCH}" ]] && BRANCH="-b ${APP_BRANCH}" -git clone --depth 1 -o upstream ${APP_REPO} ${BRANCH} /home/frappe/frappe-bench/apps/${APP_NAME} -/home/frappe/frappe-bench/env/bin/pip3 install --no-cache-dir -e /home/frappe/frappe-bench/apps/${APP_NAME} +# shellcheck disable=SC2086 +git clone --depth 1 -o upstream "${APP_REPO}" ${BRANCH} "/home/frappe/frappe-bench/apps/${APP_NAME}" +/home/frappe/frappe-bench/env/bin/pip3 install --no-cache-dir -e "/home/frappe/frappe-bench/apps/${APP_NAME}" diff --git a/tests/integration-test.sh b/tests/integration-test.sh index 4d5c32af..af7a8831 100755 --- a/tests/integration-test.sh +++ b/tests/integration-test.sh @@ -2,7 +2,6 @@ set -e -# shellcheck source=functions.sh source tests/functions.sh project_name=frappe_bench_00 diff --git a/tests/test-erpnext.sh b/tests/test-erpnext.sh index ac734164..343326f9 100755 --- a/tests/test-erpnext.sh +++ b/tests/test-erpnext.sh @@ -2,7 +2,6 @@ set -e -# shellcheck source=functions.sh source tests/functions.sh project_name="test_erpnext" diff --git a/tests/test-frappe.sh b/tests/test-frappe.sh index c4998300..c28e0cfd 100755 --- a/tests/test-frappe.sh +++ b/tests/test-frappe.sh @@ -2,7 +2,6 @@ set -e -# shellcheck source=functions.sh source tests/functions.sh project_name="test_frappe"