From 685329926624f01c9ea0e3d3140daf14bcbf1db2 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 16 Mar 2020 22:25:45 +0530 Subject: [PATCH 01/21] fix: travis wrapper script for build on master first attempt to get builds on master running --- .travis.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index c2946355..3b3b3b47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,60 +53,60 @@ jobs: - stage: "Frappe (v12)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --worker --git-branch 12 + - ./travis.py frappe --worker --git-version 12 - ./travis.py frappe --worker --tag v12 --tag-only - ./travis.py frappe --worker --tag version-12 --tag-only - stage: "Frappe (v12)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --nginx --git-branch 12 + - ./travis.py frappe --nginx --git-version 12 - ./travis.py frappe --nginx --tag v12 --tag-only - ./travis.py frappe --nginx --tag version-12 --tag-only - stage: "Frappe (v12)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --socketio --git-branch 12 + - ./travis.py frappe --socketio --git-version 12 - ./travis.py frappe --socketio --tag v12 --tag-only - ./travis.py frappe --socketio --tag version-12 --tag-only - stage: "ERPNext (v12)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --worker --git-branch 12 + - ./travis.py erpnext --worker --git-version 12 - ./travis.py erpnext --worker --tag v12 --tag-only - ./travis.py erpnext --worker --tag version-12 --tag-only - stage: "ERPNext (v12)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --nginx --git-branch 12 + - ./travis.py erpnext --nginx --git-version 12 - ./travis.py erpnext --nginx --tag v12 --tag-only - ./travis.py erpnext --nginx --tag version-12 --tag-only - stage: "Frappe (v11)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --worker --git-branch 11 + - ./travis.py frappe --worker --git-version 11 - ./travis.py frappe --worker --tag v11 --tag-only - ./travis.py frappe --worker --tag version-11 --tag-only - stage: "Frappe (v11)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext frappe --nginx --git-branch 11 + - ./travis.py erpnext frappe --nginx --git-version 11 - ./travis.py erpnext frappe --nginx --tag v11 --tag-only - ./travis.py erpnext frappe --nginx --tag version-11 --tag-only - stage: "Frappe (v11)" if: branch = master AND type != pull_request script: - - ./travis.py frappe --socketio --git-branch 11 + - ./travis.py frappe --socketio --git-version 11 - ./travis.py frappe --socketio --tag v11 --tag-only - ./travis.py frappe --socketio --tag version-11 --tag-only - stage: "ERPNext (v11)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --worker --git-branch 11 + - ./travis.py erpnext --worker --git-version 11 - ./travis.py erpnext --worker --tag v11 --tag-only - ./travis.py erpnext --worker --tag version-11 --tag-only - stage: "ERPNext (v11)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext --nginx --git-branch 11 + - ./travis.py erpnext --nginx --git-version 11 - ./travis.py erpnext --nginx --tag v11 --tag-only - - ./travis.py erpnext --nginx --tag version-11 --tag-only \ No newline at end of file + - ./travis.py erpnext --nginx --tag version-11 --tag-only From 4b1e7530e25db03e639626f22c77a2f30a2e32af Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 16 Mar 2020 22:47:40 +0530 Subject: [PATCH 02/21] fix: travis wrapper script for build on master second attempt to get builds on master running --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3b3b3b47..73f9a74b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,9 +89,9 @@ jobs: - stage: "Frappe (v11)" if: branch = master AND type != pull_request script: - - ./travis.py erpnext frappe --nginx --git-version 11 - - ./travis.py erpnext frappe --nginx --tag v11 --tag-only - - ./travis.py erpnext frappe --nginx --tag version-11 --tag-only + - ./travis.py frappe --nginx --git-version 11 + - ./travis.py frappe --nginx --tag v11 --tag-only + - ./travis.py frappe --nginx --tag version-11 --tag-only - stage: "Frappe (v11)" if: branch = master AND type != pull_request script: From 1db512d334d0e56909867430835592ca35038298 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 17 Mar 2020 00:14:16 +0530 Subject: [PATCH 03/21] fix: travis wrapper script for build on master third attempt to get builds on master running --- build/erpnext-nginx/v11.Dockerfile | 2 +- build/erpnext-nginx/v12.Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/erpnext-nginx/v11.Dockerfile b/build/erpnext-nginx/v11.Dockerfile index 7ad82a4e..be288a18 100644 --- a/build/erpnext-nginx/v11.Dockerfile +++ b/build/erpnext-nginx/v11.Dockerfile @@ -8,7 +8,7 @@ FROM frappe/frappe-nginx:v11 COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ COPY --from=0 /rsync /rsync -RUN echo -n "\nerpnext" >> /home/frappe/frappe-bench/sites/apps.txt +RUN echo -n "\nerpnext" >> /var/www/html/apps.txt VOLUME [ "/assets" ] diff --git a/build/erpnext-nginx/v12.Dockerfile b/build/erpnext-nginx/v12.Dockerfile index f5f40853..90f02904 100644 --- a/build/erpnext-nginx/v12.Dockerfile +++ b/build/erpnext-nginx/v12.Dockerfile @@ -8,7 +8,7 @@ FROM frappe/frappe-nginx:v12 COPY --from=0 /home/frappe/frappe-bench/sites/ /var/www/html/ COPY --from=0 /rsync /rsync -RUN echo -n "\nerpnext" >> /home/frappe/frappe-bench/sites/apps.txt +RUN echo -n "\nerpnext" >> /var/www/html/apps.txt VOLUME [ "/assets" ] From 8475af3785bf42aa71eddc73c0923b34e3fbf4fb Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 17 Mar 2020 22:31:37 +0530 Subject: [PATCH 04/21] fix(frappe-worker): set sites volume ownership --- build/frappe-worker/Dockerfile | 4 ++++ build/frappe-worker/v11.Dockerfile | 4 ++++ build/frappe-worker/v12.Dockerfile | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 40ce9c06..00766f57 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -40,5 +40,9 @@ COPY build/common/worker/install_app.sh /usr/local/bin/install_app WORKDIR /home/frappe/frappe-bench/sites +RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites + +VOLUME [ "/home/frappe/frappe-bench/sites" ] + ENTRYPOINT ["docker-entrypoint.sh"] CMD ["start"] diff --git a/build/frappe-worker/v11.Dockerfile b/build/frappe-worker/v11.Dockerfile index 335eb7a9..de61332c 100644 --- a/build/frappe-worker/v11.Dockerfile +++ b/build/frappe-worker/v11.Dockerfile @@ -37,5 +37,9 @@ COPY build/common/worker/install_app.sh /usr/local/bin/install_app WORKDIR /home/frappe/frappe-bench/sites +RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites + +VOLUME [ "/home/frappe/frappe-bench/sites" ] + ENTRYPOINT ["docker-entrypoint.sh"] CMD ["start"] diff --git a/build/frappe-worker/v12.Dockerfile b/build/frappe-worker/v12.Dockerfile index 2f10db39..bfdaa317 100644 --- a/build/frappe-worker/v12.Dockerfile +++ b/build/frappe-worker/v12.Dockerfile @@ -40,5 +40,9 @@ COPY build/common/worker/install_app.sh /usr/local/bin/install_app WORKDIR /home/frappe/frappe-bench/sites +RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites + +VOLUME [ "/home/frappe/frappe-bench/sites" ] + ENTRYPOINT ["docker-entrypoint.sh"] CMD ["start"] From f1393c3a9405c010679a664446aec35d80399fd2 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 20 Mar 2020 05:21:30 +0530 Subject: [PATCH 05/21] fix(frappe-nginx): fix missing /rsync fixes #160 --- build/erpnext-nginx/install_app.sh | 3 ++- build/frappe-nginx/Dockerfile | 4 +++- build/frappe-nginx/v11.Dockerfile | 4 +++- build/frappe-nginx/v12.Dockerfile | 4 +++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh index 65a1af6f..324f2bd3 100755 --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx/install_app.sh @@ -27,5 +27,6 @@ mkdir -p /home/frappe/frappe-bench/sites/assets/${APP_NAME} cp -R /home/frappe/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public/* /home/frappe/frappe-bench/sites/assets/${APP_NAME} echo "rsync -a --delete /var/www/html/assets/${APP_NAME} /assets" > /rsync +chmod +x /rsync -rm /home/frappe/frappe-bench/sites/apps.txt \ No newline at end of file +rm /home/frappe/frappe-bench/sites/apps.txt diff --git a/build/frappe-nginx/Dockerfile b/build/frappe-nginx/Dockerfile index 64df31b7..6fa6441a 100644 --- a/build/frappe-nginx/Dockerfile +++ b/build/frappe-nginx/Dockerfile @@ -32,7 +32,9 @@ COPY --from=0 /var/www/error_pages /var/www/ COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template COPY build/frappe-nginx/docker-entrypoint.sh / -RUN apt-get update && apt-get install -y rsync && apt-get clean +RUN apt-get update && apt-get install -y rsync && apt-get clean \ + && echo "#!/bin/bash" > /rsync \ + && chmod +x /rsync VOLUME [ "/assets" ] diff --git a/build/frappe-nginx/v11.Dockerfile b/build/frappe-nginx/v11.Dockerfile index e44e7663..a1852f8d 100644 --- a/build/frappe-nginx/v11.Dockerfile +++ b/build/frappe-nginx/v11.Dockerfile @@ -32,7 +32,9 @@ COPY --from=0 /var/www/error_pages /var/www/ COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template COPY build/frappe-nginx/docker-entrypoint.sh / -RUN apt-get update && apt-get install -y rsync && apt-get clean +RUN apt-get update && apt-get install -y rsync && apt-get clean \ + && echo "#!/bin/bash" > /rsync \ + && chmod +x /rsync VOLUME [ "/assets" ] diff --git a/build/frappe-nginx/v12.Dockerfile b/build/frappe-nginx/v12.Dockerfile index 29adfead..c1860a2a 100644 --- a/build/frappe-nginx/v12.Dockerfile +++ b/build/frappe-nginx/v12.Dockerfile @@ -32,7 +32,9 @@ COPY --from=0 /var/www/error_pages /var/www/ COPY build/common/nginx-default.conf.template /etc/nginx/conf.d/default.conf.template COPY build/frappe-nginx/docker-entrypoint.sh / -RUN apt-get update && apt-get install -y rsync && apt-get clean +RUN apt-get update && apt-get install -y rsync && apt-get clean \ + && echo "#!/bin/bash" > /rsync \ + && chmod +x /rsync VOLUME [ "/assets" ] From 7c356ccb273fea73805b73c10d3b094b1cbf49ae Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 20 Mar 2020 20:56:23 +0530 Subject: [PATCH 06/21] fix: make python commands modular and reusable --- build/common/commands/background.py | 9 +- build/common/commands/backup.py | 13 ++- build/common/commands/check_connection.py | 9 +- build/common/commands/console.py | 6 +- build/common/commands/doctor.py | 2 +- build/common/commands/migrate.py | 45 ++++----- build/common/commands/new.py | 114 +++++++++++----------- build/common/commands/worker.py | 9 +- 8 files changed, 112 insertions(+), 95 deletions(-) diff --git a/build/common/commands/background.py b/build/common/commands/background.py index 7065efd9..fb008ba9 100644 --- a/build/common/commands/background.py +++ b/build/common/commands/background.py @@ -1,7 +1,10 @@ import frappe from frappe.utils.scheduler import start_scheduler -print("Starting background scheduler . . .") -start_scheduler() +def main(): + print("Starting background scheduler . . .") + start_scheduler() + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/commands/backup.py b/build/common/commands/backup.py index e6fbcbb6..9fa01cfd 100644 --- a/build/common/commands/backup.py +++ b/build/common/commands/backup.py @@ -20,10 +20,13 @@ def backup(sites, with_files=False): print("private files backup taken -", odb.backup_path_private_files, "- on", now()) frappe.destroy() -installed_sites = ":".join(get_sites()) -sites = os.environ.get("SITES", installed_sites).split(":") -with_files=True if os.environ.get("WITH_FILES") else False +def main(): + installed_sites = ":".join(get_sites()) + sites = os.environ.get("SITES", installed_sites).split(":") + with_files=True if os.environ.get("WITH_FILES") else False -backup(sites, with_files) + backup(sites, with_files) + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/commands/check_connection.py b/build/common/commands/check_connection.py index 80fb7dbd..b44ea12b 100644 --- a/build/common/commands/check_connection.py +++ b/build/common/commands/check_connection.py @@ -108,6 +108,13 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True): print("Connection to redis socketio timed out") exit(1) +# Get site_config.json +def get_site_config(site_name): + site_config = None + with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + site_config = json.load(site_config_file) + return site_config + def main(): check_mariadb() check_redis_queue() @@ -116,4 +123,4 @@ def main(): print('Connections OK') if __name__ == "__main__": - main() + main() diff --git a/build/common/commands/console.py b/build/common/commands/console.py index 20b3fd34..a9ade863 100644 --- a/build/common/commands/console.py +++ b/build/common/commands/console.py @@ -20,6 +20,6 @@ def console(site): print("Apps in this namespace:\n{}".format(", ".join(all_apps))) IPython.embed(display_banner="", header="") - -site = sys.argv[-1] -console(site) +def main(): + site = sys.argv[-1] + console(site) diff --git a/build/common/commands/doctor.py b/build/common/commands/doctor.py index 36e71411..98f2509f 100644 --- a/build/common/commands/doctor.py +++ b/build/common/commands/doctor.py @@ -23,4 +23,4 @@ def main(): exit(0) if __name__ == "__main__": - main() + main() diff --git a/build/common/commands/migrate.py b/build/common/commands/migrate.py index be382bea..7f8a1602 100644 --- a/build/common/commands/migrate.py +++ b/build/common/commands/migrate.py @@ -2,12 +2,7 @@ import os, frappe, compileall, re, json from frappe.migrate import migrate from frappe.utils import get_sites - -def get_config(): - config = None - with open('common_site_config.json') as config_file: - config = json.load(config_file) - return config +from check_connection import get_config def save_config(config): with open('common_site_config.json', 'w') as f: @@ -24,24 +19,30 @@ def set_maintenance_mode(enable=True): conf.update({ "maintenance_mode": 0, "pause_scheduler": 0 }) save_config(conf) +def migrate_sites(maintenance_mode=False): + installed_sites = ":".join(get_sites()) + sites = os.environ.get("SITES", installed_sites).split(":") + if not maintenance_mode: + maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False -installed_sites = ":".join(get_sites()) -sites = os.environ.get("SITES", installed_sites).split(":") -maintenance_mode = True if os.environ.get("MAINTENANCE_MODE") else False + if maintenance_mode: + set_maintenance_mode(True) -if maintenance_mode: - set_maintenance_mode(True) + for site in sites: + print('Migrating', site) + frappe.init(site=site) + frappe.connect() + try: + migrate() + finally: + frappe.destroy() -for site in sites: - print('Migrating', site) - frappe.init(site=site) - frappe.connect() - try: - migrate() - finally: - frappe.destroy() + if maintenance_mode: + set_maintenance_mode(False) -if maintenance_mode: - set_maintenance_mode(False) +def main(): + migrate_sites() + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/commands/new.py b/build/common/commands/new.py index c951e63e..b603458c 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -1,68 +1,68 @@ import os, frappe, json from frappe.commands.site import _new_site +from check_connection import get_config, get_site_config -site_name = os.environ.get("SITE_NAME", 'site1.localhost') -mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') -mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin') -force = True if os.environ.get("FORCE", None) else False -install_apps = os.environ.get("INSTALL_APPS", None) -install_apps = install_apps.split(',') if install_apps else [] -frappe.init(site_name, new_site=True) +def main(): + site_name = os.environ.get("SITE_NAME", 'site1.localhost') + mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root') + mariadb_root_password = os.environ.get("MYSQL_ROOT_PASSWORD", 'admin') + force = True if os.environ.get("FORCE", None) else False + install_apps = os.environ.get("INSTALL_APPS", None) + install_apps = install_apps.split(',') if install_apps else [] + frappe.init(site_name, new_site=True) -_new_site( - None, - site_name, - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password, - admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), - verbose=True, - install_apps=install_apps, - source_sql=None, - force=force, - reinstall=False, -) + _new_site( + None, + site_name, + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password, + admin_password=os.environ.get("ADMIN_PASSWORD", 'admin'), + verbose=True, + install_apps=install_apps, + source_sql=None, + force=force, + reinstall=False, + ) -config = None -with open('common_site_config.json') as config_file: - config = json.load(config_file) + config = get_config() -site_config = None -with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: - site_config = json.load(site_config_file) + site_config = get_site_config(site_name) -# update User's host to '%' required to connect from any container -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') -) -os.system(command) + # update User's host to '%' required to connect from any container + command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password + ) + command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') + ) + os.system(command) -# Set db password -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password') -) -os.system(command) + # Set db password + command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password + ) + command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') + ) + os.system(command) -# Grant permission to database -command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password -) -command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') -) -os.system(command) + # Grant permission to database + command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + db_host=config.get('db_host'), + mariadb_root_username=mariadb_root_username, + mariadb_root_password=mariadb_root_password + ) + command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') + ) + os.system(command) + exit(0) -exit(0) +if __name__ == "__main__": + main() diff --git a/build/common/commands/worker.py b/build/common/commands/worker.py index 6ec0bcbf..c810adf9 100644 --- a/build/common/commands/worker.py +++ b/build/common/commands/worker.py @@ -1,7 +1,10 @@ import os, frappe from frappe.utils.background_jobs import start_worker -queue = os.environ.get("WORKER_TYPE", "default") -start_worker(queue, False) +def main(): + queue = os.environ.get("WORKER_TYPE", "default") + start_worker(queue, False) + exit(0) -exit(0) +if __name__ == "__main__": + main() From 674c7664804525f8754d3557936b9c8fe30ea762 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sat, 21 Mar 2020 15:47:35 +0530 Subject: [PATCH 07/21] feat: auto migrate on container start if AUTO_MIGRATE set --- build/common/commands/auto_migrate.py | 110 +++++++++++++++++++++++ build/common/worker/docker-entrypoint.sh | 4 + installation/docker-compose-erpnext.yml | 1 + installation/docker-compose-frappe.yml | 1 + 4 files changed, 116 insertions(+) create mode 100644 build/common/commands/auto_migrate.py diff --git a/build/common/commands/auto_migrate.py b/build/common/commands/auto_migrate.py new file mode 100644 index 00000000..55124f0b --- /dev/null +++ b/build/common/commands/auto_migrate.py @@ -0,0 +1,110 @@ +import os +import json +import semantic_version +import git + +from migrate import migrate_sites +from check_connection import get_config + +APP_VERSIONS_JSON_FILE = 'app_versions.json' +APPS_TXT_FILE = 'apps.txt' + +def save_version_file(versions): + with open(APP_VERSIONS_JSON_FILE, 'w') as f: + return json.dump(versions, f, indent=1, sort_keys=True) + +def get_apps(): + apps = [] + try: + with open(APPS_TXT_FILE) as apps_file: + for app in apps_file.readlines(): + if app.strip(): + apps.append(app.strip()) + + except FileNotFoundError as exception: + print(exception) + exit(1) + except: + print(APPS_TXT_FILE+" is not valid") + exit(1) + + return apps + +def get_container_versions(apps): + versions = {} + for app in apps: + try: + version = __import__(app).__version__ + versions.update({app:version}) + except: + pass + + try: + path = os.path.join('..','apps', app) + repo = git.Repo(path) + commit_hash = repo.head.object.hexsha + versions.update({app+'_git_hash':commit_hash}) + except: + pass + + return versions + +def get_version_file(): + versions = None + try: + with open(APP_VERSIONS_JSON_FILE) as versions_file: + versions = json.load(versions_file) + except: + pass + return versions + +def main(): + is_ready = False + apps = get_apps() + + container_versions = get_container_versions(apps) + + version_file = get_version_file() + + if not version_file: + version_file = container_versions + save_version_file(version_file) + is_ready = True + + for app in apps: + container_version = None + file_version = None + version_file_hash = None + container_hash = None + + repo = git.Repo(os.path.join('..','apps',app)) + branch = repo.active_branch.name + + if branch == 'develop': + version_file_hash = version_file.get(app+'_git_hash') + container_hash = container_versions.get(app+'_git_hash') + if container_hash and version_file_hash: + if container_hash != version_file_hash: + is_ready = True + break + + if version_file.get(app): + file_version = semantic_version.Version(version_file.get(app)) + + if container_versions.get(app): + container_version = semantic_version.Version(container_versions.get(app)) + + if file_version and container_version: + if container_version > file_version: + is_ready = True + break + + config = get_config() + + if is_ready and config.get('maintenance_mode') != 1: + migrate_sites(maintenance_mode=True) + version_file = container_versions + save_version_file(version_file) + +if __name__ == "__main__": + main() diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index bf8a90c2..26cb9ef1 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -76,6 +76,10 @@ if [ "$1" = 'start' ]; then export FRAPPE_PORT=8000 fi + if [[ ! -z "$AUTO_MIGRATE" ]]; then + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/auto_migrate.py" + fi if [[ -z "$RUN_AS_ROOT" ]]; then su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ diff --git a/installation/docker-compose-erpnext.yml b/installation/docker-compose-erpnext.yml index f932a512..8e2cbd9e 100644 --- a/installation/docker-compose-erpnext.yml +++ b/installation/docker-compose-erpnext.yml @@ -37,6 +37,7 @@ services: - REDIS_QUEUE=redis-queue:6379 - REDIS_SOCKETIO=redis-socketio:6379 - SOCKETIO_PORT=9000 + - AUTO_MIGRATE=1 volumes: - ./sites:/home/frappe/frappe-bench/sites:rw - assets-vol:/home/frappe/frappe-bench/sites/assets:rw diff --git a/installation/docker-compose-frappe.yml b/installation/docker-compose-frappe.yml index df1e7c43..f8365225 100644 --- a/installation/docker-compose-frappe.yml +++ b/installation/docker-compose-frappe.yml @@ -37,6 +37,7 @@ services: - REDIS_QUEUE=redis-queue:6379 - REDIS_SOCKETIO=redis-socketio:6379 - SOCKETIO_PORT=9000 + - AUTO_MIGRATE=1 volumes: - ./sites:/home/frappe/frappe-bench/sites:rw - assets-vol:/home/frappe/frappe-bench/sites/assets:rw From 14dc20d5103eebb9fb526f9e5bb687d30f59eaea Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Sun, 22 Mar 2020 19:08:41 +0530 Subject: [PATCH 08/21] fix: do not auto migrate on first run --- build/common/commands/auto_migrate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/build/common/commands/auto_migrate.py b/build/common/commands/auto_migrate.py index 55124f0b..1c096ef7 100644 --- a/build/common/commands/auto_migrate.py +++ b/build/common/commands/auto_migrate.py @@ -69,7 +69,6 @@ def main(): if not version_file: version_file = container_versions save_version_file(version_file) - is_ready = True for app in apps: container_version = None From 55641c07deedcb7b2678e547b2a02227ce855830 Mon Sep 17 00:00:00 2001 From: Davide Bortolami Date: Tue, 24 Mar 2020 15:00:46 +0000 Subject: [PATCH 09/21] Add contributing.md by github standards --- CONTRIBUTING.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..1174e93c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,25 @@ +# Contribution Guidelines +## Branches + +* *master*: images on the master branch are built monthly. +* *develop*: images on this branch are when commits are pushed. + +# Pull Requests + +Please send all pull request exclusively to the *develop* branch. +When the PR are merged, the merge will trigger the image build automatically. + +Please test all PR as extensively as you can, considering that the software can be run in different modes: +* with docker-compose for production +* with or without Nginx proxy +* with VScode for testing environments + +Every once in a while (or before monthly release) develop will be merged into master. + +## Reducing the number of branching and builds :evergreen_tree: :evergreen_tree: :evergreen_tree: +Please be considerate when pushing commits and opening PR for multiple branches, as the process of building images (triggered on push and PR branch push) uses energy and contributes to global warming. + +# Documentation + +You should place README.md(s) in the relevant directories, explaining what the software in that particular directory does. + From d64662d7da534bc2a58852c2b9e45955dfef9a85 Mon Sep 17 00:00:00 2001 From: Davide Bortolami Date: Tue, 24 Mar 2020 17:01:33 +0000 Subject: [PATCH 10/21] fixed --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1174e93c..d9b4633c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,11 +2,11 @@ ## Branches * *master*: images on the master branch are built monthly. -* *develop*: images on this branch are when commits are pushed. +* *develop*: images on this branch are built daily. # Pull Requests -Please send all pull request exclusively to the *develop* branch. +Please **send all pull request exclusively to the *develop*** branch. When the PR are merged, the merge will trigger the image build automatically. Please test all PR as extensively as you can, considering that the software can be run in different modes: From ebd9d877d92b622916ae7d3871ce249b88868f2c Mon Sep 17 00:00:00 2001 From: Davide Bortolami Date: Tue, 24 Mar 2020 17:09:09 +0000 Subject: [PATCH 11/21] Create stale.yml --- .github/workflows/stale.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000..7bbc0505 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,19 @@ +name: Mark stale issues and pull requests + +on: + schedule: + - cron: "0 0 * * *" + +jobs: + stale: + + runs-on: ubuntu-latest + + steps: + - uses: actions/stale@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Stale issue message' + stale-pr-message: 'Stale pull request message' + stale-issue-label: 'no-issue-activity' + stale-pr-label: 'no-pr-activity' From dbd172efc5d90410d1c1c04fa8638e02a74f84f7 Mon Sep 17 00:00:00 2001 From: Davide Bortolami Date: Tue, 24 Mar 2020 17:15:27 +0000 Subject: [PATCH 12/21] Create greetings.yml Adds greeting for first time user interacts with the repo --- greetings.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 greetings.yml diff --git a/greetings.yml b/greetings.yml new file mode 100644 index 00000000..669221c0 --- /dev/null +++ b/greetings.yml @@ -0,0 +1,15 @@ +name: Greetings + +on: [pull_request, issues] + +jobs: + greeting: + runs-on: ubuntu-latest + steps: + - uses: actions/first-interaction@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + issue-message: | + Hello! We're very happy to see your first issue. If your issue is about a problem, go back and check you have copy-pasted all the debug logs you can so we can help you as fast as possible! + pr-message: | + Hello! Thank you about this PR. Since this is your first PR, please make sure you have described the improvements and your code is well documented. From 5f187c4e3f73430cdff661d585d9112959391268 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 25 Mar 2020 06:35:49 +0530 Subject: [PATCH 13/21] feat: worker command to push backups to cloud --- README.md | 20 +++ build/common/commands/push_backup.py | 161 +++++++++++++++++++++++ build/common/worker/docker-entrypoint.sh | 6 + 3 files changed, 187 insertions(+) create mode 100644 build/common/commands/push_backup.py diff --git a/README.md b/README.md index 2b688b1f..1991c39d 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,26 @@ docker exec -it \ The backup will be available in the `sites` mounted volume. +#### Push backup to s3 compatible storage + +Environment Variables + +- `BUCKET_NAME`, Required to set bucket created on S3 compatible storage. +- `ACCESS_KEY_ID`, Required to set access key. +- `SECRET_ACCESS_KEY`, Required to set secret access key. +- `ENDPOINT_URL`, Required to set URL of S3 compatible storage. +- `BUCKET_DIR`, Required to set directory in bucket where sites from this deployment will be backed up. +- `BACKUP_LIMIT`, Optionally set this to limit number of backups in bucket directory. Defaults to 3. + +```sh +docker exec -it \ + -e "BUCKET_NAME=backups" \ + -e "ACCESS_KEY_ID=access_id_from_provider" \ + -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ + -e "ENDPOINT_URL=https://region.storage-provider.com" \ + -e "BUCKET_DIR=frappe-bench-v12" \ +``` + #### Updating and Migrating Sites Switch to the root of the `frappe_docker` directory before running the following commands: diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py new file mode 100644 index 00000000..c02e510b --- /dev/null +++ b/build/common/commands/push_backup.py @@ -0,0 +1,161 @@ +import os +import time +import boto3 + +import datetime +from glob import glob +from frappe.utils import get_sites + +def get_file_ext(): + return { + "database": "-database.sql.gz", + "private_files": "-private-files.tar", + "public_files": "-files.tar" + } + +def get_backup_details(sitename): + backup_details = dict() + file_ext = get_file_ext() + + # add trailing slash https://stackoverflow.com/a/15010678 + site_backup_path = os.path.join(os.getcwd(), sitename, "private", "backups", "") + + if os.path.exists(site_backup_path): + for filetype, ext in file_ext.items(): + site_slug = sitename.replace('.', '_') + pattern = site_backup_path + '*-' + site_slug + ext + backup_files = list(filter(os.path.isfile, glob(pattern))) + + if len(backup_files) > 0: + backup_files.sort(key=lambda file: os.stat(os.path.join(site_backup_path, file)).st_ctime) + backup_date = datetime.datetime.strptime(time.ctime(os.path.getmtime(backup_files[0])), "%a %b %d %H:%M:%S %Y") + backup_details[filetype] = { + "sitename": sitename, + "file_size_in_bytes": os.stat(backup_files[-1]).st_size, + "file_path": os.path.abspath(backup_files[-1]), + "filename": os.path.basename(backup_files[-1]), + "backup_date": backup_date.date().strftime("%Y-%m-%d %H:%M:%S") + } + + return backup_details + +def get_s3_config(): + check_environment_variables() + bucket = os.environ.get('BUCKET_NAME') + + conn = boto3.client( + 's3', + aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), + endpoint_url=os.environ.get('ENDPOINT_URL') + ) + + return conn, bucket + +def check_environment_variables(): + if not 'BUCKET_NAME' in os.environ: + print('Variable BUCKET_NAME not set') + exit(1) + + if not 'ACCESS_KEY_ID' in os.environ: + print('Variable ACCESS_KEY_ID not set') + exit(1) + + if not 'SECRET_ACCESS_KEY' in os.environ: + print('Variable SECRET_ACCESS_KEY not set') + exit(1) + + if not 'ENDPOINT_URL' in os.environ: + print('Variable ENDPOINT_URL not set') + exit(1) + + if not 'BUCKET_DIR' in os.environ: + print('Variable BUCKET_DIR not set') + exit(1) + +def upload_file_to_s3(filename, folder, conn, bucket): + + destpath = os.path.join(folder, os.path.basename(filename)) + try: + print("Uploading file:", filename) + conn.upload_file(filename, bucket, destpath) + + except Exception as e: + print("Error uploading: %s" % (e)) + exit(1) + +def delete_old_backups(limit, bucket, folder): + all_backups = list() + backup_limit = int(limit) + check_environment_variables() + bucket_dir = os.environ.get('BUCKET_DIR') + + s3 = boto3.resource( + 's3', + aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), + endpoint_url=os.environ.get('ENDPOINT_URL') + ) + + bucket = s3.Bucket(bucket) + objects = bucket.meta.client.list_objects_v2( + Bucket=bucket.name, + Delimiter='/') + + if objects: + for obj in objects.get('CommonPrefixes'): + if obj.get('Prefix') in folder: + for backup_obj in bucket.objects.filter(Prefix=obj.get('Prefix')): + try: + backup_dir = backup_obj.key.split('/')[1] + all_backups.append(backup_dir) + except expression as error: + print(error) + exit(1) + + all_backups = set(sorted(all_backups)) + if len(all_backups) > backup_limit: + latest_backup = sorted(all_backups)[0] if len(all_backups) > 0 else None + print("Deleting Backup: {0}".format(latest_backup)) + for obj in bucket.objects.filter(Prefix=bucket_dir + '/' + latest_backup): + # delete all keys that are inside the latest_backup + if bucket_dir in obj.key: + try: + delete_directory = obj.key.split('/')[1] + print('Deleteing ' + obj.key) + s3.Object(bucket.name, obj.key).delete() + except expression as error: + print(error) + exit(1) + +def main(): + details = dict() + sites = get_sites() + conn, bucket = get_s3_config() + + for site in sites: + details = get_backup_details(site) + db_file = details.get('database', {}).get('file_path') + folder = None + if db_file: + folder = os.environ.get('BUCKET_DIR') + '/' + os.path.basename(db_file)[:15] + '/' + upload_file_to_s3(db_file, folder, conn, bucket) + + public_files = details.get('public_files', {}).get('file_path') + if public_files: + folder = os.environ.get('BUCKET_DIR') + '/' + os.path.basename(public_files)[:15] + '/' + upload_file_to_s3(public_files, folder, conn, bucket) + + private_files = details.get('private_files', {}).get('file_path') + if private_files: + folder = os.environ.get('BUCKET_DIR') + '/' + os.path.basename(private_files)[:15] + '/' + upload_file_to_s3(private_files, folder, conn, bucket) + + if folder: + delete_old_backups(os.environ.get('BACKUP_LIMIT', '3'), bucket, folder) + + print('push-backup complete') + exit(0) + +if __name__ == "__main__": + main() diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index 26cb9ef1..31085d6f 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -175,6 +175,12 @@ elif [ "$1" = 'console' ]; then python /home/frappe/frappe-bench/commands/console.py "$2" fi +elif [ "$1" = 'push-backup' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/push_backup.py" + exit + else exec su frappe -c "$@" From 754ba8a91a83253fb7470dade73e53584aaa4dd7 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 25 Mar 2020 22:25:24 +0530 Subject: [PATCH 14/21] feat: restrict backups to backup limit for each site --- README.md | 7 +++ build/common/commands/push_backup.py | 69 ++++++++++++++++++---------- 2 files changed, 51 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 1991c39d..a8b406fd 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,13 @@ docker exec -it \ -e "BUCKET_DIR=frappe-bench-v12" \ ``` +Note: + +- Above example will backup files in bucket called `backup` at location `frappe-bench-v12/site.name.com/DATE_TIME/DATE_TIME-site_name_com-{filetype}.{extension}`, +- example DATE_TIME: 20200325_042020. +- example filetype: database, files or private-files +- example extension: sql.gz or tar + #### Updating and Migrating Sites Switch to the root of the `frappe_docker` directory before running the following commands: diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py index c02e510b..0e291060 100644 --- a/build/common/commands/push_backup.py +++ b/build/common/commands/push_backup.py @@ -6,6 +6,8 @@ import datetime from glob import glob from frappe.utils import get_sites +DATE_FORMAT = "%Y%m%d_%H%M%S" + def get_file_ext(): return { "database": "-database.sql.gz", @@ -84,8 +86,9 @@ def upload_file_to_s3(filename, folder, conn, bucket): print("Error uploading: %s" % (e)) exit(1) -def delete_old_backups(limit, bucket, folder): +def delete_old_backups(limit, bucket, site_name): all_backups = list() + all_backup_dates = list() backup_limit = int(limit) check_environment_variables() bucket_dir = os.environ.get('BUCKET_DIR') @@ -104,29 +107,46 @@ def delete_old_backups(limit, bucket, folder): if objects: for obj in objects.get('CommonPrefixes'): - if obj.get('Prefix') in folder: + if obj.get('Prefix') == bucket_dir + '/': for backup_obj in bucket.objects.filter(Prefix=obj.get('Prefix')): try: - backup_dir = backup_obj.key.split('/')[1] - all_backups.append(backup_dir) - except expression as error: + # backup_obj.key is bucket_dir/site/date_time/backupfile.extension + bucket_dir, site_slug, date_time, backupfile = backup_obj.key.split('/') + date_time_object = datetime.datetime.strptime( + date_time, DATE_FORMAT + ) + + if site_name in backup_obj.key: + all_backup_dates.append(date_time_object) + all_backups.append(backup_obj.key) + except IndexError as error: print(error) exit(1) - all_backups = set(sorted(all_backups)) - if len(all_backups) > backup_limit: - latest_backup = sorted(all_backups)[0] if len(all_backups) > 0 else None - print("Deleting Backup: {0}".format(latest_backup)) - for obj in bucket.objects.filter(Prefix=bucket_dir + '/' + latest_backup): - # delete all keys that are inside the latest_backup - if bucket_dir in obj.key: - try: - delete_directory = obj.key.split('/')[1] - print('Deleteing ' + obj.key) - s3.Object(bucket.name, obj.key).delete() - except expression as error: - print(error) - exit(1) + oldest_backup_date = min(all_backup_dates) + + if len(all_backups) / 3 > backup_limit: + oldest_backup = None + for backup in all_backups: + try: + # backup is bucket_dir/site/date_time/backupfile.extension + backup_dir, site_slug, backup_dt_string, filename = backup.split('/') + backup_datetime = datetime.datetime.strptime( + backup_dt_string, DATE_FORMAT + ) + if backup_datetime == oldest_backup_date: + oldest_backup = backup + + except IndexError as error: + print(error) + exit(1) + + if oldest_backup: + for obj in bucket.objects.filter(Prefix=oldest_backup): + # delete all keys that are inside the oldest_backup + if bucket_dir in obj.key: + print('Deleteing ' + obj.key) + s3.Object(bucket.name, obj.key).delete() def main(): details = dict() @@ -136,23 +156,22 @@ def main(): for site in sites: details = get_backup_details(site) db_file = details.get('database', {}).get('file_path') - folder = None + folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' if db_file: - folder = os.environ.get('BUCKET_DIR') + '/' + os.path.basename(db_file)[:15] + '/' + folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(db_file)[:15] + '/' upload_file_to_s3(db_file, folder, conn, bucket) public_files = details.get('public_files', {}).get('file_path') if public_files: - folder = os.environ.get('BUCKET_DIR') + '/' + os.path.basename(public_files)[:15] + '/' + folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(public_files)[:15] + '/' upload_file_to_s3(public_files, folder, conn, bucket) private_files = details.get('private_files', {}).get('file_path') if private_files: - folder = os.environ.get('BUCKET_DIR') + '/' + os.path.basename(private_files)[:15] + '/' + folder = os.environ.get('BUCKET_DIR') + '/' + site + '/' + os.path.basename(private_files)[:15] + '/' upload_file_to_s3(private_files, folder, conn, bucket) - if folder: - delete_old_backups(os.environ.get('BACKUP_LIMIT', '3'), bucket, folder) + delete_old_backups(os.environ.get('BACKUP_LIMIT', '3'), bucket, site) print('push-backup complete') exit(0) From 3a6f7e1934892697900be37961427f258f6bb9ba Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 27 Mar 2020 00:28:50 +0530 Subject: [PATCH 15/21] feat: resotre backups from volume or cloud --- build/common/commands/restore_backup.py | 175 +++++++++++++++++++++++ build/common/worker/docker-entrypoint.sh | 6 + 2 files changed, 181 insertions(+) create mode 100644 build/common/commands/restore_backup.py diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py new file mode 100644 index 00000000..f7e14b4a --- /dev/null +++ b/build/common/commands/restore_backup.py @@ -0,0 +1,175 @@ +import os +import datetime +import tarfile +import hashlib +import frappe +import boto3 + +from push_backup import DATE_FORMAT, check_environment_variables +from frappe.utils import get_sites, random_string +from frappe.commands.site import _new_site +from frappe.installer import make_conf, get_conf_params +from check_connection import get_site_config, get_config + +def list_directories(path): + directories = [] + for name in os.listdir(path): + if os.path.isdir(os.path.join(path, name)): + directories.append(name) + return directories + +def get_backup_dir(): + return os.path.join( + os.path.expanduser('~'), + 'backups' + ) + +def decompress_db(files_base, site): + database_file = files_base + '-database.sql.gz' + config = get_config() + site_config = get_site_config(site) + db_root_user = os.environ.get('DB_ROOT_USER', 'root') + command = 'gunzip -c {database_file} > {database_extract}'.format( + database_file=database_file, + database_extract=database_file.replace('.gz','') + ) + + print('Extract Database GZip for site {}'.format(site)) + os.system(command) + +def restore_database(files_base, site): + db_root_password = os.environ.get('MYSQL_ROOT_PASSWORD') + if not db_root_password: + print('Variable MYSQL_ROOT_PASSWORD not set') + exit(1) + + db_root_user = os.environ.get("DB_ROOT_USER", 'root') + # restore database + + database_file = files_base + '-database.sql.gz' + decompress_db(files_base, site) + config = get_config() + site_config = get_site_config(site) + + # mysql command prefix + mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password} -e '.format( + db_root_user=db_root_user, + db_host=config.get('db_host'), + db_password=db_root_password + ) + + # create db + create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( + db_name=site_config.get('db_name') + ) + os.system(create_database) + + # create user + create_user = mysql_command + "\"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') + ) + os.system(create_user) + + # grant db privileges to user + grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + db_name=site_config.get('db_name') + ) + os.system(grant_privileges) + + command = "mysql -u{db_root_user} -h{db_host} -p{db_password} '{db_name}' < {database_file}".format( + db_root_user=db_root_user, + db_host=config.get('db_host'), + db_password=db_root_password, + db_name=site_config.get('db_name'), + database_file=database_file.replace('.gz',''), + ) + + print('Restoring database for site: {}'.format(site)) + os.system(command) + +def restore_files(files_base): + public_files = files_base + '-files.tar' + # extract tar + public_tar = tarfile.open(public_files) + print('Extracting {}'.format(public_files)) + public_tar.extractall() + +def restore_private_files(files_base): + private_files = files_base + '-private-files.tar' + private_tar = tarfile.open(private_files) + print('Extracting {}'.format(private_files)) + private_tar.extractall() + +def pull_backup_from_s3(): + check_environment_variables() + + # https://stackoverflow.com/a/54672690 + s3 = boto3.resource( + 's3', + aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), + endpoint_url=os.environ.get('ENDPOINT_URL') + ) + + bucket_dir = os.environ.get('BUCKET_DIR') + bucket_name = os.environ.get('BUCKET_NAME') + bucket = s3.Bucket(bucket_name) + + # Change directory to /home/frappe/backups + os.chdir(get_backup_dir()) + + for obj in bucket.objects.filter(Prefix = bucket_dir): + backup_file = obj.key.replace(os.path.join(bucket_dir,''),'') + if not os.path.exists(os.path.dirname(backup_file)): + os.makedirs(os.path.dirname(backup_file)) + print('Downloading {}'.format(backup_file)) + bucket.download_file(obj.key, backup_file) + + os.chdir(os.path.join(os.path.expanduser('~'), 'frappe-bench', 'sites')) + +def main(): + backup_dir = get_backup_dir() + + if len(list_directories(backup_dir)) == 0: + pull_backup_from_s3() + + for site in list_directories(backup_dir): + site_slug = site.replace('.','_') + backups = [datetime.datetime.strptime(backup, DATE_FORMAT) for backup in list_directories(os.path.join(backup_dir,site))] + latest_backup = max(backups).strftime(DATE_FORMAT) + files_base = os.path.join(backup_dir, site, latest_backup, '') + files_base += latest_backup + '-' + site_slug + if site in get_sites(): + restore_database(files_base, site) + restore_private_files(files_base) + restore_files(files_base) + else: + mariadb_root_password = os.environ.get('MYSQL_ROOT_PASSWORD') + if not mariadb_root_password: + print('Variable MYSQL_ROOT_PASSWORD not set') + exit(1) + mariadb_root_username = os.environ.get('DB_ROOT_USER', 'root') + database_file = files_base + '-database.sql.gz' + + site_config = get_conf_params( + db_name='_' + hashlib.sha1(site.encode()).hexdigest()[:16], + db_password=random_string(16) + ) + + frappe.local.site = site + frappe.local.sites_path = os.getcwd() + frappe.local.site_path = os.getcwd() + '/' + site + make_conf( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_name'), + ) + + restore_database(files_base, site) + restore_private_files(files_base) + restore_files(files_base) + + exit(0) + +if __name__ == "__main__": + main() diff --git a/build/common/worker/docker-entrypoint.sh b/build/common/worker/docker-entrypoint.sh index 31085d6f..d07e3cb6 100755 --- a/build/common/worker/docker-entrypoint.sh +++ b/build/common/worker/docker-entrypoint.sh @@ -181,6 +181,12 @@ elif [ "$1" = 'push-backup' ]; then && python /home/frappe/frappe-bench/commands/push_backup.py" exit +elif [ "$1" = 'restore-backup' ]; then + + su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \ + && python /home/frappe/frappe-bench/commands/restore_backup.py" + exit + else exec su frappe -c "$@" From 4e7b7690ee0b8de2b102e433035e4e0964e86475 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 27 Mar 2020 16:07:12 +0530 Subject: [PATCH 16/21] fix: backup and restore new command FORCE=1 error fixed only push backups if exists prepare and process db restore --- build/common/commands/new.py | 21 ++++++--------------- build/common/commands/push_backup.py | 4 +++- build/common/commands/restore_backup.py | 21 +++++++++++++++++---- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index b603458c..5fbe2cb9 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -29,36 +29,27 @@ def main(): site_config = get_site_config(site_name) - # update User's host to '%' required to connect from any container - command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( + mysql_command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( db_host=config.get('db_host'), mariadb_root_username=mariadb_root_username, mariadb_root_password=mariadb_root_password ) - command += "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( + + # update User's host to '%' required to connect from any container + command = mysql_command + "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(command) # Set db password - command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password - ) - command += "\"SET PASSWORD FOR '{db_name}'@'%' = PASSWORD('{db_password}'); FLUSH PRIVILEGES;\"".format( + command = mysql_command + "\"UPDATE mysql.user SET authentication_string = PASSWORD('{db_password}') WHERE User = \'{db_name}\' AND Host = \'%\';\"".format( db_name=site_config.get('db_name'), db_password=site_config.get('db_password') ) os.system(command) # Grant permission to database - command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password - ) - command += "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( + command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') ) os.system(command) diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py index 0e291060..e795b3ef 100644 --- a/build/common/commands/push_backup.py +++ b/build/common/commands/push_backup.py @@ -92,6 +92,7 @@ def delete_old_backups(limit, bucket, site_name): backup_limit = int(limit) check_environment_variables() bucket_dir = os.environ.get('BUCKET_DIR') + oldest_backup_date = None s3 = boto3.resource( 's3', @@ -123,7 +124,8 @@ def delete_old_backups(limit, bucket, site_name): print(error) exit(1) - oldest_backup_date = min(all_backup_dates) + if len(all_backup_dates) > 0: + oldest_backup_date = min(all_backup_dates) if len(all_backups) / 3 > backup_limit: oldest_backup = None diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index f7e14b4a..82854e85 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -8,7 +8,7 @@ import boto3 from push_backup import DATE_FORMAT, check_environment_variables from frappe.utils import get_sites, random_string from frappe.commands.site import _new_site -from frappe.installer import make_conf, get_conf_params +from frappe.installer import make_conf, get_conf_params, make_site_dirs from check_connection import get_site_config, get_config def list_directories(path): @@ -44,8 +44,8 @@ def restore_database(files_base, site): exit(1) db_root_user = os.environ.get("DB_ROOT_USER", 'root') - # restore database + # restore database database_file = files_base + '-database.sql.gz' decompress_db(files_base, site) config = get_config() @@ -58,6 +58,12 @@ def restore_database(files_base, site): db_password=db_root_password ) + # drop db if exists for clean restore + drop_database = mysql_command + "\"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format( + db_name=site_config.get('db_name') + ) + os.system(drop_database) + # create db create_database = mysql_command + "\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( db_name=site_config.get('db_name') @@ -71,6 +77,13 @@ def restore_database(files_base, site): ) os.system(create_user) + # create user password + set_user_password = mysql_command + "\"UPDATE mysql.user SET authentication_string = PASSWORD('{db_password}') WHERE User = \'{db_name}\' AND Host = \'%\';\"".format( + db_name=site_config.get('db_name'), + db_password=site_config.get('db_password') + ) + os.system(set_user_password) + # grant db privileges to user grant_privileges = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( db_name=site_config.get('db_name') @@ -162,9 +175,9 @@ def main(): frappe.local.site_path = os.getcwd() + '/' + site make_conf( db_name=site_config.get('db_name'), - db_password=site_config.get('db_name'), + db_password=site_config.get('db_password'), ) - + make_site_dirs() restore_database(files_base, site) restore_private_files(files_base) restore_files(files_base) From 2422fbad26a63c7c34d90845db006cac73ef1ddc Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 27 Mar 2020 16:41:32 +0530 Subject: [PATCH 17/21] fix: backup and restore create backup dir in worker images set ownership and mount volume for backups update readme about restore backup --- README.md | 43 +++++++++++++++++++++++++++++- build/frappe-worker/Dockerfile | 6 ++--- build/frappe-worker/v11.Dockerfile | 6 ++--- build/frappe-worker/v12.Dockerfile | 6 ++--- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a8b406fd..2a9e26d0 100644 --- a/README.md +++ b/README.md @@ -236,12 +236,15 @@ Environment Variables - `BACKUP_LIMIT`, Optionally set this to limit number of backups in bucket directory. Defaults to 3. ```sh -docker exec -it \ + docker run \ -e "BUCKET_NAME=backups" \ -e "ACCESS_KEY_ID=access_id_from_provider" \ -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ -e "ENDPOINT_URL=https://region.storage-provider.com" \ -e "BUCKET_DIR=frappe-bench-v12" \ + -v ./installation/sites:/home/frappe/frappe-bench/sites \ + --network _default \ + frappe/frappe-worker:v12 push-backup ``` Note: @@ -278,6 +281,44 @@ docker exec -it \ _erpnext-python_1 docker-entrypoint.sh migrate ``` +#### Restore backups + +Environment Variables + +- `MYSQL_ROOT_PASSWORD`, Required to restore mariadb backups. +- `BUCKET_NAME`, Required to set bucket created on S3 compatible storage. +- `ACCESS_KEY_ID`, Required to set access key. +- `SECRET_ACCESS_KEY`, Required to set secret access key. +- `ENDPOINT_URL`, Required to set URL of S3 compatible storage. +- `BUCKET_DIR`, Required to set directory in bucket where sites from this deployment will be backed up. + +```sh +docker run \ + -e "MYSQL_ROOT_PASSWORD=admin" \ + -e "BUCKET_NAME=backups" \ + -e "ACCESS_KEY_ID=access_id_from_provider" \ + -e "SECRET_ACCESS_KEY=secret_access_from_provider" \ + -e "ENDPOINT_URL=https://region.storage-provider.com" \ + -e "BUCKET_DIR=frappe-bench-v12" \ + -v ./installation/sites:/home/frappe/frappe-bench/sites \ + -v ./backups:/home/frappe/backups \ + --network _default \ + frappe/frappe-worker:v12 restore-backup +``` + +Note: + +- Volume must be mounted at location `/home/frappe/backups` for restoring sites +- If no backup files are found in volume, it will use s3 credentials to pull backups +- Backup structure for mounted volume or downloaded from s3: + - /home/frappe/backups + - site1.domain.com + - 20200420_162000 + - 20200420_162000-site1_domain_com-* + - site2.domain.com + - 20200420_162000 + - 20200420_162000-site2_domain_com-* + ### Custom apps To add your own Frappe/ERPNext apps to the image, we'll need to create a custom image with the help of a unique wrapper script diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 00766f57..337f43e8 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -21,7 +21,7 @@ RUN install_packages \ RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb -RUN mkdir -p apps logs commands +RUN mkdir -p apps logs commands /home/frappe/backups RUN virtualenv env \ && . env/bin/activate \ @@ -40,9 +40,9 @@ COPY build/common/worker/install_app.sh /usr/local/bin/install_app WORKDIR /home/frappe/frappe-bench/sites -RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites +RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites /home/frappe/backups -VOLUME [ "/home/frappe/frappe-bench/sites" ] +VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups" ] ENTRYPOINT ["docker-entrypoint.sh"] CMD ["start"] diff --git a/build/frappe-worker/v11.Dockerfile b/build/frappe-worker/v11.Dockerfile index de61332c..6d9384cb 100644 --- a/build/frappe-worker/v11.Dockerfile +++ b/build/frappe-worker/v11.Dockerfile @@ -18,7 +18,7 @@ RUN install_packages \ RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb -RUN mkdir -p apps logs commands +RUN mkdir -p apps logs commands /home/frappe/backups RUN virtualenv env \ && . env/bin/activate \ @@ -37,9 +37,9 @@ COPY build/common/worker/install_app.sh /usr/local/bin/install_app WORKDIR /home/frappe/frappe-bench/sites -RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites +RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites /home/frappe/backups -VOLUME [ "/home/frappe/frappe-bench/sites" ] +VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups" ] ENTRYPOINT ["docker-entrypoint.sh"] CMD ["start"] diff --git a/build/frappe-worker/v12.Dockerfile b/build/frappe-worker/v12.Dockerfile index bfdaa317..f923e26d 100644 --- a/build/frappe-worker/v12.Dockerfile +++ b/build/frappe-worker/v12.Dockerfile @@ -21,7 +21,7 @@ RUN install_packages \ RUN wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.stretch_amd64.deb RUN dpkg -i wkhtmltox_0.12.5-1.stretch_amd64.deb && rm wkhtmltox_0.12.5-1.stretch_amd64.deb -RUN mkdir -p apps logs commands +RUN mkdir -p apps logs commands /home/frappe/backups RUN virtualenv env \ && . env/bin/activate \ @@ -40,9 +40,9 @@ COPY build/common/worker/install_app.sh /usr/local/bin/install_app WORKDIR /home/frappe/frappe-bench/sites -RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites +RUN chown -R frappe:frappe /home/frappe/frappe-bench/sites /home/frappe/backups -VOLUME [ "/home/frappe/frappe-bench/sites" ] +VOLUME [ "/home/frappe/frappe-bench/sites", "/home/frappe/backups" ] ENTRYPOINT ["docker-entrypoint.sh"] CMD ["start"] From 80fdb213d83f1cb22461ab67e60363505d8a8fcb Mon Sep 17 00:00:00 2001 From: Davide Bortolami Date: Sun, 29 Mar 2020 22:34:39 +0100 Subject: [PATCH 18/21] Add PR template, issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 34 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 23 +++++++++++++ .../question-about-using-frappe_docker.md | 12 +++++++ .github/PULL_REQUEST_TEMPLATE.md | 7 ++++ 4 files changed, 76 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..d4c7a8c1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug report +about: Report a bug encountered while using the Frappe_docker +labels: bug +--- + + + +## Description of the issue + +## Context information (for bug reports) + +## Steps to reproduce the issue + +1. +2. +3. + +### Observed result + +### Expected result + +### Stacktrace / full error message if available + +``` +(paste here) +``` diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..15c410eb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,23 @@ +--- +name: Feature request +about: Suggest an idea to improve frappe_docker +labels: enhancement +--- + + + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md b/.github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md new file mode 100644 index 00000000..02fdd694 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question-about-using-frappe_docker.md @@ -0,0 +1,12 @@ +--- +name: Question about using Frappe/Frappe Apps +about: Ask how to do something +labels: question +--- + + \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..6c9fc532 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,7 @@ +> Please provide enough information so that others can review your pull request: + + + +> Explain the **details** for making this change. What existing problem does the pull request solve? + + From 830d3959b71250dfef6a9379b74a57798d5ff712 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Mon, 30 Mar 2020 13:08:55 +0530 Subject: [PATCH 19/21] docs: add info about backup command in readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 2a9e26d0..860ceaac 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,9 @@ Environment Variables - `SITES` is list of sites separated by `:` colon to migrate. e.g. `SITES=site1.domain.com` or `SITES=site1.domain.com:site2.domain.com` By default all sites in bench will be backed up. - `WITH_FILES` if set to 1, it will backup user-uploaded files. +- By default `backup` takes mariadb dump and gzips it. Example file, `20200325_221230-test_localhost-database.sql.gz` +- If `WITH_FILES` is set then it will also backup public and private files of each site as uncompressed tarball. Example files, `20200325_221230-test_localhost-files.tar` and `20200325_221230-test_localhost-private-files.tar` +- All the files generated by backup are placed at volume location `sites/{site-name}/private/backups/*` ```sh docker exec -it \ From 87c251a464ef56de9be9d99710305e14f4a7e671 Mon Sep 17 00:00:00 2001 From: girish pasupathy Date: Thu, 9 Apr 2020 18:58:34 +0530 Subject: [PATCH 20/21] Changed X-Frappe-Site-Name header to use value from `$host` instead of `$http_host` in nginx configuration ISSUE ----- `$http_host` is used for setting header 'X-Frappe-Site-Name' which adds port number to the header along with the host value. Frappe source app.py expects the header value to contain only the host name and not the port number. So `$host` should be used instead of `$http_host` to set the 'X-Frappe-Site-Name' header `$http_host` vs `$host` in nginx -------------------------------- `$http_host` contains the host name along with port number whereas `$host` contains only the host name in lowercase without the port number. > `$host` - This variable is equal to line Host in the header of request or > name of the server processing the request if the Host header is not available. > This variable may have a different value from $http_host in such cases: > * when the Host input header is absent or has an empty value, > `$host` equals to the value of server_name directive; > * when the value of Host contains port number, `$host` doesn't include > that port number. $host's value is always lowercase since 0.8.17. > - [$host vs $http_host stackoverflow](https://stackoverflow.com/questions/15414810/whats-the-difference-of-host-and-http-host-in-nginx) From the frappe source file [app.py](https://github.com/frappe/frappe/blob/develop/frappe/app.py#L107), X-Frappe-Site-Name is used if its set. ```Python site = _site or request.headers.get('X-Frappe-Site-Name') or get_site_name(request.host) ``` Since `$host` variable will never contain port number which is not the case with `$http_host`, `$host` should be used for setting the header 'X-Frappe-Site-Name'. Otherwise we have issues with site serving. Tested the above changes in compose as well as in swarm environment. In compose, tested the site with host mapping of 80 and 8000. Works with both the host port mapping. Tested with erpnext version - v12.5.2 Changes to be committed: modified: build/common/nginx-default.conf.template --- build/common/nginx-default.conf.template | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/common/nginx-default.conf.template b/build/common/nginx-default.conf.template index f19ccee2..076b2c42 100644 --- a/build/common/nginx-default.conf.template +++ b/build/common/nginx-default.conf.template @@ -29,9 +29,9 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_set_header X-Frappe-Site-Name $http_host; + proxy_set_header X-Frappe-Site-Name $host; proxy_set_header Origin $scheme://$http_host; - proxy_set_header Host $host; + proxy_set_header Host $http_host; proxy_pass http://socketio-server; } @@ -52,8 +52,8 @@ server { location @webserver { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frappe-Site-Name $http_host; - proxy_set_header Host $host; + proxy_set_header X-Frappe-Site-Name $host; + proxy_set_header Host $http_host; proxy_set_header X-Use-X-Accel-Redirect True; proxy_read_timeout 120; proxy_redirect off; From 362075a2bc5cd99a18a1ee742b30dfa0c3d611c0 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 10 Apr 2020 14:26:00 +0530 Subject: [PATCH 21/21] fix: improve readme mention to add site for production deployment mention to start bench for development deployment fixes #181 --- README.md | 7 ++++++- development/README.md | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 860ceaac..0f090c8f 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Make sure to replace `` with the desired name you wish to set for Notes: +- New site (first site) needs to be added after starting the services. - The local deployment is for testing and REST API development purpose only - A complete development environment is available [here](Development/README.md) - The site names are limited to patterns matching \*.localhost by default @@ -140,7 +141,11 @@ docker-compose \ ``` Make sure to replace `` with any desired name you wish to set for the project. -Note: use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext. + +Notes: + +- Use `docker-compose-frappe.yml` in case you need only Frappe without ERPNext. +- New site (first site) needs to be added after starting the services. ### Docker containers diff --git a/development/README.md b/development/README.md index 2f08d3d1..00fbd1a5 100644 --- a/development/README.md +++ b/development/README.md @@ -103,6 +103,16 @@ bench set-config developer_mode 1 bench clear-cache ``` +### Start development + +Execute following command from the `frappe-bench` directory. + +```shell +bench start +``` + +Note: To start bench with debugger refer section for debugging. + ### Fixing MariaDB issues after rebuilding the container The `bench new-site` command creates a user in MariaDB with container IP as host, for this reason after rebuilding the container there is a chance that you will not be able to access MariaDB correctly with the previous configuration