From 29e727491b85d2753c697bf6b3cb7a59cf4f8b05 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 7 Jul 2020 12:18:06 +0530 Subject: [PATCH 01/13] fix: pwd socketio hostname [skip travis] --- tests/pwd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pwd.yml b/tests/pwd.yml index 328d44f2..332e5b94 100644 --- a/tests/pwd.yml +++ b/tests/pwd.yml @@ -35,7 +35,7 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.erpnext-nginx.rule=HostRegexp(`{catchall:.*}`)" - - "traefik.http.middlewares.erpnext-nginx.headers.customrequestheaders.Host=mysite.localhost" + - "traefik.http.middlewares.erpnext-nginx.headers.customrequestheaders.Host=erpnext-nginx" - "traefik.http.routers.erpnext-nginx.middlewares=erpnext-nginx" - "traefik.http.routers.erpnext-nginx.entrypoints=web" - "traefik.http.services.erpnext-nginx.loadbalancer.server.port=80" @@ -148,7 +148,7 @@ services: deploy: restart_policy: condition: none - command: ["bash", "-c", "echo mysite.localhost > /sites/currentsite.txt"] + command: ["bash", "-c", "echo erpnext-nginx > /sites/currentsite.txt"] volumes: - sites-vol:/sites:rw @@ -193,7 +193,7 @@ services: condition: none command: new environment: - - SITE_NAME=mysite.localhost + - SITE_NAME=erpnext-nginx - DB_ROOT_USER=root - MYSQL_ROOT_PASSWORD=admin - ADMIN_PASSWORD=admin From 2b2a57a85d22fa0b51edd3a9d162334738eb857b Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 9 Jul 2020 17:01:50 +0530 Subject: [PATCH 02/13] fix: use subprocess instead of os to run commands * add a wrapper method run_command * remove os.system calls * use f-strings for commands Signed-off-by: Chinmay D. Pai --- build/common/commands/new.py | 28 +++----- build/common/commands/restore_backup.py | 92 +++++++++---------------- 2 files changed, 43 insertions(+), 77 deletions(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index cab9bc61..96175e12 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -5,6 +5,7 @@ import semantic_version from frappe.commands.site import _new_site from frappe.installer import update_site_config from check_connection import get_config, get_site_config, COMMON_SITE_CONFIG_FILE +from restore_backup import run_command def get_password(env_var, default=None): @@ -91,31 +92,22 @@ def main(): if db_type == "mariadb": site_config = get_site_config(site_name) + db_name = site_config.get('db_name') + db_password = site_config.get('db_password') - mysql_command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format( - db_host=config.get('db_host'), - mariadb_root_username=mariadb_root_username, - mariadb_root_password=mariadb_root_password - ) + mysql_command = ["mysql", f"-h{db_host}", f"-u{mariadb_root_username}", f"-p{mariadb_root_password}", "-e"] # update User's host to '%' required to connect from any container - command = mysql_command + "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') - ) - os.system(command) + command = mysql_command + [f"\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\""] + run_command(command) # Set db password - command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password') - ) - os.system(command) + command = mysql_command + [f"\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\""] + run_command(command) # Grant permission to database - command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format( - db_name=site_config.get('db_name') - ) - os.system(command) + command = mysql_command + [f"\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\""] + run_command(command) if frappe.redis_server: frappe.redis_server.connection_pool.disconnect() diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index efb51545..ce085bfc 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -1,6 +1,7 @@ import os import datetime import json +import subprocess import tarfile import hashlib import frappe @@ -13,6 +14,16 @@ from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_ from check_connection import get_site_config, get_config, COMMON_SITE_CONFIG_FILE +def run_command(command): + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, error = process.communicate() + if process.returncode: + print("Something went wrong:") + print(f"error code: {process.returncode}") + print(f"stdout:\n{out}") + print(f"\nstderr:\n{error}") + + def list_directories(path): directories = [] for name in os.listdir(path): @@ -30,13 +41,10 @@ def get_backup_dir(): def decompress_db(files_base, site): database_file = files_base + '-database.sql.gz' - command = 'gunzip -c {database_file} > {database_extract}'.format( - database_file=database_file, - database_extract=database_file.replace('.gz', '') - ) + command = ["gunzip", "-c", database_file, ">", database_file.replace(".gz", "")] print('Extract Database GZip for site {}'.format(site)) - os.system(command) + run_command(command) def restore_database(files_base, site_config_path, site): @@ -203,28 +211,15 @@ def restore_postgres(config, site_config, database_file): db_name = site_config.get('db_name') db_password = site_config.get('db_password') - psql_command = "psql postgres://{root_login}:{root_password}@{db_host}:{db_port}".format( - root_login=db_root_user, - root_password=db_root_password, - db_host=db_host, - db_port=db_port - ) + psql_command = ["psql", f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}"] print('Restoring PostgreSQL') - os.system(psql_command + ' -c "DROP DATABASE IF EXISTS \"{db_name}\""'.format(db_name=db_name)) - os.system(psql_command + ' -c "DROP USER IF EXISTS {db_name}"'.format(db_name=db_name)) - os.system(psql_command + ' -c "CREATE DATABASE \"{db_name}\""'.format(db_name=db_name)) - os.system(psql_command + ' -c "CREATE user {db_name} password \'{db_password}\'"'.format( - db_name=db_name, - db_password=db_password)) - os.system(psql_command + ' -c "GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"'.format( - db_name=db_name)) - - os.system("{psql_command}/{db_name} < {database_file}".format( - psql_command=psql_command, - database_file=database_file.replace('.gz', ''), - db_name=db_name, - )) + run_command(psql_command + ["-c", f"\"DROP DATABASE IF EXISTS \"{db_name}\"\""]) + run_command(psql_command + ["-c", f"\"DROP USER IF EXISTS {db_name}\""]) + run_command(psql_command + ["-c ", f"\"CREATE DATABASE \"{db_name}\"\""]) + run_command(psql_command + ["-c", f"\"CREATE user {db_name} password '{db_password}'\""]) + run_command(psql_command + ["-c", f"\"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}\""]) + run_command([f"{psql_command}/{db_name}", "<", database_file.replace('.gz', '')]) def restore_mariadb(config, site_config, database_file): @@ -237,53 +232,32 @@ def restore_mariadb(config, site_config, database_file): db_host = site_config.get('db_host', config.get('db_host')) db_port = site_config.get('db_port', config.get('db_port')) + db_name = site_config.get('db_name') + db_password = site_config.get('db_password') # mysql command prefix - mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password}'.format( - db_root_user=db_root_user, - db_host=db_host, - db_port=db_port, - db_password=db_root_password - ) + mysql_command = ["mysql", f"-u{db_root_user}", f"-h{db_host}", f"-p{db_root_password}", f"-P{db_port}"] # drop db if exists for clean restore - drop_database = "{mysql_command} -e \"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format( - mysql_command=mysql_command, - db_name=site_config.get('db_name'), - ) - os.system(drop_database) + drop_database = mysql_command + ["-e", f"\"DROP DATABASE IF EXISTS \`{db_name}\`;\""] + run_command(drop_database) # create db - create_database = "{mysql_command} -e \"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format( - mysql_command=mysql_command, - db_name=site_config.get('db_name'), - ) - os.system(create_database) + create_database = mysql_command + ["-e", f"\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\""] + run_command(create_database) # create user - create_user = "{mysql_command} -e \"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format( - mysql_command=mysql_command, - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password'), - ) - os.system(create_user) + create_user = mysql_command + ["-e", f"\"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\""] + run_command(create_user) # grant db privileges to user - grant_privileges = "{mysql_command} -e \"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format( - mysql_command=mysql_command, - db_name=site_config.get('db_name'), - db_password=site_config.get('db_password'), - ) - os.system(grant_privileges) + grant_privileges = mysql_command + ["-e", f"\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\""] + run_command(grant_privileges) - command = "{mysql_command} '{db_name}' < {database_file}".format( - mysql_command=mysql_command, - db_name=site_config.get('db_name'), - database_file=database_file.replace('.gz', ''), - ) + command = mysql_command + [f"'{db_name}'", "<", database_file.replace(".gz", "")] print('Restoring MariaDB') - os.system(command) + run_command(command) def main(): From 5c57334bb61ecd44d8b6b69080dce9ff4748a806 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 9 Jul 2020 17:27:54 +0530 Subject: [PATCH 03/13] fix: move run_command to utils Signed-off-by: Chinmay D. Pai --- build/common/commands/new.py | 2 +- build/common/commands/restore_backup.py | 12 +----------- build/common/commands/utils.py | 12 ++++++++++++ 3 files changed, 14 insertions(+), 12 deletions(-) create mode 100644 build/common/commands/utils.py diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 96175e12..feba0401 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -5,7 +5,7 @@ import semantic_version from frappe.commands.site import _new_site from frappe.installer import update_site_config from check_connection import get_config, get_site_config, COMMON_SITE_CONFIG_FILE -from restore_backup import run_command +from utils import run_command def get_password(env_var, default=None): diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index ce085bfc..dc7333e0 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -1,7 +1,6 @@ import os import datetime import json -import subprocess import tarfile import hashlib import frappe @@ -9,21 +8,12 @@ import boto3 from new import get_password from push_backup import DATE_FORMAT, check_environment_variables +from utils import run_command from frappe.utils import get_sites, random_string from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config from check_connection import get_site_config, get_config, COMMON_SITE_CONFIG_FILE -def run_command(command): - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, error = process.communicate() - if process.returncode: - print("Something went wrong:") - print(f"error code: {process.returncode}") - print(f"stdout:\n{out}") - print(f"\nstderr:\n{error}") - - def list_directories(path): directories = [] for name in os.listdir(path): diff --git a/build/common/commands/utils.py b/build/common/commands/utils.py new file mode 100644 index 00000000..d9535c6b --- /dev/null +++ b/build/common/commands/utils.py @@ -0,0 +1,12 @@ +import subprocess + + +def run_command(command): + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, error = process.communicate() + if process.returncode: + print("Something went wrong:") + print(f"return code: {process.returncode}") + print(f"stdout:\n{out}") + print(f"\nstderr:\n{error}") + exit(process.returncode) From c67fc024521a8d822733b3de64f6f934d5afed98 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Thu, 9 Jul 2020 22:57:12 +0530 Subject: [PATCH 04/13] fix: make changes to psql restore Signed-off-by: Chinmay D. Pai --- build/common/commands/restore_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index dc7333e0..79d4ea0a 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -209,7 +209,7 @@ def restore_postgres(config, site_config, database_file): run_command(psql_command + ["-c ", f"\"CREATE DATABASE \"{db_name}\"\""]) run_command(psql_command + ["-c", f"\"CREATE user {db_name} password '{db_password}'\""]) run_command(psql_command + ["-c", f"\"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}\""]) - run_command([f"{psql_command}/{db_name}", "<", database_file.replace('.gz', '')]) + run_command(["psql", f"{psql_command[-1]}/{db_name}", "<", database_file.replace('.gz', '')]) def restore_mariadb(config, site_config, database_file): From 1d9dd966b182183f68792bb5cd4c1f194b7a3126 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Fri, 10 Jul 2020 10:07:22 +0530 Subject: [PATCH 05/13] fix: pipe uncompressed database to file Signed-off-by: Chinmay D. Pai --- build/common/commands/restore_backup.py | 13 ++++++------- build/common/commands/utils.py | 6 ++++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 79d4ea0a..ecb2e24d 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -29,18 +29,17 @@ def get_backup_dir(): ) -def decompress_db(files_base, site): - database_file = files_base + '-database.sql.gz' - command = ["gunzip", "-c", database_file, ">", database_file.replace(".gz", "")] - - print('Extract Database GZip for site {}'.format(site)) - run_command(command) +def decompress_db(database_file, site): + command = ["gunzip", "-c", database_file] + with open(database_file.replace(".gz", ""), "w") as db_file: + print('Extract Database GZip for site {}'.format(site)) + run_command(command, stdout=db_file) def restore_database(files_base, site_config_path, site): # restore database database_file = files_base + '-database.sql.gz' - decompress_db(files_base, site) + decompress_db(database_file, site) config = get_config() # Set db_type if it exists in backup site_config.json diff --git a/build/common/commands/utils.py b/build/common/commands/utils.py index d9535c6b..d962fbd5 100644 --- a/build/common/commands/utils.py +++ b/build/common/commands/utils.py @@ -1,8 +1,10 @@ import subprocess -def run_command(command): - process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +def run_command(command, stdout=None, stderr=None): + stdout = stdout or subprocess.PIPE + stderr = stderr or subprocess.PIPE + process = subprocess.Popen(command, stdout=stdout, stderr=stderr) out, error = process.communicate() if process.returncode: print("Something went wrong:") From bafa7f65987604a6f8140d6f2c605548f25da3a3 Mon Sep 17 00:00:00 2001 From: "Chinmay D. Pai" Date: Fri, 10 Jul 2020 10:27:36 +0530 Subject: [PATCH 06/13] fix: remove quotes from psql db restore Signed-off-by: Chinmay D. Pai --- build/common/commands/restore_backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index ecb2e24d..1fe46ae5 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -203,9 +203,9 @@ def restore_postgres(config, site_config, database_file): psql_command = ["psql", f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}"] print('Restoring PostgreSQL') - run_command(psql_command + ["-c", f"\"DROP DATABASE IF EXISTS \"{db_name}\"\""]) + run_command(psql_command + ["-c", f"\"DROP DATABASE IF EXISTS {db_name}\""]) run_command(psql_command + ["-c", f"\"DROP USER IF EXISTS {db_name}\""]) - run_command(psql_command + ["-c ", f"\"CREATE DATABASE \"{db_name}\"\""]) + run_command(psql_command + ["-c ", f"\"CREATE DATABASE {db_name}\""]) run_command(psql_command + ["-c", f"\"CREATE user {db_name} password '{db_password}'\""]) run_command(psql_command + ["-c", f"\"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}\""]) run_command(["psql", f"{psql_command[-1]}/{db_name}", "<", database_file.replace('.gz', '')]) From fa2aff47df15645e9fa5792c82a4b32981a01236 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 10 Jul 2020 15:04:29 +0530 Subject: [PATCH 07/13] fix: psql command with subprocess --- build/common/commands/restore_backup.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 1fe46ae5..fa8f4883 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -200,15 +200,16 @@ def restore_postgres(config, site_config, database_file): db_name = site_config.get('db_name') db_password = site_config.get('db_password') - psql_command = ["psql", f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}"] + psql_command = ["psql"] + psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}" print('Restoring PostgreSQL') - run_command(psql_command + ["-c", f"\"DROP DATABASE IF EXISTS {db_name}\""]) - run_command(psql_command + ["-c", f"\"DROP USER IF EXISTS {db_name}\""]) - run_command(psql_command + ["-c ", f"\"CREATE DATABASE {db_name}\""]) - run_command(psql_command + ["-c", f"\"CREATE user {db_name} password '{db_password}'\""]) - run_command(psql_command + ["-c", f"\"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}\""]) - run_command(["psql", f"{psql_command[-1]}/{db_name}", "<", database_file.replace('.gz', '')]) + run_command(psql_command + [psql_uri, "-c", f"DROP DATABASE IF EXISTS \"{db_name}\""]) + run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"]) + run_command(psql_command + [psql_uri, "-c", f"CREATE DATABASE \"{db_name}\""]) + run_command(psql_command + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"]) + run_command(psql_command + [psql_uri, "-c", f"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"]) + run_command(psql_command + [f"{psql_uri}/{db_name}", "<", database_file.replace('.gz', '')]) def restore_mariadb(config, site_config, database_file): From 31ff697835ef0af2be35dec43fbe9353140697f2 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 10 Jul 2020 15:42:36 +0530 Subject: [PATCH 08/13] fix: mysql command with subprocess --- build/common/commands/new.py | 6 +++--- build/common/commands/restore_backup.py | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index feba0401..f6cc00a3 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -98,15 +98,15 @@ def main(): mysql_command = ["mysql", f"-h{db_host}", f"-u{mariadb_root_username}", f"-p{mariadb_root_password}", "-e"] # update User's host to '%' required to connect from any container - command = mysql_command + [f"\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\""] + command = mysql_command + [f"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;"] run_command(command) # Set db password - command = mysql_command + [f"\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\""] + command = mysql_command + [f"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] run_command(command) # Grant permission to database - command = mysql_command + [f"\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\""] + command = mysql_command + [f"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;"] run_command(command) if frappe.redis_server: diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index fa8f4883..d2e4f0ee 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -229,22 +229,22 @@ def restore_mariadb(config, site_config, database_file): mysql_command = ["mysql", f"-u{db_root_user}", f"-h{db_host}", f"-p{db_root_password}", f"-P{db_port}"] # drop db if exists for clean restore - drop_database = mysql_command + ["-e", f"\"DROP DATABASE IF EXISTS \`{db_name}\`;\""] + drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS \`{db_name}\`;"] run_command(drop_database) # create db - create_database = mysql_command + ["-e", f"\"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\""] + create_database = mysql_command + ["-e", f"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;"] run_command(create_database) # create user - create_user = mysql_command + ["-e", f"\"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\""] + create_user = mysql_command + ["-e", f"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;"] run_command(create_user) # grant db privileges to user - grant_privileges = mysql_command + ["-e", f"\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\""] + grant_privileges = mysql_command + ["-e", f"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] run_command(grant_privileges) - command = mysql_command + [f"'{db_name}'", "<", database_file.replace(".gz", "")] + command = mysql_command + [f"{db_name}", "<", database_file.replace(".gz", "")] print('Restoring MariaDB') run_command(command) From 4598cb24f4a8ccca751898c1a7cdc3e462265bb0 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 10 Jul 2020 16:18:25 +0530 Subject: [PATCH 09/13] fix: file input with subprocess --- build/common/commands/restore_backup.py | 8 ++++---- build/common/commands/utils.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index d2e4f0ee..2ae6ca78 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -209,7 +209,8 @@ def restore_postgres(config, site_config, database_file): run_command(psql_command + [psql_uri, "-c", f"CREATE DATABASE \"{db_name}\""]) run_command(psql_command + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"]) run_command(psql_command + [psql_uri, "-c", f"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"]) - run_command(psql_command + [f"{psql_uri}/{db_name}", "<", database_file.replace('.gz', '')]) + with open(database_file.replace('.gz', ''), 'r') as db_file: + run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file) def restore_mariadb(config, site_config, database_file): @@ -244,10 +245,9 @@ def restore_mariadb(config, site_config, database_file): grant_privileges = mysql_command + ["-e", f"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] run_command(grant_privileges) - command = mysql_command + [f"{db_name}", "<", database_file.replace(".gz", "")] - print('Restoring MariaDB') - run_command(command) + with open(database_file.replace(".gz", ""), "r") as db_file: + run_command(mysql_command + [f"{db_name}", "<"], stdin=db_file) def main(): diff --git a/build/common/commands/utils.py b/build/common/commands/utils.py index d962fbd5..8ebffb63 100644 --- a/build/common/commands/utils.py +++ b/build/common/commands/utils.py @@ -1,10 +1,10 @@ import subprocess -def run_command(command, stdout=None, stderr=None): +def run_command(command, stdout=None, stdin=None, stderr=None): stdout = stdout or subprocess.PIPE stderr = stderr or subprocess.PIPE - process = subprocess.Popen(command, stdout=stdout, stderr=stderr) + process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr) out, error = process.communicate() if process.returncode: print("Something went wrong:") From 21e2b139556238d0ad21cb151016c1429ed34f51 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 10 Jul 2020 19:21:59 +0530 Subject: [PATCH 10/13] fix: mysql db commands with subprocess --- build/common/commands/new.py | 6 +++++- build/common/commands/restore_backup.py | 12 ++++++------ build/common/commands/utils.py | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/build/common/commands/new.py b/build/common/commands/new.py index f6cc00a3..3438fa89 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -97,6 +97,10 @@ def main(): mysql_command = ["mysql", f"-h{db_host}", f"-u{mariadb_root_username}", f"-p{mariadb_root_password}", "-e"] + # Drop User if exists + command = mysql_command + [f"DROP USER IF EXISTS '{db_name}'@'%'; FLUSH PRIVILEGES;"] + run_command(command) + # update User's host to '%' required to connect from any container command = mysql_command + [f"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;"] run_command(command) @@ -106,7 +110,7 @@ def main(): run_command(command) # Grant permission to database - command = mysql_command + [f"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;"] + command = mysql_command + [f"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;"] run_command(command) if frappe.redis_server: diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 2ae6ca78..9fa9ef55 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -222,7 +222,7 @@ def restore_mariadb(config, site_config, database_file): db_root_user = os.environ.get("DB_ROOT_USER", 'root') db_host = site_config.get('db_host', config.get('db_host')) - db_port = site_config.get('db_port', config.get('db_port')) + db_port = site_config.get('db_port', config.get('db_port', 3306)) db_name = site_config.get('db_name') db_password = site_config.get('db_password') @@ -230,23 +230,23 @@ def restore_mariadb(config, site_config, database_file): mysql_command = ["mysql", f"-u{db_root_user}", f"-h{db_host}", f"-p{db_root_password}", f"-P{db_port}"] # drop db if exists for clean restore - drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS \`{db_name}\`;"] + drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"] run_command(drop_database) # create db - create_database = mysql_command + ["-e", f"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;"] + create_database = mysql_command + ["-e", f"CREATE DATABASE IF NOT EXISTS `{db_name}`;"] run_command(create_database) # create user - create_user = mysql_command + ["-e", f"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;"] + create_user = mysql_command + ["-e", f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] run_command(create_user) # grant db privileges to user - grant_privileges = mysql_command + ["-e", f"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] + grant_privileges = mysql_command + ["-e", f"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"] run_command(grant_privileges) print('Restoring MariaDB') - with open(database_file.replace(".gz", ""), "r") as db_file: + with open(database_file.replace('.gz', ''), 'r') as db_file: run_command(mysql_command + [f"{db_name}", "<"], stdin=db_file) diff --git a/build/common/commands/utils.py b/build/common/commands/utils.py index 8ebffb63..c808132e 100644 --- a/build/common/commands/utils.py +++ b/build/common/commands/utils.py @@ -4,6 +4,7 @@ import subprocess def run_command(command, stdout=None, stdin=None, stderr=None): stdout = stdout or subprocess.PIPE stderr = stderr or subprocess.PIPE + stdin = stdin or subprocess.PIPE process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr) out, error = process.communicate() if process.returncode: From 36f150bd95257e7ffdbc25d515b60cda7b759cb4 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 10 Jul 2020 19:45:28 +0530 Subject: [PATCH 11/13] fix: mysql restore db command with subprocess --- build/common/commands/restore_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 9fa9ef55..7de34367 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -247,7 +247,7 @@ def restore_mariadb(config, site_config, database_file): print('Restoring MariaDB') with open(database_file.replace('.gz', ''), 'r') as db_file: - run_command(mysql_command + [f"{db_name}", "<"], stdin=db_file) + run_command(mysql_command + [f"{db_name}"], stdin=db_file) def main(): From 84fefb228301517b3aa83fec7e6e9fe8fe4bb96a Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Fri, 10 Jul 2020 23:15:36 +0530 Subject: [PATCH 12/13] refactor: add utils and constants --- .gitignore | 3 + build/common/commands/auto_migrate.py | 65 +------- build/common/commands/check_connection.py | 41 ++--- build/common/commands/constants.py | 10 ++ build/common/commands/migrate.py | 8 +- build/common/commands/new.py | 31 +--- build/common/commands/push_backup.py | 63 +------- build/common/commands/restore_backup.py | 49 ++---- build/common/commands/utils.py | 189 ++++++++++++++++++++++ build/erpnext-nginx/install_app.sh | 4 +- 10 files changed, 253 insertions(+), 210 deletions(-) create mode 100644 build/common/commands/constants.py diff --git a/.gitignore b/.gitignore index efb2a23b..c5aa2d25 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ development !development/README.md deploy_key deploy_key.pub + +# Pycharm +.idea diff --git a/build/common/commands/auto_migrate.py b/build/common/commands/auto_migrate.py index 3c53db8d..ce5f3c2f 100644 --- a/build/common/commands/auto_migrate.py +++ b/build/common/commands/auto_migrate.py @@ -1,66 +1,15 @@ import os -import json import semantic_version import git from migrate import migrate_sites -from check_connection import get_config - -APP_VERSIONS_JSON_FILE = 'app_versions.json' -APPS_TXT_FILE = 'apps.txt' - - -def save_version_file(versions): - with open(APP_VERSIONS_JSON_FILE, 'w') as f: - return json.dump(versions, f, indent=1, sort_keys=True) - - -def get_apps(): - apps = [] - try: - with open(APPS_TXT_FILE) as apps_file: - for app in apps_file.readlines(): - if app.strip(): - apps.append(app.strip()) - - except FileNotFoundError as exception: - print(exception) - exit(1) - except Exception: - print(APPS_TXT_FILE+" is not valid") - exit(1) - - return apps - - -def get_container_versions(apps): - versions = {} - for app in apps: - try: - version = __import__(app).__version__ - versions.update({app: version}) - except Exception: - pass - - try: - path = os.path.join('..', 'apps', app) - repo = git.Repo(path) - commit_hash = repo.head.object.hexsha - versions.update({app+'_git_hash': commit_hash}) - except Exception: - pass - - return versions - - -def get_version_file(): - versions = None - try: - with open(APP_VERSIONS_JSON_FILE) as versions_file: - versions = json.load(versions_file) - except Exception: - pass - return versions +from utils import ( + save_version_file, + get_apps, + get_container_versions, + get_version_file, + get_config +) def main(): diff --git a/build/common/commands/check_connection.py b/build/common/commands/check_connection.py index 2b47c753..359cc58f 100644 --- a/build/common/commands/check_connection.py +++ b/build/common/commands/check_connection.py @@ -1,15 +1,15 @@ import socket -import json import time from six.moves.urllib.parse import urlparse - -COMMON_SITE_CONFIG_FILE = 'common_site_config.json' -REDIS_QUEUE_KEY = 'redis_queue' -REDIS_CACHE_KEY = 'redis_cache' -REDIS_SOCKETIO_KEY = 'redis_socketio' -DB_HOST_KEY = 'db_host' -DB_PORT_KEY = 'db_port' -DB_PORT = 3306 +from utils import get_config +from constants import ( + REDIS_QUEUE_KEY, + REDIS_CACHE_KEY, + REDIS_SOCKETIO_KEY, + DB_HOST_KEY, + DB_PORT_KEY, + DB_PORT +) def is_open(ip, port, timeout=30): @@ -38,21 +38,6 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True): return ipup -# Check connection to servers -def get_config(): - config = None - try: - with open(COMMON_SITE_CONFIG_FILE) as config_file: - config = json.load(config_file) - except FileNotFoundError as exception: - print(exception) - exit(1) - except Exception: - print(COMMON_SITE_CONFIG_FILE+" is not valid") - exit(1) - return config - - # Check service def check_service( retry=10, @@ -133,14 +118,6 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True): exit(1) -# Get site_config.json -def get_site_config(site_name): - site_config = None - with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: - site_config = json.load(site_config_file) - return site_config - - def main(): check_service() check_redis_queue() diff --git a/build/common/commands/constants.py b/build/common/commands/constants.py new file mode 100644 index 00000000..b07e3381 --- /dev/null +++ b/build/common/commands/constants.py @@ -0,0 +1,10 @@ +REDIS_QUEUE_KEY = 'redis_queue' +REDIS_CACHE_KEY = 'redis_cache' +REDIS_SOCKETIO_KEY = 'redis_socketio' +DB_HOST_KEY = 'db_host' +DB_PORT_KEY = 'db_port' +DB_PORT = 3306 +APP_VERSIONS_JSON_FILE = 'app_versions.json' +APPS_TXT_FILE = 'apps.txt' +COMMON_SITE_CONFIG_FILE = 'common_site_config.json' +DATE_FORMAT = "%Y%m%d_%H%M%S" \ No newline at end of file diff --git a/build/common/commands/migrate.py b/build/common/commands/migrate.py index 08550a94..6e2846b0 100644 --- a/build/common/commands/migrate.py +++ b/build/common/commands/migrate.py @@ -1,14 +1,8 @@ import os import frappe -import json from frappe.utils import cint, get_sites -from check_connection import get_config, COMMON_SITE_CONFIG_FILE - - -def save_config(config): - with open(COMMON_SITE_CONFIG_FILE, 'w') as f: - return json.dump(config, f, indent=1, sort_keys=True) +from utils import get_config, save_config def set_maintenance_mode(enable=True): diff --git a/build/common/commands/new.py b/build/common/commands/new.py index 3438fa89..57c9f978 100644 --- a/build/common/commands/new.py +++ b/build/common/commands/new.py @@ -4,30 +4,13 @@ import semantic_version from frappe.commands.site import _new_site from frappe.installer import update_site_config -from check_connection import get_config, get_site_config, COMMON_SITE_CONFIG_FILE -from utils import run_command - - -def get_password(env_var, default=None): - return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default - - -def _get_password_from_secret(env_var): - """Fetches the secret value from the docker secret file - usually located inside /run/secrets/ - Arguments: - env_var {str} -- Name of the environment variable - containing the path to the secret file. - Returns: - [str] -- Secret value - """ - passwd = None - secret_file_path = os.environ.get(env_var) - if secret_file_path: - with open(secret_file_path) as secret_file: - passwd = secret_file.read().strip() - - return passwd +from constants import COMMON_SITE_CONFIG_FILE +from utils import ( + run_command, + get_config, + get_site_config, + get_password, +) def main(): diff --git a/build/common/commands/push_backup.py b/build/common/commands/push_backup.py index 2e08441a..ce3fa3d8 100644 --- a/build/common/commands/push_backup.py +++ b/build/common/commands/push_backup.py @@ -5,8 +5,12 @@ import boto3 import datetime from glob import glob from frappe.utils import get_sites - -DATE_FORMAT = "%Y%m%d_%H%M%S" +from constants import DATE_FORMAT +from utils import ( + get_s3_config, + upload_file_to_s3, + check_s3_environment_variables, +) def get_file_ext(): @@ -45,64 +49,11 @@ def get_backup_details(sitename): return backup_details -def get_s3_config(): - check_environment_variables() - bucket = os.environ.get('BUCKET_NAME') - - conn = boto3.client( - 's3', - region_name=os.environ.get('REGION'), - aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), - aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), - endpoint_url=os.environ.get('ENDPOINT_URL') - ) - - return conn, bucket - - -def check_environment_variables(): - if 'BUCKET_NAME' not in os.environ: - print('Variable BUCKET_NAME not set') - exit(1) - - if 'ACCESS_KEY_ID' not in os.environ: - print('Variable ACCESS_KEY_ID not set') - exit(1) - - if 'SECRET_ACCESS_KEY' not in os.environ: - print('Variable SECRET_ACCESS_KEY not set') - exit(1) - - if 'ENDPOINT_URL' not in os.environ: - print('Variable ENDPOINT_URL not set') - exit(1) - - if 'BUCKET_DIR' not in os.environ: - print('Variable BUCKET_DIR not set') - exit(1) - - if 'REGION' not in os.environ: - print('Variable REGION not set') - exit(1) - - -def upload_file_to_s3(filename, folder, conn, bucket): - - destpath = os.path.join(folder, os.path.basename(filename)) - try: - print("Uploading file:", filename) - conn.upload_file(filename, bucket, destpath) - - except Exception as e: - print("Error uploading: %s" % (e)) - exit(1) - - def delete_old_backups(limit, bucket, site_name): all_backups = list() all_backup_dates = list() backup_limit = int(limit) - check_environment_variables() + check_s3_environment_variables() bucket_dir = os.environ.get('BUCKET_DIR') oldest_backup_date = None diff --git a/build/common/commands/restore_backup.py b/build/common/commands/restore_backup.py index 7de34367..474ff3ee 100644 --- a/build/common/commands/restore_backup.py +++ b/build/common/commands/restore_backup.py @@ -1,25 +1,27 @@ import os import datetime -import json import tarfile import hashlib import frappe import boto3 -from new import get_password -from push_backup import DATE_FORMAT, check_environment_variables -from utils import run_command from frappe.utils import get_sites, random_string -from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config -from check_connection import get_site_config, get_config, COMMON_SITE_CONFIG_FILE - - -def list_directories(path): - directories = [] - for name in os.listdir(path): - if os.path.isdir(os.path.join(path, name)): - directories.append(name) - return directories +from frappe.installer import ( + make_conf, + get_conf_params, + make_site_dirs, + update_site_config +) +from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT +from utils import ( + run_command, + list_directories, + set_key_in_site_config, + get_site_config, + get_config, + get_password, + check_s3_environment_variables, +) def get_backup_dir(): @@ -74,23 +76,6 @@ def restore_database(files_base, site_config_path, site): set_key_in_site_config('encryption_key', site, site_config_path) -def set_key_in_site_config(key, site, site_config_path): - site_config = get_site_config_from_path(site_config_path) - value = site_config.get(key) - if value: - print('Set {key} in site config for site: {site}'.format(key=key, site=site)) - update_site_config(key, value, - site_config_path=os.path.join(os.getcwd(), site, "site_config.json")) - - -def get_site_config_from_path(site_config_path): - site_config = dict() - if os.path.exists(site_config_path): - with open(site_config_path, 'r') as sc: - site_config = json.load(sc) - return site_config - - def restore_files(files_base): public_files = files_base + '-files.tar' # extract tar @@ -107,7 +92,7 @@ def restore_private_files(files_base): def pull_backup_from_s3(): - check_environment_variables() + check_s3_environment_variables() # https://stackoverflow.com/a/54672690 s3 = boto3.resource( diff --git a/build/common/commands/utils.py b/build/common/commands/utils.py index c808132e..eaa9a2cf 100644 --- a/build/common/commands/utils.py +++ b/build/common/commands/utils.py @@ -1,5 +1,15 @@ +import json +import os import subprocess +import boto3 +import git +from frappe.installer import update_site_config +from constants import ( + APP_VERSIONS_JSON_FILE, + APPS_TXT_FILE, + COMMON_SITE_CONFIG_FILE +) def run_command(command, stdout=None, stdin=None, stderr=None): stdout = stdout or subprocess.PIPE @@ -13,3 +23,182 @@ def run_command(command, stdout=None, stdin=None, stderr=None): print(f"stdout:\n{out}") print(f"\nstderr:\n{error}") exit(process.returncode) + + +def save_version_file(versions): + with open(APP_VERSIONS_JSON_FILE, 'w') as f: + return json.dump(versions, f, indent=1, sort_keys=True) + + +def get_apps(): + apps = [] + try: + with open(APPS_TXT_FILE) as apps_file: + for app in apps_file.readlines(): + if app.strip(): + apps.append(app.strip()) + + except FileNotFoundError as exception: + print(exception) + exit(1) + except Exception: + print(APPS_TXT_FILE + " is not valid") + exit(1) + + return apps + + +def get_container_versions(apps): + versions = {} + for app in apps: + try: + version = __import__(app).__version__ + versions.update({app: version}) + except Exception: + pass + + try: + path = os.path.join('..', 'apps', app) + repo = git.Repo(path) + commit_hash = repo.head.object.hexsha + versions.update({app+'_git_hash': commit_hash}) + except Exception: + pass + + return versions + + +def get_version_file(): + versions = None + try: + with open(APP_VERSIONS_JSON_FILE) as versions_file: + versions = json.load(versions_file) + except Exception: + pass + return versions + + +def get_config(): + config = None + try: + with open(COMMON_SITE_CONFIG_FILE) as config_file: + config = json.load(config_file) + except FileNotFoundError as exception: + print(exception) + exit(1) + except Exception: + print(COMMON_SITE_CONFIG_FILE + " is not valid") + exit(1) + return config + + +def get_site_config(site_name): + site_config = None + with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file: + site_config = json.load(site_config_file) + return site_config + + +def save_config(config): + with open(COMMON_SITE_CONFIG_FILE, 'w') as f: + return json.dump(config, f, indent=1, sort_keys=True) + + +def get_password(env_var, default=None): + return os.environ.get(env_var) or get_password_from_secret(f"{env_var}_FILE") or default + + +def get_password_from_secret(env_var): + """Fetches the secret value from the docker secret file + usually located inside /run/secrets/ + Arguments: + env_var {str} -- Name of the environment variable + containing the path to the secret file. + Returns: + [str] -- Secret value + """ + passwd = None + secret_file_path = os.environ.get(env_var) + if secret_file_path: + with open(secret_file_path) as secret_file: + passwd = secret_file.read().strip() + + return passwd + + +def get_s3_config(): + check_s3_environment_variables() + bucket = os.environ.get('BUCKET_NAME') + + conn = boto3.client( + 's3', + region_name=os.environ.get('REGION'), + aws_access_key_id=os.environ.get('ACCESS_KEY_ID'), + aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'), + endpoint_url=os.environ.get('ENDPOINT_URL') + ) + + return conn, bucket + + +def upload_file_to_s3(filename, folder, conn, bucket): + + destpath = os.path.join(folder, os.path.basename(filename)) + try: + print("Uploading file:", filename) + conn.upload_file(filename, bucket, destpath) + + except Exception as e: + print("Error uploading: %s" % (e)) + exit(1) + + +def list_directories(path): + directories = [] + for name in os.listdir(path): + if os.path.isdir(os.path.join(path, name)): + directories.append(name) + return directories + + +def get_site_config_from_path(site_config_path): + site_config = dict() + if os.path.exists(site_config_path): + with open(site_config_path, 'r') as sc: + site_config = json.load(sc) + return site_config + + +def set_key_in_site_config(key, site, site_config_path): + site_config = get_site_config_from_path(site_config_path) + value = site_config.get(key) + if value: + print('Set {key} in site config for site: {site}'.format(key=key, site=site)) + update_site_config(key, value, + site_config_path=os.path.join(os.getcwd(), site, "site_config.json")) + + +def check_s3_environment_variables(): + if 'BUCKET_NAME' not in os.environ: + print('Variable BUCKET_NAME not set') + exit(1) + + if 'ACCESS_KEY_ID' not in os.environ: + print('Variable ACCESS_KEY_ID not set') + exit(1) + + if 'SECRET_ACCESS_KEY' not in os.environ: + print('Variable SECRET_ACCESS_KEY not set') + exit(1) + + if 'ENDPOINT_URL' not in os.environ: + print('Variable ENDPOINT_URL not set') + exit(1) + + if 'BUCKET_DIR' not in os.environ: + print('Variable BUCKET_DIR not set') + exit(1) + + if 'REGION' not in os.environ: + print('Variable REGION not set') + exit(1) diff --git a/build/erpnext-nginx/install_app.sh b/build/erpnext-nginx/install_app.sh index 9042c5f4..8a8d7f44 100755 --- a/build/erpnext-nginx/install_app.sh +++ b/build/erpnext-nginx/install_app.sh @@ -26,7 +26,9 @@ yarn install --production=true mkdir -p /home/frappe/frappe-bench/sites/assets/${APP_NAME} cp -R /home/frappe/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public/* /home/frappe/frappe-bench/sites/assets/${APP_NAME} -echo "rsync -a --delete /var/www/html/assets/${APP_NAME} /assets" > /rsync +# Add frappe and all the apps available under in frappe-bench here +echo "rsync -a --delete /var/www/html/assets/frappe /assets" > /rsync +echo "rsync -a --delete /var/www/html/assets/${APP_NAME} /assets" >> /rsync chmod +x /rsync rm /home/frappe/frappe-bench/sites/apps.txt From e513b9c8ceed6ec187a25388f42d70b21aba8f0b Mon Sep 17 00:00:00 2001 From: Sun Howwrongbum Date: Mon, 13 Jul 2020 23:12:29 +0530 Subject: [PATCH 13/13] feat: touch .build --- build/frappe-nginx/docker-entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/frappe-nginx/docker-entrypoint.sh b/build/frappe-nginx/docker-entrypoint.sh index c1caad54..69431038 100755 --- a/build/frappe-nginx/docker-entrypoint.sh +++ b/build/frappe-nginx/docker-entrypoint.sh @@ -13,6 +13,8 @@ rsync -a --delete /var/www/html/assets/frappe /assets chmod -R 755 /assets +touch /var/www/html/sites/.build -r $(ls -td /assets/* | head -n 1) + if [[ -z "$FRAPPE_PY" ]]; then export FRAPPE_PY=0.0.0.0 fi