Merge pull request #309 from frappe/develop

Build Frappe v12.8.1
This commit is contained in:
Revant Nandgaonkar 2020-07-14 21:53:54 +05:30 committed by GitHub
commit b4685bd2ef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 315 additions and 295 deletions

3
.gitignore vendored
View file

@ -10,3 +10,6 @@ development
!development/README.md
deploy_key
deploy_key.pub
# Pycharm
.idea

View file

@ -1,66 +1,15 @@
import os
import json
import semantic_version
import git
from migrate import migrate_sites
from check_connection import get_config
APP_VERSIONS_JSON_FILE = 'app_versions.json'
APPS_TXT_FILE = 'apps.txt'
def save_version_file(versions):
with open(APP_VERSIONS_JSON_FILE, 'w') as f:
return json.dump(versions, f, indent=1, sort_keys=True)
def get_apps():
apps = []
try:
with open(APPS_TXT_FILE) as apps_file:
for app in apps_file.readlines():
if app.strip():
apps.append(app.strip())
except FileNotFoundError as exception:
print(exception)
exit(1)
except Exception:
print(APPS_TXT_FILE+" is not valid")
exit(1)
return apps
def get_container_versions(apps):
versions = {}
for app in apps:
try:
version = __import__(app).__version__
versions.update({app: version})
except Exception:
pass
try:
path = os.path.join('..', 'apps', app)
repo = git.Repo(path)
commit_hash = repo.head.object.hexsha
versions.update({app+'_git_hash': commit_hash})
except Exception:
pass
return versions
def get_version_file():
versions = None
try:
with open(APP_VERSIONS_JSON_FILE) as versions_file:
versions = json.load(versions_file)
except Exception:
pass
return versions
from utils import (
save_version_file,
get_apps,
get_container_versions,
get_version_file,
get_config
)
def main():

View file

@ -1,15 +1,15 @@
import socket
import json
import time
from six.moves.urllib.parse import urlparse
COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
REDIS_QUEUE_KEY = 'redis_queue'
REDIS_CACHE_KEY = 'redis_cache'
REDIS_SOCKETIO_KEY = 'redis_socketio'
DB_HOST_KEY = 'db_host'
DB_PORT_KEY = 'db_port'
DB_PORT = 3306
from utils import get_config
from constants import (
REDIS_QUEUE_KEY,
REDIS_CACHE_KEY,
REDIS_SOCKETIO_KEY,
DB_HOST_KEY,
DB_PORT_KEY,
DB_PORT
)
def is_open(ip, port, timeout=30):
@ -38,21 +38,6 @@ def check_host(ip, port, retry=10, delay=3, print_attempt=True):
return ipup
# Check connection to servers
def get_config():
config = None
try:
with open(COMMON_SITE_CONFIG_FILE) as config_file:
config = json.load(config_file)
except FileNotFoundError as exception:
print(exception)
exit(1)
except Exception:
print(COMMON_SITE_CONFIG_FILE+" is not valid")
exit(1)
return config
# Check service
def check_service(
retry=10,
@ -133,14 +118,6 @@ def check_redis_socketio(retry=10, delay=3, print_attempt=True):
exit(1)
# Get site_config.json
def get_site_config(site_name):
site_config = None
with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file:
site_config = json.load(site_config_file)
return site_config
def main():
check_service()
check_redis_queue()

View file

@ -0,0 +1,10 @@
REDIS_QUEUE_KEY = 'redis_queue'
REDIS_CACHE_KEY = 'redis_cache'
REDIS_SOCKETIO_KEY = 'redis_socketio'
DB_HOST_KEY = 'db_host'
DB_PORT_KEY = 'db_port'
DB_PORT = 3306
APP_VERSIONS_JSON_FILE = 'app_versions.json'
APPS_TXT_FILE = 'apps.txt'
COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
DATE_FORMAT = "%Y%m%d_%H%M%S"

View file

@ -1,14 +1,8 @@
import os
import frappe
import json
from frappe.utils import cint, get_sites
from check_connection import get_config, COMMON_SITE_CONFIG_FILE
def save_config(config):
with open(COMMON_SITE_CONFIG_FILE, 'w') as f:
return json.dump(config, f, indent=1, sort_keys=True)
from utils import get_config, save_config
def set_maintenance_mode(enable=True):

View file

@ -4,29 +4,13 @@ import semantic_version
from frappe.commands.site import _new_site
from frappe.installer import update_site_config
from check_connection import get_config, get_site_config, COMMON_SITE_CONFIG_FILE
def get_password(env_var, default=None):
return os.environ.get(env_var) or _get_password_from_secret(f"{env_var}_FILE") or default
def _get_password_from_secret(env_var):
"""Fetches the secret value from the docker secret file
usually located inside /run/secrets/
Arguments:
env_var {str} -- Name of the environment variable
containing the path to the secret file.
Returns:
[str] -- Secret value
"""
passwd = None
secret_file_path = os.environ.get(env_var)
if secret_file_path:
with open(secret_file_path) as secret_file:
passwd = secret_file.read().strip()
return passwd
from constants import COMMON_SITE_CONFIG_FILE
from utils import (
run_command,
get_config,
get_site_config,
get_password,
)
def main():
@ -91,31 +75,26 @@ def main():
if db_type == "mariadb":
site_config = get_site_config(site_name)
db_name = site_config.get('db_name')
db_password = site_config.get('db_password')
mysql_command = 'mysql -h{db_host} -u{mariadb_root_username} -p{mariadb_root_password} -e '.format(
db_host=config.get('db_host'),
mariadb_root_username=mariadb_root_username,
mariadb_root_password=mariadb_root_password
)
mysql_command = ["mysql", f"-h{db_host}", f"-u{mariadb_root_username}", f"-p{mariadb_root_password}", "-e"]
# Drop User if exists
command = mysql_command + [f"DROP USER IF EXISTS '{db_name}'@'%'; FLUSH PRIVILEGES;"]
run_command(command)
# update User's host to '%' required to connect from any container
command = mysql_command + "\"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name')
)
os.system(command)
command = mysql_command + [f"UPDATE mysql.user SET Host = '%' where User = '{db_name}'; FLUSH PRIVILEGES;"]
run_command(command)
# Set db password
command = mysql_command + "\"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password')
)
os.system(command)
command = mysql_command + [f"ALTER USER '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"]
run_command(command)
# Grant permission to database
command = mysql_command + "\"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;\"".format(
db_name=site_config.get('db_name')
)
os.system(command)
command = mysql_command + [f"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%'; FLUSH PRIVILEGES;"]
run_command(command)
if frappe.redis_server:
frappe.redis_server.connection_pool.disconnect()

View file

@ -5,8 +5,12 @@ import boto3
import datetime
from glob import glob
from frappe.utils import get_sites
DATE_FORMAT = "%Y%m%d_%H%M%S"
from constants import DATE_FORMAT
from utils import (
get_s3_config,
upload_file_to_s3,
check_s3_environment_variables,
)
def get_file_ext():
@ -45,64 +49,11 @@ def get_backup_details(sitename):
return backup_details
def get_s3_config():
check_environment_variables()
bucket = os.environ.get('BUCKET_NAME')
conn = boto3.client(
's3',
region_name=os.environ.get('REGION'),
aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
endpoint_url=os.environ.get('ENDPOINT_URL')
)
return conn, bucket
def check_environment_variables():
if 'BUCKET_NAME' not in os.environ:
print('Variable BUCKET_NAME not set')
exit(1)
if 'ACCESS_KEY_ID' not in os.environ:
print('Variable ACCESS_KEY_ID not set')
exit(1)
if 'SECRET_ACCESS_KEY' not in os.environ:
print('Variable SECRET_ACCESS_KEY not set')
exit(1)
if 'ENDPOINT_URL' not in os.environ:
print('Variable ENDPOINT_URL not set')
exit(1)
if 'BUCKET_DIR' not in os.environ:
print('Variable BUCKET_DIR not set')
exit(1)
if 'REGION' not in os.environ:
print('Variable REGION not set')
exit(1)
def upload_file_to_s3(filename, folder, conn, bucket):
destpath = os.path.join(folder, os.path.basename(filename))
try:
print("Uploading file:", filename)
conn.upload_file(filename, bucket, destpath)
except Exception as e:
print("Error uploading: %s" % (e))
exit(1)
def delete_old_backups(limit, bucket, site_name):
all_backups = list()
all_backup_dates = list()
backup_limit = int(limit)
check_environment_variables()
check_s3_environment_variables()
bucket_dir = os.environ.get('BUCKET_DIR')
oldest_backup_date = None

View file

@ -1,24 +1,27 @@
import os
import datetime
import json
import tarfile
import hashlib
import frappe
import boto3
from new import get_password
from push_backup import DATE_FORMAT, check_environment_variables
from frappe.utils import get_sites, random_string
from frappe.installer import make_conf, get_conf_params, make_site_dirs, update_site_config
from check_connection import get_site_config, get_config, COMMON_SITE_CONFIG_FILE
def list_directories(path):
directories = []
for name in os.listdir(path):
if os.path.isdir(os.path.join(path, name)):
directories.append(name)
return directories
from frappe.installer import (
make_conf,
get_conf_params,
make_site_dirs,
update_site_config
)
from constants import COMMON_SITE_CONFIG_FILE, DATE_FORMAT
from utils import (
run_command,
list_directories,
set_key_in_site_config,
get_site_config,
get_config,
get_password,
check_s3_environment_variables,
)
def get_backup_dir():
@ -28,21 +31,17 @@ def get_backup_dir():
)
def decompress_db(files_base, site):
database_file = files_base + '-database.sql.gz'
command = 'gunzip -c {database_file} > {database_extract}'.format(
database_file=database_file,
database_extract=database_file.replace('.gz', '')
)
def decompress_db(database_file, site):
command = ["gunzip", "-c", database_file]
with open(database_file.replace(".gz", ""), "w") as db_file:
print('Extract Database GZip for site {}'.format(site))
os.system(command)
run_command(command, stdout=db_file)
def restore_database(files_base, site_config_path, site):
# restore database
database_file = files_base + '-database.sql.gz'
decompress_db(files_base, site)
decompress_db(database_file, site)
config = get_config()
# Set db_type if it exists in backup site_config.json
@ -77,23 +76,6 @@ def restore_database(files_base, site_config_path, site):
set_key_in_site_config('encryption_key', site, site_config_path)
def set_key_in_site_config(key, site, site_config_path):
site_config = get_site_config_from_path(site_config_path)
value = site_config.get(key)
if value:
print('Set {key} in site config for site: {site}'.format(key=key, site=site))
update_site_config(key, value,
site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
def get_site_config_from_path(site_config_path):
site_config = dict()
if os.path.exists(site_config_path):
with open(site_config_path, 'r') as sc:
site_config = json.load(sc)
return site_config
def restore_files(files_base):
public_files = files_base + '-files.tar'
# extract tar
@ -110,7 +92,7 @@ def restore_private_files(files_base):
def pull_backup_from_s3():
check_environment_variables()
check_s3_environment_variables()
# https://stackoverflow.com/a/54672690
s3 = boto3.resource(
@ -203,28 +185,17 @@ def restore_postgres(config, site_config, database_file):
db_name = site_config.get('db_name')
db_password = site_config.get('db_password')
psql_command = "psql postgres://{root_login}:{root_password}@{db_host}:{db_port}".format(
root_login=db_root_user,
root_password=db_root_password,
db_host=db_host,
db_port=db_port
)
psql_command = ["psql"]
psql_uri = f"postgres://{db_root_user}:{db_root_password}@{db_host}:{db_port}"
print('Restoring PostgreSQL')
os.system(psql_command + ' -c "DROP DATABASE IF EXISTS \"{db_name}\""'.format(db_name=db_name))
os.system(psql_command + ' -c "DROP USER IF EXISTS {db_name}"'.format(db_name=db_name))
os.system(psql_command + ' -c "CREATE DATABASE \"{db_name}\""'.format(db_name=db_name))
os.system(psql_command + ' -c "CREATE user {db_name} password \'{db_password}\'"'.format(
db_name=db_name,
db_password=db_password))
os.system(psql_command + ' -c "GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"'.format(
db_name=db_name))
os.system("{psql_command}/{db_name} < {database_file}".format(
psql_command=psql_command,
database_file=database_file.replace('.gz', ''),
db_name=db_name,
))
run_command(psql_command + [psql_uri, "-c", f"DROP DATABASE IF EXISTS \"{db_name}\""])
run_command(psql_command + [psql_uri, "-c", f"DROP USER IF EXISTS {db_name}"])
run_command(psql_command + [psql_uri, "-c", f"CREATE DATABASE \"{db_name}\""])
run_command(psql_command + [psql_uri, "-c", f"CREATE user {db_name} password '{db_password}'"])
run_command(psql_command + [psql_uri, "-c", f"GRANT ALL PRIVILEGES ON DATABASE \"{db_name}\" TO {db_name}"])
with open(database_file.replace('.gz', ''), 'r') as db_file:
run_command(psql_command + [f"{psql_uri}/{db_name}", "<"], stdin=db_file)
def restore_mariadb(config, site_config, database_file):
@ -236,54 +207,32 @@ def restore_mariadb(config, site_config, database_file):
db_root_user = os.environ.get("DB_ROOT_USER", 'root')
db_host = site_config.get('db_host', config.get('db_host'))
db_port = site_config.get('db_port', config.get('db_port'))
db_port = site_config.get('db_port', config.get('db_port', 3306))
db_name = site_config.get('db_name')
db_password = site_config.get('db_password')
# mysql command prefix
mysql_command = 'mysql -u{db_root_user} -h{db_host} -p{db_password}'.format(
db_root_user=db_root_user,
db_host=db_host,
db_port=db_port,
db_password=db_root_password
)
mysql_command = ["mysql", f"-u{db_root_user}", f"-h{db_host}", f"-p{db_root_password}", f"-P{db_port}"]
# drop db if exists for clean restore
drop_database = "{mysql_command} -e \"DROP DATABASE IF EXISTS \`{db_name}\`;\"".format(
mysql_command=mysql_command,
db_name=site_config.get('db_name'),
)
os.system(drop_database)
drop_database = mysql_command + ["-e", f"DROP DATABASE IF EXISTS `{db_name}`;"]
run_command(drop_database)
# create db
create_database = "{mysql_command} -e \"CREATE DATABASE IF NOT EXISTS \`{db_name}\`;\"".format(
mysql_command=mysql_command,
db_name=site_config.get('db_name'),
)
os.system(create_database)
create_database = mysql_command + ["-e", f"CREATE DATABASE IF NOT EXISTS `{db_name}`;"]
run_command(create_database)
# create user
create_user = "{mysql_command} -e \"CREATE USER IF NOT EXISTS \'{db_name}\'@\'%\' IDENTIFIED BY \'{db_password}\'; FLUSH PRIVILEGES;\"".format(
mysql_command=mysql_command,
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password'),
)
os.system(create_user)
create_user = mysql_command + ["-e", f"CREATE USER IF NOT EXISTS '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"]
run_command(create_user)
# grant db privileges to user
grant_privileges = "{mysql_command} -e \"GRANT ALL PRIVILEGES ON \`{db_name}\`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;\"".format(
mysql_command=mysql_command,
db_name=site_config.get('db_name'),
db_password=site_config.get('db_password'),
)
os.system(grant_privileges)
command = "{mysql_command} '{db_name}' < {database_file}".format(
mysql_command=mysql_command,
db_name=site_config.get('db_name'),
database_file=database_file.replace('.gz', ''),
)
grant_privileges = mysql_command + ["-e", f"GRANT ALL PRIVILEGES ON `{db_name}`.* TO '{db_name}'@'%' IDENTIFIED BY '{db_password}'; FLUSH PRIVILEGES;"]
run_command(grant_privileges)
print('Restoring MariaDB')
os.system(command)
with open(database_file.replace('.gz', ''), 'r') as db_file:
run_command(mysql_command + [f"{db_name}"], stdin=db_file)
def main():

View file

@ -0,0 +1,204 @@
import json
import os
import subprocess
import boto3
import git
from frappe.installer import update_site_config
from constants import (
APP_VERSIONS_JSON_FILE,
APPS_TXT_FILE,
COMMON_SITE_CONFIG_FILE
)
def run_command(command, stdout=None, stdin=None, stderr=None):
stdout = stdout or subprocess.PIPE
stderr = stderr or subprocess.PIPE
stdin = stdin or subprocess.PIPE
process = subprocess.Popen(command, stdout=stdout, stdin=stdin, stderr=stderr)
out, error = process.communicate()
if process.returncode:
print("Something went wrong:")
print(f"return code: {process.returncode}")
print(f"stdout:\n{out}")
print(f"\nstderr:\n{error}")
exit(process.returncode)
def save_version_file(versions):
with open(APP_VERSIONS_JSON_FILE, 'w') as f:
return json.dump(versions, f, indent=1, sort_keys=True)
def get_apps():
apps = []
try:
with open(APPS_TXT_FILE) as apps_file:
for app in apps_file.readlines():
if app.strip():
apps.append(app.strip())
except FileNotFoundError as exception:
print(exception)
exit(1)
except Exception:
print(APPS_TXT_FILE + " is not valid")
exit(1)
return apps
def get_container_versions(apps):
versions = {}
for app in apps:
try:
version = __import__(app).__version__
versions.update({app: version})
except Exception:
pass
try:
path = os.path.join('..', 'apps', app)
repo = git.Repo(path)
commit_hash = repo.head.object.hexsha
versions.update({app+'_git_hash': commit_hash})
except Exception:
pass
return versions
def get_version_file():
versions = None
try:
with open(APP_VERSIONS_JSON_FILE) as versions_file:
versions = json.load(versions_file)
except Exception:
pass
return versions
def get_config():
config = None
try:
with open(COMMON_SITE_CONFIG_FILE) as config_file:
config = json.load(config_file)
except FileNotFoundError as exception:
print(exception)
exit(1)
except Exception:
print(COMMON_SITE_CONFIG_FILE + " is not valid")
exit(1)
return config
def get_site_config(site_name):
site_config = None
with open('{site_name}/site_config.json'.format(site_name=site_name)) as site_config_file:
site_config = json.load(site_config_file)
return site_config
def save_config(config):
with open(COMMON_SITE_CONFIG_FILE, 'w') as f:
return json.dump(config, f, indent=1, sort_keys=True)
def get_password(env_var, default=None):
return os.environ.get(env_var) or get_password_from_secret(f"{env_var}_FILE") or default
def get_password_from_secret(env_var):
"""Fetches the secret value from the docker secret file
usually located inside /run/secrets/
Arguments:
env_var {str} -- Name of the environment variable
containing the path to the secret file.
Returns:
[str] -- Secret value
"""
passwd = None
secret_file_path = os.environ.get(env_var)
if secret_file_path:
with open(secret_file_path) as secret_file:
passwd = secret_file.read().strip()
return passwd
def get_s3_config():
check_s3_environment_variables()
bucket = os.environ.get('BUCKET_NAME')
conn = boto3.client(
's3',
region_name=os.environ.get('REGION'),
aws_access_key_id=os.environ.get('ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('SECRET_ACCESS_KEY'),
endpoint_url=os.environ.get('ENDPOINT_URL')
)
return conn, bucket
def upload_file_to_s3(filename, folder, conn, bucket):
destpath = os.path.join(folder, os.path.basename(filename))
try:
print("Uploading file:", filename)
conn.upload_file(filename, bucket, destpath)
except Exception as e:
print("Error uploading: %s" % (e))
exit(1)
def list_directories(path):
directories = []
for name in os.listdir(path):
if os.path.isdir(os.path.join(path, name)):
directories.append(name)
return directories
def get_site_config_from_path(site_config_path):
site_config = dict()
if os.path.exists(site_config_path):
with open(site_config_path, 'r') as sc:
site_config = json.load(sc)
return site_config
def set_key_in_site_config(key, site, site_config_path):
site_config = get_site_config_from_path(site_config_path)
value = site_config.get(key)
if value:
print('Set {key} in site config for site: {site}'.format(key=key, site=site))
update_site_config(key, value,
site_config_path=os.path.join(os.getcwd(), site, "site_config.json"))
def check_s3_environment_variables():
if 'BUCKET_NAME' not in os.environ:
print('Variable BUCKET_NAME not set')
exit(1)
if 'ACCESS_KEY_ID' not in os.environ:
print('Variable ACCESS_KEY_ID not set')
exit(1)
if 'SECRET_ACCESS_KEY' not in os.environ:
print('Variable SECRET_ACCESS_KEY not set')
exit(1)
if 'ENDPOINT_URL' not in os.environ:
print('Variable ENDPOINT_URL not set')
exit(1)
if 'BUCKET_DIR' not in os.environ:
print('Variable BUCKET_DIR not set')
exit(1)
if 'REGION' not in os.environ:
print('Variable REGION not set')
exit(1)

View file

@ -26,7 +26,9 @@ yarn install --production=true
mkdir -p /home/frappe/frappe-bench/sites/assets/${APP_NAME}
cp -R /home/frappe/frappe-bench/apps/${APP_NAME}/${APP_NAME}/public/* /home/frappe/frappe-bench/sites/assets/${APP_NAME}
echo "rsync -a --delete /var/www/html/assets/${APP_NAME} /assets" > /rsync
# Add frappe and all the apps available under in frappe-bench here
echo "rsync -a --delete /var/www/html/assets/frappe /assets" > /rsync
echo "rsync -a --delete /var/www/html/assets/${APP_NAME} /assets" >> /rsync
chmod +x /rsync
rm /home/frappe/frappe-bench/sites/apps.txt

View file

@ -13,6 +13,8 @@ rsync -a --delete /var/www/html/assets/frappe /assets
chmod -R 755 /assets
touch /var/www/html/sites/.build -r $(ls -td /assets/* | head -n 1)
if [[ -z "$FRAPPE_PY" ]]; then
export FRAPPE_PY=0.0.0.0
fi

View file

@ -35,7 +35,7 @@ services:
labels:
- "traefik.enable=true"
- "traefik.http.routers.erpnext-nginx.rule=HostRegexp(`{catchall:.*}`)"
- "traefik.http.middlewares.erpnext-nginx.headers.customrequestheaders.Host=mysite.localhost"
- "traefik.http.middlewares.erpnext-nginx.headers.customrequestheaders.Host=erpnext-nginx"
- "traefik.http.routers.erpnext-nginx.middlewares=erpnext-nginx"
- "traefik.http.routers.erpnext-nginx.entrypoints=web"
- "traefik.http.services.erpnext-nginx.loadbalancer.server.port=80"
@ -148,7 +148,7 @@ services:
deploy:
restart_policy:
condition: none
command: ["bash", "-c", "echo mysite.localhost > /sites/currentsite.txt"]
command: ["bash", "-c", "echo erpnext-nginx > /sites/currentsite.txt"]
volumes:
- sites-vol:/sites:rw
@ -193,7 +193,7 @@ services:
condition: none
command: new
environment:
- SITE_NAME=mysite.localhost
- SITE_NAME=erpnext-nginx
- DB_ROOT_USER=root
- MYSQL_ROOT_PASSWORD=admin
- ADMIN_PASSWORD=admin