Merge pull request #364 from frappe/develop

build/common/commands/constants.py

@@ -10,3 +10,4 @@ COMMON_SITE_CONFIG_FILE = 'common_site_config.json'
 DATE_FORMAT = "%Y%m%d_%H%M%S"
 RDS_DB = 'rds_db'
 RDS_PRIVILEGES = "SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER, CREATE TEMPORARY TABLES, CREATE VIEW, EVENT, TRIGGER, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EXECUTE, LOCK TABLES"
+ARCHIVE_SITES_PATH = '/home/frappe/frappe-bench/sites/archive_sites'

build/common/commands/drop.py

@@ -0,0 +1,39 @@
+import os
+import frappe
+
+from frappe.commands.site import _drop_site
+from constants import ARCHIVE_SITES_PATH
+from utils import get_password
+
+
+def main():
+    site_name = os.environ.get("SITE_NAME", 'site1.localhost')
+    db_root_username = os.environ.get("DB_ROOT_USER", 'root')
+    mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin')
+    postgres_root_password = get_password("POSTGRES_PASSWORD")
+    db_root_password = mariadb_root_password
+
+    if postgres_root_password:
+        db_root_password = postgres_root_password
+
+    force = True if os.environ.get("FORCE", None) else False
+    no_backup = True if os.environ.get("NO_BACKUP", None) else False
+    frappe.init(site_name, new_site=True)
+
+    _drop_site(
+        site=site_name,
+        root_login=db_root_username,
+        root_password=db_root_password,
+        archived_sites_path=ARCHIVE_SITES_PATH,
+        force=force,
+        no_backup=no_backup
+    )
+
+    if frappe.redis_server:
+        frappe.redis_server.connection_pool.disconnect()
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    main()

build/common/commands/new.py

@@ -19,14 +19,16 @@ def main():
     db_port = config.get('db_port', 3306)
     db_host = config.get('db_host')
     site_name = os.environ.get("SITE_NAME", 'site1.localhost')
-    mariadb_root_username = os.environ.get("DB_ROOT_USER", 'root')
+    db_root_username = os.environ.get("DB_ROOT_USER", 'root')
     mariadb_root_password = get_password("MYSQL_ROOT_PASSWORD", 'admin')
     postgres_root_password = get_password("POSTGRES_PASSWORD")
+    db_root_password = mariadb_root_password
 
     if postgres_root_password:
         db_type = 'postgres'
         db_host = os.environ.get("POSTGRES_HOST")
         db_port = 5432
+        db_root_password = postgres_root_password
         if not db_host:
             db_host = config.get('db_host')
             print('Environment variable POSTGRES_HOST not found.')
@@ -34,8 +36,8 @@ def main():
 
         sites_path = os.getcwd()
         common_site_config_path = os.path.join(sites_path, COMMON_SITE_CONFIG_FILE)
-        update_site_config("root_login", mariadb_root_username, validate = False, site_config_path = common_site_config_path)
+        update_site_config("root_login", db_root_username, validate = False, site_config_path = common_site_config_path)
-        update_site_config("root_password", postgres_root_password, validate = False, site_config_path = common_site_config_path)
+        update_site_config("root_password", db_root_password, validate = False, site_config_path = common_site_config_path)
 
     force = True if os.environ.get("FORCE", None) else False
     install_apps = os.environ.get("INSTALL_APPS", None)
@@ -46,8 +48,8 @@ def main():
         _new_site(
             None,
             site_name,
-            mariadb_root_username=mariadb_root_username,
+            mariadb_root_username=db_root_username,
-            mariadb_root_password=mariadb_root_password,
+            mariadb_root_password=db_root_password,
             admin_password=get_password("ADMIN_PASSWORD", 'admin'),
             verbose=True,
             install_apps=install_apps,
@@ -62,8 +64,8 @@ def main():
         _new_site(
             None,
             site_name,
-            mariadb_root_username=mariadb_root_username,
+            mariadb_root_username=db_root_username,
-            mariadb_root_password=mariadb_root_password,
+            mariadb_root_password=db_root_password,
             admin_password=get_password("ADMIN_PASSWORD", 'admin'),
             verbose=True,
             install_apps=install_apps,
@@ -78,7 +80,7 @@ def main():
         db_name = site_config.get('db_name')
         db_password = site_config.get('db_password')
 
-        mysql_command = ["mysql", f"-h{db_host}", f"-u{mariadb_root_username}", f"-p{mariadb_root_password}", "-e"]
+        mysql_command = ["mysql", f"-h{db_host}", f"-u{db_root_username}", f"-p{mariadb_root_password}", "-e"]
 
         # Drop User if exists
         command = mysql_command + [f"DROP USER IF EXISTS '{db_name}'@'%'; FLUSH PRIVILEGES;"]

build/common/commands/push_backup.py

@@ -74,6 +74,8 @@ def delete_old_backups(limit, bucket, site_name):
         for obj in objects.get('CommonPrefixes'):
             if obj.get('Prefix') == bucket_dir + '/':
                 for backup_obj in bucket.objects.filter(Prefix=obj.get('Prefix')):
+                    if backup_obj.get()["ContentType"] == "application/x-directory":
+                        continue
                     try:
                         # backup_obj.key is bucket_dir/site/date_time/backupfile.extension
                         bucket_dir, site_slug, date_time, backupfile = backup_obj.key.split('/')

build/common/commands/restore_backup.py

@@ -116,6 +116,8 @@ def pull_backup_from_s3():
     download_backups = []
 
     for obj in bucket.objects.filter(Prefix=bucket_dir):
+        if obj.get()["ContentType"] == "application/x-directory":
+            continue
         backup_file = obj.key.replace(os.path.join(bucket_dir, ''), '')
         backup_files.append(backup_file)
         site_name, timestamp, backup_type = backup_file.split('/')

build/common/worker/docker-entrypoint.sh

@@ -149,6 +149,18 @@ elif [ "$1" = 'new' ]; then
     python /home/frappe/frappe-bench/commands/new.py
   fi
 
+elif [ "$1" = 'drop' ]; then
+  checkConfigExists
+  checkConnection
+  if [[ -z "$RUN_AS_ROOT" ]]; then
+    su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \
+      && python /home/frappe/frappe-bench/commands/drop.py"
+    exit
+  else
+    . /home/frappe/frappe-bench/env/bin/activate
+    python /home/frappe/frappe-bench/commands/drop.py
+  fi
+
 elif [ "$1" = 'migrate' ]; then
 
   su frappe -c ". /home/frappe/frappe-bench/env/bin/activate \

build/common/worker/healthcheck.sh

@@ -11,8 +11,19 @@ fi
 
 # Set REDIS host:port
 export REDIS_CACHE=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_cache/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'`
+if [[ "$REDIS_CACHE" == *"/"* ]]; then
+  export REDIS_CACHE=`echo $REDIS_CACHE | cut -f1 -d"/"`
+fi
+
 export REDIS_QUEUE=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_queue/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'`
+if [[ "$REDIS_QUEUE" == *"/"* ]]; then
+  export REDIS_QUEUE=`echo $REDIS_QUEUE | cut -f1 -d"/"`
+fi
+
 export REDIS_SOCKETIO=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_socketio/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'`
+if [[ "$REDIS_SOCKETIO" == *"/"* ]]; then
+  export REDIS_SOCKETIO=`echo $REDIS_SOCKETIO | cut -f1 -d"/"`
+fi
 
 echo "Check $DB_HOST:$DB_PORT"
 wait-for-it $DB_HOST:$DB_PORT -t 1

build/frappe-nginx/Dockerfile

@@ -25,8 +25,12 @@ RUN mkdir -p apps sites/assets/css  \
     && cd apps \
     && git clone --depth 1 https://github.com/frappe/frappe --branch $GIT_BRANCH
 
-COPY build/frappe-nginx/build.sh /build.sh
+RUN cd /home/frappe/frappe-bench/apps/frappe \
-RUN /build.sh
+    && yarn \
+    && yarn run production \
+    && rm -fr node_modules \
+    && yarn install --production=true
+
 RUN node --version \
     && npm --version \
     && yarn --version

build/frappe-nginx/build.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-function nodeCleanUp() {
-    rm -fr node_modules
-    yarn install --production=true
-}
-
-cd /home/frappe/frappe-bench/apps/frappe
-yarn
-yarn run production
-
-if [[ "$GIT_BRANCH" =~ ^(version-12|version-11)$ ]]; then
-    nodeCleanUp
-else
-    nodeCleanUp
-    # remove this when frappe framework moves this to dependencies from devDependencies
-    yarn add node-sass
-fi

docs/site-operations.md

@@ -77,7 +77,7 @@ Change `SITES` variable to the list of sites created encapsulated in backtick an
 Reload variables with following command.
 
 ```sh
-docker-compose up --project-name <project-name> -d
+docker-compose --project-name <project-name> up -d
 ```
 
 ## Backup Sites
@@ -230,3 +230,44 @@ Notes:
 
 - Use it to install/uninstall custom apps, add system manager user, etc.
 - To run the command as non root user add the command option `--user frappe`.
+
+
+## Delete/Drop Site
+
+#### MariaDB Site
+
+```sh
+# Delete/Drop ERPNext site
+docker run \
+    -e "SITE_NAME=$SITE_NAME" \
+    -e "DB_ROOT_USER=$DB_ROOT_USER" \
+    -e "MYSQL_ROOT_PASSWORD=$MYSQL_ROOT_PASSWORD" \
+    -v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
+    --network <project-name>_default \
+    frappe/erpnext-worker:$VERSION drop
+```
+
+#### PostgreSQL Site
+
+```sh
+# Delete/Drop ERPNext site
+docker run \
+    -e "SITE_NAME=$SITE_NAME" \
+    -e "DB_ROOT_USER=$DB_ROOT_USER" \
+    -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \
+    -v <project-name>_sites-vol:/home/frappe/frappe-bench/sites \
+    --network <project-name>_default \
+    frappe/erpnext-worker:$VERSION drop
+```
+
+Environment Variables needed:
+
+- `SITE_NAME`: name of the site to be deleted. Site name is domain name that resolves. e.g. `erp.example.com` or `erp.localhost`.
+- `DB_ROOT_USER`: MariaDB/PostgreSQL Root user.
+- `MYSQL_ROOT_PASSWORD`: Root User password for MariaDB.
+- `FORCE=1`: optional variable which force deletion of the same site.
+- `NO_BACKUP=1`: option variable to skip the process of taking backup before deleting the site. 
+
+Environment Variables for PostgreSQL only:
+
+- `POSTGRES_PASSWORD`: Password for `postgres`. The database root user.

env-example

@@ -1,4 +1,5 @@
-VERSION=edge
+ERPNEXT_VERSION=edge
+FRAPPE_VERSION=edge
 MARIADB_HOST=mariadb
 MYSQL_ROOT_PASSWORD=admin
 SITES=your.domain.com

installation/docker-compose-custom.yml

@@ -46,7 +46,7 @@ services:
       - assets-vol:/home/frappe/frappe-bench/sites/assets:rw
 
   frappe-socketio:
-    image: frappe/frappe-socketio:${VERSION}
+    image: frappe/frappe-socketio:${FRAPPE_VERSION}
     restart: on-failure
     depends_on:
       - redis-socketio

installation/docker-compose-erpnext.yml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   erpnext-nginx:
-    image: frappe/erpnext-nginx:${VERSION}
+    image: frappe/erpnext-nginx:${ERPNEXT_VERSION}
     restart: on-failure
     environment:
       - FRAPPE_PY=erpnext-python
@@ -29,7 +29,7 @@ services:
       - assets-vol:/assets:rw
 
   erpnext-python:
-    image: frappe/erpnext-worker:${VERSION}
+    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
     restart: on-failure
     environment:
       - MARIADB_HOST=${MARIADB_HOST}
@@ -43,7 +43,7 @@ services:
       - assets-vol:/home/frappe/frappe-bench/sites/assets:rw
 
   frappe-socketio:
-    image: frappe/frappe-socketio:${VERSION}
+    image: frappe/frappe-socketio:${FRAPPE_VERSION}
     restart: on-failure
     depends_on:
       - redis-socketio
@@ -53,7 +53,7 @@ services:
       - sites-vol:/home/frappe/frappe-bench/sites:rw
 
   frappe-worker-default:
-    image: frappe/erpnext-worker:${VERSION}
+    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
     restart: on-failure
     command: worker
     depends_on:
@@ -66,7 +66,7 @@ services:
       - sites-vol:/home/frappe/frappe-bench/sites:rw
 
   frappe-worker-short:
-    image: frappe/erpnext-worker:${VERSION}
+    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
     restart: on-failure
     command: worker
     environment:
@@ -81,7 +81,7 @@ services:
       - sites-vol:/home/frappe/frappe-bench/sites:rw
 
   frappe-worker-long:
-    image: frappe/erpnext-worker:${VERSION}
+    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
     restart: on-failure
     command: worker
     environment:
@@ -96,7 +96,7 @@ services:
       - sites-vol:/home/frappe/frappe-bench/sites:rw
 
   frappe-schedule:
-    image: frappe/erpnext-worker:${VERSION}
+    image: frappe/erpnext-worker:${ERPNEXT_VERSION}
     restart: on-failure
     command: schedule
     depends_on:

installation/docker-compose-frappe.yml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   frappe-nginx:
-    image: frappe/frappe-nginx:${VERSION}
+    image: frappe/frappe-nginx:${FRAPPE_VERSION}
     restart: on-failure
     environment:
       - FRAPPE_PY=frappe-python
@@ -29,7 +29,7 @@ services:
       - assets-vol:/assets:rw
 
   frappe-python:
-    image: frappe/frappe-worker:${VERSION}
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
     restart: on-failure
     environment:
       - MARIADB_HOST=${MARIADB_HOST}
@@ -43,7 +43,7 @@ services:
       - assets-vol:/home/frappe/frappe-bench/sites/assets:rw
 
   frappe-socketio:
-    image: frappe/frappe-socketio:${VERSION}
+    image: frappe/frappe-socketio:${FRAPPE_VERSION}
     restart: on-failure
     depends_on:
       - redis-socketio
@@ -53,7 +53,7 @@ services:
       - sites-vol:/home/frappe/frappe-bench/sites:rw
 
   frappe-worker-default:
-    image: frappe/frappe-worker:${VERSION}
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
     restart: on-failure
     command: worker
     depends_on:
@@ -66,7 +66,7 @@ services:
       - sites-vol:/home/frappe/frappe-bench/sites:rw
 
   frappe-worker-short:
-    image: frappe/frappe-worker:${VERSION}
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
     restart: on-failure
     command: worker
     environment:
@@ -81,7 +81,7 @@ services:
       - sites-vol:/home/frappe/frappe-bench/sites:rw
 
   frappe-worker-long:
-    image: frappe/frappe-worker:${VERSION}
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
     restart: on-failure
     command: worker
     environment:
@@ -96,7 +96,7 @@ services:
       - sites-vol:/home/frappe/frappe-bench/sites:rw
 
   frappe-schedule:
-    image: frappe/frappe-worker:${VERSION}
+    image: frappe/frappe-worker:${FRAPPE_VERSION}
     restart: on-failure
     command: schedule
     depends_on:

tests/docker-test.sh

@@ -347,6 +347,7 @@ echo -e "\e[1m\e[4mMigrate command in edge container\e[0m"
 docker run -it \
     -e "MAINTENANCE_MODE=1" \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    -v frappebench00_assets-vol:/home/frappe/frappe-bench/sites/assets \
     --network frappebench00_default \
     frappe/erpnext-worker:edge migrate
 
@@ -409,6 +410,20 @@ docker run \
     --network frappebench00_default \
     frappe/erpnext-worker:edge console pgsql.localhost
 
+echo -e "\e[1m\e[4mCheck drop site: test.localhost (mariadb)\e[0m"
+docker run \
+    -e SITE_NAME=test.localhost \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge drop
+
+echo -e "\e[1m\e[4mCheck drop site: pgsql.localhost (pgsql)\e[0m"
+docker run \
+    -e SITE_NAME=pgsql.localhost \
+    -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \
+    --network frappebench00_default \
+    frappe/erpnext-worker:edge drop
+
 echo -e "\e[1m\e[4mCheck bench --help\e[0m"
 docker run \
     -v frappebench00_sites-vol:/home/frappe/frappe-bench/sites \