diff --git a/production/backup/README.md b/production/backup/README.md new file mode 100644 index 00000000..1b3b85a2 --- /dev/null +++ b/production/backup/README.md @@ -0,0 +1,414 @@ +# ERPNext Backup System - Complete Guide + +Automated backup system with Digital Ocean Spaces (S3) for ERPNext production. + +--- + +## 🚀 Quick Production Setup (5 minutes) + +### Option 1: Interactive Setup + +```bash +cd production/backup +./manage-backups.sh setup # Interactive wizard +chmod 600 backup.env +./manage-backups.sh start +./manage-backups.sh test +``` + +### Option 2: Manual Configuration + +**Edit `backup/backup.env`:** +```bash +ENV_PREFIX=production # Change from 'development' +BACKUP_SITES=your-domain.com # Change from 'erp.localhost' +S3_BACKUP_RETENTION_DAYS=30 # Increase from 5 +S3_ACCESS_KEY_ID=your-key +S3_SECRET_ACCESS_KEY=your-secret +``` + +**Deploy:** +```bash +chmod 600 backup/backup.env +./manage-backups.sh restart +./manage-backups.sh test +./manage-backups.sh list-s3 | grep "production/" +``` + +**Verify automated backups:** +- Hourly DB backup runs every hour from container start time +- Daily full backup runs at 3:00 AM +- Check status: `./manage-backups.sh status` +- Monitor logs: `./manage-backups.sh logs` + +--- + +## 📦 What Gets Backed Up? + +### Always Included +✅ **Database** - All data, doctypes, settings (compressed SQL) + +### Backup Modes + +**Full Backup** (recommended for daily): +```bash +BACKUP_WITH_FILES=1 # Everything: DB + all files + site_config +``` + +**Database Only** (recommended for frequent/hourly): +```bash +BACKUP_WITH_FILES=0 # Database + site_config only +``` + +**What's in site_config.json:** +```json +{ + "db_name": "_73c82ec6d255ebe3", + "db_password": "Dp0yVfnoBvwYpR0y", + "db_type": "mariadb" +} +``` +*Note: production.env and other secrets are NOT included* + +--- + +## ⏰ Backup Schedules + +### Single Schedule (Simple) +```bash +# Note: For dual-schedule setup, see compose.backup-s3.yaml +# Single schedule is not commonly used in production +BACKUP_CRON_SCHEDULE=@every 1h # Every hour +``` + +### Dual Schedule (Recommended for Your Use Case) + +**Frequent DB backups + Occasional full backups:** + +Edit `compose.backup-s3.yaml` to add two jobs: + +```yaml +scheduler: + labels: + # Job 1: Hourly database-only backup (official @every syntax) + ofelia.job-exec.backup-db.schedule: "@every 1h" + ofelia.job-exec.backup-db.command: "bash -c 'export BACKUP_WITH_FILES=0 && /bin/bash /usr/local/bin/backup-to-s3.sh'" + ofelia.job-exec.backup-db.user: "frappe" + ofelia.job-exec.backup-db.no-overlap: "true" + + # Job 2: Daily full backup at 3 AM (standard cron) + ofelia.job-exec.backup-full.schedule: "0 3 * * *" + ofelia.job-exec.backup-full.command: "bash -c 'export BACKUP_WITH_FILES=1 && /bin/bash /usr/local/bin/backup-to-s3.sh'" + ofelia.job-exec.backup-full.user: "frappe" + ofelia.job-exec.backup-full.no-overlap: "true" +``` + +### Common Schedules +```bash +# Interval syntax (recommended for sub-daily) +@every 1h # Every hour +@every 2h # Every 2 hours +@every 30m # Every 30 minutes + +# Standard cron format (for specific times) +0 3 * * * # Daily at 3 AM +0 3 * * 0 # Weekly on Sunday at 3 AM +0 0,12 * * * # Twice daily (midnight and noon) +``` + +**References:** +- [Go cron intervals](https://pkg.go.dev/github.com/robfig/cron/v3#hdr-Intervals) +- [Cron format generator](https://crontab.guru/) + +--- + +## 🗂️ S3 Structure & Environment Segregation + +``` +s3://erp-is-backup/ +├── production/ ← ENV_PREFIX=production +│ └── erp.example.com/ +│ └── 2025-11-20/ ← Single date folder +│ ├── 20251120_120000-database.sql.gz +│ ├── 20251120_120000-files.tar +│ └── 20251120_120000-site_config_backup.json +├── staging/ ← ENV_PREFIX=staging +├── development/ ← ENV_PREFIX=development +└── local/ ← ENV_PREFIX=local +``` + +**Set environment:** +```bash +ENV_PREFIX=production # For production +ENV_PREFIX=staging # For staging +ENV_PREFIX=local # For local dev +``` + +--- + +## ⚙️ Configuration + +**File:** `backup/backup.env` + +### Essential Settings +```bash +# Credentials (MUST CHANGE) +S3_ACCESS_KEY_ID=your_key_here +S3_SECRET_ACCESS_KEY=your_secret_here + +# S3 Config +S3_ENDPOINT_URL=https://blr1.digitaloceanspaces.com +S3_BUCKET_NAME=erp-is-backup +S3_REGION=blr1 + +# Environment +ENV_PREFIX=production + +# Schedule (configured in compose.backup-s3.yaml) +# See "Dual Schedule" section for details + +# What to backup +BACKUP_WITH_FILES=1 # 1=DB+files, 0=DB only +BACKUP_COMPRESS=1 # Compress SQL (recommended) + +# Sites +BACKUP_SITES=erp.localhost + +# Retention +BACKUP_RETENTION_DAYS=7 # Local retention +S3_BACKUP_RETENTION_DAYS=30 # S3 retention +``` + +--- + +## 🛠️ Management Commands + +```bash +cd production/backup + +# Service control +./manage-backups.sh start # Start backup services +./manage-backups.sh stop # Stop backup services +./manage-backups.sh restart # Restart services + +# Operations +./manage-backups.sh test # Run backup now +./manage-backups.sh status # Check status +./manage-backups.sh logs # View logs +./manage-backups.sh list-s3 # List S3 backups +./manage-backups.sh validate # Validate config + +# Interactive setup +./manage-backups.sh setup # Configuration wizard +``` + +--- + +## 📋 Example Configurations + +### Dual Schedule: Hourly DB + Daily Full (Recommended) +```bash +# backup.env +ENV_PREFIX=production +BACKUP_SITES=erp.localhost +BACKUP_WITH_FILES=0 # Default (overridden by cron) +BACKUP_COMPRESS=1 +BACKUP_RETENTION_DAYS=1 # Keep local 1 day +S3_BACKUP_RETENTION_DAYS=5 # Keep S3 5 days + +# Schedules are in compose.backup-s3.yaml: +# - Hourly: BACKUP_WITH_FILES=0 (DB only) +# - Daily 3AM: BACKUP_WITH_FILES=1 (Full) +``` + +### High-Traffic Production +```bash +# Note: Schedule is set in compose.backup-s3.yaml, not in backup.env +ENV_PREFIX=production +BACKUP_WITH_FILES=1 +S3_BACKUP_RETENTION_DAYS=90 + +# In compose.backup-s3.yaml: +# ofelia.job-exec.backup-db.schedule: "@every 2h" +``` + +### Staging/Dev +```bash +ENV_PREFIX=staging +BACKUP_WITH_FILES=0 # DB only +S3_BACKUP_RETENTION_DAYS=14 + +# In compose.backup-s3.yaml: +# ofelia.job-exec.backup-daily.schedule: "0 3 * * *" +``` + +--- + +## 🔄 Restore Process + +### Download from S3 +```bash +# List backups +aws s3 ls s3://erp-is-backup/production/erp.localhost/ --recursive \ + --endpoint-url=https://blr1.digitaloceanspaces.com + +# Download specific backup +aws s3 cp s3://erp-is-backup/production/erp.localhost/2025-11-20/backup.sql.gz . \ + --endpoint-url=https://blr1.digitaloceanspaces.com +``` + +### Restore Database +```bash +# Copy to container +docker cp backup.sql.gz erpnext-production-backend:/tmp/ + +# Restore +docker compose -p erpnext-production exec backend \ + bench --site erp.localhost --force restore /tmp/backup.sql.gz +``` + +### Restore with Files +```bash +docker compose -p erpnext-production exec backend \ + bench --site erp.localhost --force restore \ + --with-public-files /tmp/files.tar \ + --with-private-files /tmp/private-files.tar \ + /tmp/backup.sql.gz +``` + +--- + +## 🔍 Monitoring + +### Check Status +```bash +./manage-backups.sh status + +# Or manually +docker ps | grep backup +docker compose -p erpnext-production logs -f backup-cron +``` + +### View Recent Backups +```bash +# Local backups +docker compose -p erpnext-production exec scheduler \ + ls -lht /home/frappe/frappe-bench/sites/*/private/backups/ | head -10 + +# S3 backups +./manage-backups.sh list-s3 +``` + +--- + +## 📁 File Structure + +``` +production/ +├── backup/ ← All backup configs here +│ ├── backup.env ← Main configuration +│ ├── compose.backup-s3.yaml ← Docker compose override +│ ├── backup-to-s3.sh ← S3 backup script +│ ├── backup-site.sh ← Local backup script +│ └── manage-backups.sh ← Management helper +│ +├── backups/ ← Local backup storage +│ └── {timestamp}-{site}-*.{sql.gz|tar|json} +│ +├── scripts/ ← Other scripts +└── docs/ ← This documentation +``` + +--- + +## 🚨 Troubleshooting + +### Services Not Running +```bash +./manage-backups.sh restart +docker ps -a | grep backup +``` + +### S3 Upload Fails +- Check credentials in `backup.env` +- Verify bucket exists in Digital Ocean +- Test connection: `./manage-backups.sh validate` + +### No Backups Created +```bash +# Verify site name +docker compose -p erpnext-production exec backend bench list-apps + +# Check site in backup.env matches actual site +grep BACKUP_SITES backup.env + +# Run with debug +docker compose -p erpnext-production exec scheduler bash -c " + export BACKUP_DEBUG=1 + /bin/bash /usr/local/bin/backup-to-s3.sh +" +``` + +### AWS CLI Missing +Script auto-installs, but if issues: +```bash +docker compose -p erpnext-production exec scheduler bash -c " + pip3 install --user awscli --upgrade + export PATH=\"\$HOME/.local/bin:\$PATH\" + aws --version +" +``` + +--- + +## 🔐 Security + +```bash +# Protect credentials +chmod 600 backup/backup.env + +# Gitignore +echo "production/backup/backup.env" >> .gitignore + +# Use Digital Ocean IAM +# Create dedicated API keys with bucket-only access +``` + +--- + +## 💡 Best Practices + +1. **Test Restores** - Regularly test restore in staging +2. **Monitor Sizes** - Check backup sizes, adjust file inclusion +3. **Environment Prefix** - Always use for clarity +4. **Start Conservative** - Begin with less frequent, increase as needed +5. **Document Changes** - Note when you modify backup config + +--- + +## Storage Estimates + +| Frequency | With Files | DB Only | Monthly (30d) | +|-----------|-----------|---------|---------------| +| Hourly | ~200MB × 24 = 4.8GB/day | ~50MB × 24 = 1.2GB/day | 144GB / 36GB | +| Every 6h | ~200MB × 4 = 800MB/day | ~50MB × 4 = 200MB/day | 24GB / 6GB | +| Daily | ~200MB × 1 = 200MB/day | ~50MB × 1 = 50MB/day | 6GB / 1.5GB | + +**Your Config (Hourly DB + Daily Full):** +- Hourly DB: ~50MB × 24 = 1.2GB/day +- Daily Full: ~200MB × 1 = 200MB/day +- **Total: ~1.4GB/day = ~42GB/month** (30 day retention) + +--- + +## Support & Resources + +- [Digital Ocean Spaces Docs](https://docs.digitalocean.com/products/spaces/) +- [Ofelia Cron Scheduler](https://github.com/mcuadros/ofelia) +- [ERPNext Backup Docs](https://frappeframework.com/docs/user/en/bench/reference/backup) +- [Cron Schedule Generator](https://crontab.guru/) + +**Management Script:** +```bash +./manage-backups.sh help +``` diff --git a/production/backup/backup-to-s3.sh b/production/backup/backup-to-s3.sh new file mode 100755 index 00000000..bc32a7b2 --- /dev/null +++ b/production/backup/backup-to-s3.sh @@ -0,0 +1,433 @@ +#!/bin/bash + +############################################################################### +# ERPNext Backup to S3 (Digital Ocean Spaces) Script +# +# This script creates ERPNext backups and uploads them to S3-compatible storage +# (Digital Ocean Spaces). It handles multiple sites, retention policies, and +# provides detailed logging and error handling. +# +# Usage: /usr/local/bin/backup-to-s3.sh +# +# Environment Variables Required: +# S3_ENDPOINT_URL - S3 endpoint (e.g., https://blr1.digitaloceanspaces.com) +# S3_BUCKET_NAME - S3 bucket name +# AWS_ACCESS_KEY_ID - S3 access key +# AWS_SECRET_ACCESS_KEY - S3 secret key +# BACKUP_SITES - Space-separated list of sites to backup +# +# Optional Environment Variables: +# BACKUP_WITH_FILES - Include files (default: 1) +# BACKUP_COMPRESS - Compress backups (default: 1) +# BACKUP_RETENTION_DAYS - Local retention in days (default: 7) +# S3_BACKUP_RETENTION_DAYS - S3 retention in days (default: 30) +# BACKUP_DEBUG - Enable debug logging (default: 0) +# S3_REGION - S3 region (default: blr1) +# S3_STORAGE_CLASS - S3 storage class (default: STANDARD) +############################################################################### + +set -euo pipefail + +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging functions +log_info() { echo -e "${GREEN}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; } +log_debug() { [[ "${BACKUP_DEBUG:-0}" == "1" ]] && echo -e "${BLUE}[DEBUG]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" || true; } + +# Configuration with defaults +BACKUP_WITH_FILES="${BACKUP_WITH_FILES:-1}" +BACKUP_COMPRESS="${BACKUP_COMPRESS:-1}" +BACKUP_RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-7}" +S3_BACKUP_RETENTION_DAYS="${S3_BACKUP_RETENTION_DAYS:-30}" +BACKUP_DEBUG="${BACKUP_DEBUG:-0}" +S3_REGION="${S3_REGION:-blr1}" +S3_STORAGE_CLASS="${S3_STORAGE_CLASS:-STANDARD}" +BACKUP_SITES="${BACKUP_SITES:-}" +ENV_PREFIX="${ENV_PREFIX:-production}" + +# Counters +TOTAL_BACKUPS=0 +SUCCESSFUL_BACKUPS=0 +FAILED_BACKUPS=0 +UPLOADED_FILES=0 + +############################################################################### +# Function: check_prerequisites +# Description: Verify all required tools and environment variables are present +############################################################################### +check_prerequisites() { + log_info "Checking prerequisites..." + + # Check for bench command + if ! command -v bench &> /dev/null; then + log_error "Required command 'bench' not found. Is this running in ERPNext container?" + exit 1 + fi + + # Check for AWS CLI, install if missing + if ! command -v aws &> /dev/null; then + log_info "AWS CLI not found. Installing..." + # Install AWS CLI + pip3 install --user --no-warn-script-location awscli > /tmp/aws-install.log 2>&1 + export PATH="$HOME/.local/bin:$PATH" + + # Verify installation + if command -v aws &> /dev/null || [ -f "$HOME/.local/bin/aws" ]; then + log_info "AWS CLI installed successfully" + else + log_error "Failed to install AWS CLI" + log_debug "Install log: $(cat /tmp/aws-install.log)" + exit 1 + fi + fi + + # Check required environment variables + local required_vars=( + "S3_ENDPOINT_URL" + "S3_BUCKET_NAME" + "AWS_ACCESS_KEY_ID" + "AWS_SECRET_ACCESS_KEY" + "BACKUP_SITES" + ) + + for var in "${required_vars[@]}"; do + if [[ -z "${!var:-}" ]]; then + log_error "Required environment variable '$var' is not set" + exit 1 + fi + done + + log_debug "S3_ENDPOINT_URL: $S3_ENDPOINT_URL" + log_debug "S3_BUCKET_NAME: $S3_BUCKET_NAME" + log_debug "S3_REGION: $S3_REGION" + log_debug "BACKUP_SITES: $BACKUP_SITES" + + log_info "Prerequisites check passed" +} + +############################################################################### +# Function: configure_aws_cli +# Description: Configure AWS CLI for Digital Ocean Spaces +############################################################################### +configure_aws_cli() { + log_info "Configuring AWS CLI for Digital Ocean Spaces..." + + mkdir -p "$HOME/.aws" + + cat > "$HOME/.aws/credentials" < "$HOME/.aws/config" < /dev/null; then + log_info "S3 connection successful" + return 0 + else + log_error "Failed to connect to S3 bucket: ${S3_BUCKET_NAME}" + log_error "Endpoint: ${S3_ENDPOINT_URL}" + return 1 + fi +} + +############################################################################### +# Function: create_backup +# Description: Create backup for a specific site +# Arguments: $1 - site name +############################################################################### +create_backup() { + local site="$1" + local backup_dir="/home/frappe/frappe-bench/sites/${site}/private/backups" + + log_info "Creating backup for site: ${site}" + + # Build bench backup command + local bench_cmd="bench --site ${site} backup" + + # Add options based on configuration + if [[ "$BACKUP_WITH_FILES" == "1" ]]; then + bench_cmd="${bench_cmd} --with-files" + log_debug "Backup mode: Full (database + files)" + else + log_debug "Backup mode: Database only" + fi + + [[ "$BACKUP_COMPRESS" == "1" ]] && bench_cmd="${bench_cmd} --compress" + + log_debug "Executing: ${bench_cmd}" + + # Create marker file to identify new backups + local marker="/tmp/backup-marker-${site}-$$.tmp" + touch "$marker" + sleep 1 + + # Execute backup (redirect output to avoid polluting file list) + if eval "$bench_cmd" > /tmp/backup-output-$$.log 2>&1; then + log_info "Backup created successfully for ${site}" + + # Find new backup files + local new_files + new_files=$(find "$backup_dir" -type f -newer "$marker" 2>/dev/null || true) + rm -f "$marker" + + if [[ -z "$new_files" ]]; then + log_warn "No new backup files found for ${site}" + cat /tmp/backup-output-$$.log >&2 + rm -f /tmp/backup-output-$$.log + return 1 + fi + + rm -f /tmp/backup-output-$$.log + echo "$new_files" + return 0 + else + log_error "Backup failed for ${site}" + cat /tmp/backup-output-$$.log >&2 + rm -f "$marker" /tmp/backup-output-$$.log + return 1 + fi +} + +############################################################################### +# Function: upload_to_s3 +# Description: Upload backup files to S3 +# Arguments: $1 - site name, $2 - file path +############################################################################### +upload_to_s3() { + local site="$1" + local file_path="$2" + local file_name=$(basename "$file_path") + local timestamp=$(date '+%Y-%m-%d') + # S3 path structure: s3://bucket/{env}/{site}/{YYYY-MM-DD}/{filename} + local s3_path="s3://${S3_BUCKET_NAME}/${ENV_PREFIX}/${site}/${timestamp}/${file_name}" + + log_info "Uploading ${file_name} to S3..." + log_debug "S3 path: ${s3_path}" + + if aws s3 cp "$file_path" "$s3_path" \ + --endpoint-url="${S3_ENDPOINT_URL}" \ + --storage-class="${S3_STORAGE_CLASS}" \ + --no-progress 2>&1 | while IFS= read -r line; do log_debug "$line"; done; then + + local file_size=$(stat -f%z "$file_path" 2>/dev/null || stat -c%s "$file_path" 2>/dev/null || echo "0") + local file_size_hr=$(numfmt --to=iec-i --suffix=B "$file_size" 2>/dev/null || echo "${file_size} bytes") + + log_info "✓ Uploaded ${file_name} (${file_size_hr})" + ((UPLOADED_FILES++)) + return 0 + else + log_error "✗ Failed to upload ${file_name}" + return 1 + fi +} + +############################################################################### +# Function: cleanup_old_local_backups +# Description: Remove old local backup files +# Arguments: $1 - site name +############################################################################### +cleanup_old_local_backups() { + local site="$1" + local backup_dir="/home/frappe/frappe-bench/sites/${site}/private/backups" + + log_info "Cleaning up old local backups for ${site} (keeping ${BACKUP_RETENTION_DAYS} days)..." + + if [[ ! -d "$backup_dir" ]]; then + log_warn "Backup directory not found: ${backup_dir}" + return 0 + fi + + local deleted_count=0 + while IFS= read -r -d '' file; do + rm -f "$file" + ((deleted_count++)) + log_debug "Deleted old local backup: $(basename "$file")" + done < <(find "$backup_dir" -type f -mtime "+${BACKUP_RETENTION_DAYS}" -print0 2>/dev/null || true) + + if [[ $deleted_count -gt 0 ]]; then + log_info "Deleted ${deleted_count} old local backup file(s)" + else + log_debug "No old local backups to delete" + fi +} + +############################################################################### +# Function: cleanup_old_s3_backups +# Description: Remove old S3 backup files +# Arguments: $1 - site name +############################################################################### +cleanup_old_s3_backups() { + local site="$1" + local cutoff_date=$(date -u -d "${S3_BACKUP_RETENTION_DAYS} days ago" +%s 2>/dev/null || date -u -v-${S3_BACKUP_RETENTION_DAYS}d +%s 2>/dev/null || echo "0") + + log_info "Cleaning up old S3 backups for ${site} (keeping ${S3_BACKUP_RETENTION_DAYS} days)..." + + local deleted_count=0 + local s3_prefix="s3://${S3_BUCKET_NAME}/${ENV_PREFIX}/${site}/" + + # List all objects and filter by date + while IFS= read -r line; do + local object_date=$(echo "$line" | awk '{print $1, $2}') + local object_key=$(echo "$line" | awk '{$1=$2=$3=""; print $0}' | sed 's/^[ \t]*//') + + if [[ -z "$object_key" ]]; then + continue + fi + + local object_timestamp=$(date -u -d "$object_date" +%s 2>/dev/null || date -u -j -f "%Y-%m-%d %H:%M:%S" "$object_date" +%s 2>/dev/null || echo "0") + + if [[ "$object_timestamp" -lt "$cutoff_date" ]]; then + if aws s3 rm "s3://${S3_BUCKET_NAME}/${object_key}" --endpoint-url="${S3_ENDPOINT_URL}" &>/dev/null; then + ((deleted_count++)) + log_debug "Deleted old S3 backup: ${object_key}" + fi + fi + done < <(aws s3 ls "${s3_prefix}" --endpoint-url="${S3_ENDPOINT_URL}" --recursive 2>/dev/null || true) + + if [[ $deleted_count -gt 0 ]]; then + log_info "Deleted ${deleted_count} old S3 backup file(s)" + else + log_debug "No old S3 backups to delete" + fi +} + +############################################################################### +# Function: process_site_backup +# Description: Complete backup workflow for a single site +# Arguments: $1 - site name +############################################################################### +process_site_backup() { + local site="$1" + ((TOTAL_BACKUPS++)) + + log_info "==========================================" + log_info "Processing backup for: ${site}" + log_info "==========================================" + + # Create backup + local backup_files + if ! backup_files=$(create_backup "$site"); then + log_error "Backup creation failed for ${site}" + ((FAILED_BACKUPS++)) + return 1 + fi + + # Upload each backup file to S3 + local upload_success=true + while IFS= read -r file; do + # Skip empty lines and non-file paths + [[ -z "$file" ]] && continue + [[ ! -f "$file" ]] && continue + + if ! upload_to_s3 "$site" "$file"; then + upload_success=false + fi + done <<< "$backup_files" + + if [[ "$upload_success" == "true" ]]; then + ((SUCCESSFUL_BACKUPS++)) + log_info "✓ Backup completed successfully for ${site}" + else + ((FAILED_BACKUPS++)) + log_error "✗ Some uploads failed for ${site}" + fi + + # Cleanup old backups + cleanup_old_local_backups "$site" + cleanup_old_s3_backups "$site" + + return 0 +} + +############################################################################### +# Function: send_notification +# Description: Send notification about backup status (placeholder for future) +############################################################################### +send_notification() { + local status="$1" + local message="$2" + + # TODO: Implement email or Slack notifications + log_debug "Notification: ${status} - ${message}" +} + +############################################################################### +# Main execution +############################################################################### +main() { + local start_time=$(date +%s) + + log_info "==========================================" + log_info "ERPNext Backup to S3 Started" + log_info "==========================================" + + # Check prerequisites + check_prerequisites + + # Configure AWS CLI + configure_aws_cli + + # Test S3 connection + if ! test_s3_connection; then + log_error "Cannot proceed without S3 connection" + exit 1 + fi + + # Process each site + IFS=' ' read -ra SITES <<< "$BACKUP_SITES" + for site in "${SITES[@]}"; do + [[ -z "$site" ]] && continue + process_site_backup "$site" || true + done + + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + + # Summary + log_info "==========================================" + log_info "Backup Summary" + log_info "==========================================" + log_info "Total sites processed: ${TOTAL_BACKUPS}" + log_info "Successful backups: ${SUCCESSFUL_BACKUPS}" + log_info "Failed backups: ${FAILED_BACKUPS}" + log_info "Files uploaded to S3: ${UPLOADED_FILES}" + log_info "Duration: ${duration} seconds" + log_info "==========================================" + + if [[ $FAILED_BACKUPS -gt 0 ]]; then + send_notification "WARNING" "Some backups failed. Check logs for details." + exit 1 + else + send_notification "SUCCESS" "All backups completed successfully." + exit 0 + fi +} + +# Execute main function +main "$@" diff --git a/production/backup/compose.backup-s3.yaml b/production/backup/compose.backup-s3.yaml new file mode 100644 index 00000000..0eb5908d --- /dev/null +++ b/production/backup/compose.backup-s3.yaml @@ -0,0 +1,59 @@ +services: + # Ofelia cron scheduler for automated backups + backup-cron: + image: mcuadros/ofelia:latest + container_name: ${PROJECT_NAME:-erpnext-production}-backup-cron + restart: unless-stopped + depends_on: + - scheduler + command: daemon --docker + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: + - bench-network + + # Scheduler service with backup job labels + scheduler: + env_file: + - ./backup/backup.env + environment: + # S3/Digital Ocean Spaces configuration + - S3_ENDPOINT_URL=${S3_ENDPOINT_URL:-} + - S3_BUCKET_NAME=${S3_BUCKET_NAME:-} + - S3_REGION=${S3_REGION:-blr1} + - AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID:-} + - AWS_SECRET_ACCESS_KEY=${S3_SECRET_ACCESS_KEY:-} + # Environment segregation + - ENV_PREFIX=${ENV_PREFIX:-production} + # Backup configuration + - BACKUP_SITES=${BACKUP_SITES:-erp.localhost} + - BACKUP_WITH_FILES=${BACKUP_WITH_FILES:-1} + - BACKUP_COMPRESS=${BACKUP_COMPRESS:-1} + - BACKUP_RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-7} + - S3_BACKUP_RETENTION_DAYS=${S3_BACKUP_RETENTION_DAYS:-30} + - BACKUP_DEBUG=${BACKUP_DEBUG:-0} + - S3_STORAGE_CLASS=${S3_STORAGE_CLASS:-STANDARD} + volumes: + - ./backup/backup-to-s3.sh:/usr/local/bin/backup-to-s3.sh:ro + labels: + # Ofelia job configuration for automated S3 backups + ofelia.enabled: "true" + + # Job 1: Hourly database-only backup (official @every syntax) + # See: https://pkg.go.dev/github.com/robfig/cron/v3#hdr-Intervals + ofelia.job-exec.backup-db-hourly.schedule: "@every 1h" + ofelia.job-exec.backup-db-hourly.command: "bash -c 'export BACKUP_WITH_FILES=0 && /bin/bash /usr/local/bin/backup-to-s3.sh'" + ofelia.job-exec.backup-db-hourly.user: "frappe" + ofelia.job-exec.backup-db-hourly.no-overlap: "true" + + # Job 2: Daily full backup at 3 AM (standard 5-field cron) + # See: https://pkg.go.dev/github.com/robfig/cron/v3#hdr-CRON_Expression_Format + ofelia.job-exec.backup-full-daily.schedule: "0 3 * * *" + ofelia.job-exec.backup-full-daily.command: "bash -c 'export BACKUP_WITH_FILES=1 && /bin/bash /usr/local/bin/backup-to-s3.sh'" + ofelia.job-exec.backup-full-daily.user: "frappe" + ofelia.job-exec.backup-full-daily.no-overlap: "true" + +networks: + bench-network: + external: true + name: ${PROJECT_NAME:-erpnext-production} diff --git a/production/backup/manage-backups.sh b/production/backup/manage-backups.sh new file mode 100755 index 00000000..29cb2a4d --- /dev/null +++ b/production/backup/manage-backups.sh @@ -0,0 +1,389 @@ +#!/bin/bash + +############################################################################### +# ERPNext Backup Management Helper Script +# Manages periodic backup setup with Digital Ocean Spaces (S3) +############################################################################### + +set -euo pipefail + +# Colors +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' + +echo_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +echo_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +echo_error() { echo -e "${RED}[ERROR]${NC} $1"; } +echo_step() { echo -e "${BLUE}[STEP]${NC} $1"; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PRODUCTION_DIR="$(dirname "$SCRIPT_DIR")" +BACKUP_DIR="$SCRIPT_DIR" + +cd "$BACKUP_DIR" + +show_help() { + cat </dev/null || { + echo_error "Failed to load backup.env - check for syntax errors" + return 1 + } + set +a + + local errors=0 + + # Check required variables + if [[ "${S3_ACCESS_KEY_ID:-}" == "CHANGEME"* ]] || [[ -z "${S3_ACCESS_KEY_ID:-}" ]]; then + echo_error "S3_ACCESS_KEY_ID not configured" + ((errors++)) + fi + + if [[ "${S3_SECRET_ACCESS_KEY:-}" == "CHANGEME"* ]] || [[ -z "${S3_SECRET_ACCESS_KEY:-}" ]]; then + echo_error "S3_SECRET_ACCESS_KEY not configured" + ((errors++)) + fi + + if [[ -z "${S3_ENDPOINT_URL:-}" ]]; then + echo_error "S3_ENDPOINT_URL not configured" + ((errors++)) + fi + + if [[ -z "${S3_BUCKET_NAME:-}" ]]; then + echo_error "S3_BUCKET_NAME not configured" + ((errors++)) + fi + + if [[ -z "${BACKUP_SITES:-}" ]]; then + echo_warn "BACKUP_SITES not configured, will default to 'erp.localhost'" + fi + + if [[ $errors -gt 0 ]]; then + echo_error "Configuration has $errors error(s). Please fix backup.env" + return 1 + fi + + echo_info "✓ Configuration valid" + return 0 +} + +setup_backup_config() { + echo_step "Backup Configuration Setup" + echo "" + + # Check if backup.env exists + if [[ -f "backup.env" ]]; then + read -p "backup.env exists. Overwrite? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo_info "Keeping existing configuration. Edit backup.env manually if needed." + return 0 + fi + fi + + # Infrastructure constants (hardcoded) + local BUCKET_NAME="erp-is-backup" + local REGION="blr1" + + # Interactive prompts + echo_warn "⚠️ SECURITY: Never commit credentials to git!" + echo_info "Enter your Digital Ocean Spaces credentials:" + echo "" + + read -p "Spaces Access Key ID: " ACCESS_KEY + read -sp "Spaces Secret Access Key: " SECRET_KEY + echo "" + echo "" + + read -p "Environment (production/staging/development) [default: development]: " ENV_PREFIX + ENV_PREFIX=${ENV_PREFIX:-development} + + read -p "Site name to backup (default: erp.localhost): " SITE_NAME + SITE_NAME=${SITE_NAME:-erp.localhost} + + echo "" + echo_info "Backup schedules (automated via Docker Compose):" + echo " • Hourly: Database only (every hour)" + echo " • Daily: Full backup with files (3:00 AM)" + echo "" + + read -p "Local retention days (default: 1): " LOCAL_RETENTION + LOCAL_RETENTION=${LOCAL_RETENTION:-1} + + read -p "S3 retention days (default: 5): " S3_RETENTION + S3_RETENTION=${S3_RETENTION:-5} + + # Create backup.env + cat > backup.env < /dev/null 2>&1; then + echo_error "Docker Compose configuration validation failed" + exit 1 + fi + + echo_info "Starting backup-cron and updating scheduler with backup volume..." + # Force recreate backup-cron to ensure it picks up latest schedule labels + docker compose --project-name erpnext-production \ + -f "$PRODUCTION_DIR/production.yaml" \ + -f "$BACKUP_DIR/compose.backup-s3.yaml" \ + up -d --force-recreate backup-cron scheduler + + # Wait for services to be ready + sleep 2 + + # Verify backup script is mounted + if docker compose --project-name erpnext-production \ + -f "$PRODUCTION_DIR/production.yaml" \ + exec -T scheduler test -f /usr/local/bin/backup-to-s3.sh 2>/dev/null; then + echo_info "✓ Backup script mounted successfully" + else + echo_error "Backup script not found in scheduler container" + exit 1 + fi + + echo_info "✓ Backup services started" + echo_info " - Hourly DB backup: Every hour" + echo_info " - Daily full backup: 3:00 AM" +} + +stop_backup_services() { + echo_step "Stopping backup services..." + + docker compose --project-name erpnext-production \ + -f "$PRODUCTION_DIR/production.yaml" \ + -f "$BACKUP_DIR/compose.backup-s3.yaml" \ + stop backup-cron + + echo_info "✓ Backup services stopped" +} + +restart_backup_services() { + stop_backup_services + sleep 2 + start_backup_services +} + +run_test_backup() { + echo_step "Running test backup..." + + validate_config || exit 1 + + # Check if scheduler has backup script mounted + echo_info "Checking backup script availability..." + if ! docker compose --project-name erpnext-production \ + -f "$PRODUCTION_DIR/production.yaml" \ + exec -T scheduler test -f /usr/local/bin/backup-to-s3.sh 2>/dev/null; then + echo_warn "Backup script not mounted. Starting backup services first..." + start_backup_services + fi + + echo_info "Executing backup script manually..." + docker compose --project-name erpnext-production \ + -f "$PRODUCTION_DIR/production.yaml" \ + -f "$BACKUP_DIR/compose.backup-s3.yaml" \ + exec scheduler /bin/bash /usr/local/bin/backup-to-s3.sh + + echo_info "✓ Test backup completed" +} + +show_status() { + echo_step "Backup Service Status" + echo "" + + # Check if containers are running + if docker ps --filter "name=erpnext-production-backup-cron" --format "{{.Status}}" | grep -q "Up"; then + echo_info "✓ Backup cron service: Running" + # Show cron jobs + echo_info " Active jobs:" + docker exec erpnext-production-backup-cron-1 ofelia jobs 2>/dev/null | grep -E "backup-db|backup-full" || echo " (unable to list jobs)" + else + echo_warn "✗ Backup cron service: Not running" + echo_info " Run: $0 start" + fi + + if docker ps --filter "name=erpnext-production-scheduler" --format "{{.Status}}" | grep -q "Up"; then + echo_info "✓ Scheduler service: Running" + + # Check if backup script is mounted + if docker compose --project-name erpnext-production \ + -f "$PRODUCTION_DIR/production.yaml" \ + exec -T scheduler test -f /usr/local/bin/backup-to-s3.sh 2>/dev/null; then + echo_info " ✓ Backup script mounted" + else + echo_warn " ✗ Backup script NOT mounted - run: $0 restart" + fi + else + echo_warn "✗ Scheduler service: Not running" + fi + + echo "" + echo_step "Recent backups in container:" + docker compose --project-name erpnext-production exec scheduler \ + ls -lht /home/frappe/frappe-bench/sites/*/private/backups/ 2>/dev/null | head -10 || echo_warn "No backups found" +} + +show_logs() { + echo_step "Following backup logs (Ctrl+C to exit)..." + docker compose --project-name erpnext-production logs -f backup-cron scheduler +} + +list_s3_backups() { + echo_step "Listing S3 backups..." + + validate_config || exit 1 + + # Variables already loaded by validate_config + + docker compose --project-name erpnext-production \ + -f "$PRODUCTION_DIR/production.yaml" \ + exec scheduler bash -c " + export AWS_ACCESS_KEY_ID='${S3_ACCESS_KEY_ID}' + export AWS_SECRET_ACCESS_KEY='${S3_SECRET_ACCESS_KEY}' + + if ! command -v aws &>/dev/null; then + pip3 install --user awscli --upgrade + export PATH=\"\$HOME/.local/bin:\$PATH\" + fi + + aws s3 ls s3://${S3_BUCKET_NAME}/ --recursive --endpoint-url=${S3_ENDPOINT_URL} --human-readable + " +} + +# Main command dispatcher +case "${1:-help}" in + setup) + setup_backup_config + echo "" + echo_info "Next steps:" + echo " 1. Review backup.env" + echo " 2. Run: $0 validate" + echo " 3. Run: $0 start" + ;; + start) + start_backup_services + ;; + stop) + stop_backup_services + ;; + restart) + restart_backup_services + ;; + test) + run_test_backup + ;; + status) + show_status + ;; + logs) + show_logs + ;; + list-s3) + list_s3_backups + ;; + validate) + validate_config + ;; + help|--help|-h) + show_help + ;; + *) + echo_error "Unknown command: $1" + show_help + exit 1 + ;; +esac