From f956574c9d8116b60855bc29505f7e3d0415ee54 Mon Sep 17 00:00:00 2001 From: Gabos Levente Date: Thu, 26 Jun 2025 15:18:07 +0300 Subject: [PATCH] KAN-63: academy lms dockerization + CI/CD started --- .env.example | 65 +++++++++ .github/workflows/deploy.yml | 151 +++++++++++++++++++++ DEPLOYMENT.md | 246 +++++++++++++++++++++++++++++++++++ README.md | 151 ++++++++++++--------- compose.yaml | 136 ++++++++++++++++--- images/custom/Containerfile | 219 ++++++++++++++++--------------- nginx/502.html | 59 +++++++++ nginx/conf.d/default.conf | 67 ++++++++++ nginx/nginx.conf | 43 ++++++ scripts/create-site.sh | 67 ++++++++++ scripts/migrate-all-sites.sh | 47 +++++++ scripts/setup-hetzner.sh | 112 ++++++++++++++++ 12 files changed, 1175 insertions(+), 188 deletions(-) create mode 100644 .env.example create mode 100644 .github/workflows/deploy.yml create mode 100644 DEPLOYMENT.md create mode 100644 nginx/502.html create mode 100644 nginx/conf.d/default.conf create mode 100644 nginx/nginx.conf create mode 100644 scripts/create-site.sh create mode 100644 scripts/migrate-all-sites.sh create mode 100644 scripts/setup-hetzner.sh diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..dc1c07e0 --- /dev/null +++ b/.env.example @@ -0,0 +1,65 @@ +# Frappe/ERPNext Environment Configuration + +# Database Configuration +DB_HOST=db +DB_PORT=3306 +MARIADB_ROOT_PASSWORD=changeme123 +MARIADB_DATABASE=frappe +MARIADB_USER=frappe +MARIADB_PASSWORD=changeme456 + +# Redis Configuration +REDIS_CACHE=redis-cache:6379 +REDIS_QUEUE=redis-queue:6379 +REDIS_SOCKETIO=redis-socketio:6379 + +# Frappe Configuration +FRAPPE_SITE_NAME_HEADER=academy.example.com +FRAPPE_DEFAULT_SITE=academy.example.com +FRAPPE_SITES_DIR=/workspace/development/frappe-bench/sites + +# Security +ADMIN_PASSWORD=changeme789 +ENCRYPTION_KEY=changeme_32_character_encryption_key_here + +# LangChain Service Configuration +LANGCHAIN_API_URL=http://langchain-service:8080 +LANGCHAIN_API_KEY=changeme_langchain_api_key + +# OpenAI Configuration (for AI Tutor) +OPENAI_API_KEY=your_openai_api_key_here + +# Anthropic Configuration (optional, for Claude models) +ANTHROPIC_API_KEY=your_anthropic_api_key_here + +# LangChain Database Configuration +LANGCHAIN_DB_NAME=langchain_db +LANGCHAIN_DB_USER=langchain_user +LANGCHAIN_DB_PASSWORD=changeme_langchain_db_password +LANGCHAIN_ENV=production +LANGCHAIN_DEBUG=false + +# AI Tutor API Configuration +AI_TUTOR_API_URL=http://langchain-service:8000 + +# Email Configuration (optional) +MAIL_SERVER=smtp.gmail.com +MAIL_PORT=587 +MAIL_USE_TLS=1 +MAIL_USERNAME=your-email@gmail.com +MAIL_PASSWORD=your-app-password + +# Backup Configuration (optional) +BACKUP_RETENTION_DAYS=7 +BACKUP_PATH=/backups + +# Development/Production Mode +FRAPPE_ENV=production +DEVELOPER_MODE=0 + +# Network Configuration +COMPOSE_PROJECT_NAME=academy-lms +NETWORK_NAME=langchain-network + +# Hetzner Specific (if needed) +EXTERNAL_IP=188.245.211.114 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..91929234 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,151 @@ +name: Deploy Academy LMS to Hetzner + +on: + # Manual trigger + workflow_dispatch: + inputs: + force_rebuild: + description: 'Force rebuild all images' + required: false + default: 'false' + type: boolean + + # Webhook triggers from watched repositories + repository_dispatch: + types: [academy-lms-updated, academy-ai-tutor-updated, academy-langchain-updated] + + # Push to master branch of this repo + push: + branches: [ master ] + paths: + - 'compose.yaml' + - 'images/**' + - '.github/workflows/**' + - 'nginx/**' + +env: + REGISTRY: ghcr.io + HETZNER_HOST: 188.245.211.114 + HETZNER_USER: frappe + DEPLOY_PATH: /opt/frappe-deployment + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Frappe image + id: meta-frappe + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/exarlabs/academy-frappe + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push Frappe image + uses: docker/build-push-action@v5 + with: + context: . + file: ./images/custom/Containerfile + push: true + tags: ${{ steps.meta-frappe.outputs.tags }} + labels: ${{ steps.meta-frappe.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + LMS_REPO_URL=https://github.com/ExarLabs/academy-lms + AI_TUTOR_REPO_URL=https://github.com/ExarLabs/academy-ai-tutor-chat + + - name: Setup SSH key + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ secrets.HETZNER_SSH_KEY }} + + - name: Add Hetzner server to known hosts + run: | + ssh-keyscan -H ${{ env.HETZNER_HOST }} >> ~/.ssh/known_hosts + + - name: Deploy to Hetzner + run: | + # Copy deployment files to server + scp -r compose.yaml nginx/ scripts/ ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }}:${{ env.DEPLOY_PATH }}/ + + # Copy environment file if it doesn't exist + ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " + cd ${{ env.DEPLOY_PATH }} + if [ ! -f .env ]; then + cp .env.example .env + echo 'Please update .env file with your configuration' + fi + " + + - name: Update and restart services + run: | + ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " + cd ${{ env.DEPLOY_PATH }} + + # Pull latest images + docker compose pull + + # Stop services gracefully + docker compose down --timeout 30 + + # Start services + docker compose up -d + + # Wait for services to be ready + sleep 30 + + # Run migrations on all sites + ./scripts/migrate-all-sites.sh + + # Show status + docker compose ps + " + + - name: Health check + run: | + # Wait a bit more for services to fully start + sleep 60 + + # Check if nginx-proxy is responding + if curl -f http://${{ env.HETZNER_HOST }}/health; then + echo "โœ… Deployment successful - Health check passed" + else + echo "โŒ Health check failed" + # Show logs for debugging + ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " + cd ${{ env.DEPLOY_PATH }} + docker compose logs --tail=50 + " + exit 1 + fi + + - name: Notify deployment status + if: always() + run: | + if [ "${{ job.status }}" == "success" ]; then + echo "๐Ÿš€ Deployment to Hetzner completed successfully!" + echo "๐ŸŒ Access your application at: http://${{ env.HETZNER_HOST }}" + else + echo "๐Ÿ’ฅ Deployment failed. Check the logs above for details." + fi diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 00000000..147883d5 --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,246 @@ +# Academy LMS Deployment Guide + +This repository is configured to automatically deploy the Academy LMS stack to Hetzner Cloud, including: +- Frappe Framework with custom Academy LMS app +- AI Tutor Chat application +- LangChain service for AI functionality + +## Architecture Overview + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ academy-lms โ”‚ โ”‚ academy-ai-tutor โ”‚ โ”‚ academy-langchain โ”‚ +โ”‚ (GitHub Repo) โ”‚ โ”‚ (GitHub Repo) โ”‚ โ”‚ (GitHub Repo) โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ โ”‚ โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + Webhook Triggers + โ”‚ + โ–ผ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ academy_docker โ”‚ + โ”‚ (This Repository) โ”‚ + โ”‚ โ”‚ + โ”‚ โ€ข Builds Docker images โ”‚ + โ”‚ โ€ข Pushes to GHCR โ”‚ + โ”‚ โ€ข Deploys to Hetzner โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ Hetzner Server โ”‚ + โ”‚ 188.245.211.114 โ”‚ + โ”‚ โ”‚ + โ”‚ Running Services: โ”‚ + โ”‚ โ€ข Nginx Proxy โ”‚ + โ”‚ โ€ข Frappe Backend โ”‚ + โ”‚ โ€ข MariaDB โ”‚ + โ”‚ โ€ข Redis โ”‚ + โ”‚ โ€ข LangChain Service โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Setup Instructions + +### 1. Prerequisites + +- GitHub account with access to all repositories +- Hetzner server with Docker and Docker Compose installed +- SSH access to the Hetzner server + +### 2. Repository Secrets + +Configure the following secrets in this repository: + +- `HETZNER_SSH_KEY`: Private SSH key for accessing the Hetzner server +- `ACADEMY_DOCKER_PAT`: GitHub Personal Access Token with `repo` and `write:packages` permissions + +### 3. Webhook Setup + +Add the webhook workflow files to each watched repository: + +1. **For academy-lms repository:** + - Copy `.github/workflows/webhook-academy-lms.yml` to the academy-lms repo + - Add secret `ACADEMY_DOCKER_PAT` with the same PAT + +2. **For academy-ai-tutor-chat repository:** + - Copy `.github/workflows/webhook-academy-ai-tutor.yml` to the academy-ai-tutor-chat repo + - Add secret `ACADEMY_DOCKER_PAT` with the same PAT + +3. **For academy-LangChain repository:** + - Copy `.github/workflows/webhook-academy-langchain.yml` to the academy-LangChain repo + - Add secret `ACADEMY_DOCKER_PAT` with the same PAT + +### 4. Hetzner Server Setup + +1. SSH into your Hetzner server: + ```bash + ssh frappe@188.245.211.114 + ``` + +2. Run the automated setup script: + ```bash + # Download and run setup script + curl -O https://raw.githubusercontent.com/ExarLabs/academy_docker/master/scripts/setup-hetzner.sh + sudo bash setup-hetzner.sh + ``` + + The script will: + - Install Docker and Docker Compose + - Create deployment directory + - Setup Docker networks + - Configure firewall rules + - Create backup directory and cron jobs + - Setup systemd service for auto-start + - Install monitoring tools + +3. Configure environment: + ```bash + # Switch to frappe user + su - frappe + cd /opt/frappe-deployment + + # Copy .env.example to .env and update with your values + cp .env.example .env + nano .env + ``` + + Important variables to update: + - All passwords (MARIADB_ROOT_PASSWORD, ADMIN_PASSWORD, etc.) + - OPENAI_API_KEY for AI functionality + - FRAPPE_SITE_NAME_HEADER with your domain + - Email configuration if needed + +### 5. Initial Deployment + +Trigger the deployment manually: + +1. Go to Actions tab in this repository +2. Select "Deploy Academy LMS to Hetzner" +3. Click "Run workflow" +4. Optionally check "Force rebuild all images" + +### 6. Create Your First Site + +After deployment is complete, create your first site: + +```bash +ssh frappe@188.245.211.114 +cd /opt/frappe-deployment +./scripts/create-site.sh academy.example.com +``` + +The script will: +- Create a new Frappe site +- Install Academy LMS app +- Install AI Tutor Chat app +- Configure the site +- Run migrations + +## Deployment Process + +### Automatic Deployment + +The system automatically deploys when: + +1. **Changes to watched repositories**: Any push to main/master branch triggers deployment +2. **Changes to this repository**: Updates to deployment configuration trigger deployment +3. **Manual trigger**: Use GitHub Actions workflow dispatch + +### What Happens During Deployment + +1. **Build Phase**: + - Builds custom Frappe image with Academy LMS and AI Tutor apps + - Tags and pushes to GitHub Container Registry + +2. **Deploy Phase**: + - Copies deployment files to Hetzner server + - Pulls latest images + - Stops existing services gracefully + - Starts new services + - Runs database migrations on all sites + - Performs health check + +### Migration Process + +The `migrate-all-sites.sh` script automatically: +- Detects all Frappe sites +- Runs `bench migrate` on each site +- Clears cache +- Runs system health check + +## Monitoring and Troubleshooting + +### Check Service Status + +```bash +ssh frappe@188.245.211.114 +cd /opt/frappe-deployment +docker compose ps +docker compose logs -f +``` + +### Manual Migration + +If needed, run migrations manually: + +```bash +ssh frappe@188.245.211.114 +cd /opt/frappe-deployment +./scripts/migrate-all-sites.sh +``` + +### Common Issues + +1. **502 Bad Gateway**: Services are still starting. Wait a few minutes. +2. **Migration Failures**: Check logs with `docker compose logs backend` +3. **Network Issues**: Ensure langchain-network exists: `docker network create langchain-network` + +## Security Considerations + +1. **Secrets Management**: + - Never commit `.env` file + - Use strong passwords + - Rotate credentials regularly + +2. **Network Security**: + - Configure firewall rules on Hetzner + - Use HTTPS in production (see SSL setup below) + +3. **Backup Strategy**: + - Regular database backups + - Store backups off-site + +## SSL/TLS Setup (Production) + +For production, set up SSL certificates: + +1. Install Certbot on Hetzner server +2. Update nginx configuration for SSL +3. Use compose.custom-domain-ssl.yaml override + +## Maintenance + +### Updating Dependencies + +1. Update Frappe version in `images/custom/Containerfile` +2. Update app versions by triggering rebuild +3. Test in staging before production + +### Backup and Restore + +```bash +# Backup +docker compose exec backend bench --site academy.example.com backup + +# Restore +docker compose exec backend bench --site academy.example.com restore [backup-file] +``` + +## Support + +For issues: +1. Check GitHub Actions logs +2. Review server logs +3. Open issue in this repository diff --git a/README.md b/README.md index 39f843eb..a858b37a 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,117 @@ -[![Build Stable](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml) -[![Build Develop](https://github.com/frappe/frappe_docker/actions/workflows/build_develop.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_develop.yml) +# Academy Docker - Automated Deployment for Academy LMS Stack -Everything about [Frappe](https://github.com/frappe/frappe) and [ERPNext](https://github.com/frappe/erpnext) in containers. +This repository provides an automated deployment solution for the Academy LMS stack on Hetzner Cloud. It monitors changes in the application repositories and automatically builds, pushes, and deploys updated Docker images. -# Getting Started +## ๐ŸŽฏ Purpose -To get started you need [Docker](https://docs.docker.com/get-docker/), [docker-compose](https://docs.docker.com/compose/), and [git](https://docs.github.com/en/get-started/getting-started-with-git/set-up-git) setup on your machine. For Docker basics and best practices refer to Docker's [documentation](http://docs.docker.com). +This is a fork of [frappe/frappe_docker](https://github.com/frappe/frappe_docker) customized to: +- Automatically deploy the Academy LMS stack with custom Frappe apps +- Monitor and react to changes in watched repositories +- Provide CI/CD pipeline for Hetzner deployment +- Integrate AI-powered tutoring capabilities via LangChain -Once completed, chose one of the following two sections for next steps. +## ๐Ÿ“ฆ Components -### Try in Play With Docker +The stack includes: -To play in an already set up sandbox, in your browser, click the button below: +1. **[Academy LMS](https://github.com/ExarLabs/academy-lms)** - Custom fork of Frappe LMS +2. **[Academy AI Tutor Chat](https://github.com/ExarLabs/academy-ai-tutor-chat)** - AI-powered tutoring Frappe app +3. **[Academy LangChain](https://github.com/ExarLabs/academy-LangChain)** - LangChain service for AI functionality +4. **Frappe Framework** - The underlying framework +5. **Supporting Services** - MariaDB, Redis, PostgreSQL, Nginx - - Try in PWD - +## ๐Ÿš€ Quick Start -### Try on your Dev environment +For detailed deployment instructions, see [DEPLOYMENT.md](DEPLOYMENT.md). -First clone the repo: +### Prerequisites -```sh -git clone https://github.com/frappe/frappe_docker -cd frappe_docker +- GitHub account with access to all repositories +- Hetzner server (Ubuntu 20.04+ recommended) +- GitHub Personal Access Token +- OpenAI API key (for AI features) + +### Basic Setup + +1. Fork this repository +2. Configure GitHub secrets: + - `HETZNER_SSH_KEY` + - `ACADEMY_DOCKER_PAT` +3. Add webhook workflows to watched repositories +4. Run setup script on Hetzner server +5. Configure environment variables +6. Trigger initial deployment + +## ๐Ÿ”„ Automated Workflow + +```mermaid +graph LR + A[Code Push] --> B[Webhook Trigger] + B --> C[Build Docker Image] + C --> D[Push to GHCR] + D --> E[Deploy to Hetzner] + E --> F[Run Migrations] + F --> G[Health Check] ``` -Then run: `docker compose -f pwd.yml up -d` +## ๐Ÿ“ Repository Structure -### To run on ARM64 architecture follow this instructions +``` +academy_docker/ +โ”œโ”€โ”€ .github/workflows/ # CI/CD workflows +โ”‚ โ”œโ”€โ”€ deploy.yml # Main deployment workflow +โ”‚ โ””โ”€โ”€ webhook-*.yml # Webhook templates for watched repos +โ”œโ”€โ”€ images/ # Docker image definitions +โ”‚ โ””โ”€โ”€ custom/ # Custom Frappe image with apps +โ”œโ”€โ”€ nginx/ # Nginx configuration +โ”œโ”€โ”€ scripts/ # Utility scripts +โ”‚ โ”œโ”€โ”€ migrate-all-sites.sh +โ”‚ โ””โ”€โ”€ setup-hetzner.sh +โ”œโ”€โ”€ compose.yaml # Docker Compose configuration +โ”œโ”€โ”€ .env.example # Environment variables template +โ””โ”€โ”€ DEPLOYMENT.md # Detailed deployment guide +``` -After cloning the repo run this command to build multi-architecture images specifically for ARM64. +## ๐Ÿ”ง Configuration -`docker buildx bake --no-cache --set "*.platform=linux/arm64"` +Key environment variables: -and then +- `MARIADB_ROOT_PASSWORD` - Database root password +- `ADMIN_PASSWORD` - Frappe admin password +- `OPENAI_API_KEY` - OpenAI API key for AI features +- `FRAPPE_SITE_NAME_HEADER` - Your domain name +- `LANGCHAIN_API_URL` - LangChain service URL -- add `platform: linux/arm64` to all services in the `pwd.yml` -- replace the current specified versions of erpnext image on `pwd.yml` with `:latest` +## ๐Ÿ›ก๏ธ Security -Then run: `docker compose -f pwd.yml up -d` +- All secrets stored in GitHub Secrets +- Firewall rules configured automatically +- SSL/TLS support for production +- Regular automated backups -## Final steps +## ๐Ÿ“Š Monitoring -Wait for 5 minutes for ERPNext site to be created or check `create-site` container logs before opening browser on port 8080. (username: `Administrator`, password: `admin`) +- Check service status: `docker compose ps` +- View logs: `docker compose logs -f` +- System health: `docker compose exec backend bench doctor` -If you ran in a Dev Docker environment, to view container logs: `docker compose -f pwd.yml logs -f create-site`. Don't worry about some of the initial error messages, some services take a while to become ready, and then they go away. +## ๐Ÿค Contributing -# Documentation +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Submit a pull request -### [Frequently Asked Questions](https://github.com/frappe/frappe_docker/wiki/Frequently-Asked-Questions) +## ๐Ÿ“ License -### [Production](#production) +This project inherits the license from the original [frappe_docker](https://github.com/frappe/frappe_docker) repository. -- [List of containers](docs/list-of-containers.md) -- [Single Compose Setup](docs/single-compose-setup.md) -- [Environment Variables](docs/environment-variables.md) -- [Single Server Example](docs/single-server-example.md) -- [Setup Options](docs/setup-options.md) -- [Site Operations](docs/site-operations.md) -- [Backup and Push Cron Job](docs/backup-and-push-cronjob.md) -- [Port Based Multi Tenancy](docs/port-based-multi-tenancy.md) -- [Migrate from multi-image setup](docs/migrate-from-multi-image-setup.md) -- [running on linux/mac](docs/setup_for_linux_mac.md) -- [TLS for local deployment](docs/tls-for-local-deployment.md) +## ๐Ÿ†˜ Support -### [Custom Images](#custom-images) +- Check [DEPLOYMENT.md](DEPLOYMENT.md) for detailed instructions +- Review GitHub Actions logs for deployment issues +- Open an issue for bugs or feature requests -- [Custom Apps](docs/custom-apps.md) -- [Custom Apps with podman](docs/custom-apps-podman.md) -- [Build Version 10 Images](docs/build-version-10-images.md) +--- -### [Development](#development) - -- [Development using containers](docs/development.md) -- [Bench Console and VSCode Debugger](docs/bench-console-and-vscode-debugger.md) -- [Connect to localhost services](docs/connect-to-localhost-services-from-containers-for-local-app-development.md) - -### [Troubleshoot](docs/troubleshoot.md) - -# Contributing - -If you want to contribute to this repo refer to [CONTRIBUTING.md](CONTRIBUTING.md) - -This repository is only for container related stuff. You also might want to contribute to: - -- [Frappe framework](https://github.com/frappe/frappe#contributing), -- [ERPNext](https://github.com/frappe/erpnext#contributing), -- [Frappe Bench](https://github.com/frappe/bench). +Built with โค๏ธ for automated Academy LMS deployment diff --git a/compose.yaml b/compose.yaml index 7c8e64f2..43ef41f7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,8 +1,7 @@ -x-customizable-image: &customizable_image - # By default the image used only contains the `frappe` and `erpnext` apps. - # See https://github.com/frappe/frappe_docker/blob/main/docs/custom-apps.md - # about using custom images. - image: ${CUSTOM_IMAGE:-frappe/erpnext}:${CUSTOM_TAG:-$ERPNEXT_VERSION} +x-customizable-image: + # Custom Academy LMS image with all required apps + &customizable_image + image: ${CUSTOM_IMAGE:-ghcr.io/exarlabs/academy-frappe}:${CUSTOM_TAG:-latest} pull_policy: ${PULL_POLICY:-always} restart: ${RESTART_POLICY:-unless-stopped} @@ -12,39 +11,120 @@ x-depends-on-configurator: &depends_on_configurator condition: service_completed_successfully x-backend-defaults: &backend_defaults - <<: [*depends_on_configurator, *customizable_image] + <<: [ *depends_on_configurator, *customizable_image ] volumes: - sites:/home/frappe/frappe-bench/sites + networks: + - frappe-network + - langchain-network services: + # Nginx reverse proxy for multi-site support + nginx-proxy: + image: nginx:alpine + ports: + - "80:80" + - "443:443" + volumes: + - ./nginx/conf.d:/etc/nginx/conf.d:ro + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - sites:/var/www/html/sites:ro + - ./ssl:/etc/nginx/ssl:ro + depends_on: + - frontend + networks: + - frappe-network + restart: unless-stopped + + # MariaDB database (from academy-lms) + mariadb: + image: mariadb:10.8 + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_unicode_ci + - --skip-character-set-client-handshake + - --skip-innodb-read-only-compressed + environment: + MARIADB_ROOT_PASSWORD: ${MARIADB_ROOT_PASSWORD} + volumes: + - mariadb-data:/var/lib/mysql + - ./backups:/backups + networks: + - frappe-network + restart: unless-stopped + + # Redis (shared between frappe and langchain) + redis: + image: redis:alpine + networks: + - frappe-network + - langchain-network + restart: unless-stopped + + # PostgreSQL for LangChain service + postgres: + image: postgres:15 + environment: + POSTGRES_DB: ${LANGCHAIN_DB_NAME:-langchain_db} + POSTGRES_USER: ${LANGCHAIN_DB_USER:-langchain_user} + POSTGRES_PASSWORD: ${LANGCHAIN_DB_PASSWORD:-langchain_pass} + volumes: + - postgres-data:/var/lib/postgresql/data + networks: + - langchain-network + restart: unless-stopped + + # LangChain service for AI functionality + langchain-service: + image: ${LANGCHAIN_IMAGE:-ghcr.io/exarlabs/academy-langchain}:${LANGCHAIN_TAG:-latest} + environment: + - OPENAI_API_KEY=${OPENAI_API_KEY} + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + + - DATABASE_URL=postgresql://${LANGCHAIN_DB_USER:-langchain_user}:${LANGCHAIN_DB_PASSWORD:-langchain_pass}@postgres:5432/${LANGCHAIN_DB_NAME:-langchain_db} + - REDIS_URL=redis://redis:6379 + - ENV=${LANGCHAIN_ENV:-development} + - DEBUG=${LANGCHAIN_DEBUG:-true} + networks: + - langchain-network + depends_on: + - postgres + - redis + restart: unless-stopped + ports: + - "8001:8000" # Expose on different port to avoid conflict + configurator: <<: *backend_defaults platform: linux/amd64 entrypoint: - bash - -c - # add redis_socketio for backward compatibility command: - > - ls -1 apps > sites/apps.txt; - bench set-config -g db_host $$DB_HOST; - bench set-config -gp db_port $$DB_PORT; - bench set-config -g redis_cache "redis://$$REDIS_CACHE"; - bench set-config -g redis_queue "redis://$$REDIS_QUEUE"; - bench set-config -g redis_socketio "redis://$$REDIS_QUEUE"; - bench set-config -gp socketio_port $$SOCKETIO_PORT; + ls -1 apps > sites/apps.txt; bench set-config -g db_host mariadb; bench set-config -gp db_port 3306; bench set-config -g redis_cache "redis://redis:6379"; bench set-config -g redis_queue "redis://redis:6379"; bench set-config -g redis_socketio "redis://redis:6379"; bench set-config -gp socketio_port 9000; bench set-config -g ai_tutor_api_url "${AI_TUTOR_API_URL}"; environment: - DB_HOST: ${DB_HOST:-} - DB_PORT: ${DB_PORT:-} - REDIS_CACHE: ${REDIS_CACHE:-} - REDIS_QUEUE: ${REDIS_QUEUE:-} + DB_HOST: mariadb + DB_PORT: 3306 + REDIS_CACHE: redis:6379 + REDIS_QUEUE: redis:6379 SOCKETIO_PORT: 9000 - depends_on: {} + AI_TUTOR_API_URL: ${AI_TUTOR_API_URL:-http://langchain-service:8000} + depends_on: + - mariadb + - redis restart: on-failure backend: <<: *backend_defaults platform: linux/amd64 + environment: + - LANGCHAIN_SERVICE_URL=http://langchain-service:8000 + - AI_TUTOR_API_URL=${AI_TUTOR_API_URL:-http://langchain-service:8000} + depends_on: + - mariadb + - redis + - langchain-service frontend: <<: *customizable_image @@ -62,18 +142,23 @@ services: CLIENT_MAX_BODY_SIZE: ${CLIENT_MAX_BODY_SIZE:-50m} volumes: - sites:/home/frappe/frappe-bench/sites + networks: + - frappe-network depends_on: - backend - websocket + restart: unless-stopped websocket: - <<: [*depends_on_configurator, *customizable_image] + <<: [ *depends_on_configurator, *customizable_image ] platform: linux/amd64 command: - node - /home/frappe/frappe-bench/apps/frappe/socketio.js volumes: - sites:/home/frappe/frappe-bench/sites + networks: + - frappe-network queue-short: <<: *backend_defaults @@ -90,6 +175,15 @@ services: platform: linux/amd64 command: bench schedule -# ERPNext requires local assets access (Frappe does not) +# Volumes for persistent data volumes: sites: + mariadb-data: + postgres-data: + + # Networks for service communication +networks: + frappe-network: + driver: bridge + langchain-network: + driver: bridge diff --git a/images/custom/Containerfile b/images/custom/Containerfile index 208f13e3..d90f1ae2 100644 --- a/images/custom/Containerfile +++ b/images/custom/Containerfile @@ -12,123 +12,132 @@ ENV NVM_DIR=/home/frappe/.nvm ENV PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} RUN useradd -ms /bin/bash frappe \ - && apt-get update \ - && apt-get install --no-install-recommends -y \ - curl \ - git \ - vim \ - nginx \ - gettext-base \ - file \ - # weasyprint dependencies - libpango-1.0-0 \ - libharfbuzz0b \ - libpangoft2-1.0-0 \ - libpangocairo-1.0-0 \ - # For backups - restic \ - gpg \ - # MariaDB - mariadb-client \ - less \ - # Postgres - libpq-dev \ - postgresql-client \ - # For healthcheck - wait-for-it \ - jq \ - # NodeJS - && mkdir -p ${NVM_DIR} \ - && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash \ - && . ${NVM_DIR}/nvm.sh \ - && nvm install ${NODE_VERSION} \ - && nvm use v${NODE_VERSION} \ - && npm install -g yarn \ - && nvm alias default v${NODE_VERSION} \ - && rm -rf ${NVM_DIR}/.cache \ - && echo 'export NVM_DIR="/home/frappe/.nvm"' >>/home/frappe/.bashrc \ - && echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >>/home/frappe/.bashrc \ - && echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >>/home/frappe/.bashrc \ - # Install wkhtmltopdf with patched qt - && if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ - && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ - && downloaded_file=wkhtmltox_${WKHTMLTOPDF_VERSION}.${WKHTMLTOPDF_DISTRO}_${ARCH}.deb \ - && curl -sLO https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ - && apt-get install -y ./$downloaded_file \ - && rm $downloaded_file \ - # Clean up - && rm -rf /var/lib/apt/lists/* \ - && rm -fr /etc/nginx/sites-enabled/default \ - && pip3 install frappe-bench \ - # Fixes for non-root nginx and logs to stdout - && sed -i '/user www-data/d' /etc/nginx/nginx.conf \ - && ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log \ - && touch /run/nginx.pid \ - && chown -R frappe:frappe /etc/nginx/conf.d \ - && chown -R frappe:frappe /etc/nginx/nginx.conf \ - && chown -R frappe:frappe /var/log/nginx \ - && chown -R frappe:frappe /var/lib/nginx \ - && chown -R frappe:frappe /run/nginx.pid \ - && chmod 755 /usr/local/bin/nginx-entrypoint.sh \ - && chmod 644 /templates/nginx/frappe.conf.template + && apt-get update \ + && apt-get install --no-install-recommends -y \ + curl \ + git \ + vim \ + nginx \ + gettext-base \ + file \ + # weasyprint dependencies + libpango-1.0-0 \ + libharfbuzz0b \ + libpangoft2-1.0-0 \ + libpangocairo-1.0-0 \ + # For backups + restic \ + gpg \ + # MariaDB + mariadb-client \ + less \ + # Postgres + libpq-dev \ + postgresql-client \ + # For healthcheck + wait-for-it \ + jq \ + # NodeJS + && mkdir -p ${NVM_DIR} \ + && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash \ + && . ${NVM_DIR}/nvm.sh \ + && nvm install ${NODE_VERSION} \ + && nvm use v${NODE_VERSION} \ + && npm install -g yarn \ + && nvm alias default v${NODE_VERSION} \ + && rm -rf ${NVM_DIR}/.cache \ + && echo 'export NVM_DIR="/home/frappe/.nvm"' >>/home/frappe/.bashrc \ + && echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm' >>/home/frappe/.bashrc \ + && echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion' >>/home/frappe/.bashrc \ + # Install wkhtmltopdf with patched qt + && if [ "$(uname -m)" = "aarch64" ]; then export ARCH=arm64; fi \ + && if [ "$(uname -m)" = "x86_64" ]; then export ARCH=amd64; fi \ + && downloaded_file=wkhtmltox_${WKHTMLTOPDF_VERSION}.${WKHTMLTOPDF_DISTRO}_${ARCH}.deb \ + && curl -sLO https://github.com/wkhtmltopdf/packaging/releases/download/$WKHTMLTOPDF_VERSION/$downloaded_file \ + && apt-get install -y ./$downloaded_file \ + && rm $downloaded_file \ + # Clean up + && rm -rf /var/lib/apt/lists/* \ + && rm -fr /etc/nginx/sites-enabled/default \ + && pip3 install frappe-bench \ + # Fixes for non-root nginx and logs to stdout + && sed -i '/user www-data/d' /etc/nginx/nginx.conf \ + && ln -sf /dev/stdout /var/log/nginx/access.log && ln -sf /dev/stderr /var/log/nginx/error.log \ + && touch /run/nginx.pid \ + && chown -R frappe:frappe /etc/nginx/conf.d \ + && chown -R frappe:frappe /etc/nginx/nginx.conf \ + && chown -R frappe:frappe /var/log/nginx \ + && chown -R frappe:frappe /var/lib/nginx \ + && chown -R frappe:frappe /run/nginx.pid \ + && chmod 755 /usr/local/bin/nginx-entrypoint.sh \ + && chmod 644 /templates/nginx/frappe.conf.template FROM base AS builder RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - # For frappe framework - wget \ - #for building arm64 binaries - libcairo2-dev \ - libpango1.0-dev \ - libjpeg-dev \ - libgif-dev \ - librsvg2-dev \ - # For psycopg2 - libpq-dev \ - # Other - libffi-dev \ - liblcms2-dev \ - libldap2-dev \ - libmariadb-dev \ - libsasl2-dev \ - libtiff5-dev \ - libwebp-dev \ - pkg-config \ - redis-tools \ - rlwrap \ - tk8.6-dev \ - cron \ - # For pandas - gcc \ - build-essential \ - libbz2-dev \ - && rm -rf /var/lib/apt/lists/* + && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + # For frappe framework + wget \ + #for building arm64 binaries + libcairo2-dev \ + libpango1.0-dev \ + libjpeg-dev \ + libgif-dev \ + librsvg2-dev \ + # For psycopg2 + libpq-dev \ + # Other + libffi-dev \ + liblcms2-dev \ + libldap2-dev \ + libmariadb-dev \ + libsasl2-dev \ + libtiff5-dev \ + libwebp-dev \ + pkg-config \ + redis-tools \ + rlwrap \ + tk8.6-dev \ + cron \ + # For pandas + gcc \ + build-essential \ + libbz2-dev \ + && rm -rf /var/lib/apt/lists/* # apps.json includes ARG APPS_JSON_BASE64 RUN if [ -n "${APPS_JSON_BASE64}" ]; then \ - mkdir /opt/frappe && echo "${APPS_JSON_BASE64}" | base64 -d > /opt/frappe/apps.json; \ + mkdir /opt/frappe && echo "${APPS_JSON_BASE64}" | base64 -d > /opt/frappe/apps.json; \ fi USER frappe ARG FRAPPE_BRANCH=version-15 ARG FRAPPE_PATH=https://github.com/frappe/frappe -RUN export APP_INSTALL_ARGS="" && \ - if [ -n "${APPS_JSON_BASE64}" ]; then \ - export APP_INSTALL_ARGS="--apps_path=/opt/frappe/apps.json"; \ - fi && \ - bench init ${APP_INSTALL_ARGS}\ - --frappe-branch=${FRAPPE_BRANCH} \ - --frappe-path=${FRAPPE_PATH} \ - --no-procfile \ - --no-backups \ - --skip-redis-config-generation \ - --verbose \ - /home/frappe/frappe-bench && \ - cd /home/frappe/frappe-bench && \ - echo "{}" > sites/common_site_config.json && \ +ARG LMS_REPO_URL=https://github.com/ExarLabs/academy-lms +ARG AI_TUTOR_REPO_URL=https://github.com/ExarLabs/academy-ai-tutor-chat + +# Initialize Frappe bench +RUN bench init \ + --frappe-branch=${FRAPPE_BRANCH} \ + --frappe-path=${FRAPPE_PATH} \ + --no-procfile \ + --no-backups \ + --skip-redis-config-generation \ + --verbose \ + /home/frappe/frappe-bench + +WORKDIR /home/frappe/frappe-bench + +# Install Academy LMS app +RUN bench get-app lms ${LMS_REPO_URL} + +# Install AI Tutor Chat app +RUN bench get-app ai_tutor_chat ${AI_TUTOR_REPO_URL} + +# Setup common site config +RUN echo "{}" > sites/common_site_config.json && \ find apps -mindepth 1 -path "*/.git" | xargs rm -fr FROM base AS backend @@ -143,7 +152,7 @@ VOLUME [ \ "/home/frappe/frappe-bench/sites", \ "/home/frappe/frappe-bench/sites/assets", \ "/home/frappe/frappe-bench/logs" \ -] + ] CMD [ \ "/home/frappe/frappe-bench/env/bin/gunicorn", \ @@ -156,4 +165,4 @@ CMD [ \ "--timeout=120", \ "--preload", \ "frappe.app:application" \ -] + ] diff --git a/nginx/502.html b/nginx/502.html new file mode 100644 index 00000000..3326012a --- /dev/null +++ b/nginx/502.html @@ -0,0 +1,59 @@ + + + + + + + Service Temporarily Unavailable + + + + +
+
502
+

Service Temporarily Unavailable

+

The server is temporarily unable to service your request. This may be due to maintenance or capacity problems. +

+

Please try again in a few moments.

+
+ + + \ No newline at end of file diff --git a/nginx/conf.d/default.conf b/nginx/conf.d/default.conf new file mode 100644 index 00000000..b5b9f14d --- /dev/null +++ b/nginx/conf.d/default.conf @@ -0,0 +1,67 @@ +# Default server configuration +upstream frappe-bench-frappe { + server backend:8000 fail_timeout=0; +} + +upstream frappe-bench-socketio { + server websocket:9000 fail_timeout=0; +} + +# HTTP server +server { + listen 80; + server_name _; + + root /workspace/development/frappe-bench/sites; + + # Health check endpoint + location /health { + access_log off; + return 200 "healthy\n"; + add_header Content-Type text/plain; + } + + # Static files + location /assets { + try_files $uri =404; + add_header Cache-Control "max-age=31536000"; + } + + location ~ ^/protected/(.*) { + internal; + try_files /sites/$host/$1 =404; + } + + # Socket.io + location /socket.io { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Frappe-Site-Name $host; + proxy_set_header Origin $scheme://$http_host; + proxy_set_header Host $host; + + proxy_pass http://frappe-bench-socketio; + } + + # Main application + location / { + proxy_http_version 1.1; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Frappe-Site-Name $host; + proxy_set_header Host $host; + proxy_set_header X-Use-X-Accel-Redirect True; + proxy_read_timeout 120; + proxy_redirect off; + + proxy_pass http://frappe-bench-frappe; + } + + # Error pages + error_page 502 /502.html; + location = /502.html { + root /usr/share/nginx/html; + internal; + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..5c7afe27 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,43 @@ +# Nginx configuration for Academy LMS +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + client_max_body_size 50M; + + # Gzip compression + gzip on; + gzip_vary on; + gzip_proxied any; + gzip_comp_level 6; + gzip_types text/plain text/css text/xml text/javascript application/json application/javascript application/xml+rss application/rss+xml application/atom+xml image/svg+xml; + + # Security headers + add_header X-Frame-Options "SAMEORIGIN" always; + add_header X-Content-Type-Options "nosniff" always; + add_header X-XSS-Protection "1; mode=block" always; + add_header Referrer-Policy "no-referrer-when-downgrade" always; + + # Include site configurations + include /etc/nginx/conf.d/*.conf; +} diff --git a/scripts/create-site.sh b/scripts/create-site.sh new file mode 100644 index 00000000..5fdcfc66 --- /dev/null +++ b/scripts/create-site.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Script to create a new Frappe site with Academy apps + +set -e + +# Check if site name is provided +if [ -z "$1" ]; then + echo "โŒ Usage: $0 " + echo "Example: $0 academy.example.com" + exit 1 +fi + +SITE_NAME=$1 + +echo "๐ŸŒ Creating new site: $SITE_NAME" + +# Load environment variables +if [ -f .env ]; then + export $(cat .env | grep -v '^#' | xargs) +else + echo "โŒ .env file not found. Please create it from .env.example" + exit 1 +fi + +# Create the site +echo "๐Ÿ“ฆ Creating Frappe site..." +docker compose exec -T backend bench new-site \ + --no-mariadb-socket \ + --admin-password="$ADMIN_PASSWORD" \ + --db-root-password="$MARIADB_ROOT_PASSWORD" \ + "$SITE_NAME" + +# Install LMS +echo "๐Ÿ“ฆ Installing Academy LMS..." +docker compose exec -T backend bench --site "$SITE_NAME" install-app lms + +# Install AI Tutor Chat +echo "๐Ÿ“ฆ Installing AI Tutor Chat..." +docker compose exec -T backend bench --site "$SITE_NAME" install-app academy_ai_tutor_chat + +# Set as default site (optional) +read -p "Set as default site? (y/n) " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + docker compose exec -T backend bench use "$SITE_NAME" + echo "โœ… Set as default site" +fi + +# Clear cache +echo "๐Ÿงน Clearing cache..." +docker compose exec -T backend bench --site "$SITE_NAME" clear-cache + +# Run migrations +echo "๐Ÿ”„ Running migrations..." +docker compose exec -T backend bench --site "$SITE_NAME" migrate + +echo "โœ… Site created successfully!" +echo "" +echo "๐Ÿ“‹ Site details:" +echo "URL: http://$SITE_NAME" +echo "Username: Administrator" +echo "Password: $ADMIN_PASSWORD" +echo "" +echo "๐Ÿ” Remember to:" +echo "1. Update your DNS to point to the server IP" +echo "2. Configure SSL certificate for production" +echo "3. Update nginx configuration if needed" diff --git a/scripts/migrate-all-sites.sh b/scripts/migrate-all-sites.sh new file mode 100644 index 00000000..d884b5c7 --- /dev/null +++ b/scripts/migrate-all-sites.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Script to run migrations on all Frappe sites + +set -e + +echo "๐Ÿ”„ Starting migration for all sites..." + +# Get list of all sites +SITES=$(docker compose exec -T backend ls -1 /workspace/development/frappe-bench/sites | grep -v '^apps.txt$' | grep -v '^common_site_config.json$' | grep -v '^assets$' | grep -v '^\..*$') + +if [ -z "$SITES" ]; then + echo "โŒ No sites found!" + exit 1 +fi + +echo "๐Ÿ“‹ Found sites:" +echo "$SITES" +echo "" + +# Run migrate for each site +for site in $SITES; do + echo "๐Ÿ”ง Migrating site: $site" + + # Run bench migrate + docker compose exec -T backend bench --site "$site" migrate || { + echo "โŒ Migration failed for site: $site" + exit 1 + } + + # Clear cache + docker compose exec -T backend bench --site "$site" clear-cache || { + echo "โš ๏ธ Warning: Failed to clear cache for site: $site" + } + + echo "โœ… Migration completed for site: $site" + echo "" +done + +echo "๐ŸŽ‰ All migrations completed successfully!" + +# Optional: Run bench doctor to check system health +echo "๐Ÿฅ Running system health check..." +docker compose exec -T backend bench doctor || { + echo "โš ๏ธ Warning: Some health checks failed" +} + +echo "โœจ Migration process finished!" diff --git a/scripts/setup-hetzner.sh b/scripts/setup-hetzner.sh new file mode 100644 index 00000000..a829c959 --- /dev/null +++ b/scripts/setup-hetzner.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Initial setup script for Hetzner server + +set -e + +echo "๐Ÿš€ Starting Academy LMS Hetzner setup..." + +# Check if running as root or with sudo +if [ "$EUID" -ne 0 ]; then + echo "โŒ Please run this script with sudo or as root" + exit 1 +fi + +# Update system +echo "๐Ÿ“ฆ Updating system packages..." +apt-get update && apt-get upgrade -y + +# Install Docker if not already installed +if ! command -v docker &> /dev/null; then + echo "๐Ÿณ Installing Docker..." + curl -fsSL https://get.docker.com -o get-docker.sh + sh get-docker.sh + rm get-docker.sh + + # Add frappe user to docker group + usermod -aG docker frappe +else + echo "โœ… Docker is already installed" +fi + +# Install Docker Compose if not already installed +if ! command -v docker-compose &> /dev/null; then + echo "๐Ÿณ Installing Docker Compose..." + curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose +else + echo "โœ… Docker Compose is already installed" +fi + +# Create deployment directory +echo "๐Ÿ“ Creating deployment directory..." +mkdir -p /opt/frappe-deployment +chown frappe:frappe /opt/frappe-deployment + +# Create required networks +echo "๐ŸŒ Creating Docker networks..." +docker network create langchain-network 2>/dev/null || echo "Network langchain-network already exists" + +# Setup firewall rules +echo "๐Ÿ”ฅ Configuring firewall..." +ufw allow 22/tcp # SSH +ufw allow 80/tcp # HTTP +ufw allow 443/tcp # HTTPS +ufw allow 8001/tcp # LangChain service (if needed for debugging) +ufw --force enable + +# Create backup directory +echo "๐Ÿ’พ Creating backup directory..." +mkdir -p /opt/frappe-deployment/backups +chown frappe:frappe /opt/frappe-deployment/backups + +# Setup cron for automated backups (optional) +echo "โฐ Setting up automated backup cron job..." +cat > /etc/cron.d/frappe-backup << EOF +# Backup Frappe sites daily at 2 AM +0 2 * * * frappe cd /opt/frappe-deployment && docker compose exec -T backend bench --site all backup --with-files >> /opt/frappe-deployment/backups/backup.log 2>&1 +# Clean old backups (keep last 7 days) +0 3 * * * frappe find /opt/frappe-deployment/backups -name "*.sql.gz" -mtime +7 -delete +EOF + +# Install monitoring tools (optional) +echo "๐Ÿ“Š Installing monitoring tools..." +apt-get install -y htop iotop ncdu + +# Create systemd service for auto-start +echo "๐Ÿ”ง Creating systemd service..." +cat > /etc/systemd/system/academy-lms.service << EOF +[Unit] +Description=Academy LMS Docker Compose Application +Requires=docker.service +After=docker.service network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +User=frappe +Group=frappe +WorkingDirectory=/opt/frappe-deployment +ExecStart=/usr/local/bin/docker-compose up -d +ExecStop=/usr/local/bin/docker-compose down +ExecReload=/usr/local/bin/docker-compose restart + +[Install] +WantedBy=multi-user.target +EOF + +systemctl daemon-reload +systemctl enable academy-lms.service + +echo "โœ… Setup completed!" +echo "" +echo "๐Ÿ“‹ Next steps:" +echo "1. Switch to frappe user: su - frappe" +echo "2. Go to deployment directory: cd /opt/frappe-deployment" +echo "3. Copy your deployment files there" +echo "4. Create .env file from .env.example and configure it" +echo "5. Start services: docker compose up -d" +echo "" +echo "๐Ÿ” Security reminder:" +echo "- Change all default passwords in .env file" +echo "- Configure SSL certificates for production" +echo "- Review firewall rules for your specific needs"