From e8e16b0e40022fe1697e2d89aaccda131f846d33 Mon Sep 17 00:00:00 2001 From: Gabos Levente Date: Fri, 27 Jun 2025 10:11:21 +0300 Subject: [PATCH] KAN-63: create and upload langchain image --- .env.example | 70 ++++---------- .github/workflows/deploy.yml | 178 +++++++++++++++++++++-------------- DEPLOYMENT.md | 41 ++++++++ LOCAL_TESTING.md | 139 +++++++++++++++++++++++++++ compose.yaml | 2 +- 5 files changed, 306 insertions(+), 124 deletions(-) create mode 100644 LOCAL_TESTING.md diff --git a/.env.example b/.env.example index dc1c07e0..5b753253 100644 --- a/.env.example +++ b/.env.example @@ -1,65 +1,27 @@ -# Frappe/ERPNext Environment Configuration +# Academy LMS Environment Configuration -# Database Configuration -DB_HOST=db -DB_PORT=3306 +# REQUIRED: Database Configuration MARIADB_ROOT_PASSWORD=changeme123 -MARIADB_DATABASE=frappe -MARIADB_USER=frappe -MARIADB_PASSWORD=changeme456 -# Redis Configuration -REDIS_CACHE=redis-cache:6379 -REDIS_QUEUE=redis-queue:6379 -REDIS_SOCKETIO=redis-socketio:6379 - -# Frappe Configuration -FRAPPE_SITE_NAME_HEADER=academy.example.com -FRAPPE_DEFAULT_SITE=academy.example.com -FRAPPE_SITES_DIR=/workspace/development/frappe-bench/sites - -# Security -ADMIN_PASSWORD=changeme789 -ENCRYPTION_KEY=changeme_32_character_encryption_key_here - -# LangChain Service Configuration -LANGCHAIN_API_URL=http://langchain-service:8080 -LANGCHAIN_API_KEY=changeme_langchain_api_key - -# OpenAI Configuration (for AI Tutor) +# REQUIRED: AI Configuration OPENAI_API_KEY=your_openai_api_key_here +ANTHROPIC_API_KEY=your_anthropic_api_key_here # Optional, but recommended -# Anthropic Configuration (optional, for Claude models) -ANTHROPIC_API_KEY=your_anthropic_api_key_here - -# LangChain Database Configuration +# REQUIRED: LangChain Database LANGCHAIN_DB_NAME=langchain_db LANGCHAIN_DB_USER=langchain_user LANGCHAIN_DB_PASSWORD=changeme_langchain_db_password + +# Service URLs (usually don't need to change) +AI_TUTOR_API_URL=http://langchain-service:8000 + +# Optional: Frappe Site Configuration +FRAPPE_SITE_NAME_HEADER=$$host # Use $$host for multi-site support + +# Optional: Environment Settings LANGCHAIN_ENV=production LANGCHAIN_DEBUG=false -# AI Tutor API Configuration -AI_TUTOR_API_URL=http://langchain-service:8000 - -# Email Configuration (optional) -MAIL_SERVER=smtp.gmail.com -MAIL_PORT=587 -MAIL_USE_TLS=1 -MAIL_USERNAME=your-email@gmail.com -MAIL_PASSWORD=your-app-password - -# Backup Configuration (optional) -BACKUP_RETENTION_DAYS=7 -BACKUP_PATH=/backups - -# Development/Production Mode -FRAPPE_ENV=production -DEVELOPER_MODE=0 - -# Network Configuration -COMPOSE_PROJECT_NAME=academy-lms -NETWORK_NAME=langchain-network - -# Hetzner Specific (if needed) -EXTERNAL_IP=188.245.211.114 +# Optional: Custom Image Tags (for development) +# CUSTOM_TAG=develop +# LANGCHAIN_TAG=develop diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 91929234..3e32480d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -9,6 +9,14 @@ on: required: false default: 'false' type: boolean + environment: + description: 'Deployment environment' + required: false + default: 'production' + type: choice + options: + - production + - test # Webhook triggers from watched repositories repository_dispatch: @@ -32,6 +40,8 @@ env: jobs: build-and-deploy: runs-on: ubuntu-latest + # Use environment for secrets management + environment: ${{ github.event.inputs.environment || 'production' }} permissions: contents: read packages: write @@ -54,7 +64,7 @@ jobs: id: meta-frappe uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/exarlabs/academy-frappe + images: ${{ env.REGISTRY }}/exarlabs/ignis-academy-lms tags: | type=ref,event=branch type=ref,event=pr @@ -75,77 +85,107 @@ jobs: LMS_REPO_URL=https://github.com/ExarLabs/academy-lms AI_TUTOR_REPO_URL=https://github.com/ExarLabs/academy-ai-tutor-chat - - name: Setup SSH key - uses: webfactory/ssh-agent@v0.8.0 + - name: Clone and prepare LangChain service + run: | + git clone https://github.com/ExarLabs/academy-LangChain.git langchain-temp + + - name: Extract metadata for LangChain image + id: meta-langchain + uses: docker/metadata-action@v5 with: - ssh-private-key: ${{ secrets.HETZNER_SSH_KEY }} + images: ${{ env.REGISTRY }}/exarlabs/academy-langchain + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha,prefix={{branch}}- + type=raw,value=latest,enable={{is_default_branch}} - - name: Add Hetzner server to known hosts - run: | - ssh-keyscan -H ${{ env.HETZNER_HOST }} >> ~/.ssh/known_hosts + - name: Build and push LangChain image + uses: docker/build-push-action@v5 + with: + context: ./langchain-temp + push: true + tags: ${{ steps.meta-langchain.outputs.tags }} + labels: ${{ steps.meta-langchain.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max - - name: Deploy to Hetzner - run: | - # Copy deployment files to server - scp -r compose.yaml nginx/ scripts/ ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }}:${{ env.DEPLOY_PATH }}/ + # - name: Setup SSH key + # uses: webfactory/ssh-agent@v0.8.0 + # with: + # ssh-private-key: ${{ secrets.HETZNER_SSH_KEY }} + + # - name: Add Hetzner server to known hosts + # run: | + # ssh-keyscan -H ${{ env.HETZNER_HOST }} >> ~/.ssh/known_hosts + + # - name: Deploy to Hetzner + # run: | + # # Copy deployment files to server + # scp -r compose.yaml nginx/ scripts/ ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }}:${{ env.DEPLOY_PATH }}/ - # Copy environment file if it doesn't exist - ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " - cd ${{ env.DEPLOY_PATH }} - if [ ! -f .env ]; then - cp .env.example .env - echo 'Please update .env file with your configuration' - fi - " - - - name: Update and restart services - run: | - ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " - cd ${{ env.DEPLOY_PATH }} - - # Pull latest images - docker compose pull - - # Stop services gracefully - docker compose down --timeout 30 - - # Start services - docker compose up -d - - # Wait for services to be ready - sleep 30 - - # Run migrations on all sites - ./scripts/migrate-all-sites.sh - - # Show status - docker compose ps - " - - - name: Health check - run: | - # Wait a bit more for services to fully start - sleep 60 + # # Copy environment file if it doesn't exist + # ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " + # cd ${{ env.DEPLOY_PATH }} + # if [ ! -f .env ]; then + # cp .env.example .env + # echo 'Please update .env file with your configuration' + # fi + # " - # Check if nginx-proxy is responding - if curl -f http://${{ env.HETZNER_HOST }}/health; then - echo "✅ Deployment successful - Health check passed" - else - echo "❌ Health check failed" - # Show logs for debugging - ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " - cd ${{ env.DEPLOY_PATH }} - docker compose logs --tail=50 - " - exit 1 - fi + # # Login to private registry + # ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " + # echo '${{ secrets.GITHUB_TOKEN }}' | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin + # " - - name: Notify deployment status - if: always() - run: | - if [ "${{ job.status }}" == "success" ]; then - echo "🚀 Deployment to Hetzner completed successfully!" - echo "🌐 Access your application at: http://${{ env.HETZNER_HOST }}" - else - echo "💥 Deployment failed. Check the logs above for details." - fi + # - name: Update and restart services + # run: | + # ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " + # cd ${{ env.DEPLOY_PATH }} + + # # Pull latest images + # docker compose pull + + # # Stop services gracefully + # docker compose down --timeout 30 + + # # Start services + # docker compose up -d + + # # Wait for services to be ready + # sleep 30 + + # # Run migrations on all sites + # ./scripts/migrate-all-sites.sh + + # # Show status + # docker compose ps + # " + + # - name: Health check + # run: | + # # Wait a bit more for services to fully start + # sleep 60 + + # # Check if nginx-proxy is responding + # if curl -f http://${{ env.HETZNER_HOST }}/health; then + # echo "✅ Deployment successful - Health check passed" + # else + # echo "❌ Health check failed" + # # Show logs for debugging + # ssh ${{ env.HETZNER_USER }}@${{ env.HETZNER_HOST }} " + # cd ${{ env.DEPLOY_PATH }} + # docker compose logs --tail=50 + # " + # exit 1 + # fi + + # - name: Notify deployment status + # if: always() + # run: | + # if [ "${{ job.status }}" == "success" ]; then + # echo "🚀 Deployment to Hetzner completed successfully!" + # echo "🌐 Access your application at: http://${{ env.HETZNER_HOST }}" + # else + # echo "💥 Deployment failed. Check the logs above for details." + # fi diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 147883d5..557e7176 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -56,6 +56,16 @@ Configure the following secrets in this repository: - `HETZNER_SSH_KEY`: Private SSH key for accessing the Hetzner server - `ACADEMY_DOCKER_PAT`: GitHub Personal Access Token with `repo` and `write:packages` permissions +For environment variables, you can either: +- Use a `.env` file on the server (default approach) +- Use GitHub Secrets for sensitive values (recommended for production) + +If using GitHub Secrets, add these: +- `MARIADB_ROOT_PASSWORD`: Strong password for MariaDB root user +- `OPENAI_API_KEY`: Your OpenAI API key for AI features +- `ANTHROPIC_API_KEY`: Your Anthropic API key (optional) +- `LANGCHAIN_DB_PASSWORD`: Password for LangChain PostgreSQL database + ### 3. Webhook Setup Add the webhook workflow files to each watched repository: @@ -203,6 +213,7 @@ cd /opt/frappe-deployment - Never commit `.env` file - Use strong passwords - Rotate credentials regularly + - Consider using GitHub Secrets for production deployments 2. **Network Security**: - Configure firewall rules on Hetzner @@ -212,6 +223,36 @@ cd /opt/frappe-deployment - Regular database backups - Store backups off-site +### Private Registry Setup + +By default, images are pushed to GitHub Container Registry (ghcr.io). To use private images: + +1. **Make packages private in GitHub**: + - Go to your repository settings + - Navigate to "Packages" in the sidebar + - Find your package (e.g., `ignis-academy-lms`, `academy-langchain`) + - Click on "Package settings" + - Change visibility to "Private" + +2. **Enable registry authentication in deployment**: + - The workflow already includes code for private registry login + - Uncomment the "Login to private registry" section in `.github/workflows/deploy.yml` + - The `GITHUB_TOKEN` automatically has `packages:read` permission for private packages in the same org + +3. **Manual registry login on Hetzner** (if needed): + ```bash + # For GitHub Container Registry + echo $GITHUB_TOKEN | docker login ghcr.io -u USERNAME --password-stdin + + # Or use a Personal Access Token + echo $PAT | docker login ghcr.io -u USERNAME --password-stdin + ``` + +4. **Alternative private registries**: + - Docker Hub: Update `REGISTRY` to `docker.io` + - Harbor: Update `REGISTRY` to your Harbor URL + - GitLab: Update `REGISTRY` to `registry.gitlab.com` + ## SSL/TLS Setup (Production) For production, set up SSL certificates: diff --git a/LOCAL_TESTING.md b/LOCAL_TESTING.md new file mode 100644 index 00000000..c738b498 --- /dev/null +++ b/LOCAL_TESTING.md @@ -0,0 +1,139 @@ +# Local Testing Guide for Pre-built Images + +This guide explains how to run the Academy LMS stack locally using images from GitHub Container Registry. + +## Prerequisites + +1. Docker and Docker Compose installed +2. Access to the GitHub Container Registry (for private images) + +## Quick Start + +### 1. Clone this repository + +```bash +git clone https://github.com/ExarLabs/academy_docker.git +cd academy_docker +``` + +### 2. Set up environment variables + +```bash +# Copy the example environment file +cp .env.example .env + +# Edit the .env file with your actual values +nano .env +``` + +**Required variables to update:** +- `MARIADB_ROOT_PASSWORD` - Set a secure password +- `OPENAI_API_KEY` - Your OpenAI API key +- `ANTHROPIC_API_KEY` - Your Anthropic API key (optional) +- `LANGCHAIN_DB_PASSWORD` - Password for LangChain database + +### 3. Login to GitHub Container Registry (if images are private) + +```bash +# Using GitHub Personal Access Token +echo $GITHUB_PAT | docker login ghcr.io -u YOUR_GITHUB_USERNAME --password-stdin + +# Or using GitHub CLI +gh auth token | docker login ghcr.io -u YOUR_GITHUB_USERNAME --password-stdin +``` + +### 4. Pull and run the services + +```bash +# Pull the latest images +docker compose pull + +# Start all services +docker compose up -d + +# Check status +docker compose ps + +# View logs +docker compose logs -f +``` + +### 5. Create your first site (after services are running) + +```bash +# Create a new site +docker compose exec backend bench new-site academy.local \ + --admin-password admin \ + --db-root-password $MARIADB_ROOT_PASSWORD + +# Install the LMS app +docker compose exec backend bench --site academy.local install-app lms + +# Install the AI Tutor Chat app +docker compose exec backend bench --site academy.local install-app academy_ai_tutor_chat + +# Set the site as default +docker compose exec backend bench use academy.local +``` + +### 6. Access the application + +1. Add to your hosts file: + - Windows: `C:\Windows\System32\drivers\etc\hosts` + - Linux/Mac: `/etc/hosts` + + Add this line: + ``` + 127.0.0.1 academy.local + ``` + +2. Open in browser: http://academy.local + +## What's included? + +The `compose.yaml` file includes: +- **Frappe/ERPNext** with Academy LMS and AI Tutor apps (single image: `ghcr.io/exarlabs/ignis-academy-lms`) +- **LangChain service** for AI functionality (image: `ghcr.io/exarlabs/academy-langchain`) +- **MariaDB** for Frappe database +- **PostgreSQL** for LangChain database +- **Redis** for caching and queues +- **Nginx** reverse proxy + +## Troubleshooting + +### "manifest unknown" error + +This means the image hasn't been built yet. Either: +1. Wait for GitHub Actions to build and push the images +2. Build locally (see development guide) + +### Environment variable warnings + +Ensure your `.env` file exists and contains all required variables. Use `.env.example` as reference. + +### Cannot access the site + +1. Check if all services are running: `docker compose ps` +2. Ensure you've added the hostname to your hosts file +3. Check nginx logs: `docker compose logs nginx-proxy` + +### Database connection errors + +1. Ensure MariaDB is fully started before creating sites +2. Check the password in `.env` matches what you use in commands + +## Stopping the services + +```bash +# Stop all services +docker compose down + +# Stop and remove all data (careful!) +docker compose down -v +``` + +## Next Steps + +- For production deployment, see [DEPLOYMENT.md](DEPLOYMENT.md) +- For development setup, see [README.md](README.md) +- For environment configuration, see [docs/environment-secrets-explained.md](docs/environment-secrets-explained.md) diff --git a/compose.yaml b/compose.yaml index 43ef41f7..b8d24069 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,7 +1,7 @@ x-customizable-image: # Custom Academy LMS image with all required apps &customizable_image - image: ${CUSTOM_IMAGE:-ghcr.io/exarlabs/academy-frappe}:${CUSTOM_TAG:-latest} + image: ${CUSTOM_IMAGE:-ghcr.io/exarlabs/ignis-academy-lms}:${CUSTOM_TAG:-latest} pull_policy: ${PULL_POLICY:-always} restart: ${RESTART_POLICY:-unless-stopped}