diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 1f1e0fcf..12fc6039 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -2,6 +2,7 @@ version: "3.7" services: mariadb: image: docker.io/mariadb:10.6 + platform: linux/amd64 command: - --character-set-server=utf8mb4 - --collation-server=utf8mb4_unicode_ci @@ -38,12 +39,15 @@ services: redis-cache: image: docker.io/redis:alpine + platform: linux/amd64 redis-queue: image: docker.io/redis:alpine + platform: linux/amd64 frappe: image: docker.io/frappe/bench:latest + platform: linux/amd64 command: sleep infinity environment: - SHELL=/bin/bash diff --git a/.env.example b/.env.example index f458f2e3..bc9207c0 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ -# Reference: https://github.com/frappe/frappe_docker/blob/main/docs/images-and-compose-files.md +# Reference: https://github.com/frappe/frappe_docker/blob/main/docs/environment-variables.md -ERPNEXT_VERSION=v15.12.2 +ERPNEXT_VERSION=v15.44.0 DB_PASSWORD=123 @@ -23,6 +23,9 @@ DB_PASSWORD=123 # and do want to access it by `127.0.0.1` host. Than you would set this variable to `mysite`. FRAPPE_SITE_NAME_HEADER= +# Default value is `8080`. +HTTP_PUBLISH_PORT= + # Default value is `127.0.0.1`. Set IP address as our trusted upstream address. # UPSTREAM_REAL_IP_ADDRESS= diff --git a/.github/workflows/build_bench.yml b/.github/workflows/build_bench.yml index da5a0762..d45d7cef 100644 --- a/.github/workflows/build_bench.yml +++ b/.github/workflows/build_bench.yml @@ -34,7 +34,7 @@ jobs: run: echo "LATEST_BENCH_RELEASE=$(curl -s 'https://api.github.com/repos/frappe/bench/releases/latest' | jq -r '.tag_name')" >> "$GITHUB_ENV" - name: Build and test - uses: docker/bake-action@v4.1.0 + uses: docker/bake-action@v5.10.0 with: targets: bench-test @@ -47,7 +47,7 @@ jobs: - name: Push if: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} - uses: docker/bake-action@v4.1.0 + uses: docker/bake-action@v5.10.0 with: targets: bench push: true diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index b519ec9a..dac92bf1 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -37,15 +37,22 @@ jobs: image: registry:2 ports: - 5000:5000 + strategy: + matrix: + arch: [amd64, arm64] steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Setup Buildx uses: docker/setup-buildx-action@v3 with: driver-opts: network=host + platforms: linux/${{ matrix.arch }} - name: Get latest versions run: python3 ./.github/scripts/get_latest_tags.py --repo ${{ inputs.repo }} --version ${{ inputs.version }} @@ -56,7 +63,7 @@ jobs: echo "NODE_VERSION=${{ inputs.node_version }}" >> "$GITHUB_ENV" - name: Build - uses: docker/bake-action@v4.1.0 + uses: docker/bake-action@v5.10.0 with: push: true env: @@ -84,6 +91,6 @@ jobs: - name: Push if: ${{ inputs.push }} - uses: docker/bake-action@v4.1.0 + uses: docker/bake-action@v5.10.0 with: push: true diff --git a/.github/workflows/pre-commit-autoupdate.yml b/.github/workflows/pre-commit-autoupdate.yml new file mode 100644 index 00000000..5612c735 --- /dev/null +++ b/.github/workflows/pre-commit-autoupdate.yml @@ -0,0 +1,26 @@ +name: Autoupdate pre-commit hooks + +on: + schedule: + # Every day at 7 am + - cron: 0 7 * * * + +jobs: + pre-commit-autoupdate: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Update pre-commit hooks + uses: vrslev/pre-commit-autoupdate@v1.0.0 + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + branch: pre-commit-autoupdate + title: "chore(deps): Update pre-commit hooks" + commit-message: "chore(deps): Update pre-commit hooks" + body: Update pre-commit hooks + labels: dependencies,development + delete-branch: True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..a4b37b29 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,53 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: trailing-whitespace + - id: end-of-file-fixer + + - repo: https://github.com/asottile/pyupgrade + rev: v3.19.0 + hooks: + - id: pyupgrade + args: [--py37-plus] + + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black + + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v4.0.0-alpha.8 + hooks: + - id: prettier + + - repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + args: + - -L + - "ro" + + - repo: local + hooks: + - id: shfmt + name: shfmt + language: golang + additional_dependencies: [mvdan.cc/sh/v3/cmd/shfmt@latest] + entry: shfmt + args: [-w] + types: [shell] + + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck + args: [-x] diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..0e01dd85 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socioeconomic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at hello@frappe.io. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/README.md b/README.md new file mode 100644 index 00000000..b68905a5 --- /dev/null +++ b/README.md @@ -0,0 +1,88 @@ +[![Build Stable](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_stable.yml) +[![Build Develop](https://github.com/frappe/frappe_docker/actions/workflows/build_develop.yml/badge.svg)](https://github.com/frappe/frappe_docker/actions/workflows/build_develop.yml) + +Everything about [Frappe](https://github.com/frappe/frappe) and [ERPNext](https://github.com/frappe/erpnext) in containers. + +# Getting Started + +To get started you need [Docker](https://docs.docker.com/get-docker/), [docker-compose](https://docs.docker.com/compose/), and [git](https://docs.github.com/en/get-started/getting-started-with-git/set-up-git) setup on your machine. For Docker basics and best practices refer to Docker's [documentation](http://docs.docker.com). + +Once completed, chose one of the following two sections for next steps. + +### Try in Play With Docker + +To play in an already set up sandbox, in your browser, click the button below: + + + Try in PWD + + +### Try on your Dev environment + +First clone the repo: + +```sh +git clone https://github.com/frappe/frappe_docker +cd frappe_docker +``` + +Then run: `docker compose -f pwd.yml up -d` + +### To run on ARM64 architecture follow this instructions + +After cloning the repo run this command to build multi-architecture images specifically for ARM64. + +`docker buildx bake --no-cache --set *.platform=linux/arm64` + +and then + +- add `platform: linux/arm64` to all services in the pwd.yaml +- replace the current specified versions of erpnext image on `pwd.yml` with `:latest` + +Then run: `docker compose -f pwd.yml up -d` + +## Final steps + +Wait for 5 minutes for ERPNext site to be created or check `create-site` container logs before opening browser on port 8080. (username: `Administrator`, password: `admin`) + +If you ran in a Dev Docker environment, to view container logs: `docker compose -f pwd.yml logs -f create-site`. Don't worry about some of the initial error messages, some services take a while to become ready, and then they go away. + +# Documentation + +### [Frequently Asked Questions](https://github.com/frappe/frappe_docker/wiki/Frequently-Asked-Questions) + +### [Production](#production) + +- [List of containers](docs/list-of-containers.md) +- [Single Compose Setup](docs/single-compose-setup.md) +- [Environment Variables](docs/environment-variables.md) +- [Single Server Example](docs/single-server-example.md) +- [Setup Options](docs/setup-options.md) +- [Site Operations](docs/site-operations.md) +- [Backup and Push Cron Job](docs/backup-and-push-cronjob.md) +- [Port Based Multi Tenancy](docs/port-based-multi-tenancy.md) +- [Migrate from multi-image setup](docs/migrate-from-multi-image-setup.md) +- [running on linux/mac](docs/setup_for_linux_mac.md) + +### [Custom Images](#custom-images) + +- [Custom Apps](docs/custom-apps.md) +- [Build Version 10 Images](docs/build-version-10-images.md) + +### [Development](#development) + +- [Development using containers](docs/development.md) +- [Bench Console and VSCode Debugger](docs/bench-console-and-vscode-debugger.md) +- [Connect to localhost services](docs/connect-to-localhost-services-from-containers-for-local-app-development.md) + +### [Troubleshoot](docs/troubleshoot.md) + +# Contributing + +If you want to contribute to this repo refer to [CONTRIBUTING.md](CONTRIBUTING.md) + +This repository is only for container related stuff. You also might want to contribute to: + +- [Frappe framework](https://github.com/frappe/frappe#contributing), +- [ERPNext](https://github.com/frappe/erpnext#contributing), +- [Frappe Bench](https://github.com/frappe/bench). diff --git a/compose.yaml b/compose.yaml index fd91f9d2..21839d0c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,5 +1,9 @@ x-customizable-image: &customizable_image - image: zapal/erp:latest + # By default the image used only contains the `frappe` and `erpnext` apps. + # See https://github.com/frappe/frappe_docker/blob/main/docs/custom-apps.md + # about using custom images. + image: ${CUSTOM_IMAGE:-zapal/erp}:${CUSTOM_TAG:-$ERPNEXT_VERSION} + pull_policy: ${PULL_POLICY:-always} x-depends-on-configurator: &depends_on_configurator depends_on: @@ -14,6 +18,7 @@ x-backend-defaults: &backend_defaults services: configurator: <<: *backend_defaults + platform: linux/amd64 entrypoint: - bash - -c @@ -28,18 +33,20 @@ services: bench set-config -g redis_socketio "redis://$$REDIS_QUEUE"; bench set-config -gp socketio_port $$SOCKETIO_PORT; environment: - DB_HOST: ${DB_HOST} - DB_PORT: ${DB_PORT} - REDIS_CACHE: ${REDIS_CACHE} - REDIS_QUEUE: ${REDIS_QUEUE} + DB_HOST: ${DB_HOST:-} + DB_PORT: ${DB_PORT:-} + REDIS_CACHE: ${REDIS_CACHE:-} + REDIS_QUEUE: ${REDIS_QUEUE:-} SOCKETIO_PORT: 9000 depends_on: {} backend: <<: *backend_defaults + platform: linux/amd64 frontend: <<: *customizable_image + platform: linux/amd64 command: - nginx-entrypoint.sh environment: @@ -59,6 +66,7 @@ services: websocket: <<: [*depends_on_configurator, *customizable_image] + platform: linux/amd64 command: - node - /home/zapal/frappe-bench/apps/frappe/socketio.js @@ -67,14 +75,17 @@ services: queue-short: <<: *backend_defaults + platform: linux/amd64 command: bench worker --queue short,default queue-long: <<: *backend_defaults + platform: linux/amd64 command: bench worker --queue long,default,short scheduler: <<: *backend_defaults + platform: linux/amd64 command: bench schedule volumes: diff --git a/devcontainer-example/devcontainer.json b/devcontainer-example/devcontainer.json new file mode 100644 index 00000000..2e60f65d --- /dev/null +++ b/devcontainer-example/devcontainer.json @@ -0,0 +1,32 @@ +{ + "name": "Frappe Bench", + "forwardPorts": [8000, 9000, 6787], + "remoteUser": "frappe", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-vscode.live-server", + "grapecity.gc-excelviewer", + "mtxr.sqltools", + "visualstudioexptteam.vscodeintellicode" + ], + "settings": { + "terminal.integrated.profiles.linux": { + "frappe bash": { + "path": "/bin/bash" + } + }, + "terminal.integrated.defaultProfile.linux": "frappe bash", + "debug.node.autoAttach": "disabled" + } + } + }, + "dockerComposeFile": "./docker-compose.yml", + "service": "frappe", + "workspaceFolder": "/workspace/development", + "shutdownAction": "stopCompose", + "mounts": [ + "source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh,target=/home/frappe/.ssh,type=bind,consistency=cached" + ] +} diff --git a/development/.vscode/settings.json b/development/.vscode/settings.json new file mode 100644 index 00000000..1490b727 --- /dev/null +++ b/development/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.defaultInterpreterPath": "${workspaceFolder}/frappe-bench/env/bin/python" +} diff --git a/docker-bake.hcl b/docker-bake.hcl index 6ed20308..633897cf 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -69,12 +69,21 @@ target "bench-test" { # Base for all other targets group "default" { - targets = ["erp"] + targets = ["erp", "base", "build"] } function "tag" { params = [repo, version] - result = ["${REGISTRY_USER}/${repo}:latest", "${REGISTRY_USER}/${repo}:${version}"] + result = [ + # Push frappe or erpnext branch as tag + "${REGISTRY_USER}/${repo}:${version}", + # If `version` param is develop (development build) then use tag `latest` + "${version}" == "develop" ? "${REGISTRY_USER}/${repo}:latest" : "${REGISTRY_USER}/${repo}:${version}", + # Make short tag for major version if possible. For example, from v13.16.0 make v13. + can(regex("(v[0-9]+)[.]", "${version}")) ? "${REGISTRY_USER}/${repo}:${regex("(v[0-9]+)[.]", "${version}")[0]}" : "", + # Make short tag for major version if possible. For example, from v13.16.0 make version-13. + can(regex("(v[0-9]+)[.]", "${version}")) ? "${REGISTRY_USER}/${repo}:version-${regex("([0-9]+)[.]", "${version}")[0]}" : "", + ] } target "default-args" { @@ -100,3 +109,19 @@ target "erp" { target = "erp" tags = tag("erp", "${ERPNEXT_VERSION}") } + +target "base" { + inherits = ["default-args"] + context = "." + dockerfile = "images/production/Containerfile" + target = "base" + tags = tag("base", "${FRAPPE_VERSION}") +} + +target "build" { + inherits = ["default-args"] + context = "." + dockerfile = "images/production/Containerfile" + target = "build" + tags = tag("build", "${ERPNEXT_VERSION}") +} diff --git a/docs/custom-apps.md b/docs/custom-apps.md new file mode 100644 index 00000000..d979aa5d --- /dev/null +++ b/docs/custom-apps.md @@ -0,0 +1,156 @@ +### Load custom apps through apps.json file + +Base64 encoded string of `apps.json` file needs to be passed in as build arg environment variable. + +Create following `apps.json` file: + +```json +[ + { + "url": "https://github.com/frappe/erpnext", + "branch": "version-15" + }, + { + "url": "https://github.com/frappe/payments", + "branch": "version-15" + }, + { + "url": "https://{{ PAT }}@git.example.com/project/repository.git", + "branch": "main" + } +] +``` + +Note: + +- `url` needs to be http(s) git url with personal access tokens without username eg:- `http://{{PAT}}@github.com/project/repository.git` in case of private repo. +- Add dependencies manually in `apps.json` e.g. add `erpnext` if you are installing `hrms`. +- Use fork repo or branch for ERPNext in case you need to use your fork or test a PR. + +Generate base64 string from json file: + +```shell +export APPS_JSON_BASE64=$(base64 -w 0 /path/to/apps.json) +``` + +Test the Previous Step: Decode the Base64-encoded Environment Variable + +To verify the previous step, decode the `APPS_JSON_BASE64` environment variable (which is Base64-encoded) into a JSON file. Follow the steps below: + +1. Use the following command to decode and save the output into a JSON file named apps-test-output.json: + +```shell +echo -n ${APPS_JSON_BASE64} | base64 -d > apps-test-output.json +``` + +2. Open the apps-test-output.json file to review the JSON output and ensure that the content is correct. + +### Clone frappe_docker and switch directory + +```shell +git clone https://github.com/frappe/frappe_docker +cd frappe_docker +``` + +### Configure build + +Common build args. + +- `FRAPPE_PATH`, customize the source repo for frappe framework. Defaults to `https://github.com/frappe/frappe` +- `FRAPPE_BRANCH`, customize the source repo branch for frappe framework. Defaults to `version-15`. +- `APPS_JSON_BASE64`, correct base64 encoded JSON string generated from `apps.json` file. + +Notes + +- Use `buildah` or `docker` as per your setup. +- Make sure `APPS_JSON_BASE64` variable has correct base64 encoded JSON string. It is consumed as build arg, base64 encoding ensures it to be friendly with environment variables. Use `jq empty apps.json` to validate `apps.json` file. +- Make sure the `--tag` is valid image name that will be pushed to registry. See section [below](#use-images) for remarks about its use. +- `.git` directories for all apps are removed from the image. + +### Quick build image + +This method uses pre-built `frappe/base:${FRAPPE_BRANCH}` and `frappe/build:${FRAPPE_BRANCH}` image layers which come with required Python and NodeJS runtime. It speeds up the build time. + +It uses `images/layered/Containerfile`. + +```shell +docker build \ + --build-arg=FRAPPE_PATH=https://github.com/frappe/frappe \ + --build-arg=FRAPPE_BRANCH=version-15 \ + --build-arg=APPS_JSON_BASE64=$APPS_JSON_BASE64 \ + --tag=ghcr.io/user/repo/custom:1.0.0 \ + --file=images/layered/Containerfile . +``` + +### Custom build image + +This method builds the base and build layer every time, it allows to customize Python and NodeJS runtime versions. It takes more time to build. + +It uses `images/custom/Containerfile`. + +```shell +docker build \ + --build-arg=FRAPPE_PATH=https://github.com/frappe/frappe \ + --build-arg=FRAPPE_BRANCH=version-15 \ + --build-arg=PYTHON_VERSION=3.11.9 \ + --build-arg=NODE_VERSION=18.20.2 \ + --build-arg=APPS_JSON_BASE64=$APPS_JSON_BASE64 \ + --tag=ghcr.io/user/repo/custom:1.0.0 \ + --file=images/custom/Containerfile . +``` + +Custom build args, + +- `PYTHON_VERSION`, use the specified python version for base image. Default is `3.11.6`. +- `NODE_VERSION`, use the specified nodejs version, Default `18.18.2`. +- `DEBIAN_BASE` use the base Debian version, defaults to `bookworm`. +- `WKHTMLTOPDF_VERSION`, use the specified qt patched `wkhtmltopdf` version. Default is `0.12.6.1-3`. +- `WKHTMLTOPDF_DISTRO`, use the specified distro for debian package. Default is `bookworm`. + +### Push image to use in yaml files + +Login to `docker` or `buildah` + +```shell +docker login +``` + +Push image + +```shell +docker push ghcr.io/user/repo/custom:1.0.0 +``` + +### Use Images + +In the [compose.yaml](../compose.yaml), you can set the image name and tag through environment variables, making it easier to customize. + +```yaml +x-customizable-image: &customizable_image + image: ${CUSTOM_IMAGE:-frappe/erpnext}:${CUSTOM_TAG:-${ERPNEXT_VERSION:?No ERPNext version or tag set}} + pull_policy: ${PULL_POLICY:-always} +``` + +The environment variables can be set in the shell or in the .env file as [setup-options.md](setup-options.md) describes. + +- `CUSTOM_IMAGE`: The name of your custom image. Defaults to `frappe/erpnext` if not set. +- `CUSTOM_TAG`: The tag for your custom image. Must be set if `CUSTOM_IMAGE` is used. Defaults to the value of `ERPNEXT_VERSION` if not set. +- `PULL_POLICY`: The Docker pull policy. Defaults to `always`. Recommended set to `never` for local images, so prevent `docker` from trying to download the image when it has been built locally. +- `HTTP_PUBLISH_PORT`: The port to publish through no SSL channel. Default depending on deployment, it may be `80` if SSL activated or `8080` if not. +- `HTTPS_PUBLISH_PORT`: The secure port to publish using SSL. Default is `443`. + +Make sure image name is correct to be pushed to registry. After the images are pushed, you can pull them to servers to be deployed. If the registry is private, additional auth is needed. + +#### Example + +If you built an image with the tag `ghcr.io/user/repo/custom:1.0.0`, you would set the environment variables as follows: + +```bash +export CUSTOM_IMAGE='ghcr.io/user/repo/custom' +export CUSTOM_TAG='1.0.0' +docker compose -f compose.yaml \ + -f overrides/compose.mariadb.yaml \ + -f overrides/compose.redis.yaml \ + -f overrides/compose.https.yaml \ + config > ~/gitops/docker-compose.yaml +``` diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 00000000..89b20ec2 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,417 @@ +# Getting Started + +## Prerequisites + +In order to start developing you need to satisfy the following prerequisites: + +- Docker +- docker-compose +- user added to docker group + +It is recommended you allocate at least 4GB of RAM to docker: + +- [Instructions for Windows](https://docs.docker.com/docker-for-windows/#resources) +- [Instructions for macOS](https://docs.docker.com/desktop/settings/mac/#advanced) + +Here is a screenshot showing the relevant setting in the Help Manual +![image](images/Docker%20Manual%20Screenshot%20-%20Resources%20section.png) +Here is a screenshot showing the settings in Docker Desktop on Mac +![images](images/Docker%20Desktop%20Screenshot%20-%20Resources%20section.png) + +## Bootstrap Containers for development + +Clone and change directory to frappe_docker directory + +```shell +git clone https://github.com/frappe/frappe_docker.git +cd frappe_docker +``` + +Copy example devcontainer config from `devcontainer-example` to `.devcontainer` + +```shell +cp -R devcontainer-example .devcontainer +``` + +Copy example vscode config for devcontainer from `development/vscode-example` to `development/.vscode`. This will setup basic configuration for debugging. + +```shell +cp -R development/vscode-example development/.vscode +``` + +## Use VSCode Remote Containers extension + +For most people getting started with Frappe development, the best solution is to use [VSCode Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). + +Before opening the folder in container, determine the database that you want to use. The default is MariaDB. +If you want to use PostgreSQL instead, edit `.devcontainer/docker-compose.yml` and uncomment the section for `postgresql` service, and you may also want to comment `mariadb` as well. + +VSCode should automatically inquire you to install the required extensions, that can also be installed manually as follows: + +- Install Dev Containers for VSCode + - through command line `code --install-extension ms-vscode-remote.remote-containers` + - clicking on the Install button in the Vistual Studio Marketplace: [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) + - View: Extensions command in VSCode (Windows: Ctrl+Shift+X; macOS: Cmd+Shift+X) then search for extension `ms-vscode-remote.remote-containers` + +After the extensions are installed, you can: + +- Open frappe_docker folder in VS Code. + - `code .` +- Launch the command, from Command Palette (Ctrl + Shift + P) `Dev Containers: Reopen in Container`. You can also click in the bottom left corner to access the remote container menu. + +Notes: + +- The `development` directory is ignored by git. It is mounted and available inside the container. Create all your benches (installations of bench, the tool that manages frappe) inside this directory. +- Node v14 and v10 are installed. Check with `nvm ls`. Node v14 is used by default. + +### Setup first bench + +> Jump to [scripts](#setup-bench--new-site-using-script) section to setup a bench automatically. Alternatively, you can setup a bench manually using below guide. + +Run the following commands in the terminal inside the container. You might need to create a new terminal in VSCode. + +NOTE: Prior to doing the following, make sure the user is **frappe**. + +```shell +bench init --skip-redis-config-generation frappe-bench +cd frappe-bench +``` + +To setup frappe framework version 14 bench set `PYENV_VERSION` environment variable to `3.10.5` (default) and use NodeJS version 16 (default), + +```shell +# Use default environments +bench init --skip-redis-config-generation --frappe-branch version-14 frappe-bench +# Or set environment versions explicitly +nvm use v16 +PYENV_VERSION=3.10.13 bench init --skip-redis-config-generation --frappe-branch version-14 frappe-bench +# Switch directory +cd frappe-bench +``` + +To setup frappe framework version 13 bench set `PYENV_VERSION` environment variable to `3.9.17` and use NodeJS version 14, + +```shell +nvm use v14 +PYENV_VERSION=3.9.17 bench init --skip-redis-config-generation --frappe-branch version-13 frappe-bench +cd frappe-bench +``` + +### Setup hosts + +We need to tell bench to use the right containers instead of localhost. Run the following commands inside the container: + +```shell +bench set-config -g db_host mariadb +bench set-config -g redis_cache redis://redis-cache:6379 +bench set-config -g redis_queue redis://redis-queue:6379 +bench set-config -g redis_socketio redis://redis-queue:6379 +``` + +For any reason the above commands fail, set the values in `common_site_config.json` manually. + +```json +{ + "db_host": "mariadb", + "redis_cache": "redis://redis-cache:6379", + "redis_queue": "redis://redis-queue:6379", + "redis_socketio": "redis://redis-queue:6379" +} +``` + +### Edit Honcho's Procfile + +Note : With the option '--skip-redis-config-generation' during bench init, these actions are no more needed. But at least, take a look to ProcFile to see what going on when bench launch honcho on start command + +Honcho is the tool used by Bench to manage all the processes Frappe requires. Usually, these all run in localhost, but in this case, we have external containers for Redis. For this reason, we have to stop Honcho from trying to start Redis processes. + +Honcho is installed in global python environment along with bench. To make it available locally you've to install it in every `frappe-bench/env` you create. Install it using command `./env/bin/pip install honcho`. It is required locally if you wish to use is as part of VSCode launch configuration. + +Open the Procfile file and remove the three lines containing the configuration from Redis, either by editing manually the file: + +```shell +code Procfile +``` + +Or running the following command: + +```shell +sed -i '/redis/d' ./Procfile +``` + +### Create a new site with bench + +You can create a new site with the following command: + +```shell +bench new-site --no-mariadb-socket sitename +``` + +sitename MUST end with .localhost for trying deployments locally. + +for example: + +```shell +bench new-site --no-mariadb-socket development.localhost +``` + +The same command can be run non-interactively as well: + +```shell +bench new-site --mariadb-root-password 123 --admin-password admin --no-mariadb-socket development.localhost +``` + +The command will ask the MariaDB root password. The default root password is `123`. +This will create a new site and a `development.localhost` directory under `frappe-bench/sites`. +The option `--no-mariadb-socket` will configure site's database credentials to work with docker. +You may need to configure your system /etc/hosts if you're on Linux, Mac, or its Windows equivalent. + +To setup site with PostgreSQL as database use option `--db-type postgres` and `--db-host postgresql`. (Available only v12 onwards, currently NOT available for ERPNext). + +Example: + +```shell +bench new-site --db-type postgres --db-host postgresql mypgsql.localhost +``` + +To avoid entering postgresql username and root password, set it in `common_site_config.json`, + +```shell +bench config set-common-config -c root_login postgres +bench config set-common-config -c root_password '"123"' +``` + +Note: If PostgreSQL is not required, the postgresql service / container can be stopped. + +### Set bench developer mode on the new site + +To develop a new app, the last step will be setting the site into developer mode. Documentation is available at [this link](https://frappe.io/docs/user/en/guides/app-development/how-enable-developer-mode-in-frappe). + +```shell +bench --site development.localhost set-config developer_mode 1 +bench --site development.localhost clear-cache +``` + +### Install an app + +To install an app we need to fetch it from the appropriate git repo, then install in on the appropriate site: + +You can check [VSCode container remote extension documentation](https://code.visualstudio.com/docs/remote/containers#_sharing-git-credentials-with-your-container) regarding git credential sharing. + +To install custom app + +```shell +# --branch is optional, use it to point to branch on custom app repository +bench get-app --branch version-12 https://github.com/myusername/myapp +bench --site development.localhost install-app myapp +``` + +At the time of this writing, the Payments app has been factored out of the Version 14 ERPNext app and is now a separate app. ERPNext will not install it. + +```shell +bench get-app --branch version-14 --resolve-deps erpnext +bench --site development.localhost install-app erpnext +``` + +To install ERPNext (from the version-13 branch): + +```shell +bench get-app --branch version-13 erpnext +bench --site development.localhost install-app erpnext +``` + +Note: Both frappe and erpnext must be on branch with same name. e.g. version-14 + +### Start Frappe without debugging + +Execute following command from the `frappe-bench` directory. + +```shell +bench start +``` + +You can now login with user `Administrator` and the password you choose when creating the site. +Your website will now be accessible at location [development.localhost:8000](http://development.localhost:8000) +Note: To start bench with debugger refer section for debugging. + +### Setup bench / new site using script + +Most developers work with numerous clients and versions. Moreover, apps may be required to be installed by everyone on the team working for a client. + +This is simplified using a script to automate the process of creating a new bench / site and installing the required apps. The `Administrator` password for created sites is `admin`. + +Sample `apps-example.json` is used by default, it installs erpnext on current stable release. To install custom apps, copy the `apps-example.json` to custom json file and make changes to list of apps. Pass this file to the `installer.py` script. + +> You may have apps in private repos which may require ssh access. You may use SSH from your home directory on linux (configurable in docker-compose.yml). + +```shell +python installer.py #pass --db-type postgres for postgresdb +``` + +For command help + +```shell +python installer.py --help +usage: installer.py [-h] [-j APPS_JSON] [-b BENCH_NAME] [-s SITE_NAME] [-r FRAPPE_REPO] [-t FRAPPE_BRANCH] [-p PY_VERSION] [-n NODE_VERSION] [-v] [-a ADMIN_PASSWORD] [-d DB_TYPE] + +options: + -h, --help show this help message and exit + -j APPS_JSON, --apps-json APPS_JSON + Path to apps.json, default: apps-example.json + -b BENCH_NAME, --bench-name BENCH_NAME + Bench directory name, default: frappe-bench + -s SITE_NAME, --site-name SITE_NAME + Site name, should end with .localhost, default: development.localhost + -r FRAPPE_REPO, --frappe-repo FRAPPE_REPO + frappe repo to use, default: https://github.com/frappe/frappe + -t FRAPPE_BRANCH, --frappe-branch FRAPPE_BRANCH + frappe repo to use, default: version-15 + -p PY_VERSION, --py-version PY_VERSION + python version, default: Not Set + -n NODE_VERSION, --node-version NODE_VERSION + node version, default: Not Set + -v, --verbose verbose output + -a ADMIN_PASSWORD, --admin-password ADMIN_PASSWORD + admin password for site, default: admin + -d DB_TYPE, --db-type DB_TYPE + Database type to use (e.g., mariadb or postgres) +``` + +A new bench and / or site is created for the client with following defaults. + +- MariaDB root password: `123` +- Admin password: `admin` + +> To use Postegres DB, comment the mariabdb service and uncomment postegres service. + +### Start Frappe with Visual Studio Code Python Debugging + +To enable Python debugging inside Visual Studio Code, you must first install the `ms-python.python` extension inside the container. This should have already happened automatically, but depending on your VSCode config, you can force it by: + +- Click on the extension icon inside VSCode +- Search `ms-python.python` +- Click on `Install on Dev Container: Frappe Bench` +- Click on 'Reload' + +We need to start bench separately through the VSCode debugger. For this reason, **instead** of running `bench start` you should run the following command inside the frappe-bench directory: + +```shell +honcho start \ + socketio \ + watch \ + schedule \ + worker_short \ + worker_long +``` + +Alternatively you can use the VSCode launch configuration "Honcho SocketIO Watch Schedule Worker" which launches the same command as above. + +This command starts all processes with the exception of Redis (which is already running in separate container) and the `web` process. The latter can can finally be started from the debugger tab of VSCode by clicking on the "play" button. + +You can now login with user `Administrator` and the password you choose when creating the site, if you followed this guide's unattended install that password is going to be `admin`. + +To debug workers, skip starting worker with honcho and start it with VSCode debugger. + +For advance vscode configuration in the devcontainer, change the config files in `development/.vscode`. + +## Developing using the interactive console + +You can launch a simple interactive shell console in the terminal with: + +```shell +bench --site development.localhost console +``` + +More likely, you may want to launch VSCode interactive console based on Jupyter kernel. + +Launch VSCode command palette (cmd+shift+p or ctrl+shift+p), run the command `Python: Select interpreter to start Jupyter server` and select `/workspace/development/frappe-bench/env/bin/python`. + +The first step is installing and updating the required software. Usually the frappe framework may require an older version of Jupyter, while VSCode likes to move fast, this can [cause issues](https://github.com/jupyter/jupyter_console/issues/158). For this reason we need to run the following command. + +```shell +/workspace/development/frappe-bench/env/bin/python -m pip install --upgrade jupyter ipykernel ipython +``` + +Then, run the command `Python: Show Python interactive window` from the VSCode command palette. + +Replace `development.localhost` with your site and run the following code in a Jupyter cell: + +```python +import frappe + +frappe.init(site='development.localhost', sites_path='/workspace/development/frappe-bench/sites') +frappe.connect() +frappe.local.lang = frappe.db.get_default('lang') +frappe.db.connect() +``` + +The first command can take a few seconds to be executed, this is to be expected. + +## Manually start containers + +In case you don't use VSCode, you may start the containers manually with the following command: + +### Running the containers + +```shell +docker-compose -f .devcontainer/docker-compose.yml up -d +``` + +And enter the interactive shell for the development container with the following command: + +```shell +docker exec -e "TERM=xterm-256color" -w /workspace/development -it devcontainer-frappe-1 bash +``` + +## Use additional services during development + +Add any service that is needed for development in the `.devcontainer/docker-compose.yml` then rebuild and reopen in devcontainer. + +e.g. + +```yaml +... +services: + ... + postgresql: + image: postgres:11.8 + environment: + POSTGRES_PASSWORD: 123 + volumes: + - postgresql-data:/var/lib/postgresql/data + ports: + - 5432:5432 + +volumes: + ... + postgresql-data: +``` + +Access the service by service name from the `frappe` development container. The above service will be accessible via hostname `postgresql`. If ports are published on to host, access it via `localhost:5432`. + +## Using Cypress UI tests + +To run cypress based UI tests in a docker environment, follow the below steps: + +1. Install and setup X11 tooling on VM using the script `install_x11_deps.sh` + +```shell + sudo bash ./install_x11_deps.sh +``` + +This script will install required deps, enable X11Forwarding and restart SSH daemon and export `DISPLAY` variable. + +2. Run X11 service `startx` or `xquartz` +3. Start docker compose services. +4. SSH into ui-tester service using `docker exec..` command +5. Export CYPRESS_baseUrl and other required env variables +6. Start Cypress UI console by issuing `cypress run command` + +> More references : [Cypress Official Documentation](https://www.cypress.io/blog/2019/05/02/run-cypress-with-a-single-docker-command) + +> Ensure DISPLAY environment is always exported. + +## Using Mailpit to test mail services + +To use Mailpit just uncomment the service in the docker-compose.yml file. +The Interface is then available under port 8025 and the smtp service can be used as mailpit:1025. diff --git a/docs/environment-variables.md b/docs/environment-variables.md new file mode 100644 index 00000000..b1a44d9f --- /dev/null +++ b/docs/environment-variables.md @@ -0,0 +1,56 @@ +## Environment Variables + +All of the commands are directly passed to container as per type of service. Only environment variables used in image are for `nginx-entrypoint.sh` command. They are as follows: + +- `BACKEND`: Set to `{host}:{port}`, defaults to `0.0.0.0:8000` +- `SOCKETIO`: Set to `{host}:{port}`, defaults to `0.0.0.0:9000` +- `UPSTREAM_REAL_IP_ADDRESS`: Set Nginx config for [ngx_http_realip_module#set_real_ip_from](http://nginx.org/en/docs/http/ngx_http_realip_module.html#set_real_ip_from), defaults to `127.0.0.1` +- `UPSTREAM_REAL_IP_HEADER`: Set Nginx config for [ngx_http_realip_module#real_ip_header](http://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_header), defaults to `X-Forwarded-For` +- `UPSTREAM_REAL_IP_RECURSIVE`: Set Nginx config for [ngx_http_realip_module#real_ip_recursive](http://nginx.org/en/docs/http/ngx_http_realip_module.html#real_ip_recursive) Set defaults to `off` +- `FRAPPE_SITE_NAME_HEADER`: Set proxy header `X-Frappe-Site-Name` and serve site named in the header, defaults to `$host`, i.e. find site name from host header. More details [below](#frappe_site_name_header) +- `PROXY_READ_TIMEOUT`: Upstream gunicorn service timeout, defaults to `120` +- `CLIENT_MAX_BODY_SIZE`: Max body size for uploads, defaults to `50m` + +To bypass `nginx-entrypoint.sh`, mount desired `/etc/nginx/conf.d/default.conf` and run `nginx -g 'daemon off;'` as container command. + +## Configuration + +We use environment variables to configure our setup. docker-compose uses variables from the `environment:` section of the services defined within and the`.env` file, if present. Variables defined in the `.env` file are referenced via `${VARIABLE_NAME}` within the docker-compose `.yml` file. `example.env` contains a non-exhaustive list of possible configuration variables. To get started, copy `example.env` to `.env`. + +### `FRAPPE_VERSION` + +Frappe framework release. You can find all releases [here](https://github.com/frappe/frappe/releases). + +### `DB_PASSWORD` + +Password for MariaDB (or Postgres) database. + +### `DB_HOST` + +Hostname for MariaDB (or Postgres) database. Set only if external service for database is used or the container can not be reached by its service name (db) by other containers. + +### `DB_PORT` + +Port for MariaDB (3306) or Postgres (5432) database. Set only if external service for database is used. + +### `REDIS_CACHE` + +Hostname for redis server to store cache. Set only if external service for redis is used or the container can not be reached by its service name (redis-cache) by other containers. + +### `REDIS_QUEUE` + +Hostname for redis server to store queue data and socketio. Set only if external service for redis is used or the container can not be reached by its service name (redis-queue) by other containers. + +### `ERPNEXT_VERSION` + +ERPNext [release](https://github.com/frappe/erpnext/releases). This variable is required if you use ERPNext override. + +### `LETSENCRYPT_EMAIL` + +Email that used to register https certificate. This one is required only if you use HTTPS override. + +### `FRAPPE_SITE_NAME_HEADER` + +This environment variable is not required. Default value is `$$host` which resolves site by host. For example, if your host is `example.com`, site's name should be `example.com`, or if host is `127.0.0.1` (local debugging), it should be `127.0.0.1` This variable allows to override described behavior. Let's say you create site named `mysite` and do want to access it by `127.0.0.1` host. Than you would set this variable to `mysite`. + +There is other variables not mentioned here. They're somewhat internal and you don't have to worry about them except you want to change main compose file. diff --git a/docs/error-nginx-entrypoint-windows.md b/docs/error-nginx-entrypoint-windows.md new file mode 100644 index 00000000..7be1c2ca --- /dev/null +++ b/docs/error-nginx-entrypoint-windows.md @@ -0,0 +1,12 @@ +# Resolving Docker `nginx-entrypoint.sh` Script Not Found Error on Windows + +If you're encountering the error `exec /usr/local/bin/nginx-entrypoint.sh: no such file or directory` in a Docker container on Windows, follow these steps to resolve the issue. + +## 1. Check Line Endings + +On Windows, files often have `CRLF` line endings, while Linux systems expect `LF`. This can cause issues when executing shell scripts in Linux containers. + +- **Convert Line Endings using `dos2unix`:** + ```bash + dos2unix resources/nginx-entrypoint.sh + ``` diff --git a/docs/list-of-containers.md b/docs/list-of-containers.md new file mode 100644 index 00000000..5ed757d5 --- /dev/null +++ b/docs/list-of-containers.md @@ -0,0 +1,58 @@ +# Images + +There are 3 images that you can find in `/images` directory: + +- `bench`. It is used for development. [Learn more how to start development](development.md). +- `production`. + - Multi-purpose Python backend. Runs [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/) with [gunicorn](https://gunicorn.org), queues (via `bench worker`), or schedule (via `bench schedule`). + - Contains JS and CSS assets and routes incoming requests using [nginx](https://www.nginx.com). + - Processes realtime websocket requests using [Socket.IO](https://socket.io). +- `custom`. It is used to build bench using `apps.json` file set with `--apps_path` during bench initialization. `apps.json` is a json array. e.g. `[{"url":"{{repo_url}}","branch":"{{repo_branch}}"}]` + +Image has everything we need to be able to run all processes that Frappe framework requires (take a look at [Bench Procfile reference](https://frappeframework.com/docs/v14/user/en/bench/resources/bench-procfile)). We follow [Docker best practices](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#decouple-applications) and split these processes to different containers. + +> We use [multi-stage builds](https://docs.docker.com/develop/develop-images/multistage-build/) and [Docker Buildx](https://docs.docker.com/engine/reference/commandline/buildx/) to reuse as much things as possible and make our builds more efficient. + +# Compose files + +After building the images we have to run the containers. The best and simplest way to do this is to use [compose files](https://docs.docker.com/compose/compose-file/). + +We have one main compose file, `compose.yaml`. Services described, networking, volumes are also handled there. + +## Services + +All services are described in `compose.yaml` + +- `configurator`. Updates `common_site_config.json` so Frappe knows how to access db and redis. It is executed on every `docker-compose up` (and exited immediately). Other services start after this container exits successfully. +- `backend`. [Werkzeug server](https://werkzeug.palletsprojects.com/en/2.0.x/). +- `db`. Optional service that runs [MariaDB](https://mariadb.com) if you also use `overrides/compose.mariadb.yaml` or [Postgres](https://www.postgresql.org) if you also use `overrides/compose.postgres.yaml`. +- `redis`. Optional service that runs [Redis](https://redis.io) server with cache, [Socket.IO](https://socket.io) and queues data. +- `frontend`. [nginx](https://www.nginx.com) server that serves JS/CSS assets and routes incoming requests. +- `proxy`. [Traefik](https://traefik.io/traefik/) proxy. It is here for complicated setups or HTTPS override (with `overrides/compose.https.yaml`). +- `websocket`. Node server that runs [Socket.IO](https://socket.io). +- `queue-short`, `queue-long`. Python servers that run job queues using [rq](https://python-rq.org). +- `scheduler`. Python server that runs tasks on schedule using [schedule](https://schedule.readthedocs.io/en/stable/). + +## Overrides + +We have several [overrides](https://docs.docker.com/compose/extends/): + +- `overrides/compose.proxy.yaml`. Adds traefik proxy to setup. +- `overrides/compose.noproxy.yaml`. Publishes `frontend` ports directly without any proxy. +- `overrides/compose.https.yaml`. Automatically sets up Let's Encrypt certificate and redirects all requests to directed to http, to https. +- `overrides/compose.mariadb.yaml`. Adds `db` service and sets its image to MariaDB. +- `overrides/compose.postgres.yaml`. Adds `db` service and sets its image to Postgres. Note that ERPNext currently doesn't support Postgres. +- `overrides/compose.redis.yaml`. Adds `redis` service and sets its image to `redis`. + +It is quite simple to run overrides. All we need to do is to specify compose files that should be used by docker-compose. For example, we want ERPNext: + +```bash +# Point to main compose file (compose.yaml) and add one more. +docker-compose -f compose.yaml -f overrides/compose.redis.yaml config +``` + +⚠ Make sure to use docker-compose v2 (run `docker-compose -v` to check). If you want to use v1 make sure the correct `$`-signs as they get duplicated by the `config` command! + +That's it! Of course, we also have to setup `.env` before all of that, but that's not the point. + +Check [environment variables](environment-variables.md) for more. diff --git a/docs/setup_for_linux_mac.md b/docs/setup_for_linux_mac.md new file mode 100644 index 00000000..b26cef48 --- /dev/null +++ b/docs/setup_for_linux_mac.md @@ -0,0 +1,229 @@ +# How to install ERPNext on linux/mac using Frappe_docker ? + +step1: clone the repo + +``` +git clone https://github.com/frappe/frappe_docker +``` + +step2: add platform: linux/amd64 to all services in the /pwd.yaml + +here is the update pwd.yml file + +```yml +version: "3" + +services: + backend: + image: frappe/erpnext:v15 + platform: linux/amd64 + deploy: + restart_policy: + condition: on-failure + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + configurator: + image: frappe/erpnext:v15 + platform: linux/amd64 + deploy: + restart_policy: + condition: none + entrypoint: + - bash + - -c + # add redis_socketio for backward compatibility + command: + - > + ls -1 apps > sites/apps.txt; + bench set-config -g db_host $$DB_HOST; + bench set-config -gp db_port $$DB_PORT; + bench set-config -g redis_cache "redis://$$REDIS_CACHE"; + bench set-config -g redis_queue "redis://$$REDIS_QUEUE"; + bench set-config -g redis_socketio "redis://$$REDIS_QUEUE"; + bench set-config -gp socketio_port $$SOCKETIO_PORT; + environment: + DB_HOST: db + DB_PORT: "3306" + REDIS_CACHE: redis-cache:6379 + REDIS_QUEUE: redis-queue:6379 + SOCKETIO_PORT: "9000" + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + create-site: + image: frappe/erpnext:v15 + platform: linux/amd64 + deploy: + restart_policy: + condition: none + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + entrypoint: + - bash + - -c + command: + - > + wait-for-it -t 120 db:3306; + wait-for-it -t 120 redis-cache:6379; + wait-for-it -t 120 redis-queue:6379; + export start=`date +%s`; + until [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".db_host // empty"` ]] && \ + [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".redis_cache // empty"` ]] && \ + [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".redis_queue // empty"` ]]; + do + echo "Waiting for sites/common_site_config.json to be created"; + sleep 5; + if (( `date +%s`-start > 120 )); then + echo "could not find sites/common_site_config.json with required keys"; + exit 1 + fi + done; + echo "sites/common_site_config.json found"; + bench new-site --no-mariadb-socket --admin-password=admin --db-root-password=admin --install-app erpnext --set-default frontend; + + db: + image: mariadb:10.6 + platform: linux/amd64 + healthcheck: + test: mysqladmin ping -h localhost --password=admin + interval: 1s + retries: 20 + deploy: + restart_policy: + condition: on-failure + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_unicode_ci + - --skip-character-set-client-handshake + - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 + environment: + MYSQL_ROOT_PASSWORD: admin + volumes: + - db-data:/var/lib/mysql + + frontend: + image: frappe/erpnext:v15 + platform: linux/amd64 + depends_on: + - websocket + deploy: + restart_policy: + condition: on-failure + command: + - nginx-entrypoint.sh + environment: + BACKEND: backend:8000 + FRAPPE_SITE_NAME_HEADER: frontend + SOCKETIO: websocket:9000 + UPSTREAM_REAL_IP_ADDRESS: 127.0.0.1 + UPSTREAM_REAL_IP_HEADER: X-Forwarded-For + UPSTREAM_REAL_IP_RECURSIVE: "off" + PROXY_READ_TIMEOUT: 120 + CLIENT_MAX_BODY_SIZE: 50m + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + ports: + - "8080:8080" + + queue-long: + image: frappe/erpnext:v15 + platform: linux/amd64 + deploy: + restart_policy: + condition: on-failure + command: + - bench + - worker + - --queue + - long,default,short + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + queue-short: + image: frappe/erpnext:v15 + platform: linux/amd64 + deploy: + restart_policy: + condition: on-failure + command: + - bench + - worker + - --queue + - short,default + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + redis-queue: + image: redis:6.2-alpine + platform: linux/amd64 + deploy: + restart_policy: + condition: on-failure + volumes: + - redis-queue-data:/data + + redis-cache: + image: redis:6.2-alpine + platform: linux/amd64 + deploy: + restart_policy: + condition: on-failure + volumes: + - redis-cache-data:/data + + scheduler: + image: frappe/erpnext:v15 + platform: linux/amd64 + deploy: + restart_policy: + condition: on-failure + command: + - bench + - schedule + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + websocket: + image: frappe/erpnext:v15 + platform: linux/amd64 + deploy: + restart_policy: + condition: on-failure + command: + - node + - /home/frappe/frappe-bench/apps/frappe/socketio.js + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + +volumes: + db-data: + redis-queue-data: + redis-cache-data: + sites: + logs: +``` + +step3: run the docker + +``` +cd frappe_docker +``` + +``` +docker-compose -f ./pwd.yml up +``` + +--- + +Wait for couple of minutes. + +Open localhost:8080 diff --git a/docs/single-server-example.md b/docs/single-server-example.md new file mode 100644 index 00000000..1bc2c2b1 --- /dev/null +++ b/docs/single-server-example.md @@ -0,0 +1,288 @@ +### Single Server Example + +In this use case we have a single server with a static IP attached to it. It can be used in scenarios where one powerful VM has multiple benches and applications or one entry level VM with single site. For single bench, single site setup follow only up to the point where first bench and first site is added. If you choose this setup you can only scale vertically. If you need to scale horizontally you'll need to backup the sites and restore them on to cluster setup. + +We will setup the following: + +- Install docker and docker compose v2 on linux server. +- Install traefik service for internal load balancer and letsencrypt. +- Install MariaDB with containers. +- Setup project called `erpnext-one` and create sites `one.example.com` and `two.example.com` in the project. +- Setup project called `erpnext-two` and create sites `three.example.com` and `four.example.com` in the project. + +Explanation: + +Single instance of **Traefik** will be installed and act as internal loadbalancer for multiple benches and sites hosted on the server. It can also load balance other applications along with frappe benches, e.g. wordpress, metabase, etc. We only expose the ports `80` and `443` once with this instance of traefik. Traefik will also take care of letsencrypt automation for all sites installed on the server. _Why choose Traefik over Nginx Proxy Manager?_ Traefik doesn't need additional DB service and can store certificates in a json file in a volume. + +Single instance of **MariaDB** will be installed and act as database service for all the benches/projects installed on the server. + +Each instance of ERPNext project (bench) will have its own redis, socketio, gunicorn, nginx, workers and scheduler. It will connect to internal MariaDB by connecting to MariaDB network. It will expose sites to public through Traefik by connecting to Traefik network. + +### Install Docker + +Easiest way to install docker is to use the [convenience script](https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script). + +```shell +curl -fsSL https://get.docker.com | bash +``` + +Note: The documentation assumes Ubuntu LTS server is used. Use any distribution as long as the docker convenience script works. If the convenience script doesn't work, you'll need to install docker manually. + +### Install Compose V2 + +Refer [original documentation](https://docs.docker.com/compose/cli-command/#install-on-linux) for updated version. + +```shell +DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} +mkdir -p $DOCKER_CONFIG/cli-plugins +curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose +chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose +``` + +### Prepare + +Clone `frappe_docker` repo for the needed YAMLs and change the current working directory of your shell to the cloned repo. + +```shell +git clone https://github.com/frappe/frappe_docker +cd frappe_docker +``` + +Create configuration and resources directory + +```shell +mkdir ~/gitops +``` + +The `~/gitops` directory will store all the resources that we use for setup. We will also keep the environment files in this directory as there will be multiple projects with different environment variables. You can create a private repo for this directory and track the changes there. + +### Install Traefik + +Basic Traefik setup using docker compose. + +Create a file called `traefik.env` in `~/gitops` + +```shell +echo 'TRAEFIK_DOMAIN=traefik.example.com' > ~/gitops/traefik.env +echo 'EMAIL=admin@example.com' >> ~/gitops/traefik.env +echo 'HASHED_PASSWORD='$(openssl passwd -apr1 changeit | sed -e s/\\$/\\$\\$/g) >> ~/gitops/traefik.env +``` + +Note: + +- Change the domain from `traefik.example.com` to the one used in production. DNS entry needs to point to the Server IP. +- Change the letsencrypt notification email from `admin@example.com` to correct email. +- Change the password from `changeit` to more secure. + +env file generated at location `~/gitops/traefik.env` will look like following: + +```env +TRAEFIK_DOMAIN=traefik.example.com +EMAIL=admin@example.com +HASHED_PASSWORD=$apr1$K.4gp7RT$tj9R2jHh0D4Gb5o5fIAzm/ +``` + +If Container does not deploy put the HASHED_PASSWORD in ''. + +Deploy the traefik container with letsencrypt SSL + +```shell +docker compose --project-name traefik \ + --env-file ~/gitops/traefik.env \ + -f overrides/compose.traefik.yaml \ + -f overrides/compose.traefik-ssl.yaml up -d +``` + +This will make the traefik dashboard available on `traefik.example.com` and all certificates will reside in the Docker volume `cert-data`. + +For LAN setup deploy the traefik container without overriding `overrides/compose.traefik-ssl.yaml`. + +### Install MariaDB + +Basic MariaDB setup using docker compose. + +Create a file called `mariadb.env` in `~/gitops` + +```shell +echo "DB_PASSWORD=changeit" > ~/gitops/mariadb.env +``` + +Note: + +- Change the password from `changeit` to more secure. + +env file generated at location `~/gitops/mariadb.env` will look like following: + +```env +DB_PASSWORD=changeit +``` + +Note: Change the password from `changeit` to more secure one. + +Deploy the mariadb container + +```shell +docker compose --project-name mariadb --env-file ~/gitops/mariadb.env -f overrides/compose.mariadb-shared.yaml up -d +``` + +This will make `mariadb-database` service available under `mariadb-network`. Data will reside in `/data/mariadb`. + +### Install ERPNext + +#### Create first bench + +Create first bench called `erpnext-one` with `one.example.com` and `two.example.com` + +Create a file called `erpnext-one.env` in `~/gitops` + +```shell +cp example.env ~/gitops/erpnext-one.env +sed -i 's/DB_PASSWORD=123/DB_PASSWORD=changeit/g' ~/gitops/erpnext-one.env +sed -i 's/DB_HOST=/DB_HOST=mariadb-database/g' ~/gitops/erpnext-one.env +sed -i 's/DB_PORT=/DB_PORT=3306/g' ~/gitops/erpnext-one.env +sed -i 's/SITES=`erp.example.com`/SITES=\`one.example.com\`,\`two.example.com\`/g' ~/gitops/erpnext-one.env +echo 'ROUTER=erpnext-one' >> ~/gitops/erpnext-one.env +echo "BENCH_NETWORK=erpnext-one" >> ~/gitops/erpnext-one.env +``` + +Note: + +- Change the password from `changeit` to the one set for MariaDB compose in the previous step. + +env file is generated at location `~/gitops/erpnext-one.env`. + +Create a yaml file called `erpnext-one.yaml` in `~/gitops` directory: + +```shell +docker compose --project-name erpnext-one \ + --env-file ~/gitops/erpnext-one.env \ + -f compose.yaml \ + -f overrides/compose.redis.yaml \ + -f overrides/compose.multi-bench.yaml \ + -f overrides/compose.multi-bench-ssl.yaml config > ~/gitops/erpnext-one.yaml +``` + +For LAN setup do not override `compose.multi-bench-ssl.yaml`. + +Use the above command after any changes are made to `erpnext-one.env` file to regenerate `~/gitops/erpnext-one.yaml`. e.g. after changing version to migrate the bench. + +Deploy `erpnext-one` containers: + +```shell +docker compose --project-name erpnext-one -f ~/gitops/erpnext-one.yaml up -d +``` + +Create sites `one.example.com` and `two.example.com`: + +```shell +# one.example.com +docker compose --project-name erpnext-one exec backend \ + bench new-site --no-mariadb-socket --mariadb-root-password changeit --install-app erpnext --admin-password changeit one.example.com +``` + +You can stop here and have a single bench single site setup complete. Continue to add one more site to the current bench. + +```shell +# two.example.com +docker compose --project-name erpnext-one exec backend \ + bench new-site --no-mariadb-socket --mariadb-root-password changeit --install-app erpnext --admin-password changeit two.example.com +``` + +#### Create second bench + +Setting up additional bench is optional. Continue only if you need multi bench setup. + +Create second bench called `erpnext-two` with `three.example.com` and `four.example.com` + +Create a file called `erpnext-two.env` in `~/gitops` + +```shell +curl -sL https://raw.githubusercontent.com/frappe/frappe_docker/main/example.env -o ~/gitops/erpnext-two.env +sed -i 's/DB_PASSWORD=123/DB_PASSWORD=changeit/g' ~/gitops/erpnext-two.env +sed -i 's/DB_HOST=/DB_HOST=mariadb-database/g' ~/gitops/erpnext-two.env +sed -i 's/DB_PORT=/DB_PORT=3306/g' ~/gitops/erpnext-two.env +echo "ROUTER=erpnext-two" >> ~/gitops/erpnext-two.env +echo "SITES=\`three.example.com\`,\`four.example.com\`" >> ~/gitops/erpnext-two.env +echo "BENCH_NETWORK=erpnext-two" >> ~/gitops/erpnext-two.env +``` + +Note: + +- Change the password from `changeit` to the one set for MariaDB compose in the previous step. + +env file is generated at location `~/gitops/erpnext-two.env`. + +Create a yaml file called `erpnext-two.yaml` in `~/gitops` directory: + +```shell +docker compose --project-name erpnext-two \ + --env-file ~/gitops/erpnext-two.env \ + -f compose.yaml \ + -f overrides/compose.redis.yaml \ + -f overrides/compose.multi-bench.yaml \ + -f overrides/compose.multi-bench-ssl.yaml config > ~/gitops/erpnext-two.yaml +``` + +Use the above command after any changes are made to `erpnext-two.env` file to regenerate `~/gitops/erpnext-two.yaml`. e.g. after changing version to migrate the bench. + +Deploy `erpnext-two` containers: + +```shell +docker compose --project-name erpnext-two -f ~/gitops/erpnext-two.yaml up -d +``` + +Create sites `three.example.com` and `four.example.com`: + +```shell +# three.example.com +docker compose --project-name erpnext-two exec backend \ + bench new-site --no-mariadb-socket --mariadb-root-password changeit --install-app erpnext --admin-password changeit three.example.com +# four.example.com +docker compose --project-name erpnext-two exec backend \ + bench new-site --no-mariadb-socket --mariadb-root-password changeit --install-app erpnext --admin-password changeit four.example.com +``` + +#### Create custom domain to existing site + +In case you need to point custom domain to existing site follow these steps. +Also useful if custom domain is required for LAN based access. + +Create environment file + +```shell +echo "ROUTER=custom-one-example" > ~/gitops/custom-one-example.env +echo "SITES=\`custom-one.example.com\`" >> ~/gitops/custom-one-example.env +echo "BASE_SITE=one.example.com" >> ~/gitops/custom-one-example.env +echo "BENCH_NETWORK=erpnext-one" >> ~/gitops/custom-one-example.env +``` + +Note: + +- Change the file name from `custom-one-example.env` to a logical one. +- Change `ROUTER` variable from `custom-one.example.com` to the one being added. +- Change `SITES` variable from `custom-one.example.com` to the one being added. You can add multiple sites quoted in backtick (`) and separated by commas. +- Change `BASE_SITE` variable from `one.example.com` to the one which is being pointed to. +- Change `BENCH_NETWORK` variable from `erpnext-one` to the one which was created with the bench. + +env file is generated at location mentioned in command. + +Generate yaml to reverse proxy: + +```shell +docker compose --project-name custom-one-example \ + --env-file ~/gitops/custom-one-example.env \ + -f overrides/compose.custom-domain.yaml \ + -f overrides/compose.custom-domain-ssl.yaml config > ~/gitops/custom-one-example.yaml +``` + +For LAN setup do not override `compose.custom-domain-ssl.yaml`. + +Deploy `erpnext-two` containers: + +```shell +docker compose --project-name custom-one-example -f ~/gitops/custom-one-example.yaml up -d +``` + +### Site operations + +Refer: [site operations](./site-operations.md) diff --git a/docs/troubleshoot.md b/docs/troubleshoot.md new file mode 100644 index 00000000..7ddbe657 --- /dev/null +++ b/docs/troubleshoot.md @@ -0,0 +1,76 @@ +1. [Fixing MariaDB issues after rebuilding the container](#fixing-mariadb-issues-after-rebuilding-the-container) +1. [docker-compose does not recognize variables from `.env` file](#docker-compose-does-not-recognize-variables-from-env-file) +1. [Windows Based Installation](#windows-based-installation) + +### Fixing MariaDB issues after rebuilding the container + +For any reason after rebuilding the container if you are not be able to access MariaDB correctly (i.e. `Access denied for user [...]`) with the previous configuration. Follow these instructions. + +First test for network issues. Manually connect to the database through the `backend` container: + +``` +docker exec -it frappe_docker-backend-1 bash +mysql -uroot -padmin -hdb +``` + +Replace `root` with the database root user name, `admin` with the root password, and `db` with the service name specified in the docker-compose `.yml` configuration file. If the connection to the database is successful, then the network configuration is correct and you can proceed to the next step. Otherwise, modify the docker-compose `.yml` configuration file, in the `configurator` service's `environment` section, to use the container names (`frappe_docker-db-1`, `frappe_docker-redis-cache-1`, `frappe_docker-redis-queue-1` or as otherwise shown with `docker ps`) instead of the service names and rebuild the containers. + +Then, the parameter `'db_name'@'%'` needs to be set in MariaDB and permission to the site database suitably assigned to the user. + +This step has to be repeated for all sites available under the current bench. +Example shows the queries to be executed for site `localhost` + +Open sites/localhost/site_config.json: + +```shell +code sites/localhost/site_config.json +``` + +and take note of the parameters `db_name` and `db_password`. + +Enter MariaDB Interactive shell: + +```shell +mysql -uroot -padmin -hdb +``` + +The parameter `'db_name'@'%'` must not be duplicated. Verify that it is unique with the command: + +``` +SELECT User, Host FROM mysql.user; +``` + +Delete duplicated entries, if found, with the following: + +``` +DROP USER 'db_name'@'host'; +``` + +Modify permissions by executing following queries replacing `db_name` and `db_password` with the values found in site_config.json. + +```sql +UPDATE mysql.global_priv SET Host = '%' where User = 'db_name'; FLUSH PRIVILEGES; +SET PASSWORD FOR 'db_name'@'%' = PASSWORD('db_password'); FLUSH PRIVILEGES; +GRANT ALL PRIVILEGES ON `db_name`.* TO 'db_name'@'%' IDENTIFIED BY 'db_password' WITH GRANT OPTION; FLUSH PRIVILEGES; +EXIT; +``` + +Note: For MariaDB 10.3 and older use `mysql.user` instead of `mysql.global_priv`. + +### docker-compose does not recognize variables from `.env` file + +If you are using old version of `docker-compose` the .env file needs to be located in directory from where the docker-compose command is executed. There may also be difference in official `docker-compose` and the one packaged by distro. Use `--env-file=.env` if available to explicitly specify the path to file. + +### Windows Based Installation + +- Set environment variable `COMPOSE_CONVERT_WINDOWS_PATHS` e.g. `set COMPOSE_CONVERT_WINDOWS_PATHS=1` +- While using docker machine, port-forward the ports of VM to ports of host machine. (ports 8080/8000/9000) +- Name all the sites ending with `.localhost`. and access it via browser locally. e.g. `http://site1.localhost` + +### Redo installation + +- If you have made changes and just want to start over again (abandoning all changes), remove all docker + - containers + - images + - volumes +- Install a fresh diff --git a/images/bench/Dockerfile b/images/bench/Dockerfile index d83e437d..49186483 100644 --- a/images/bench/Dockerfile +++ b/images/bench/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bookworm-slim as bench +FROM debian:bookworm-slim AS bench ARG GIT_REPO=https://github.com/frappe/bench.git ARG GIT_BRANCH=v5.x @@ -67,6 +67,7 @@ RUN apt-get update \ xz-utils \ tk-dev \ liblzma-dev \ + file \ && rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ @@ -95,8 +96,8 @@ WORKDIR /home/frappe # Install Python via pyenv ENV PYTHON_VERSION_V14=3.10.13 ENV PYTHON_VERSION=3.11.6 -ENV PYENV_ROOT /home/frappe/.pyenv -ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH +ENV PYENV_ROOT=/home/frappe/.pyenv +ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH # From https://github.com/pyenv/pyenv#basic-github-checkout RUN git clone --depth 1 https://github.com/pyenv/pyenv.git .pyenv \ @@ -111,7 +112,7 @@ RUN git clone --depth 1 https://github.com/pyenv/pyenv.git .pyenv \ # Clone and install bench in the local user home directory # For development, bench source is located in ~/.bench -ENV PATH /home/frappe/.local/bin:$PATH +ENV PATH=/home/frappe/.local/bin:$PATH # Skip editable-bench warning # https://github.com/frappe/bench/commit/20560c97c4246b2480d7358c722bc9ad13606138 RUN git clone ${GIT_REPO} --depth 1 -b ${GIT_BRANCH} .bench \ @@ -122,8 +123,8 @@ RUN git clone ${GIT_REPO} --depth 1 -b ${GIT_BRANCH} .bench \ # Install Node via nvm ENV NODE_VERSION_14=16.20.2 ENV NODE_VERSION=18.18.2 -ENV NVM_DIR /home/frappe/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} +ENV NVM_DIR=/home/frappe/.nvm +ENV PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash \ && . ${NVM_DIR}/nvm.sh \ @@ -142,7 +143,7 @@ RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | EXPOSE 8000-8005 9000-9005 6787 -FROM bench as bench-test +FROM bench AS bench-test # Print version and verify bashrc is properly sourced so that everything works # in the interactive shell and Dockerfile diff --git a/images/custom/Containerfile b/images/custom/Containerfile index e1ed7f32..f7186294 100644 --- a/images/custom/Containerfile +++ b/images/custom/Containerfile @@ -6,7 +6,7 @@ ARG WKHTMLTOPDF_VERSION=0.12.6.1-3 ARG WKHTMLTOPDF_DISTRO=bookworm ARG NODE_VERSION=18.18.2 ENV NVM_DIR=/home/zapal/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} +ENV PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} RUN useradd -ms /bin/bash zapal \ && apt-get update \ @@ -16,6 +16,7 @@ RUN useradd -ms /bin/bash zapal \ vim \ nginx \ gettext-base \ + file \ # weasyprint dependencies libpango-1.0-0 \ libharfbuzz0b \ @@ -77,6 +78,12 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ # For frappe framework wget \ + #for building arm64 binaries + libcairo2-dev \ + libpango1.0-dev \ + libjpeg-dev \ + libgif-dev \ + librsvg2-dev \ # For psycopg2 libpq-dev \ # Other @@ -112,28 +119,32 @@ ARG ERPNEXT_BRANCH=version-15 ARG HRMS_REPO=https://github.com/zapal-tech/erp-hrms ARG HRMS_BRANCH=version-15 ARG INSIGHTS_REPO=https://github.com/zapal-tech/erp-insights -ARG INSIGHTS_BRANCH=develop +ARG INSIGHTS_BRANCH=version-3 -RUN bench init \ - --frappe-branch=${FRAPPE_BRANCH} \ - --frappe-path=${FRAPPE_PATH} \ - --no-procfile \ - --no-backups \ - --skip-redis-config-generation \ - --verbose \ - /home/zapal/frappe-bench +RUN export APP_INSTALL_ARGS="" && \ + if [ -n "${APPS_JSON_BASE64}" ]; then \ + export APP_INSTALL_ARGS="--apps_path=/opt/frappe/apps.json"; \ + fi && \ + bench init ${APP_INSTALL_ARGS}\ + --frappe-branch=${FRAPPE_BRANCH} \ + --frappe-path=${FRAPPE_PATH} \ + --no-procfile \ + --no-backups \ + --skip-redis-config-generation \ + --verbose \ + /home/frappe/frappe-bench && \ + cd /home/frappe/frappe-bench && \ + echo "frappe\nhrms\nerpnext" > sites/apps.txt + echo "{}" > sites/common_site_config.json && \ + find apps -mindepth 1 -path "*/.git" | xargs rm -fr WORKDIR /home/zapal/frappe-bench RUN bench get-app --branch=${ERPNEXT_BRANCH} --resolve-deps erpnext ${ERPNEXT_REPO} RUN bench get-app --branch=${HRMS_BRANCH} --resolve-deps hrms ${HRMS_REPO} -# RUN bench get-app --branch=${INSIGHTS_BRANCH} --resolve-deps insights ${INSIGHTS_REPO} +RUN bench get-app --branch=${INSIGHTS_BRANCH} --resolve-deps insights ${INSIGHTS_REPO} -RUN echo "frappe\nhrms\nerpnext" > sites/apps.txt -RUN echo "{}" > sites/common_site_config.json -RUN find apps -mindepth 1 -path "*/.git" | xargs rm -fr - -FROM base as erp +FROM base as backend USER zapal @@ -143,7 +154,11 @@ COPY /usr/local/bin/nginx-entrypoint.sh /usr/local/bin/nginx-entrypoint.sh WORKDIR /home/zapal/frappe-bench -VOLUME ["/home/zapal/frappe-bench", "/home/zapal/frappe-bench/logs"] +VOLUME [ \ + "/home/zapal/frappe-bench/sites", \ + "/home/zapal/frappe-bench/sites/assets", \ + "/home/zapal/frappe-bench/logs" \ +] CMD [ \ "/home/zapal/frappe-bench/env/bin/gunicorn", \ diff --git a/images/layered/Containerfile b/images/layered/Containerfile new file mode 100644 index 00000000..12a089ee --- /dev/null +++ b/images/layered/Containerfile @@ -0,0 +1,58 @@ +ARG FRAPPE_BRANCH=version-15 + +FROM frappe/build:${FRAPPE_BRANCH} AS builder + +ARG FRAPPE_BRANCH=version-15 +ARG FRAPPE_PATH=https://github.com/frappe/frappe +ARG APPS_JSON_BASE64 + +USER root + +RUN if [ -n "${APPS_JSON_BASE64}" ]; then \ + mkdir /opt/frappe && echo "${APPS_JSON_BASE64}" | base64 -d > /opt/frappe/apps.json; \ + fi + +USER frappe + +RUN export APP_INSTALL_ARGS="" && \ + if [ -n "${APPS_JSON_BASE64}" ]; then \ + export APP_INSTALL_ARGS="--apps_path=/opt/frappe/apps.json"; \ + fi && \ + bench init ${APP_INSTALL_ARGS}\ + --frappe-branch=${FRAPPE_BRANCH} \ + --frappe-path=${FRAPPE_PATH} \ + --no-procfile \ + --no-backups \ + --skip-redis-config-generation \ + --verbose \ + /home/frappe/frappe-bench && \ + cd /home/frappe/frappe-bench && \ + echo "{}" > sites/common_site_config.json && \ + find apps -mindepth 1 -path "*/.git" | xargs rm -fr + +FROM frappe/base:${FRAPPE_BRANCH} AS backend + +USER frappe + +COPY --from=builder --chown=frappe:frappe /home/frappe/frappe-bench /home/frappe/frappe-bench + +WORKDIR /home/frappe/frappe-bench + +VOLUME [ \ + "/home/frappe/frappe-bench/sites", \ + "/home/frappe/frappe-bench/sites/assets", \ + "/home/frappe/frappe-bench/logs" \ +] + +CMD [ \ + "/home/frappe/frappe-bench/env/bin/gunicorn", \ + "--chdir=/home/frappe/frappe-bench/sites", \ + "--bind=0.0.0.0:8000", \ + "--threads=4", \ + "--workers=2", \ + "--worker-class=gthread", \ + "--worker-tmp-dir=/dev/shm", \ + "--timeout=120", \ + "--preload", \ + "frappe.app:application" \ +] diff --git a/images/production/Containerfile b/images/production/Containerfile index 107b747a..daa610ed 100644 --- a/images/production/Containerfile +++ b/images/production/Containerfile @@ -6,7 +6,7 @@ ARG WKHTMLTOPDF_VERSION=0.12.6.1-3 ARG WKHTMLTOPDF_DISTRO=bookworm ARG NODE_VERSION=18.18.2 ENV NVM_DIR=/home/zapal/.nvm -ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} +ENV PATH=${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} COPY resources/nginx-template.conf /templates/nginx/erp.conf.template COPY resources/nginx-entrypoint.sh /usr/local/bin/nginx-entrypoint.sh @@ -19,6 +19,7 @@ RUN useradd -ms /bin/bash zapal \ vim \ nginx \ gettext-base \ + file \ # weasyprint dependencies libpango-1.0-0 \ libharfbuzz0b \ @@ -71,7 +72,7 @@ RUN useradd -ms /bin/bash zapal \ && chmod 755 /usr/local/bin/nginx-entrypoint.sh \ && chmod 644 /templates/nginx/erp.conf.template -FROM base AS builder +FROM base AS build RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ @@ -99,6 +100,8 @@ RUN apt-get update \ USER zapal +FROM build AS builder + ARG FRAPPE_BRANCH=version-15 ARG FRAPPE_PATH=https://github.com/zapal-tech/erp-frappe ARG ERPNEXT_REPO=https://github.com/zapal-tech/erp-erpnext @@ -106,7 +109,7 @@ ARG ERPNEXT_BRANCH=version-15 ARG HRMS_REPO=https://github.com/zapal-tech/erp-hrms ARG HRMS_BRANCH=version-15 ARG INSIGHTS_REPO=https://github.com/zapal-tech/erp-insights -ARG INSIGHTS_BRANCH=develop +ARG INSIGHTS_BRANCH=version-3 RUN bench init \ --frappe-branch=${FRAPPE_BRANCH} \ @@ -115,27 +118,32 @@ RUN bench init \ --no-backups \ --skip-redis-config-generation \ --verbose \ - /home/zapal/frappe-bench + /home/zapal/frappe-bench && \ + cd /home/frappe/frappe-bench && \ + bench get-app --branch=${ERPNEXT_BRANCH} --resolve-deps erpnext ${ERPNEXT_REPO} && \ + echo "frappe\nhrms\nerpnext" > sites/apps.txt && \ + echo "{}" > sites/common_site_config.json && \ + find apps -mindepth 1 -path "*/.git" | xargs rm -fr -WORKDIR /home/zapal/frappe-bench +FROM base AS erpnext + +USER zapal RUN cd /home/zapal/frappe-bench && bench get-app --branch=${ERPNEXT_BRANCH} --resolve-deps erpnext ${ERPNEXT_REPO} RUN cd /home/zapal/frappe-bench && bench get-app --branch=${HRMS_BRANCH} --resolve-deps hrms ${HRMS_REPO} -# RUN cd /home/zapal/frappe-bench && bench get-app --branch=${INSIGHTS_BRANCH} --resolve-deps insights ${INSIGHTS_REPO} +RUN cd /home/zapal/frappe-bench && bench get-app --branch=${INSIGHTS_BRANCH} --resolve-deps insights ${INSIGHTS_REPO} -RUN echo "frappe\nhrms\nerpnext" > sites/apps.txt -RUN echo "{}" > sites/common_site_config.json -RUN find apps -mindepth 1 -path "*/.git" | xargs rm -fr - -FROM base as erp - -USER zapal +RUN echo "echo \"Commands restricted in production container, Read FAQ before you proceed: https://frappe.fyi/ctr-faq\"" >> ~/.bashrc COPY --from=builder --chown=zapal:zapal /home/zapal/frappe-bench /home/zapal/frappe-bench WORKDIR /home/zapal/frappe-bench -VOLUME ["/home/zapal/frappe-bench", "/home/zapal/frappe-bench/logs"] +VOLUME [ \ + "/home/zapal/frappe-bench/sites", \ + "/home/zapal/frappe-bench/sites/assets", \ + "/home/zapal/frappe-bench/logs" \ +] CMD [ \ "/home/zapal/frappe-bench/env/bin/gunicorn", \ diff --git a/overrides/compose.https.yaml b/overrides/compose.https.yaml index a34e2d3c..8d78602a 100644 --- a/overrides/compose.https.yaml +++ b/overrides/compose.https.yaml @@ -8,7 +8,7 @@ services: - traefik.http.routers.frontend-http.rule=Host(${SITES:?List of sites not set}) proxy: - image: traefik:2.5 + image: traefik:v2.11 command: - --providers.docker=true - --providers.docker.exposedbydefault=false @@ -21,8 +21,8 @@ services: - --certificatesResolvers.main-resolver.acme.email=${LETSENCRYPT_EMAIL:?No Let's Encrypt email set} - --certificatesResolvers.main-resolver.acme.storage=/letsencrypt/acme.json ports: - - 80:80 - - 443:443 + - ${HTTP_PUBLISH_PORT:-80}:80 + - ${HTTPS_PUBLISH_PORT:-443}:443 volumes: - cert-data:/letsencrypt - /var/run/docker.sock:/var/run/docker.sock:ro diff --git a/overrides/compose.mariadb-shared.yaml b/overrides/compose.mariadb-shared.yaml index 13bfc656..8872b48f 100644 --- a/overrides/compose.mariadb-shared.yaml +++ b/overrides/compose.mariadb-shared.yaml @@ -8,7 +8,7 @@ services: healthcheck: test: mysqladmin ping -h localhost --password=${DB_PASSWORD:-changeit} interval: 1s - retries: 15 + retries: 20 command: - --character-set-server=utf8mb4 - --collation-server=utf8mb4_unicode_ci diff --git a/overrides/compose.mariadb.yaml b/overrides/compose.mariadb.yaml index cd719b62..3dee180f 100644 --- a/overrides/compose.mariadb.yaml +++ b/overrides/compose.mariadb.yaml @@ -12,7 +12,7 @@ services: healthcheck: test: mysqladmin ping -h localhost --password=${DB_PASSWORD} interval: 1s - retries: 15 + retries: 20 command: - --character-set-server=utf8mb4 - --collation-server=utf8mb4_unicode_ci diff --git a/overrides/compose.noproxy.yaml b/overrides/compose.noproxy.yaml index 5c4f83c0..23239ff2 100644 --- a/overrides/compose.noproxy.yaml +++ b/overrides/compose.noproxy.yaml @@ -1,4 +1,4 @@ services: frontend: ports: - - 8080:8080 + - ${HTTP_PUBLISH_PORT:-8080}:8080 diff --git a/overrides/compose.proxy.yaml b/overrides/compose.proxy.yaml index d2dc319a..32ce9fab 100644 --- a/overrides/compose.proxy.yaml +++ b/overrides/compose.proxy.yaml @@ -7,13 +7,13 @@ services: - traefik.http.routers.frontend-http.rule=HostRegexp(`{any:.+}`) proxy: - image: traefik:2.5 + image: traefik:v2.11 command: - --providers.docker - --providers.docker.exposedbydefault=false - --entrypoints.web.address=:80 ports: - - 80:80 + - ${HTTP_PUBLISH_PORT:-80}:80 volumes: - /var/run/docker.sock:/var/run/docker.sock:ro userns_mode: host diff --git a/overrides/compose.traefik-ssl.yaml b/overrides/compose.traefik-ssl.yaml new file mode 100644 index 00000000..be996f54 --- /dev/null +++ b/overrides/compose.traefik-ssl.yaml @@ -0,0 +1,48 @@ +services: + traefik: + labels: + # https-redirect middleware to redirect HTTP to HTTPS + # It can be re-used by other stacks in other Docker Compose files + - traefik.http.middlewares.https-redirect.redirectscheme.scheme=https + - traefik.http.middlewares.https-redirect.redirectscheme.permanent=true + # traefik-http to use the middleware to redirect to https + - traefik.http.routers.traefik-public-http.middlewares=https-redirect + # traefik-https the actual router using HTTPS + # Uses the environment variable DOMAIN + - traefik.http.routers.traefik-public-https.rule=Host(`${TRAEFIK_DOMAIN}`) + - traefik.http.routers.traefik-public-https.entrypoints=https + - traefik.http.routers.traefik-public-https.tls=true + # Use the special Traefik service api@internal with the web UI/Dashboard + - traefik.http.routers.traefik-public-https.service=api@internal + # Use the "le" (Let's Encrypt) resolver created below + - traefik.http.routers.traefik-public-https.tls.certresolver=le + # Enable HTTP Basic auth, using the middleware created above + - traefik.http.routers.traefik-public-https.middlewares=admin-auth + command: + # Enable Docker in Traefik, so that it reads labels from Docker services + - --providers.docker=true + # Do not expose all Docker services, only the ones explicitly exposed + - --providers.docker.exposedbydefault=false + # Create an entrypoint http listening on port 80 + - --entrypoints.http.address=:80 + # Create an entrypoint https listening on port 443 + - --entrypoints.https.address=:443 + # Create the certificate resolver le for Let's Encrypt, uses the environment variable EMAIL + - --certificatesresolvers.le.acme.email=${EMAIL:?No EMAIL set} + # Store the Let's Encrypt certificates in the mounted volume + - --certificatesresolvers.le.acme.storage=/certificates/acme.json + # Use the TLS Challenge for Let's Encrypt + - --certificatesresolvers.le.acme.tlschallenge=true + # Enable the access log, with HTTP requests + - --accesslog + # Enable the Traefik log, for configurations and errors + - --log + # Enable the Dashboard and API + - --api + ports: + - ${HTTPS_PUBLISH_PORT:-443}:443 + volumes: + - cert-data:/certificates + +volumes: + cert-data: diff --git a/overrides/compose.traefik.yaml b/overrides/compose.traefik.yaml index f3e64f6c..25d362af 100644 --- a/overrides/compose.traefik.yaml +++ b/overrides/compose.traefik.yaml @@ -2,7 +2,7 @@ version: "3.3" services: traefik: - image: "traefik:v2.6" + image: "traefik:v2.11" restart: unless-stopped labels: # Enable Traefik for this service, to make it available in the public network @@ -35,7 +35,7 @@ services: # Enable the Dashboard and API - --api ports: - - 80:80 + - ${HTTP_PUBLISH_PORT:-80}:80 volumes: - /var/run/docker.sock:/var/run/docker.sock:ro networks: diff --git a/pwd.yml b/pwd.yml new file mode 100644 index 00000000..dc4cbcbf --- /dev/null +++ b/pwd.yml @@ -0,0 +1,189 @@ +version: "3" + +services: + backend: + image: frappe/erpnext:v15.44.0 + deploy: + restart_policy: + condition: on-failure + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + configurator: + image: frappe/erpnext:v15.44.0 + deploy: + restart_policy: + condition: none + entrypoint: + - bash + - -c + # add redis_socketio for backward compatibility + command: + - > + ls -1 apps > sites/apps.txt; + bench set-config -g db_host $$DB_HOST; + bench set-config -gp db_port $$DB_PORT; + bench set-config -g redis_cache "redis://$$REDIS_CACHE"; + bench set-config -g redis_queue "redis://$$REDIS_QUEUE"; + bench set-config -g redis_socketio "redis://$$REDIS_QUEUE"; + bench set-config -gp socketio_port $$SOCKETIO_PORT; + environment: + DB_HOST: db + DB_PORT: "3306" + REDIS_CACHE: redis-cache:6379 + REDIS_QUEUE: redis-queue:6379 + SOCKETIO_PORT: "9000" + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + create-site: + image: frappe/erpnext:v15.44.0 + deploy: + restart_policy: + condition: none + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + entrypoint: + - bash + - -c + command: + - > + wait-for-it -t 120 db:3306; + wait-for-it -t 120 redis-cache:6379; + wait-for-it -t 120 redis-queue:6379; + export start=`date +%s`; + until [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".db_host // empty"` ]] && \ + [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".redis_cache // empty"` ]] && \ + [[ -n `grep -hs ^ sites/common_site_config.json | jq -r ".redis_queue // empty"` ]]; + do + echo "Waiting for sites/common_site_config.json to be created"; + sleep 5; + if (( `date +%s`-start > 120 )); then + echo "could not find sites/common_site_config.json with required keys"; + exit 1 + fi + done; + echo "sites/common_site_config.json found"; + bench new-site --mariadb-user-host-login-scope='%' --admin-password=admin --db-root-username=root --db-root-password=admin --install-app erpnext --set-default frontend; + + db: + image: mariadb:10.6 + healthcheck: + test: mysqladmin ping -h localhost --password=admin + interval: 1s + retries: 20 + deploy: + restart_policy: + condition: on-failure + command: + - --character-set-server=utf8mb4 + - --collation-server=utf8mb4_unicode_ci + - --skip-character-set-client-handshake + - --skip-innodb-read-only-compressed # Temporary fix for MariaDB 10.6 + environment: + MYSQL_ROOT_PASSWORD: admin + MARIADB_ROOT_PASSWORD: admin + volumes: + - db-data:/var/lib/mysql + + frontend: + image: frappe/erpnext:v15.44.0 + depends_on: + - websocket + deploy: + restart_policy: + condition: on-failure + command: + - nginx-entrypoint.sh + environment: + BACKEND: backend:8000 + FRAPPE_SITE_NAME_HEADER: frontend + SOCKETIO: websocket:9000 + UPSTREAM_REAL_IP_ADDRESS: 127.0.0.1 + UPSTREAM_REAL_IP_HEADER: X-Forwarded-For + UPSTREAM_REAL_IP_RECURSIVE: "off" + PROXY_READ_TIMEOUT: 120 + CLIENT_MAX_BODY_SIZE: 50m + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + ports: + - "8080:8080" + + queue-long: + image: frappe/erpnext:v15.44.0 + deploy: + restart_policy: + condition: on-failure + command: + - bench + - worker + - --queue + - long,default,short + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + queue-short: + image: frappe/erpnext:v15.44.0 + deploy: + restart_policy: + condition: on-failure + command: + - bench + - worker + - --queue + - short,default + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + redis-queue: + image: redis:6.2-alpine + deploy: + restart_policy: + condition: on-failure + volumes: + - redis-queue-data:/data + + redis-cache: + image: redis:6.2-alpine + deploy: + restart_policy: + condition: on-failure + volumes: + - redis-cache-data:/data + + scheduler: + image: frappe/erpnext:v15.44.0 + deploy: + restart_policy: + condition: on-failure + command: + - bench + - schedule + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + + websocket: + image: frappe/erpnext:v15.44.0 + deploy: + restart_policy: + condition: on-failure + command: + - node + - /home/frappe/frappe-bench/apps/frappe/socketio.js + volumes: + - sites:/home/frappe/frappe-bench/sites + - logs:/home/frappe/frappe-bench/logs + +volumes: + db-data: + redis-queue-data: + redis-cache-data: + sites: + logs: diff --git a/requirements-test.txt b/requirements-test.txt index 8075a1ec..40543aab 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1 @@ -pytest==8.0.0 +pytest==8.3.3