diff --git a/.github/scripts/get_latest_tags.py b/.github/scripts/get_latest_tags.py index c1f2cdc5..b2e083c4 100644 --- a/.github/scripts/get_latest_tags.py +++ b/.github/scripts/get_latest_tags.py @@ -19,6 +19,8 @@ def get_latest_tag(repo: Repo, version: MajorVersion) -> str: refs = subprocess.check_output( ( "git", + "-c", + "versionsort.suffix=-", "ls-remote", "--refs", "--tags", diff --git a/.github/workflows/build_bench.yml b/.github/workflows/build_bench.yml index 004c630e..be55b53c 100644 --- a/.github/workflows/build_bench.yml +++ b/.github/workflows/build_bench.yml @@ -32,7 +32,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Build and test - uses: docker/bake-action@v2.1.0 + uses: docker/bake-action@v2.2.0 with: targets: bench-test @@ -45,7 +45,7 @@ jobs: - name: Push if: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} - uses: docker/bake-action@v2.1.0 + uses: docker/bake-action@v2.2.0 with: targets: bench push: true diff --git a/.github/workflows/build_stable.yml b/.github/workflows/build_stable.yml index 4188ac5c..d11cc87e 100644 --- a/.github/workflows/build_stable.yml +++ b/.github/workflows/build_stable.yml @@ -68,7 +68,7 @@ jobs: name: Update example.env and pwd.yml runs-on: ubuntu-latest if: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} - needs: v13 + needs: v14 steps: - name: Checkout @@ -77,10 +77,10 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.10" - name: Get latest versions - run: python3 ./.github/scripts/get_latest_tags.py --repo erpnext --version 13 + run: python3 ./.github/scripts/get_latest_tags.py --repo erpnext --version 14 - name: Update run: | @@ -106,7 +106,7 @@ jobs: name: Release Helm runs-on: ubuntu-latest if: ${{ github.repository == 'frappe/frappe_docker' && github.event_name != 'pull_request' }} - needs: v13 + needs: v14 steps: - name: Setup deploy key @@ -123,4 +123,4 @@ jobs: run: | git clone git@github.com:frappe/helm.git && cd helm pip install -r release_wizard/requirements.txt - ./release_wizard/wizard 13 patch --remote origin --ci + ./release_wizard/wizard 14 patch --remote origin --ci diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 22da458b..b522aa80 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -43,7 +43,7 @@ jobs: run: python3 ./.github/scripts/get_latest_tags.py --repo ${{ inputs.repo }} --version ${{ inputs.version }} - name: Build - uses: docker/bake-action@v2.1.0 + uses: docker/bake-action@v2.2.0 with: push: true env: @@ -52,7 +52,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.10" - name: Install Docker Compose v2 uses: ndeloof/install-compose-action@4a33bc31f327b8231c4f343f6fba704fedc0fa23 @@ -74,6 +74,6 @@ jobs: - name: Push if: ${{ inputs.push }} - uses: docker/bake-action@v2.1.0 + uses: docker/bake-action@v2.2.0 with: push: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ddae51d2..97b1b04a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,7 +18,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.10" # For shfmt pre-commit hook - name: Setup Go diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd5d88ae..34eb3b62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,13 +8,13 @@ repos: - id: end-of-file-fixer - repo: https://github.com/asottile/pyupgrade - rev: v2.34.0 + rev: v2.37.3 hooks: - id: pyupgrade args: [--py37-plus] - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 22.8.0 hooks: - id: black @@ -29,9 +29,12 @@ repos: - id: prettier - repo: https://github.com/codespell-project/codespell - rev: v2.1.0 + rev: v2.2.1 hooks: - id: codespell + args: + - -L + - "ro" - repo: local hooks: diff --git a/compose.yaml b/compose.yaml index 2916c38e..97c4d1d8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,6 +8,7 @@ x-backend-defaults: &backend_defaults image: frappe/frappe-worker:${FRAPPE_VERSION:?No Frappe version set} volumes: - sites:/home/frappe/frappe-bench/sites + - assets:/home/frappe/frappe-bench/sites/assets:ro services: configurator: @@ -24,9 +25,6 @@ services: backend: <<: *backend_defaults - volumes: - - sites:/home/frappe/frappe-bench/sites - - assets:/home/frappe/frappe-bench/sites/assets:ro frontend: image: frappe/frappe-nginx:${FRAPPE_VERSION} diff --git a/development/README.md b/development/README.md index d13a32c7..8263dd1a 100644 --- a/development/README.md +++ b/development/README.md @@ -64,11 +64,23 @@ Notes: Run the following commands in the terminal inside the container. You might need to create a new terminal in VSCode. ```shell -bench init --skip-redis-config-generation --frappe-branch version-13 frappe-bench +bench init --skip-redis-config-generation frappe-bench cd frappe-bench ``` -Note: For version 12 use Python 3.7 by passing option to `bench init` command, e.g. `bench init --skip-redis-config-generation --frappe-branch version-12 --python python3.7 frappe-bench` +For version 13 use Python 3.9 by passing option to `bench init` command, + +```shell +bench init --skip-redis-config-generation --frappe-branch version-13 --python python3.9 frappe-bench +cd frappe-bench +``` + +For version 12 use Python 3.7 by passing option to `bench init` command, + +```shell +bench init --skip-redis-config-generation --frappe-branch version-12 --python python3.7 frappe-bench +cd frappe-bench +``` ### Setup hosts @@ -98,6 +110,8 @@ Note : With the option '--skip-redis-config-generation' during bench init, these Honcho is the tool used by Bench to manage all the processes Frappe requires. Usually, these all run in localhost, but in this case, we have external containers for Redis. For this reason, we have to stop Honcho from trying to start Redis processes. +Honcho is installed in global python environment along with bench. To make it available locally you've to install it in every `frappe-bench/env` you create. Install it using command `./env/bin/pip install honcho`. It is required locally if you wish to use is as part of VSCode launch configuration. + Open the Procfile file and remove the three lines containing the configuration from Redis, either by editing manually the file: ```shell diff --git a/docker-bake.hcl b/docker-bake.hcl index cd47b824..bf8eeca7 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -73,8 +73,8 @@ target "default-args" { BENCH_REPO = "${BENCH_REPO}" FRAPPE_VERSION = "${FRAPPE_VERSION}" ERPNEXT_VERSION = "${ERPNEXT_VERSION}" - # If `ERPNEXT_VERSION` variable contains "v12" use Python 3.7. Else — 3.9. - PYTHON_VERSION = can(regex("v12", "${ERPNEXT_VERSION}")) ? "3.7" : "3.9" + # If `ERPNEXT_VERSION` variable contains "v12" use Python 3.7. If "v13" — 3.9. Else 3.10. + PYTHON_VERSION = can(regex("v12", "${ERPNEXT_VERSION}")) ? "3.7" : can(regex("v13", "${ERPNEXT_VERSION}")) ? "3.9" : "3.10" } } diff --git a/docs/backup-and-push-cronjob.md b/docs/backup-and-push-cronjob.md index 745758fd..c6051ff8 100644 --- a/docs/backup-and-push-cronjob.md +++ b/docs/backup-and-push-cronjob.md @@ -5,7 +5,7 @@ Create backup service or stack. version: "3.7" services: backup: - image: frappe/erpnext-worker:v13 + image: frappe/erpnext-worker:v14 entrypoint: ["bash", "-c"] command: | for SITE in $(/home/frappe/frappe-bench/env/bin/python -c "import frappe;print(' '.join(frappe.utils.get_sites()))") diff --git a/docs/compose/compose.multi-bench.yaml b/docs/compose/compose.multi-bench.yaml index 17145c65..b83567b3 100644 --- a/docs/compose/compose.multi-bench.yaml +++ b/docs/compose/compose.multi-bench.yaml @@ -2,7 +2,6 @@ services: frontend: networks: - traefik-public - - mariadb-network - bench-network labels: - traefik.enable=true @@ -13,27 +12,34 @@ services: - traefik.http.routers.${ROUTER}-http.rule=Host(${SITES?SITES not set}) configurator: networks: + - bench-network - mariadb-network backend: networks: - mariadb-network + - bench-network websocket: networks: + - bench-network - mariadb-network scheduler: networks: + - bench-network - mariadb-network queue-default: networks: + - bench-network - mariadb-network queue-short: networks: - - mariadb-network + - bench-network queue-long: networks: + - bench-network - mariadb-network redis: networks: + - bench-network - mariadb-network networks: diff --git a/docs/setup-options.md b/docs/setup-options.md index 3c6436d2..bc74d4d1 100644 --- a/docs/setup-options.md +++ b/docs/setup-options.md @@ -17,7 +17,7 @@ Copy the example docker environment file to `.env`: cp example.env .env ``` -Note: To know more about environment variable [read here](./images-and-compose-files#configuration). Set the necessary variables in the `.env` file. +Note: To know more about environment variable [read here](./images-and-compose-files.md#configuration). Set the necessary variables in the `.env` file. ## Generate docker-compose.yml for variety of setups diff --git a/docs/single-server-example.md b/docs/single-server-example.md index 96c44032..e125d2d3 100644 --- a/docs/single-server-example.md +++ b/docs/single-server-example.md @@ -129,7 +129,7 @@ This will make `mariadb-database` service available under `mariadb-network`. Dat #### Create first bench -Create second bench called `erpnext-one` with `one.example.com` and `two.example.com` +Create first bench called `erpnext-one` with `one.example.com` and `two.example.com` Create a file called `erpnext-one.env` in `~/gitops` diff --git a/example.env b/example.env index ec796ffc..969cc19a 100644 --- a/example.env +++ b/example.env @@ -1,9 +1,9 @@ # Reference: https://github.com/frappe/frappe_docker/blob/main/docs/images-and-compose-files.md -FRAPPE_VERSION=v13.34.0 +FRAPPE_VERSION=v14.6.0 # Only with ERPNext override -ERPNEXT_VERSION=v13.34.1 +ERPNEXT_VERSION=v14.1.0 DB_PASSWORD=123 diff --git a/images/bench/Dockerfile b/images/bench/Dockerfile index a963425e..49de9d97 100644 --- a/images/bench/Dockerfile +++ b/images/bench/Dockerfile @@ -13,9 +13,6 @@ RUN apt-get update \ postgresql-client \ gettext-base \ wget \ - # for ERPNext v12 - # TODO: Remove after v12 is deprecated - python2 \ # for PDF libssl-dev \ fonts-cantarell \ @@ -51,6 +48,22 @@ RUN apt-get update \ ssh-client \ # VSCode container requirements net-tools \ + # For pyenv build dependencies + # https://github.com/frappe/frappe_docker/issues/840#issuecomment-1185206895 + make \ + # For pandas + libbz2-dev \ + # For bench execute + libsqlite3-dev \ + # For other dependencies + zlib1g-dev \ + libreadline-dev \ + llvm \ + libncurses5-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + liblzma-dev \ && rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ @@ -76,18 +89,16 @@ USER frappe WORKDIR /home/frappe # Install Python via pyenv -# Python 3.7 sits here for ERPNext version-12 -# TODO: Remove Python 3.7 when version-12 will not be supported -ENV PYTHON_VERSION_V12=3.7.12 -ENV PYTHON_VERSION=3.9.9 +ENV PYTHON_VERSION_V13=3.9.9 +ENV PYTHON_VERSION=3.10.5 ENV PYENV_ROOT /home/frappe/.pyenv ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH # From https://github.com/pyenv/pyenv#basic-github-checkout RUN git clone --depth 1 https://github.com/pyenv/pyenv.git .pyenv \ - && pyenv install $PYTHON_VERSION_V12 \ + && pyenv install $PYTHON_VERSION_V13 \ && pyenv install $PYTHON_VERSION \ - && pyenv global $PYTHON_VERSION $PYTHON_VERSION_V12 \ + && pyenv global $PYTHON_VERSION $PYTHON_VERSION_v13 \ && sed -Ei -e '/^([^#]|$)/ {a export PYENV_ROOT="/home/frappe/.pyenv" a export PATH="$PYENV_ROOT/bin:$PATH" a ' -e ':a' -e '$!{n;ba};}' ~/.profile \ && echo 'eval "$(pyenv init --path)"' >>~/.profile \ && echo 'eval "$(pyenv init -)"' >>~/.bashrc @@ -104,14 +115,11 @@ RUN git clone ${GIT_REPO} --depth 1 -b ${GIT_BRANCH} .bench \ # Install Node via nvm ENV NODE_VERSION=14.18.1 -ENV NODE_VERSION_FRAPPEV11=10.24.1 ENV NVM_DIR /home/frappe/.nvm ENV PATH ${NVM_DIR}/versions/node/v${NODE_VERSION}/bin/:${PATH} RUN wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash \ && . ${NVM_DIR}/nvm.sh \ - && nvm install ${NODE_VERSION_FRAPPEV11} \ - && nvm use v${NODE_VERSION_FRAPPEV11} \ && npm install -g yarn \ && nvm install ${NODE_VERSION} \ && nvm use v${NODE_VERSION} \ diff --git a/images/nginx/Dockerfile b/images/nginx/Dockerfile index 22100df6..eb4da5cc 100644 --- a/images/nginx/Dockerfile +++ b/images/nginx/Dockerfile @@ -32,9 +32,14 @@ RUN install-app frappe FROM assets_builder as erpnext_assets +ARG PAYMENTS_VERSION=develop +ARG PAYMENTS_REPO=https://github.com/frappe/payments ARG ERPNEXT_VERSION ARG ERPNEXT_REPO=https://github.com/frappe/erpnext -RUN git clone --depth 1 -b ${ERPNEXT_VERSION} ${ERPNEXT_REPO} apps/erpnext \ +RUN if [ -z "${ERPNEXT_VERSION##*v14*}" ] || [ "$ERPNEXT_VERSION" = "develop" ]; then \ + git clone --depth 1 -b ${PAYMENTS_VERSION} ${PAYMENTS_REPO} apps/payments && install-app payments; \ + fi \ + && git clone --depth 1 -b ${ERPNEXT_VERSION} ${ERPNEXT_REPO} apps/erpnext \ && install-app erpnext @@ -47,7 +52,7 @@ RUN git clone --depth 1 ${BENCH_REPO} /tmp/bench \ && mv /tmp/bench/bench/config/templates/502.html /out/ -FROM nginxinc/nginx-unprivileged:1.23.0-alpine as frappe +FROM nginxinc/nginx-unprivileged:1.23.1-alpine as frappe # https://github.com/nginxinc/docker-nginx-unprivileged/blob/main/stable/alpine/20-envsubst-on-templates.sh COPY nginx-template.conf /etc/nginx/templates/default.conf.template diff --git a/images/worker/Dockerfile b/images/worker/Dockerfile index 77fb2588..40c09945 100644 --- a/images/worker/Dockerfile +++ b/images/worker/Dockerfile @@ -54,10 +54,15 @@ RUN --mount=type=cache,target=/root/.cache/pip \ FROM frappe_builder as erpnext_builder +ARG PAYMENTS_VERSION=develop +ARG PAYMENTS_REPO=https://github.com/frappe/payments ARG ERPNEXT_VERSION ARG ERPNEXT_REPO=https://github.com/frappe/erpnext RUN --mount=type=cache,target=/root/.cache/pip \ - git clone --depth 1 -b ${ERPNEXT_VERSION} ${ERPNEXT_REPO} apps/erpnext \ + if [ -z "${ERPNEXT_VERSION##*v14*}" ] || [ "$ERPNEXT_VERSION" = "develop" ]; then \ + git clone --depth 1 -b ${PAYMENTS_VERSION} ${PAYMENTS_REPO} apps/payments && install-app payments; \ + fi \ + && git clone --depth 1 -b ${ERPNEXT_VERSION} ${ERPNEXT_REPO} apps/erpnext \ && install-app erpnext @@ -122,8 +127,7 @@ USER frappe # Split frappe and erpnext to reduce image size (because of frappe-bench/env/ directory) FROM configured_base as erpnext -COPY --from=frappe_builder /home/frappe/frappe-bench/apps/frappe /home/frappe/frappe-bench/apps/frappe -COPY --from=erpnext_builder /home/frappe/frappe-bench/apps/erpnext /home/frappe/frappe-bench/apps/erpnext +COPY --from=erpnext_builder /home/frappe/frappe-bench/apps /home/frappe/frappe-bench/apps COPY --from=erpnext_builder /home/frappe/frappe-bench/env /home/frappe/frappe-bench/env COPY --from=erpnext_builder /home/frappe/frappe-bench/sites/apps.txt /home/frappe/frappe-bench/sites/ diff --git a/pwd.yml b/pwd.yml index fb7b56de..e6dd5cfe 100644 --- a/pwd.yml +++ b/pwd.yml @@ -2,7 +2,7 @@ version: "3" services: backend: - image: frappe/erpnext-worker:v13.34.1 + image: frappe/erpnext-worker:v14.1.0 deploy: restart_policy: condition: on-failure @@ -11,21 +11,21 @@ services: - assets:/home/frappe/frappe-bench/sites/assets configurator: - image: frappe/erpnext-worker:v13.34.1 + image: frappe/erpnext-worker:v14.1.0 command: - configure.py environment: DB_HOST: db DB_PORT: "3306" - REDIS_CACHE: redis:6379/0 - REDIS_QUEUE: redis:6379/1 - REDIS_SOCKETIO: redis:6379/2 + REDIS_CACHE: redis-cache:6379 + REDIS_QUEUE: redis-queue:6379 + REDIS_SOCKETIO: redis-socketio:6379 SOCKETIO_PORT: "9000" volumes: - sites:/home/frappe/frappe-bench/sites create-site: - image: frappe/erpnext-worker:v13.34.1 + image: frappe/erpnext-worker:v14.1.0 deploy: restart_policy: condition: on-failure @@ -38,7 +38,9 @@ services: command: - > wait-for-it -t 120 db:3306; - wait-for-it -t 120 redis:6379; + wait-for-it -t 120 redis-cache:6379; + wait-for-it -t 120 redis-queue:6379; + wait-for-it -t 120 redis-socketio:6379; export start=`date +%s`; until [[ -n `grep -hs ^ common_site_config.json | jq -r ".db_host // empty"` ]] && \ [[ -n `grep -hs ^ common_site_config.json | jq -r ".redis_cache // empty"` ]] && \ @@ -52,7 +54,7 @@ services: fi done; echo "common_site_config.json found"; - bench new-site frontend --admin-password=admin --db-root-password=admin --install-app erpnext --set-default; + bench new-site frontend --admin-password=admin --db-root-password=admin --install-app payments --install-app erpnext --set-default; db: image: mariadb:10.6 @@ -74,7 +76,7 @@ services: - db-data:/var/lib/mysql frontend: - image: frappe/erpnext-nginx:v13.34.1 + image: frappe/erpnext-nginx:v14.1.0 deploy: restart_policy: condition: on-failure @@ -92,7 +94,7 @@ services: - "8080:8080" queue-default: - image: frappe/erpnext-worker:v13.34.1 + image: frappe/erpnext-worker:v14.1.0 deploy: restart_policy: condition: on-failure @@ -105,7 +107,7 @@ services: - sites:/home/frappe/frappe-bench/sites queue-long: - image: frappe/erpnext-worker:v13.34.1 + image: frappe/erpnext-worker:v14.1.0 deploy: restart_policy: condition: on-failure @@ -118,7 +120,7 @@ services: - sites:/home/frappe/frappe-bench/sites queue-short: - image: frappe/erpnext-worker:v13.34.1 + image: frappe/erpnext-worker:v14.1.0 deploy: restart_policy: condition: on-failure @@ -130,16 +132,32 @@ services: volumes: - sites:/home/frappe/frappe-bench/sites - redis: + redis-queue: image: redis:6.2-alpine deploy: restart_policy: condition: on-failure volumes: - - redis-data:/data + - redis-queue-data:/data + + redis-cache: + image: redis:6.2-alpine + deploy: + restart_policy: + condition: on-failure + volumes: + - redis-cache-data:/data + + redis-socketio: + image: redis:6.2-alpine + deploy: + restart_policy: + condition: on-failure + volumes: + - redis-socketio-data:/data scheduler: - image: frappe/erpnext-worker:v13.34.1 + image: frappe/erpnext-worker:v14.1.0 deploy: restart_policy: condition: on-failure @@ -150,7 +168,7 @@ services: - sites:/home/frappe/frappe-bench/sites websocket: - image: frappe/frappe-socketio:v13.34.0 + image: frappe/frappe-socketio:v14.6.0 deploy: restart_policy: condition: on-failure @@ -160,5 +178,7 @@ services: volumes: assets: db-data: - redis-data: + redis-queue-data: + redis-cache-data: + redis-socketio-data: sites: diff --git a/requirements-dev.txt b/requirements-dev.txt index 01b43fee..10e75138 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ frappe @ git+https://github.com/frappe/frappe.git boto3-stubs[s3] -black==22.3.0 +black==22.8.0 diff --git a/requirements-test.txt b/requirements-test.txt index d00689e0..e0716850 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1 +1 @@ -pytest==7.1.2 +pytest==7.1.3 diff --git a/tests/conftest.py b/tests/conftest.py index 2b531f33..f9885dd1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -81,7 +81,7 @@ def erpnext_setup(compose: Compose): @pytest.fixture(scope="class") def erpnext_site(compose: Compose): site_name = "test_erpnext_site" - compose.bench( + args = [ "new-site", site_name, "--mariadb-root-password", @@ -90,7 +90,14 @@ def erpnext_site(compose: Compose): "admin", "--install-app", "erpnext", - ) + ] + erpnext_version = os.environ.get("ERPNEXT_VERSION") + if erpnext_version in [ + "develop", + "version-14", + ] or erpnext_version.startswith("v14"): + args.append("--install-app=payments") + compose.bench(*args) compose("restart", "backend") yield site_name