From 5bd528319b7e600e1cfdacf185e4194a60d346f0 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 21 Mar 2023 14:47:37 +0000 Subject: [PATCH] Provide a changelog for minor releases. The github release will now: * Provide the changelog message from the newsfragment of the PR that triggered the backport. * Provide a github link to the PR/issue of the PR that was backported. Switch to building multi-arch images. The images build for pull requests, master and production are now multi-arch images for the architectures: * linux/amd64 * linux/arm64/v8 * linux/arm/v7 Enhance CI/CD workflow with retry functionality. All steps for building images are now automatically retried. If a build temporarily fails due to a network error, the retried step will still succeed. --- .github/workflows/arm.yml | 91 ----- .github/workflows/build_test_deploy.yml | 343 +++++++++++-------- .github/workflows/{x64.yml => multiarch.yml} | 17 +- docs/arm_images.rst | 76 ---- docs/compose/requirements.rst | 10 +- docs/index.rst | 1 - docs/setup.rst | 7 +- towncrier/newsfragments/2653.feature | 12 + 8 files changed, 230 insertions(+), 327 deletions(-) delete mode 100644 .github/workflows/arm.yml rename .github/workflows/{x64.yml => multiarch.yml} (88%) delete mode 100644 docs/arm_images.rst create mode 100644 towncrier/newsfragments/2653.feature diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml deleted file mode 100644 index fc24fbe9..00000000 --- a/.github/workflows/arm.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: start-linux-arm -on: - push: - branches: - - '1.9' - - master - -concurrency: ci-arm-${{ github.ref }} - -# REQUIRED global variables -# DOCKER_ORG, docker org used for pushing images. -env: - DOCKER_ORG: ghcr.io/mailu - -jobs: -# This job calculates all global job variables that are required by all the subsequent jobs. -# All subsequent jobs will retrieve and use these variables. This way the variables only have to be derived once. - derive-variables: - name: derive variables - runs-on: ubuntu-latest - outputs: - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - DOCKER_ORG: ${{ env.DOCKER_ORG_DERIVED }} - BRANCH: ${{ env.BRANCH }} - DEPLOY: ${{ env.DEPLOY }} - RELEASE: ${{ env.RELEASE }} - steps: - - uses: actions/checkout@v3 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to pr-xxxx - - name: Derive MAILU_VERSION and DEPLOY/RELEASE for other branches than testing - if: env.BRANCH != 'testing' - shell: bash - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG_DERIVED=${{ env.DOCKER_ORG }}" >> $GITHUB_ENV - echo "DEPLOY=true" >> $GITHUB_ENV - echo "RELEASE=false" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION and DEPLOY/RELEASE for normal release x.y - if: env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - echo "RELEASE=true" >> $GITHUB_ENV - echo "DEPLOY=true" >> $GITHUB_ENV - echo "RELEASE=true" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: env.BRANCH == 'master' - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - echo "DEPLOY=true" >> $GITHUB_ENV - echo "RELEASE=false" >> $GITHUB_ENV - - build-test-deploy: - needs: - - derive-variables - uses: ./.github/workflows/build_test_deploy.yml - with: - architecture: 'linux/arm64/v8,linux/arm/v7' - mailu_version: ${{needs.derive-variables.outputs.MAILU_VERSION}}-arm - pinned_mailu_version: ${{needs.derive-variables.outputs.PINNED_MAILU_VERSION}}-arm - docker_org: ${{needs.derive-variables.outputs.DOCKER_ORG}} - branch: ${{needs.derive-variables.outputs.BRANCH}} - deploy: ${{needs.derive-variables.outputs.DEPLOY}} - release: ${{needs.derive-variables.outputs.RELEASE}} - secrets: inherit - -################################################ -# Code block that is used as one liner for the step: -# Derive PINNED_MAILU_VERSION and DEPLOY/RELEASE for normal release x.y -##!/bin/bash -#version=$( git tag --sort=version:refname --list "{{ env.MAILU_VERSION }}.*" | tail -1 ) -#root_version=${version%.*} -#patch_version=${version##*.} -#if [ "$patch_version" == "" ] -#then -# pinned_version={{ env.MAILU_VERSION }}.0 -#else -# pinned_version=$root_version.$(expr $patch_version + 1) -#fi -#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index e3c2f920..01c6bd12 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -4,6 +4,10 @@ # Username of docker login for logging in docker for pulling images (higher pull rate limit) # ${{ secrets.Docker_Password }} # Password of docker login for logging in docker for pulling images (higher pull rate limit) +# ${{ secrets.Docker_Login2 }} +# Second Username of docker login for logging in docker for pulling images (higher pull rate limit) +# ${{ secrets.Docker_Password2 }} +# Second Password of docker login for logging in docker for pulling images (higher pull rate limit) ################################################ name: build-test-deploy @@ -11,9 +15,9 @@ on: workflow_call: inputs: architecture: - description: 'The architecture of the images that will be build.' + description: 'The architecture(s) of the images that will be build. linux/amd64 or linux/arm64/v8,linux/arm/v7 or linux/amd64,linux/arm64/v8,linux/arm/v7' required: false - default: 'linux/amd64' + default: 'linux/amd64,linux/arm64/v8,linux/arm/v7' type: string mailu_version: description: 'The main version that is build. E.g. master or x.y.' @@ -45,9 +49,9 @@ on: workflow_dispatch: inputs: architecture: - description: 'The architecture of the images that will be build.' + description: 'The architecture(s) of the images that will be build. linux/amd64 or linux/arm64/v8,linux/arm/v7 or linux/amd64,linux/arm64/v8,linux/arm/v7' required: false - default: 'linux/amd64' + default: 'linux/amd64,linux/arm64/v8,linux/arm/v7' type: string mailu_version: description: 'The main version that is build. E.g. master or x.y.' @@ -100,7 +104,7 @@ jobs: ## This job builds the base image. The base image is used by all other images. build-base-image-x64: name: Build base image x64 - if: inputs.architecture == 'linux/amd64' + if: contains(inputs.architecture, 'linux/amd64') needs: - targets runs-on: ubuntu-latest @@ -121,44 +125,53 @@ jobs: - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - - name: Build all docker images + - name: Get uuid + id: uuid + run: | + echo uuid=$RANDOM >> $GITHUB_OUTPUT + - name: Build docker base image with retry env: DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} LABEL_VERSION: ${{ env.MAILU_VERSION }} PINNED_LABEL_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + ARCH: 'linux/amd64' + BUILDER: ${{ steps.uuid.outputs.uuid }} + DOCKER_LOGIN: ${{ secrets.Docker_Login }} + DOCKER_PASSW: ${{ secrets.Docker_Password }} + uses: nick-fields/retry@v2 with: - files: ${{env.HCL_FILE}} - targets: base - load: false - push: false - set: | - *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }},mode=max - *.platform=${{ inputs.architecture }} + timeout_minutes: 5 + retry_wait_seconds: 30 + max_attempts: 3 + shell: bash + command: | + set -euxo pipefail \ + ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ + ; echo "$DOCKER_PASSW" | docker login --username "$DOCKER_LOGIN" --password-stdin \ + ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ + || echo "builder does not exist" \ + ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \ + ; /usr/bin/docker buildx bake --file ./tests/build.hcl --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }},mode=max --set *.platform=${{ env.ARCH }} base \ + ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} + - name: cleanup docker buildx instance after failure of build step + if: ${{ failure() }} + shell: bash + env: + BUILDER: ${{ steps.uuid.outputs.uuid }} + run: | + /usr/bin/docker buildx rm builder-${{ env.BUILDER }} ## This job builds the base image. The base image is used by all other images. build-base-image-arm: name: Build base image arm - if: inputs.architecture != 'linux/amd64' + if: contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7') needs: - targets runs-on: self-hosted @@ -177,47 +190,55 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - - name: Build all docker images + - name: Get uuid + id: uuid + run: | + echo uuid=$RANDOM >> $GITHUB_OUTPUT + - name: Build docker base image with retry env: DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + MAILU_VERSION: ${{ env.MAILU_VERSION }}-arm + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}-arm LABEL_VERSION: ${{ env.MAILU_VERSION }} PINNED_LABEL_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + ARCH: linux/arm64/v8,linux/arm/v7 + BUILDER: ${{ steps.uuid.outputs.uuid }} + DOCKER_LOGIN2: ${{ secrets.Docker_Login2 }} + DOCKER_PASSW2: ${{ secrets.Docker_Password2 }} + uses: nick-fields/retry@v2 with: - files: ${{env.HCL_FILE}} - targets: base - load: false - push: false - set: | - *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm - *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max - *.platform=${{ inputs.architecture }} + timeout_minutes: 10 + retry_wait_seconds: 30 + max_attempts: 10 + shell: bash + command: | + set -euxo pipefail \ + ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ + ; echo "$DOCKER_PASSW2" | docker login --username "$DOCKER_LOGIN2" --password-stdin \ + ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ + || echo "builder does not exist" \ + ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \ + ; /usr/bin/docker buildx bake --file ./tests/build.hcl --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max --set *.platform=${{ env.ARCH }} base \ + ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} + - name: cleanup docker buildx instance after failure of build step + if: ${{ failure() }} + shell: bash + env: + BUILDER: ${{ steps.uuid.outputs.uuid }} + run: | + /usr/bin/docker buildx rm builder-${{ env.BUILDER }} + # This job builds all the images. The build cache is stored in the github actions cache. # In further jobs, this cache is used to quickly rebuild the images. build: name: Build images for linux/amd64 - if: inputs.architecture == 'linux/amd64' + if: contains(inputs.architecture, 'linux/amd64') needs: - targets - build-base-image-x64 @@ -243,46 +264,54 @@ jobs: - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - - name: Build all docker images + - name: Get uuid + id: uuid + run: | + echo uuid=$RANDOM >> $GITHUB_OUTPUT + - name: Build docker image with retry env: DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }}-build PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}-build LABEL_VERSION: ${{ env.MAILU_VERSION }} PINNED_LABEL_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + ARCH: 'linux/amd64' + BUILDER: ${{ steps.uuid.outputs.uuid }} + DOCKER_LOGIN: ${{ secrets.Docker_Login }} + DOCKER_PASSW: ${{ secrets.Docker_Password }} + uses: nick-fields/retry@v2 with: - files: ${{env.HCL_FILE}} - targets: ${{ matrix.target }} - load: false - push: true - set: | - *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache - *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max - *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - *.platform=${{ inputs.architecture }} + timeout_minutes: 5 + retry_wait_seconds: 30 + max_attempts: 3 + shell: bash + command: | + set -euxo pipefail \ + ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ + ; echo "$DOCKER_PASSW" | docker login --username "$DOCKER_LOGIN" --password-stdin \ + ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ + || echo "builder does not exist" \ + ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \ + ; /usr/bin/docker buildx bake --push --file ./tests/build.hcl --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} --set *.platform=${{ env.ARCH }} ${{ matrix.target }} \ + ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} + - name: cleanup docker buildx instance after failure of build step + if: ${{ failure() }} + shell: bash + env: + BUILDER: ${{ steps.uuid.outputs.uuid }} + run: | + /usr/bin/docker buildx rm builder-${{ env.BUILDER }} # This job builds all the images. The build cache is stored in the github actions cache. # In further jobs, this cache is used to quickly rebuild the images. build-arm: name: Build images for ARM64 & ARM/V7 - if: inputs.architecture != 'linux/amd64' + if: contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7') needs: - targets - build-base-image-arm @@ -306,47 +335,54 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - - name: Build all docker images + #This is to prevent to shared runners from generating the same uuid + - name: Get unique random number + id: uuid + run: | + echo uuid=$RANDOM >> $GITHUB_OUTPUT + - name: Build docker image with retry env: DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} - MAILU_VERSION: ${{ env.MAILU_VERSION }}-build - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}-build + MAILU_VERSION: ${{ env.MAILU_VERSION }}-arm-build + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}-arm-build LABEL_VERSION: ${{ env.MAILU_VERSION }} PINNED_LABEL_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + ARCH: linux/arm64/v8,linux/arm/v7 + BUILDER: ${{ steps.uuid.outputs.uuid }} + DOCKER_LOGIN2: ${{ secrets.Docker_Login2 }} + DOCKER_PASSW2: ${{ secrets.Docker_Password2 }} + uses: nick-fields/retry@v2 with: - files: ${{env.HCL_FILE}} - targets: ${{ matrix.target }} - load: false - push: true - set: | - *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm - *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max - *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm - *.platform=${{ inputs.architecture }} + timeout_minutes: 10 + retry_wait_seconds: 30 + max_attempts: 10 + shell: bash + command: | + set -euxo pipefail \ + ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ + ; echo "$DOCKER_PASSW2" | docker login --username "$DOCKER_LOGIN2" --password-stdin \ + ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ + || echo "builder does not exist" \ + ; /usr/bin/docker buildx create --name builder-${{ env.BUILDER }} --driver docker-container --use \ + ; /usr/bin/docker buildx bake --file ./tests/build.hcl --set *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm --set *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max --set *.platform=${{ env.ARCH }} base \ + ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} + - name: cleanup docker buildx instance after failure of build step + if: ${{ failure() }} + shell: bash + env: + BUILDER: ${{ steps.uuid.outputs.uuid }} + run: | + /usr/bin/docker buildx rm builder-${{ env.BUILDER }} # This job runs all the tests. tests: name: tests - if: inputs.architecture == 'linux/amd64' + if: contains(inputs.architecture, 'linux/amd64') runs-on: ubuntu-latest permissions: contents: read @@ -406,6 +442,8 @@ jobs: if: inputs.deploy == 'true' runs-on: ubuntu-latest needs: + - build + - build-arm - tests strategy: fail-fast: false @@ -436,7 +474,19 @@ jobs: uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - - name: Push image to Github (ghcr.io) + - name: Push multiarch image to Github (ghcr.io) + if: contains(inputs.architecture, 'linux/amd64') && contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7') + shell: bash + run: | + if [ '${{ env.MAILU_VERSION }}' == 'master' ]; then pinned_mailu_version='master'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi; + docker buildx imagetools create \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \ + ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }}-build \ + ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }}-arm-build + - name: Push x64 image to Github (ghcr.io) + if: contains(inputs.architecture, 'linux/amd64') && !contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7') shell: bash run: | if [ '${{ env.MAILU_VERSION }}' == 'master' ]; then pinned_mailu_version='master'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi; @@ -445,52 +495,16 @@ jobs: --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \ ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }}-build - - deploy-arm: - name: Deploy images for arm - # Deploying is not required for staging - if: inputs.deploy == 'true' && inputs.architecture != 'linux/amd64' - runs-on: self-hosted - needs: - - build-arm - strategy: - fail-fast: false - matrix: - target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "oletools", "postfix", "dovecot", "unbound", "nginx"] - steps: - - uses: actions/checkout@v3 - - name: Retrieve global variables + - name: Push arm image to Github (ghcr.io) + if: contains(inputs.architecture, 'linux/arm64/v8,linux/arm/v7') && !contains(inputs.architecture, 'linux/amd64') shell: bash run: | - echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV - echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - uses: crazy-max/ghaction-github-runtime@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Helper to convert docker org to lowercase - id: string - uses: ASzc/change-string-case-action@v5 - with: - string: ${{ github.repository_owner }} - - name: Push image to Github (ghcr.io) - shell: bash - run: | - if [ '${{ env.MAILU_VERSION }}' == 'master-arm' ]; then pinned_mailu_version='master-arm'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi; + if [ '${{ env.MAILU_VERSION }}' == 'master' ]; then pinned_mailu_version='master'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi; docker buildx imagetools create \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \ - ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }}-build + ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }}-arm-build #This job creates a tagged release. A tag is created for the pinned version x.y.z. The GH release refers to this tag. tag-release: @@ -503,22 +517,57 @@ jobs: with: # fetch-depth 0 is required to also retrieve all tags. fetch-depth: 0 + # A bug in actions/checkout@v3 results in all files having mtime of the job running. + - name: Restore Timestamps + uses: chetan/git-restore-mtime-action@v1 - name: Retrieve global variables shell: bash run: | - echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - name: Create tag for branch x.y. shell: bash run: | echo git tag ${{ env.PINNED_MAILU_VERSION }} $(/usr/bin/git rev-parse HEAD) git tag ${{ env.PINNED_MAILU_VERSION }} $(/usr/bin/git rev-parse HEAD) git push origin ${{ env.PINNED_MAILU_VERSION }} + - name: Show list of changelog files (we pick the newest) + shell: bash + run: | + ls -Artl towncrier/newsfragments + - name: Get latest changelog + id: changelog + shell: bash + run: | + pushd . && cd towncrier/newsfragments && ls -Art | tail -n 1 | cut -d. -f1 | xargs -0I % echo "issue=%" >> $GITHUB_OUTPUT && popd + pushd . && cd towncrier/newsfragments && ls -Art | tail -n 1 | xargs cat | xargs -0I % echo "content=%" >> $GITHUB_OUTPUT && popd + - name: Construct message for release + shell: bash + env: + issue: "${{ steps.changelog.outputs.issue }}" + changelog: "${{ steps.changelog.outputs.content }}" + run: | + message="Changelog :mailbox: + --------- + + ${{ env.changelog }} + + + This release was triggered by PR/Issue [${{ env.issue }}](https://github.com/Mailu/Mailu/issues/${{ env.issue }}). + + + The release notes of the original main release can be accessed via menu item 'Release notes' on [mailu.io](https://mailu.io/). + + Update + ------ + The main version X.Y (e.g. 1.9) will always reflect the latest version of the branch. To update your Mailu installation simply pull the latest images \`docker compose pull && docker compose up -d\`. + + The pinned version X.Y.Z (e.g. 1.9.1) is not updated. It is pinned to the commit that was used for creating this release. You can use a pinned version to make sure your Mailu installation is not suddenly updated when recreating containers. The pinned version allows the user to manually update. It also allows to go back to a previous pinned version. + " && echo "$message" >> release_note.md + - name: Show release note + shell: bash + run: | + cat release_note.md - name: Create release for tag x.y.z. uses: ncipollo/release-action@v1 with: - bodyFile: "RELEASE_TEMPLATE.md" + bodyFile: "release_note.md" tag: ${{ env.PINNED_MAILU_VERSION }} token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/x64.yml b/.github/workflows/multiarch.yml similarity index 88% rename from .github/workflows/x64.yml rename to .github/workflows/multiarch.yml index 8a47b6a7..5cee1638 100644 --- a/.github/workflows/x64.yml +++ b/.github/workflows/multiarch.yml @@ -1,4 +1,4 @@ -name: start-linux-amd64 +name: start-linux-multiarch on: push: branches: @@ -6,8 +6,9 @@ on: - staging - '1.9' - master + - test-* -concurrency: ci-x64-${{ github.ref }} +concurrency: ci-multiarch-${{ github.ref }} # REQUIRED global variables # DOCKER_ORG, docker org used for pushing images. @@ -57,7 +58,7 @@ jobs: echo "DEPLOY=true" >> $GITHUB_ENV echo "RELEASE=false" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION and DEPLOY/RELEASE for normal release x.y - if: env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' + if: env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' && !(contains(env.BRANCH, 'test-')) shell: bash run: | version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV @@ -81,13 +82,21 @@ jobs: echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV echo "DEPLOY=true" >> $GITHUB_ENV echo "RELEASE=false" >> $GITHUB_ENV + - name: Derive for branch test-* + if: contains(env.BRANCH, 'test-') + run: | + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG_DERIVED=${{ env.DOCKER_ORG }}" >> $GITHUB_ENV + echo "DEPLOY=true" >> $GITHUB_ENV + echo "RELEASE=false" >> $GITHUB_ENV build-test-deploy: needs: - derive-variables uses: ./.github/workflows/build_test_deploy.yml with: - architecture: 'linux/amd64' + architecture: 'linux/amd64,linux/arm64/v8,linux/arm/v7' mailu_version: ${{needs.derive-variables.outputs.MAILU_VERSION}} pinned_mailu_version: ${{needs.derive-variables.outputs.PINNED_MAILU_VERSION}} docker_org: ${{needs.derive-variables.outputs.DOCKER_ORG}} diff --git a/docs/arm_images.rst b/docs/arm_images.rst deleted file mode 100644 index 72cbdb56..00000000 --- a/docs/arm_images.rst +++ /dev/null @@ -1,76 +0,0 @@ -.. _arm_images: - -Arm images for Mailu -==================== - -Using Mailu arm images ----------------------- - -The Mailu project makes use of github actions for automatic CI/CD. -Github actions only has x64 (amd64) runners. This means we can build the arm images -using QEMU, but we cannot test the images. For this reason the arm images have -a BETA status. We only guarantee that the images could be built. - - -We strongly recommend to make use of the pinned version (tag 1.9.10 vs tag 1.9). -Pinned versions (tag x.y.z) are not updated. This allows upgrading manually by changing the -tag to the next pinned version. - - -Whenever images are deployed for master and for releases (branch x.y), -images are also built for arm. - -The images are pushed with -arm appended to the tag. For example: - -- admin:master-arm -- admin:1.10-arm - -To use these images, simply use setup.mailu.io for generating the docker-composse.yml -file and mailu.env file. Then in the docker-compose.yml file append -arm to the tags of -all images from the mailu docker repository. - -Build manually --------------- -It is possible to build the images manually. There are two possibilities for this. - -Github actions -`````````````` -The main workflow build-test-deploy can be triggered manually. -Via the parameter ``architecture`` the target platform can be specified. -Use the value ``'linux/arm64,linux/arm/v7'``. - - -To use it: - -1. Fork the Mailu github project. -2. In the settings of your forked project, configure the secrets Docker_Login and Docker_Password. For more information on these secrets, see the comments in the build-test-deploy.yml file. -3. In the forked project, trigger the workflow build-test-deploy manually. -4. For the parameter architecture use the value ``'linux/arm64,linux/arm/v7'``. - - -Manually -```````` -It is also possible to build the images manually on bare-metal. -The buildx file ``tests/build.hcl`` can be used for this. - - -To build manually: - -1. Install QEMU static binaries. This is only required, if you don't built on an arm machine. For Ubuntu install qemu-user-static. -2. Clone the Mailu github project. -3. Export the parameters. -4. Create a buildx builder instance -5. Run buildx overriding the architecture. - -For example: - -.. code-block:: bash - - docker login - Username: Foo - Password: Bar - export DOCKER_ORG="Foo" - export MAILU_VERSION="master-arm" - export MAILU_PINNED_VERSION="hash" - docker buildx create --use - docker buildx bake -f tests/build.hcl --push --set *.platform=linux/arm64,linux/arm/v7 diff --git a/docs/compose/requirements.rst b/docs/compose/requirements.rst index 3eacc017..0ea87686 100644 --- a/docs/compose/requirements.rst +++ b/docs/compose/requirements.rst @@ -23,8 +23,8 @@ the latest Linux kernel. The minimal required memory and swap are: Pick a distribution ------------------- -The mail server runs as a set of Docker containers, so it is almost operating -system agnostic as long as a fairly recent Linux kernel is running and +The mail server runs as a set of Docker containers, so it is almost operating +system agnostic as long as a fairly recent Linux kernel is running and the Docker API (>= 1.11) is available. Because most of our tests run on Debian Jessie and Debian Stretch, we recommend @@ -59,8 +59,8 @@ Mailu uses Docker port forwarding from the host to make services available to external users. First, your host should have a public IP address configured (see ``/etc/network/interfaces``) or your router should forward connections to its internal IP address. Due to spam problems and -reputation services, it is highly recommended that you use a dedicated IP -address for your mail server and that you have a dedicated hostname +reputation services, it is highly recommended that you use a dedicated IP +address for your mail server and that you have a dedicated hostname with forward and reverse DNS entries for this IP address. Also, your host must not listen on ports ``25``, ``80``, ``110``, ``143``, @@ -114,7 +114,7 @@ Once everything is setup, you should be able to run the following commands OS/Arch: linux/arm64 Context: default Experimental: true - + Server: Docker Engine - Community Engine: Version: 20.10.22 diff --git a/docs/index.rst b/docs/index.rst index ae148f71..867e5485 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,7 +60,6 @@ the version of Mailu that you are running. dns reverse database - arm_images .. toctree:: :maxdepth: 2 diff --git a/docs/setup.rst b/docs/setup.rst index e7ede399..58970337 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -17,11 +17,12 @@ long-term professional support, you should probably turn to them instead. Prepare the environment ----------------------- -Mailu images are designed to work on x86 or equivalent hardware, so it +Mailu ships multi-arch images which are designed to work on linux/amd64, +linux/arm64v8 or linux/armv7 hardware, so it should run on pretty much any cloud server as long as enough power is -provided. For non x86 machines, see :ref:`arm_images` +provided. -You are free to choose any operating system that runs Docker (>= 1.11), +You are free to choose any linux operating system that runs Docker (>= 1.11), then chose between various flavors including Docker Compose, Kubernetes and Rancher. diff --git a/towncrier/newsfragments/2653.feature b/towncrier/newsfragments/2653.feature new file mode 100644 index 00000000..429480aa --- /dev/null +++ b/towncrier/newsfragments/2653.feature @@ -0,0 +1,12 @@ +Provide a changelog for minor releases. The github release will now: +* Provide the changelog message from the newsfragment of the PR that triggered the backport. +* Provide a github link to the PR/issue of the PR that was backported. + +Switch to building multi-arch images. The images build for pull requests, master and production +are now multi-arch images for the architectures: +* linux/amd64 +* linux/arm64/v8 +* linux/arm/v7 + +Enhance CI/CD workflow with retry functionality. All steps for building images are now automatically +retried. If a build temporarily fails due to a network error, the retried step will still succeed. \ No newline at end of file