diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml deleted file mode 100644 index 8348503b..00000000 --- a/.github/workflows/CI.yml +++ /dev/null @@ -1,678 +0,0 @@ -name: CI -on: - push: - branches: - - staging - - testing - - '1.8' - - '1.9' - - master - # test branches, e.g. test-debian - - test-* - -concurrency: ci-${{ github.ref }} - -############################################### -# REQUIRED secrets -# DOCKER_UN: ${{ secrets.Docker_Login }} -# Username of docker login for pushing the images to repo $DOCKER_ORG and $DOCKER_ORG_TESTS -# DOCKER_PW: ${{ secrets.Docker_Password }} -# Password of docker login for pushing the images to repo $DOCKER_ORG and $DOCKER_ORG_TESTS -# DOCKER_ORG: ${{ secrets.DOCKER_ORG }} -# The docker repository where the images are pushed to. -# DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} -# The docker repository for test images. Only used for the branch TESTING (BORS try). -# Add the above secrets to your github repo to determine where the images will be pushed. -################################################ -# Code block that is used as one liner. -##!/bin/bash -#version=$( git tag --sort=version:refname --list "{{ env.MAILU_VERSION }}.*" | tail -1 ) -#root_version=${version%.*} -#patch_version=${version##*.} -#if [ "$patch_version" == "" ] -#then -# pinned_version={{ env.MAILU_VERSION }}.0 -#else -# pinned_version=$root_version.$(expr $patch_version + 1) -#fi -#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - -jobs: - build: - name: Build images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing - if: ${{ env.BRANCH == 'testing' }} - shell: bash - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - run: | - echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for other branches than testing - if: ${{ env.BRANCH != 'testing' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging - if: ${{ env.BRANCH == 'staging' }} - shell: bash - run: | - echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: ${{ env.BRANCH == 'master' }} - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - name: Create folder for storing images - run: | - sudo mkdir -p /images - sudo chmod 777 /images - - name: Configure images folder for caching - uses: actions/cache@v2 - with: - path: /images - key: ${{ env.BRANCH }}-${{ github.run_id }}-${{ github.run_number }} - - name: Check docker-compose version - run: docker-compose -v - - name: Login docker - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - - name: Build all docker images - env: - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - run: docker-compose -f tests/build.yml build - - name: Save all docker images - run: docker save ${{ env.DOCKER_ORG }}/admin ${{ env.DOCKER_ORG }}/clamav ${{ env.DOCKER_ORG }}/docs ${{ env.DOCKER_ORG }}/dovecot ${{ env.DOCKER_ORG }}/fetchmail ${{ env.DOCKER_ORG }}/nginx ${{ env.DOCKER_ORG }}/none ${{ env.DOCKER_ORG }}/postfix ${{ env.DOCKER_ORG }}/radicale ${{ env.DOCKER_ORG }}/rainloop ${{ env.DOCKER_ORG }}/roundcube ${{ env.DOCKER_ORG }}/rspamd ${{ env.DOCKER_ORG }}/setup ${{ env.DOCKER_ORG }}/traefik-certdumper ${{ env.DOCKER_ORG }}/unbound -o /images/images.tar.gz - - test-core: - name: Perform core tests - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing - if: ${{ env.BRANCH == 'testing' }} - shell: bash - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - run: | - echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for other branches than testing - if: ${{ env.BRANCH != 'testing' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging - if: ${{ env.BRANCH == 'staging' }} - shell: bash - run: | - echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: ${{ env.BRANCH == 'master' }} - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - name: Create folder for storing images - run: | - sudo mkdir -p /images - sudo chmod 777 /images - - name: Configure images folder for caching - uses: actions/cache@v2 - with: - path: /images - key: ${{ env.BRANCH }}-${{ github.run_id }}-${{ github.run_number }} - - name: Load docker images - run: docker load -i /images/images.tar.gz - - name: Install python packages - run: python3 -m pip install -r tests/requirements.txt - - name: Copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: Test core suite - run: python tests/compose/test.py core 2 - env: - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - - test-fetchmail: - name: Perform fetchmail tests - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing - if: ${{ env.BRANCH == 'testing' }} - shell: bash - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - run: | - echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for other branches than testing - if: ${{ env.BRANCH != 'testing' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging - if: ${{ env.BRANCH == 'staging' }} - shell: bash - run: | - echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: ${{ env.BRANCH == 'master' }} - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - name: Create folder for storing images - run: | - sudo mkdir -p /images - sudo chmod 777 /images - - name: Configure images folder for caching - uses: actions/cache@v2 - with: - path: /images - key: ${{ env.BRANCH }}-${{ github.run_id }}-${{ github.run_number }} - - name: Load docker images - run: docker load -i /images/images.tar.gz - - name: Install python packages - run: python3 -m pip install -r tests/requirements.txt - - name: Copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: Test fetch - run: python tests/compose/test.py fetchmail 2 - env: - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - - test-filters: - name: Perform filter tests - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing - if: ${{ env.BRANCH == 'testing' }} - shell: bash - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - run: | - echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for other branches than testing - if: ${{ env.BRANCH != 'testing' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging - if: ${{ env.BRANCH == 'staging' }} - shell: bash - run: | - echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: ${{ env.BRANCH == 'master' }} - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - name: Create folder for storing images - run: | - sudo mkdir -p /images - sudo chmod 777 /images - - name: Configure images folder for caching - uses: actions/cache@v2 - with: - path: /images - key: ${{ env.BRANCH }}-${{ github.run_id }}-${{ github.run_number }} - - name: Load docker images - run: docker load -i /images/images.tar.gz - - name: Install python packages - run: python3 -m pip install -r tests/requirements.txt - - name: Copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: Test clamvav - run: python tests/compose/test.py filters 3 - env: - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - - test-rainloop: - name: Perform rainloop tests - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing - if: ${{ env.BRANCH == 'testing' }} - shell: bash - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - run: | - echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for other branches than testing - if: ${{ env.BRANCH != 'testing' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging - if: ${{ env.BRANCH == 'staging' }} - shell: bash - run: | - echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: ${{ env.BRANCH == 'master' }} - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - name: Create folder for storing images - run: | - sudo mkdir -p /images - sudo chmod 777 /images - - name: Configure images folder for caching - uses: actions/cache@v2 - with: - path: /images - key: ${{ env.BRANCH }}-${{ github.run_id }}-${{ github.run_number }} - - name: Load docker images - run: docker load -i /images/images.tar.gz - - name: Install python packages - run: python3 -m pip install -r tests/requirements.txt - - name: Copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: Test rainloop - run: python tests/compose/test.py rainloop 2 - env: - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - - test-roundcube: - name: Perform roundcube tests - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing - if: ${{ env.BRANCH == 'testing' }} - shell: bash - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - run: | - echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for other branches than testing - if: ${{ env.BRANCH != 'testing' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging - if: ${{ env.BRANCH == 'staging' }} - shell: bash - run: | - echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: ${{ env.BRANCH == 'master' }} - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - name: Create folder for storing images - run: | - sudo mkdir -p /images - sudo chmod 777 /images - - name: Configure images folder for caching - uses: actions/cache@v2 - with: - path: /images - key: ${{ env.BRANCH }}-${{ github.run_id }}-${{ github.run_number }} - - name: Load docker images - run: docker load -i /images/images.tar.gz - - name: Install python packages - run: python3 -m pip install -r tests/requirements.txt - - name: Copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: Test roundcube - run: python tests/compose/test.py roundcube 2 - env: - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - - test-webdav: - name: Perform webdav tests - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing - if: ${{ env.BRANCH == 'testing' }} - shell: bash - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - run: | - echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for other branches than testing - if: ${{ env.BRANCH != 'testing' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging - if: ${{ env.BRANCH == 'staging' }} - shell: bash - run: | - echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: ${{ env.BRANCH == 'master' }} - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - name: Create folder for storing images - run: | - sudo mkdir -p /images - sudo chmod 777 /images - - name: Configure images folder for caching - uses: actions/cache@v2 - with: - path: /images - key: ${{ env.BRANCH }}-${{ github.run_id }}-${{ github.run_number }} - - name: Load docker images - run: docker load -i /images/images.tar.gz - - name: Install python packages - run: python3 -m pip install -r tests/requirements.txt - - name: Copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: Test webdav - run: python tests/compose/test.py webdav 2 - env: - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - - deploy: - name: Deploy images - runs-on: ubuntu-latest - needs: - - build - - test-core - - test-fetchmail - - test-filters - - test-rainloop - - test-roundcube - - test-webdav - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing - if: ${{ env.BRANCH == 'testing' }} - shell: bash - env: - COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - run: | - echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for other branches than testing - if: ${{ env.BRANCH != 'testing' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging - if: ${{ env.BRANCH == 'staging' }} - shell: bash - run: | - echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for master - if: ${{ env.BRANCH == 'master' }} - shell: bash - env: - GITHUB_SHA: ${{ env.GITHUB_SHA }} - run: | - echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - - name: Create folder for storing images - run: | - sudo mkdir -p /images - sudo chmod 777 /images - - name: Configure images folder for caching - # For staging we do not deploy images. So we do not have to load them from cache. - if: ${{ env.BRANCH != 'staging' }} - uses: actions/cache@v2 - with: - path: /images - key: ${{ env.BRANCH }}-${{ github.run_id }}-${{ github.run_number }} - - name: Load docker images - if: ${{ env.BRANCH != 'staging' }} - run: docker load -i /images/images.tar.gz - - name: Deploy built docker images - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - DOCKER_ORG: ${{ env.DOCKER_ORG }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - BRANCH: ${{ env.BRANCH }} - run: bash tests/deploy.sh - - tag-release: - runs-on: ubuntu-latest - needs: - - deploy - steps: - - uses: actions/checkout@v2 - with: - # fetch-depth 0 is required to also retrieve all tags. - fetch-depth: 0 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: Derive MAILU_VERSION amd DOCKER_ORG - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' }} - shell: bash - env: - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: | - echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for normal release x.y - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' }} - shell: bash - run: | - version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - - name: Create tag for branch x.y. - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' && env.PINNED_MAILU_VERSION != '' }} - shell: bash - run: | - echo git tag ${{ env.PINNED_MAILU_VERSION }} $(/usr/bin/git rev-parse HEAD) - git tag ${{ env.PINNED_MAILU_VERSION }} $(/usr/bin/git rev-parse HEAD) - git push origin ${{ env.PINNED_MAILU_VERSION }} - - name: Create release for tag x.y.z. - if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' && env.PINNED_MAILU_VERSION != '' }} - uses: ncipollo/release-action@v1 - with: - bodyFile: "RELEASE_TEMPLATE.md" - tag: ${{ env.PINNED_MAILU_VERSION }} - token: ${{ secrets.GITHUB_TOKEN }} - - # This job is watched by bors. It only complets if building,testing and deploy worked. - ci-success: - name: CI-Done - #Returns true when none of the **previous** steps have failed or have been canceled. - if: ${{ success() }} - needs: - - deploy - runs-on: ubuntu-latest - steps: - - name: CI/CD succeeded. - run: exit 0 diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml new file mode 100644 index 00000000..afc2b4bc --- /dev/null +++ b/.github/workflows/arm.yml @@ -0,0 +1,101 @@ +name: start-linux-arm +on: + push: + branches: + - '1.9' + - master + +concurrency: ci-arm-${{ github.ref }} + +############################################### +# REQUIRED secrets +# ${{ secrets.Docker_Login }} +# Username of docker login for pushing the images to repo env.DOCKER_ORG and env.DOCKER_ORG_TESTS +# ${{ secrets.Docker_Password }} +# Password of docker login for pushing the images to repo env.DOCKER_ORG and env.DOCKER_ORG_TESTS +# Add the above secrets to your github repo to determine where the images will be pushed. +################################################ +# REQUIRED global variables +# DOCKER_ORG, docker org used for pushing release images (branch x.y and master) +# DOCKER_ORG_TEST, docker org used for pushing images for testing (branch testing). +env: + DOCKER_ORG: mailu + DOCKER_ORG_TEST: mailuci + +jobs: +# This job calculates all global job variables that are required by all the subsequent jobs. +# All subsequent jobs will retrieve and use these variables. This way the variables only have to be derived once. + derive-variables: + name: derive variables + runs-on: ubuntu-latest + outputs: + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + DOCKER_ORG: ${{ env.DOCKER_ORG_DERIVED }} + BRANCH: ${{ env.BRANCH }} + DEPLOY: ${{ env.DEPLOY }} + RELEASE: ${{ env.RELEASE }} + steps: + - uses: actions/checkout@v3 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + #For branch TESTING, we set the image tag to pr-xxxx + - name: Derive MAILU_VERSION and DEPLOY/RELEASE for other branches than testing + if: env.BRANCH != 'testing' + shell: bash + run: | + echo "MAILU_VERSION=${{ env.BRANCH }}-arm" >> $GITHUB_ENV + echo "DOCKER_ORG_DERIVED=${{ env.DOCKER_ORG }}" >> $GITHUB_ENV + echo "DEPLOY=true" >> $GITHUB_ENV + echo "RELEASE=false" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION and DEPLOY/RELEASE for normal release x.y + if: env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' + shell: bash + run: | + version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version-arm" >> $GITHUB_ENV + echo "RELEASE=true" >> $GITHUB_ENV + echo "DEPLOY=true" >> $GITHUB_ENV + echo "RELEASE=true" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging for master + if: env.BRANCH == 'master' + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA-arm" >> $GITHUB_ENV + echo "DEPLOY=true" >> $GITHUB_ENV + echo "RELEASE=false" >> $GITHUB_ENV + + build-test-deploy: + needs: + - derive-variables + uses: ./.github/workflows/build_test_deploy.yml + with: + architecture: 'linux/arm64,linux/arm/v7' + mailu_version: ${{needs.derive-variables.outputs.MAILU_VERSION}} + pinned_mailu_version: ${{needs.derive-variables.outputs.PINNED_MAILU_VERSION}} + docker_org: ${{needs.derive-variables.outputs.DOCKER_ORG}} + branch: ${{needs.derive-variables.outputs.BRANCH}} + deploy: ${{needs.derive-variables.outputs.DEPLOY}} + release: ${{needs.derive-variables.outputs.RELEASE}} + secrets: inherit + +################################################ +# Code block that is used as one liner for the step: +# Derive PINNED_MAILU_VERSION and DEPLOY/RELEASE for normal release x.y +##!/bin/bash +#version=$( git tag --sort=version:refname --list "{{ env.MAILU_VERSION }}.*" | tail -1 ) +#root_version=${version%.*} +#patch_version=${version##*.} +#if [ "$patch_version" == "" ] +#then +# pinned_version={{ env.MAILU_VERSION }}.0 +#else +# pinned_version=$root_version.$(expr $patch_version + 1) +#fi +#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml new file mode 100644 index 00000000..f78ed2c5 --- /dev/null +++ b/.github/workflows/build_test_deploy.yml @@ -0,0 +1,355 @@ +name: build-test-deploy +on: + workflow_call: + inputs: + architecture: + description: 'The architecture of the images that will be build.' + required: false + default: 'linux/amd64' + type: string + mailu_version: + description: 'The main version that is build. E.g. master or x.y.' + required: true + type: string + pinned_mailu_version: + description: 'The specific version that is build. E.g. commit hash or x.y.z.' + required: true + type: string + docker_org: + description: 'The docker organisation where the images are pushed to.' + required: true + type: string + branch: + description: 'The branch that triggered this workflow.' + required: true + type: string + deploy: + description: Deploy to docker hub. Happens for all branches but staging. Use string true or false. + default: true + required: false + type: string + release: + description: Tag and create the github release. Use string true or false. + default: false + required: false + type: string + + workflow_dispatch: + inputs: + architecture: + description: 'The architecture of the images that will be build.' + required: false + default: 'linux/amd64' + type: string + mailu_version: + description: 'The main version that is build. E.g. master or x.y.' + required: true + type: string + pinned_mailu_version: + description: 'The specific version that is build. E.g. commit hash or x.y.z.' + required: true + type: string + docker_org: + description: 'The docker organisation where the images are pushed to.' + required: true + type: string + branch: + description: 'The branch that triggered this workflow.' + required: true + type: string + deploy: + description: Deploy to docker hub. Happens for all branches but staging + default: true + required: false + type: boolean + release: + description: 'Tag and create the github release. Only happens for branch x.y (release branch)' + default: false + required: false + type: boolean +env: + HCL_FILE: ./tests/build.hcl + +jobs: +# This job calculates what images must be build. It reads the build.hcl file and then outputs all targets (images) in it. +# This is used by the next build job. + targets: + name: create targets + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.targets.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Create matrix + id: targets + run: | + echo ::set-output name=matrix::$(docker buildx bake -f ${{env.HCL_FILE}} --print | jq -cr '.group.default.targets') + - name: Show matrix + run: | + echo ${{ steps.targets.outputs.matrix }} + +# This job builds all the images. The build cache is stored in the github actions cache. +# In further jobs, this cache is used to quickly rebuild the images. + build: + name: Build images + needs: + - targets + strategy: + fail-fast: false + matrix: + target: ${{ fromJson(needs.targets.outputs.matrix) }} + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v3 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - uses: crazy-max/ghaction-github-runtime@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} + - name: Build all docker images + env: + DOCKER_ORG: ${{ env.DOCKER_ORG }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + uses: docker/bake-action@v2 + with: + files: ${{env.HCL_FILE}} + targets: ${{ matrix.target }} + load: false + push: false + set: | + *.cache-from=type=gha,scope=build-${{ matrix.target }} + *.cache-to=type=gha,scope=build-${{ matrix.target }},mode=max + *.platform=${{ inputs.architecture }} + +# This job runs all the tests. + tests: + name: tests + if: inputs.architecture == 'linux/amd64' + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + needs: + - targets + - build + strategy: + fail-fast: false + matrix: + target: ["core", "fetchmail", "filters", "rainloop", "roundcube", "webdav"] + time: ["2"] + include: + - target: "filters" + time: "3" + exclude: + - target: "filters" + time: "2" + steps: + - uses: actions/checkout@v3 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - uses: crazy-max/ghaction-github-runtime@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} + - name: Build docker images for testing from cache + env: + DOCKER_ORG: ${{ env.DOCKER_ORG }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + uses: docker/bake-action@v2 + with: + files: ${{env.HCL_FILE}} + load: true + push: false + set: | + *.cache-from=type=gha,scope=build-docs + *.cache-from=type=gha,scope=build-setup + *.cache-from=type=gha,scope=build-admin + *.cache-from=type=gha,scope=build-antispam + *.cache-from=type=gha,scope=build-front + *.cache-from=type=gha,scope=build-imap + *.cache-from=type=gha,scope=build-smtp + *.cache-from=type=gha,scope=build-rainloop + *.cache-from=type=gha,scope=build-roundcube + *.cache-from=type=gha,scope=build-antivirus + *.cache-from=type=gha,scope=build-fetchmail + *.cache-from=type=gha,scope=build-resolver + *.cache-from=type=gha,scope=build-traefik-certdumper + *.cache-from=type=gha,scope=build-webdav + *.platform=${{ inputs.architecture }} + - name: Install python packages + run: python3 -m pip install -r tests/requirements.txt + - name: Copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: Test ${{ matrix.target }} + run: python tests/compose/test.py ${{ matrix.target }} ${{ matrix.time }} + env: + DOCKER_ORG: ${{ env.DOCKER_ORG }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + +# This job deploys the docker images to the docker repository. The build.hcl file contains logic that determines what tags are pushed. +# E.g. for master only the :master and :latest tags are pushed. + deploy: + name: Deploy images + # Deploying is not required for staging + if: inputs.deploy == 'true' + runs-on: ubuntu-latest + needs: + - build + - tests + steps: + - uses: actions/checkout@v3 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - uses: crazy-max/ghaction-github-runtime@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} + - name: Deploy images to docker hub. Build.hcl contains the logic for the tags that are pushed. + env: + DOCKER_ORG: ${{ env.DOCKER_ORG }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + uses: docker/bake-action@v2 + with: + files: ${{env.HCL_FILE}} + push: true + set: | + *.cache-from=type=gha,scope=build-docs + *.cache-from=type=gha,scope=build-setup + *.cache-from=type=gha,scope=build-admin + *.cache-from=type=gha,scope=build-antispam + *.cache-from=type=gha,scope=build-front + *.cache-from=type=gha,scope=build-imap + *.cache-from=type=gha,scope=build-smtp + *.cache-from=type=gha,scope=build-rainloop + *.cache-from=type=gha,scope=build-roundcube + *.cache-from=type=gha,scope=build-antivirus + *.cache-from=type=gha,scope=build-fetchmail + *.cache-from=type=gha,scope=build-resolver + *.cache-from=type=gha,scope=build-traefik-certdumper + *.cache-from=type=gha,scope=build-webdav + *.platform=${{ inputs.architecture }} + + deploy-arm: + name: Deploy images for arm + # Deploying is not required for staging + if: inputs.deploy == 'true' && inputs.architecture != 'linux/amd64' + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v3 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - uses: crazy-max/ghaction-github-runtime@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} + - name: Deploy images to docker hub. Build.hcl contains the logic for the tags that are pushed. + env: + DOCKER_ORG: ${{ env.DOCKER_ORG }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + uses: docker/bake-action@v2 + with: + files: ${{env.HCL_FILE}} + push: true + set: | + *.cache-from=type=gha,scope=build-docs + *.cache-from=type=gha,scope=build-setup + *.cache-from=type=gha,scope=build-admin + *.cache-from=type=gha,scope=build-antispam + *.cache-from=type=gha,scope=build-front + *.cache-from=type=gha,scope=build-imap + *.cache-from=type=gha,scope=build-smtp + *.cache-from=type=gha,scope=build-rainloop + *.cache-from=type=gha,scope=build-roundcube + *.cache-from=type=gha,scope=build-antivirus + *.cache-from=type=gha,scope=build-fetchmail + *.cache-from=type=gha,scope=build-resolver + *.cache-from=type=gha,scope=build-traefik-certdumper + *.cache-from=type=gha,scope=build-webdav + *.platform=${{ inputs.architecture }} + +#This job creates a tagged release. A tag is created for the pinned version x.y.z. The GH release refers to this tag. + tag-release: + if: inputs.release == 'true' + runs-on: ubuntu-latest + needs: + - deploy + steps: + - uses: actions/checkout@v3 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Create tag for branch x.y. + shell: bash + run: | + echo git tag ${{ env.PINNED_MAILU_VERSION }} $(/usr/bin/git rev-parse HEAD) + git tag ${{ env.PINNED_MAILU_VERSION }} $(/usr/bin/git rev-parse HEAD) + git push origin ${{ env.PINNED_MAILU_VERSION }} + - name: Create release for tag x.y.z. + uses: ncipollo/release-action@v1 + with: + bodyFile: "RELEASE_TEMPLATE.md" + tag: ${{ env.PINNED_MAILU_VERSION }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/x64.yml b/.github/workflows/x64.yml new file mode 100644 index 00000000..e1691756 --- /dev/null +++ b/.github/workflows/x64.yml @@ -0,0 +1,134 @@ +name: start-linux-amd64 +on: + push: + branches: + - testing + - staging + - '1.9' + - master + +concurrency: ci-x64-${{ github.ref }} + +############################################### +# REQUIRED secrets +# ${{ secrets.Docker_Login }} +# Username of docker login for pushing the images to repo env.DOCKER_ORG and env.DOCKER_ORG_TESTS +# ${{ secrets.Docker_Password }} +# Password of docker login for pushing the images to repo env.DOCKER_ORG and env.DOCKER_ORG_TESTS +# Add the above secrets to your github repo to determine where the images will be pushed. +################################################ +# REQUIRED global variables +# DOCKER_ORG, docker org used for pushing release images (branch x.y and master) +# DOCKER_ORG_TEST, docker org used for pushing images for testing (branch testing). +env: + DOCKER_ORG: mailu + DOCKER_ORG_TEST: mailuci + +jobs: +# This job calculates all global job variables that are required by all the subsequent jobs. +# All subsequent jobs will retrieve and use these variables. This way the variables only have to be derived once. + derive-variables: + name: derive variables + runs-on: ubuntu-latest + outputs: + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + DOCKER_ORG: ${{ env.DOCKER_ORG_DERIVED }} + BRANCH: ${{ env.BRANCH }} + DEPLOY: ${{ env.DEPLOY }} + RELEASE: ${{ env.RELEASE }} + steps: + - uses: actions/checkout@v3 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + #For branch TESTING, we set the image tag to pr-xxxx + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION and DEPLOY/RELEASE for branch testing + if: env.BRANCH == 'testing' + shell: bash + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: | + echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG_DERIVED=${{ env.DOCKER_ORG_TEST }}" >> $GITHUB_ENV + echo "DEPLOY=true" >> $GITHUB_ENV + echo "RELEASE=false" >> $GITHUB_ENV + - name: Derive MAILU_VERSION and DEPLOY/RELEASE for other branches than testing + if: env.BRANCH != 'testing' + shell: bash + run: | + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG_DERIVED=${{ env.DOCKER_ORG }}" >> $GITHUB_ENV + echo "DEPLOY=true" >> $GITHUB_ENV + echo "RELEASE=false" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION and DEPLOY/RELEASE for normal release x.y + if: env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' + shell: bash + run: | + version=$( git tag --sort=version:refname --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + echo "RELEASE=true" >> $GITHUB_ENV + echo "DEPLOY=true" >> $GITHUB_ENV + echo "RELEASE=true" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION and DEPLOY/RELEASE for staging + id: deploy + if: env.BRANCH == 'staging' + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + echo "DEPLOY=false" >> $GITHUB_ENV + echo "RELEASE=false" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging for master + if: env.BRANCH == 'master' + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV + echo "DEPLOY=true" >> $GITHUB_ENV + echo "RELEASE=false" >> $GITHUB_ENV + + build-test-deploy: + needs: + - derive-variables + uses: ./.github/workflows/build_test_deploy.yml + with: + architecture: 'linux/amd64' + mailu_version: ${{needs.derive-variables.outputs.MAILU_VERSION}} + pinned_mailu_version: ${{needs.derive-variables.outputs.PINNED_MAILU_VERSION}} + docker_org: ${{needs.derive-variables.outputs.DOCKER_ORG}} + branch: ${{needs.derive-variables.outputs.BRANCH}} + deploy: ${{needs.derive-variables.outputs.DEPLOY}} + release: ${{needs.derive-variables.outputs.RELEASE}} + secrets: inherit + + # This job is watched by bors. It only complets if building,testing and deploy worked. + ci-success: + name: CI-Done + #Returns true when none of the **previous** steps have failed or have been canceled. + if: success() + needs: + - build-test-deploy + runs-on: ubuntu-latest + steps: + - name: CI/CD succeeded. + run: exit 0 + +################################################ +# Code block that is used as one liner for the step: +# Derive PINNED_MAILU_VERSION and DEPLOY/RELEASE for normal release x.y +##!/bin/bash +#version=$( git tag --sort=version:refname --list "{{ env.MAILU_VERSION }}.*" | tail -1 ) +#root_version=${version%.*} +#patch_version=${version##*.} +#if [ "$patch_version" == "" ] +#then +# pinned_version={{ env.MAILU_VERSION }}.0 +#else +# pinned_version=$root_version.$(expr $patch_version + 1) +#fi +#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV \ No newline at end of file diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 0952e62f..a9cc61c2 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -1,8 +1,7 @@ # First stage to build assets ARG DISTRO=alpine:3.14.5 -ARG ARCH="" -FROM ${ARCH}node:16 as assets +FROM node:16-alpine3.16 as assets COPY package.json ./ RUN set -eu \ @@ -22,7 +21,6 @@ RUN set -eu \ # Actual application FROM $DISTRO ARG VERSION -COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static ENV TZ Etc/UTC diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index f6077c58..13030ac5 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -15,7 +15,9 @@ RUN apk add --no-cache \ RUN pip3 install socrate==0.2.0 # Shared layer between dovecot and postfix -RUN pip3 install "podop>0.2.5" +RUN apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ + && pip3 install "podop>0.2.5" \ + && apk del .build-deps # Image specific layers under this line RUN apk add --no-cache \ diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index e7438d04..563bb228 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -16,7 +16,9 @@ RUN apk add --no-cache \ RUN pip3 install socrate==0.2.0 # Shared layer between dovecot and postfix -RUN pip3 install "podop>0.2.5" +RUN apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ + && pip3 install "podop>0.2.5" \ + && apk del .build-deps # Image specific layers under this line # Building pycares from source requires py3-wheel and libffi-dev packages diff --git a/docs/arm_images.rst b/docs/arm_images.rst new file mode 100644 index 00000000..2a1a1574 --- /dev/null +++ b/docs/arm_images.rst @@ -0,0 +1,76 @@ +.. _arm_images: + +Arm images for Mailu +==================== + +Using Mailu arm images +---------------------- + +The Mailu project makes use of github actions for automatic CI/CD. +Github actions only has x64 (amd64) runners. This means we can build the arm images +using QEMU, but we cannot test the images. For this reason the arm images have +a BETA status. We only guarantee that the images could be built. + + +We strongly recommend to make use of the pinned version (tag 1.9.10 vs tag 1.9). +Pinned versions (tag x.y.z) are not updated. This allows upgrading manually by changing the +tag to the next pinned version. + + +Whenever images are deployed for master and for releases (branch x.y), +images are also built for arm. + +The images are pushed with -arm appended to the tag. For example: + +- admin:master-arm +- admin:1.10-arm + +To use these images, simply use setup.mailu.io for generating the docker-composse.yml +file and mailu.env file. Then in the docker-compose.yml file append -arm to the tags of +all images from the mailu docker repository. + +Build manually +-------------- +It is possible to build the images manually. There are two possiblities for this. + +Github actions +`````````````` +The main workflow build-test-deploy can be triggered manually. +Via the parameter ``architecture`` the target platform can be specified. +Use the value ``'linux/arm64,linux/arm/v7'``. + + +To use it: + +1. Fork the Mailu github project. +2. In the settings of your forked project, configure the secrets Docker_Login and Docker_Password. For more information on these secrets, see the comments in the build-test-deploy.yml file. +3. In the forked project, trigger the workflow build-test-deploy manually. +4. For the parameter architecture use the value ``'linux/arm64,linux/arm/v7'``. + + +Manually +```````` +It is also possible to build the images manually on bare-metal. +The buildx file ``tests/build.hcl`` can be used for this. + + +To build manually: + +1. Install QEMU static binaries. This is only required, if you don't built on an arm machine. For Ubuntu install qemu-user-static. +2. Clone the Mailu github project. +3. Export the parameters. +4. Create a buildx builder instance +5. Run buildx overriding the architecture. + +For example: + +.. code-block:: bash + + docker login + Username: Foo + Password: Bar + export DOCKER_ORG="Foo" + export MAILU_VERSION="master-arm" + export MAILU_PINNED_VERSION="hash" + docker buildx create --use + docker buildx bake -f tests/build.hcl --push --set *.platform=linux/arm64,linux/arm/v7 diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index cef71c6c..42b6e4a2 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -106,26 +106,29 @@ The development environment is quite similar to the production one. Building images ``````````````` -We supply a separate ``test/build.yml`` file for convenience. +We supply a separate ``test/build.hcl`` file for convenience. After cloning the git repository to your workstation, you can build the images: .. code-block:: bash cd Mailu - docker-compose -f tests/build.yml build + docker buildx bake -f tests/build.hcl --load -The ``build.yml`` file has two variables: +The ``build.hcl`` file has three variables: #. ``$DOCKER_ORG``: First part of the image tag. Defaults to *mailu* and needs to be changed only when pushing to your own Docker hub account. -#. ``$VERSION``: Last part of the image tag. Defaults to *local* to differentiate from pulled +#. ``$MAILU_VERSION``: Last part of the image tag. Defaults to *local* to differentiate from pulled images. +#. ``$MAILU_PINNED_VERSION``: Last part of the image tag for x.y.z images. Defaults to *local* to differentiate from pulled + images. + To re-build only specific containers at a later time. .. code-block:: bash - docker-compose -f tests/build.yml build admin webdav + docker buildx bake -f tests/build.hcl admin webdav If you have to push the images to Docker Hub for testing in Docker Swarm or a remote host, you have to define ``DOCKER_ORG`` (usually your Docker user-name) and login to @@ -137,15 +140,15 @@ the hub. Username: Foo Password: Bar export DOCKER_ORG="Foo" - export VERSION="feat-extra-app" - docker-compose -f tests/build.yml build - docker-compose -f tests/build.yml push + export MAILU_VERSION="feat-extra-app" + export MAILU_PINNED_VERSION="feat-extra-app" + docker buildx bake -f tests/build.hcl --push Running containers `````````````````` To run the newly created images: ``cd`` to your project directory. Edit ``.env`` to set -``VERSION`` to the same value as used during the build, which defaults to ``local``. +``VERSION`` to the same value as used during the build (for MAILU_VERSION), which defaults to ``local``. After that you can run: .. code-block:: bash @@ -277,7 +280,7 @@ Merge conflicts Before proceeding, check the PR page in the bottom. It should not indicate a merge conflict. If there are merge conflicts, you have 2 options: -#. Do a review "request changes" and ask the author to resolve the merge conflict. +#. Do a review "request changes" and ask the author to resolve the merge conflict. #. Solve the merge conflict yourself on Github, using the web editor. If it can't be done in the web editor, go for option 1. Unless you want to go through the trouble of diff --git a/docs/index.rst b/docs/index.rst index 6a3c42b7..4e4bb416 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,7 +60,7 @@ the version of Mailu that you are running. dns reverse database - rpi_build + arm_images .. toctree:: :maxdepth: 2 diff --git a/docs/rpi_build.rst b/docs/rpi_build.rst deleted file mode 100644 index 07be77de..00000000 --- a/docs/rpi_build.rst +++ /dev/null @@ -1,25 +0,0 @@ -.. _rpi_build: - -Building for a Raspberry Pi -=========================== - -The build does not need to be done on the Pi. - -To do so, go to ``tests/`` and call ``./build_arm.sh``, it will build all -necessary images for arm. - -To push the locally built images to a remote server, run ``./deploy_to_pi.sh``. -Docker 18.09+ is needed to use ``-H ssh://@``. - -Adjustments ------------ - -``build_arm.sh`` uses some variables passed as ``build-arg`` to docker-compose: - -- ``ALPINE_VER``: version of ALPINE to use -- ``DISTRO``: is the main distro used. Dockerfiles are set on Alpine 3.14, and - build script overrides for ``balenalib/rpi-alpine:3.14`` -- ``QEMU``: Used by webmails dockerfiles. It will add ``qemu-arm-static`` only - if ``QEMU`` is set to ``arm`` -- ``ARCH``: Architecture to use for ``admin``, and ``webmails`` as their images - are available for those architectures. diff --git a/docs/setup.rst b/docs/setup.rst index 79b4fc7b..653cdce3 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -19,7 +19,7 @@ Prepare the environment Mailu images are designed to work on x86 or equivalent hardware, so it should run on pretty much any cloud server as long as enough power is -provided. For non x86 machines, see :ref:`rpi_build` +provided. For non x86 machines, see :ref:`arm_images` You are free to choose any operating system that runs Docker (>= 1.11), then chose between various flavors including Docker Compose, Kubernetes diff --git a/tests/build.hcl b/tests/build.hcl new file mode 100644 index 00000000..4db636df --- /dev/null +++ b/tests/build.hcl @@ -0,0 +1,185 @@ +# build.hcl +# For more information on buildx bake file definition see: +# https://github.com/docker/buildx/blob/master/docs/guides/bake/file-definition.md +# +# NOTE: You can only run this from the Mailu root folder. +# Make sure the context is Mailu (project folder) and not Mailu/tests +#----------------------------------------------------------------------------------------- +# (Environment) input variables +# If the env var is not set, then the default value is used +#----------------------------------------------------------------------------------------- +variable "DOCKER_ORG" { + default = "mailu" +} +variable "DOCKER_PREFIX" { + default = "" +} +variable "PINNED_MAILU_VERSION" { + default = "local" +} +variable "MAILU_VERSION" { + default = "local" +} + +#----------------------------------------------------------------------------------------- +# Grouping of targets to build. All these images are built when using: +# docker buildx bake -f tests\build.hcl +#----------------------------------------------------------------------------------------- +group "default" { + targets = [ + "docs", + "setup", + + "admin", + "antispam", + "front", + "imap", + "smtp", + + "rainloop", + "roundcube", + + "antivirus", + "fetchmail", + "resolver", + "traefik-certdumper", + "webdav" + ] +} + +#----------------------------------------------------------------------------------------- +# Default settings that will be inherited by all targets (images to build). +#----------------------------------------------------------------------------------------- +target "defaults" { + platforms = [ "linux/amd64"] + dockerfile = "Dockerfile" + args = { + VERSION = "${PINNED_MAILU_VERSION}" + } +} + +#----------------------------------------------------------------------------------------- +# User defined functions +#------------------------------------------------------------------------------------------ +# Derive all tags +function "tag" { + params = [image_name] + result = [ notequal("master",MAILU_VERSION) && notequal("master-arm",MAILU_VERSION) ? "${DOCKER_ORG}/${DOCKER_PREFIX}${image_name}:${PINNED_MAILU_VERSION}": "", + "${DOCKER_ORG}/${DOCKER_PREFIX}${image_name}:${MAILU_VERSION}", + "${DOCKER_ORG}/${DOCKER_PREFIX}${image_name}:latest" + ] +} + +#----------------------------------------------------------------------------------------- +# All individual targets (images to build) +# Build an individual target using. +# docker buildx bake -f tests\build.hcl +# E.g. to build target docs +# docker buildx bake -f tests\build.hcl docs +#----------------------------------------------------------------------------------------- + +# ----------------------------------------------------------------------------------------- +# Documentation and setup images +# ----------------------------------------------------------------------------------------- +target "docs" { + inherits = ["defaults"] + context = "docs" + tags = tag("docs") + args = { + version = "${MAILU_VERSION}" + pinned_version = "${PINNED_MAILU_VERSION}" + } +} + +target "setup" { + inherits = ["defaults"] + context="setup" + tags = tag("setup") +} + +# ----------------------------------------------------------------------------------------- +# Core images +# ----------------------------------------------------------------------------------------- +target "none" { + inherits = ["defaults"] + context="core/none" + tags = tag("none") +} + +target "admin" { + inherits = ["defaults"] + context="core/admin" + tags = tag("admin") +} + +target "antispam" { + inherits = ["defaults"] + context="core/rspamd" + tags = tag("rspamd") +} + +target "front" { + inherits = ["defaults"] + context="core/nginx" + tags = tag("nginx") +} + +target "imap" { + inherits = ["defaults"] + context="core/dovecot" + tags = tag("dovecot") +} + +target "smtp" { + inherits = ["defaults"] + context="core/postfix" + tags = tag("postfix") +} + +# ----------------------------------------------------------------------------------------- +# Webmail images +# ----------------------------------------------------------------------------------------- +target "rainloop" { + inherits = ["defaults"] + context="webmails/rainloop" + tags = tag("rainloop") +} + +target "roundcube" { + inherits = ["defaults"] + context="webmails/roundcube" + tags = tag("roundcube") +} + +# ----------------------------------------------------------------------------------------- +# Optional images +# ----------------------------------------------------------------------------------------- +target "antivirus" { + inherits = ["defaults"] + context="optional/clamav" + tags = tag("clamav") +} + +target "fetchmail" { + inherits = ["defaults"] + context="optional/fetchmail" + tags = tag("fetchmail") +} + +target "resolver" { + inherits = ["defaults"] + context="optional/unbound" + tags = tag("unbound") +} + +target "traefik-certdumper" { + inherits = ["defaults"] + context="optional/traefik-certdumper" + tags = tag("traefik-certdumper") +} + +target "webdav" { + inherits = ["defaults"] + context="optional/radicale" + tags = tag("radicale") +} \ No newline at end of file diff --git a/tests/build.yml b/tests/build.yml deleted file mode 100644 index da351ac8..00000000 --- a/tests/build.yml +++ /dev/null @@ -1,109 +0,0 @@ -version: '3' - -services: - - front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} - build: - context: ../core/nginx - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - resolver: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}unbound:${PINNED_MAILU_VERSION:-local} - build: - context: ../optional/unbound - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} - build: - context: ../core/dovecot - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} - build: - context: ../core/postfix - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} - build: - context: ../core/rspamd - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - antivirus: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}clamav:${PINNED_MAILU_VERSION:-local} - build: - context: ../optional/clamav - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - webdav: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}radicale:${PINNED_MAILU_VERSION:-local} - build: - context: ../optional/radicale - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - traefik-certdumper: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}traefik-certdumper:${PINNED_MAILU_VERSION:-local} - build: - context: ../optional/traefik-certdumper - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} - build: - context: ../core/admin - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - roundcube: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}roundcube:${PINNED_MAILU_VERSION:-local} - build: - context: ../webmails/roundcube - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - rainloop: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rainloop:${PINNED_MAILU_VERSION:-local} - build: - context: ../webmails/rainloop - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - fetchmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${PINNED_MAILU_VERSION:-local} - build: - context: ../optional/fetchmail - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - none: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}none:${PINNED_MAILU_VERSION:-local} - build: - context: ../core/none - args: - VERSION: ${PINNED_MAILU_VERSION:-local} - - docs: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}docs:${PINNED_MAILU_VERSION:-local} - build: - context: ../docs - args: - version: ${MAILU_VERSION:-local} - pinned_version: ${PINNED_MAILU_VERSION:-local} - - setup: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}setup:${PINNED_MAILU_VERSION:-local} - build: - context: ../setup - args: - VERSION: ${PINNED_MAILU_VERSION:-local} diff --git a/tests/build_arm.sh b/tests/build_arm.sh deleted file mode 100755 index 32dba421..00000000 --- a/tests/build_arm.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -x - -ALPINE_VER="3.14" -DISTRO="balenalib/rpi-alpine:$ALPINE_VER" -# Used for webmails -QEMU="arm" -ARCH="arm32v7/" - -# use qemu-*-static from docker container -docker run --rm --privileged multiarch/qemu-user-static:register -docker-compose -f build.yml build \ - --build-arg DISTRO=$DISTRO \ - --build-arg ARCH=$ARCH \ - --build-arg QEMU=$QEMU \ - --parallel $@ diff --git a/tests/compose/core/docker-compose.yml b/tests/compose/core/docker-compose.yml index 06f91f73..1f9d6730 100644 --- a/tests/compose/core/docker-compose.yml +++ b/tests/compose/core/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -47,7 +47,7 @@ services: - resolver imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -57,7 +57,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -66,7 +66,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/compose/fetchmail/docker-compose.yml b/tests/compose/fetchmail/docker-compose.yml index ed296d3e..c1a1a55c 100644 --- a/tests/compose/fetchmail/docker-compose.yml +++ b/tests/compose/fetchmail/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -47,7 +47,7 @@ services: - 192.168.203.254 imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -57,7 +57,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -66,7 +66,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -80,7 +80,7 @@ services: fetchmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${MAILU_VERSION:-local} restart: always env_file: mailu.env diff --git a/tests/compose/filters/docker-compose.yml b/tests/compose/filters/docker-compose.yml index 09d948df..41908a40 100644 --- a/tests/compose/filters/docker-compose.yml +++ b/tests/compose/filters/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -47,7 +47,7 @@ services: - resolver imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -57,7 +57,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -66,7 +66,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -78,7 +78,7 @@ services: # Optional services antivirus: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}clamav:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}clamav:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/compose/rainloop/docker-compose.yml b/tests/compose/rainloop/docker-compose.yml index ca8d70ba..4683e7c5 100644 --- a/tests/compose/rainloop/docker-compose.yml +++ b/tests/compose/rainloop/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -47,7 +47,7 @@ services: - 192.168.203.254 imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -57,7 +57,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -66,7 +66,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -88,7 +88,7 @@ services: # Webmail webmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rainloop:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rainloop:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/compose/roundcube/docker-compose.yml b/tests/compose/roundcube/docker-compose.yml index 7ac94810..f2c43686 100644 --- a/tests/compose/roundcube/docker-compose.yml +++ b/tests/compose/roundcube/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -47,7 +47,7 @@ services: - 192.168.203.254 imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -57,7 +57,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -66,7 +66,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -88,7 +88,7 @@ services: # Webmail webmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}roundcube:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}roundcube:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/compose/webdav/docker-compose.yml b/tests/compose/webdav/docker-compose.yml index 7c62c90a..5dfa6bc8 100644 --- a/tests/compose/webdav/docker-compose.yml +++ b/tests/compose/webdav/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -47,7 +47,7 @@ services: - resolver imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -57,7 +57,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -66,7 +66,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -79,7 +79,7 @@ services: # Optional services webdav: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}radicale:${PINNED_MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}radicale:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/deploy.sh b/tests/deploy.sh deleted file mode 100755 index db843b9c..00000000 --- a/tests/deploy.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -# Skip deploy for staging branch -[ "$BRANCH" = "staging" ] && exit 0 - -docker login -u $DOCKER_UN -p $DOCKER_PW - -if [ "$BRANCH" = "testing" ] -then - docker-compose -f tests/build.yml push - exit 0 -fi - -#Deploy for main releases -#Images are built with tag PINNED_MAILU_VERSION (x.y.z). -#We are tagging them as well with MAILU_VERSION (x.y) -#After that, both tags are pushed to the docker repository. -if [ "$PINNED_MAILU_VERSION" != "" ] && [ "$BRANCH" != "master" ] -then - images=$(docker-compose -f tests/build.yml config | awk -F ':' '/image:/{ print $2 }') - for image in $images - do - docker tag "${image}":"${PINNED_MAILU_VERSION}" "${image}":${MAILU_VERSION} - done -#Push PINNED_MAILU_VERSION images - docker-compose -f tests/build.yml push -#Push MAILU_VERSION images - PINNED_MAILU_VERSION=$MAILU_VERSION - docker-compose -f tests/build.yml push - exit 0 -fi - -#Deploy for master. For master we only publish images with tag master -#Images are built with tag PINNED_MAILU_VERSION (commit hash). -#We are tagging them as well with MAILU_VERSION (master) -#Then we publish the images with tag master -if [ "$PINNED_MAILU_VERSION" != "" ] && [ "$BRANCH" == "master" ] -then - images=$(docker-compose -f tests/build.yml config | awk -F ':' '/image:/{ print $2 }') - for image in $images - do - docker tag "${image}":"${PINNED_MAILU_VERSION}" "${image}":${MAILU_VERSION} - done -#Push MAILU_VERSION images - PINNED_MAILU_VERSION=$MAILU_VERSION - docker-compose -f tests/build.yml push - exit 0 -fi - -#Fallback in case $PINNED_MAILU_VERSION is empty. This should never execute. -docker-compose -f tests/build.yml push diff --git a/tests/deploy_to_pi.sh b/tests/deploy_to_pi.sh deleted file mode 100755 index dab5a53f..00000000 --- a/tests/deploy_to_pi.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -x -# Small script to deploy locally built images to a remote docker -compose_options=$1 -images=$(docker-compose $1 images | awk 'NR > 2 { printf $2":"$3" " }') -docker save $images | pigz - > mail.local.tgz -echo "now run 'docker -H \"ssh://user@host\" load -i mail.local.tgz" diff --git a/towncrier/newsfragments/2383.misc b/towncrier/newsfragments/2383.misc new file mode 100644 index 00000000..edcc9a9a --- /dev/null +++ b/towncrier/newsfragments/2383.misc @@ -0,0 +1,9 @@ +Switch from docker build to buildx for CI/CD. +- The main workflow file has been optimised and simplified. +- Images are built in parallel when building locally resulting in faster build times. +- The github action workflow is about 50% faster. +- Arm images are built as well. These images are not tested due to restrictions of github actions (no arm runners). The tags of the images have -arm appended to it. +- Arm images can also be built locally. +- Reusable workflow is introduced for building, testing and deploying the images. + This allows the workflow to be reused for other purposes in the future. +- Workflow can be manually triggered. This allows forked Mailu projects to also use the workflow for building images. diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index 5abcba73..ebe8b5fe 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -1,12 +1,6 @@ # NOTE: only add qemu-arm-static if building for arm -ARG ARCH="" -ARG QEMU=other -FROM ${ARCH}php:8.0-apache as build_arm -ONBUILD COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static -FROM ${ARCH}php:8.0-apache as build_other +FROM php:8.0-apache - -FROM build_${QEMU} ARG VERSION ENV TZ Etc/UTC