From 3f91dcb7afecd14c876a5c7ce81ecdffef880f72 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 6 Jul 2021 13:48:53 +0200 Subject: [PATCH 01/25] compile scheme list using a generator --- core/admin/mailu/models.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index b5ba29c0..a86e005f 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -507,13 +507,13 @@ class User(Base, Email): if cls._ctx: return cls._ctx - schemes = passlib.registry.list_crypt_handlers() - # scrypt throws a warning if the native wheels aren't found - schemes.remove('scrypt') - # we can't leave plaintext schemes as they will be misidentified - for scheme in schemes: - if scheme.endswith('plaintext'): - schemes.remove(scheme) + # compile schemes + # - scrypt throws a warning if the native wheels aren't found + # - we can't leave plaintext schemes as they will be misidentified + schemes = [ + scheme for scheme in passlib.registry.list_crypt_handlers() + if not (scheme == 'scrypt' or scheme.endswith('plaintext')) + ] cls._ctx = passlib.context.CryptContext( schemes=schemes, default='bcrypt_sha256', From c2c3030a2f80bf0ae85d186d30a47ca97b9ce81b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Jul 2021 20:54:36 +0200 Subject: [PATCH 02/25] rephrased comments --- core/admin/mailu/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a86e005f..43f7196a 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -508,8 +508,8 @@ class User(Base, Email): return cls._ctx # compile schemes - # - scrypt throws a warning if the native wheels aren't found - # - we can't leave plaintext schemes as they will be misidentified + # - skip scrypt (throws a warning if the native wheels aren't found) + # - skip plaintext schemes (will be misidentified) schemes = [ scheme for scheme in passlib.registry.list_crypt_handlers() if not (scheme == 'scrypt' or scheme.endswith('plaintext')) From 5301f0c20046da4f51e7a5b9408b47abb4b4f622 Mon Sep 17 00:00:00 2001 From: Georg Date: Wed, 22 Sep 2021 10:53:28 +0300 Subject: [PATCH 03/25] Change memory requirements Running with ClamAV requires atleast 3GB of memory otherwise ClamAV updates fail and fill the disk https://github.com/Mailu/Mailu/issues/470 --- docs/compose/requirements.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/compose/requirements.rst b/docs/compose/requirements.rst index a5fd3d6b..1db200b0 100644 --- a/docs/compose/requirements.rst +++ b/docs/compose/requirements.rst @@ -5,8 +5,8 @@ Hardware considerations ----------------------- You should make sure that your hardware (virtual or physical) is compatible with -the latest Linux kernel. Also, you should have at least 2GB of total memory and -1GB of free memory when running Mailu. +the latest Linux kernel. Also, you should have at least 3GB of total memory and +1GB of free memory/swap when running Mailu. Pick a distribution ------------------- From e2512c7cdcd67e753d3371861680042e1c845235 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Mon, 15 Nov 2021 15:34:38 +0000 Subject: [PATCH 04/25] Testing images are pushed to DOCKER_ORG_TESTS again. --- .github/workflows/CI.yml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e2a535dd..5f298224 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -19,9 +19,9 @@ on: ############################################### # REQUIRED secrets # DOCKER_UN: ${{ secrets.Docker_Login }} -# Username of docker login for pushing the images to repo $DOCKER_ORG +# Username of docker login for pushing the images to repo $DOCKER_ORG and $DOCKER_ORG_TESTS # DOCKER_PW: ${{ secrets.Docker_Password }} -# Password of docker login for pushing the images to repo $DOCKER_ORG +# Password of docker login for pushing the images to repo $DOCKER_ORG and $DOCKER_ORG_TESTS # DOCKER_ORG: ${{ secrets.DOCKER_ORG }} # The docker repository where the images are pushed to. # DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} @@ -47,6 +47,7 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }} >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash @@ -54,6 +55,7 @@ jobs: MAILU_BRANCH: ${{ env.BRANCH }} run: | echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }} >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -69,16 +71,16 @@ jobs: env: DOCKER_UN: ${{ secrets.Docker_Login }} DOCKER_PW: ${{ secrets.Docker_Password }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - name: Build all docker images env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} run: docker-compose -f tests/build.yml build - name: Save all docker images - run: docker save ${{ secrets.DOCKER_ORG }}/admin ${{ secrets.DOCKER_ORG }}/clamav ${{ secrets.DOCKER_ORG }}/docs ${{ secrets.DOCKER_ORG }}/dovecot ${{ secrets.DOCKER_ORG }}/fetchmail ${{ secrets.DOCKER_ORG }}/nginx ${{ secrets.DOCKER_ORG }}/none ${{ secrets.DOCKER_ORG }}/postfix ${{ secrets.DOCKER_ORG }}/postgresql ${{ secrets.DOCKER_ORG }}/radicale ${{ secrets.DOCKER_ORG }}/rainloop ${{ secrets.DOCKER_ORG }}/roundcube ${{ secrets.DOCKER_ORG }}/rspamd ${{ secrets.DOCKER_ORG }}/setup ${{ secrets.DOCKER_ORG }}/traefik-certdumper ${{ secrets.DOCKER_ORG }}/unbound -o /images/images.tar.gz + run: docker save ${{ env.DOCKER_ORG }}/admin ${{ env.DOCKER_ORG }}/clamav ${{ env.DOCKER_ORG }}/docs ${{ env.DOCKER_ORG }}/dovecot ${{ env.DOCKER_ORG }}/fetchmail ${{ env.DOCKER_ORG }}/nginx ${{ env.DOCKER_ORG }}/none ${{ env.DOCKER_ORG }}/postfix ${{ env.DOCKER_ORG }}/postgresql ${{ env.DOCKER_ORG }}/radicale ${{ env.DOCKER_ORG }}/rainloop ${{ env.DOCKER_ORG }}/roundcube ${{ env.DOCKER_ORG }}/rspamd ${{ env.DOCKER_ORG }}/setup ${{ env.DOCKER_ORG }}/traefik-certdumper ${{ env.DOCKER_ORG }}/unbound -o /images/images.tar.gz test-core: name: Perform core tests @@ -125,7 +127,7 @@ jobs: env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} test-fetchmail: name: Perform fetchmail tests @@ -172,7 +174,7 @@ jobs: env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} test-filters: name: Perform filter tests @@ -219,7 +221,7 @@ jobs: env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} test-rainloop: name: Perform rainloop tests @@ -266,7 +268,7 @@ jobs: env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} test-roundcube: name: Perform roundcube tests @@ -313,7 +315,7 @@ jobs: env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} test-webdav: name: Perform webdav tests @@ -360,7 +362,7 @@ jobs: env: MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} deploy: name: Deploy images @@ -412,8 +414,7 @@ jobs: env: DOCKER_UN: ${{ secrets.Docker_Login }} DOCKER_PW: ${{ secrets.Docker_Password }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} + DOCKER_ORG: ${{ env.DOCKER_ORG }} MAILU_VERSION: ${{ env.MAILU_VERSION }} TRAVIS_BRANCH: ${{ env.BRANCH }} run: bash tests/deploy.sh From b39169269800d5a7c5c57aa94587b9f152c25771 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Mon, 15 Nov 2021 15:44:31 +0000 Subject: [PATCH 05/25] It is handy to close strings. --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5f298224..578ed1a9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -47,7 +47,7 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }} >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash @@ -55,7 +55,7 @@ jobs: MAILU_BRANCH: ${{ env.BRANCH }} run: | echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }} >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images From b20d0a83d5b6f369749320fff7ca1289c3160bc9 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Mon, 15 Nov 2021 16:17:31 +0000 Subject: [PATCH 06/25] Doh! --- .github/workflows/CI.yml | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 578ed1a9..dba30722 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -55,7 +55,7 @@ jobs: MAILU_BRANCH: ${{ env.BRANCH }} run: | echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -100,13 +100,15 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: MAILU_BRANCH: ${{ env.BRANCH }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -126,7 +128,7 @@ jobs: run: python tests/compose/test.py core 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ env.DOCKER_ORG }} test-fetchmail: @@ -147,13 +149,15 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: MAILU_BRANCH: ${{ env.BRANCH }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -194,13 +198,15 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: MAILU_BRANCH: ${{ env.BRANCH }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -241,13 +247,15 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: MAILU_BRANCH: ${{ env.BRANCH }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -267,7 +275,7 @@ jobs: run: python tests/compose/test.py rainloop 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ env.DOCKER_ORG }} test-roundcube: @@ -288,13 +296,15 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: MAILU_BRANCH: ${{ env.BRANCH }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -314,7 +324,7 @@ jobs: run: python tests/compose/test.py roundcube 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ env.DOCKER_ORG }} test-webdav: @@ -335,13 +345,15 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: MAILU_BRANCH: ${{ env.BRANCH }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -389,6 +401,7 @@ jobs: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash @@ -396,10 +409,11 @@ jobs: MAILU_BRANCH: ${{ env.BRANCH }} run: | echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images - sudo chmod 777 /images + sudo chmod 777 /images - name: Configure images folder for caching # For staging we do not deploy images. So we do not have to load them from cache. if: ${{ env.BRANCH != 'staging' }} @@ -424,7 +438,7 @@ jobs: name: CI-Done #Returns true when none of the **previous** steps have failed or been canceled. if: ${{ success() }} - needs: + needs: - deploy runs-on: ubuntu-latest steps: From 56dd70cf4a24decd35b2b817242a752c478289a7 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 17 Nov 2021 20:00:04 +0000 Subject: [PATCH 07/25] Implement versioning for CI/CD workflow (see #1182). --- .github/workflows/CI.yml | 332 +++++++++++++++++++++---- RELEASE_TEMPLATE.md | 10 + core/admin/Dockerfile | 4 + core/dovecot/Dockerfile | 5 +- core/nginx/Dockerfile | 4 + core/postfix/Dockerfile | 5 + core/rspamd/Dockerfile | 5 +- docs/Dockerfile | 4 + optional/clamav/Dockerfile | 4 + optional/fetchmail/Dockerfile | 4 + optional/postgresql/Dockerfile | 4 + optional/radicale/Dockerfile | 4 + optional/traefik-certdumper/Dockerfile | 4 + optional/unbound/Dockerfile | 4 + setup/Dockerfile | 4 + tests/build.yml | 109 +++++--- tests/deploy.sh | 46 +++- towncrier/newsfragments/1182.feature | 10 + webmails/rainloop/Dockerfile | 4 + webmails/roundcube/Dockerfile | 5 +- 20 files changed, 482 insertions(+), 89 deletions(-) create mode 100644 RELEASE_TEMPLATE.md create mode 100644 towncrier/newsfragments/1182.feature diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index dba30722..2a0ff932 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -9,10 +9,6 @@ on: - '1.7' - '1.8' - master - # version tags, e.g. 1.7.1 - - '[1-9].[0-9].[0-9]' - # pre-releases, e.g. 1.8-pre1 - - 1.8-pre[0-9] # test branches, e.g. test-debian - test-* @@ -28,6 +24,18 @@ on: # The docker repository for test images. Only used for the branch TESTING (BORS try). # Add the above secrets to your github repo to determine where the images will be pushed. ################################################ +# Code block that is used as one liner. +##!/bin/bash +#version=$( git tag --list "{{ env.MAILU_VERSION }}.*" | tail -1 ) +#root_version=${version:0:3} +#patch_version=${version:4:4} +#if [ "$patch_version" == "" ] +#then +# pinned_version={{ env.MAILU_VERSION }}.0 +#else +# pinned_version=$root_version.$(expr $patch_version + 1) +#fi +#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV jobs: build: @@ -35,27 +43,49 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 - name: Extract branch name shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION for branch testing + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing if: ${{ env.BRANCH == 'testing' }} shell: bash env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: - MAILU_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging + if: ${{ env.BRANCH == 'staging' }} + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for master + if: ${{ env.BRANCH == 'master' }} + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -76,7 +106,7 @@ jobs: - name: Build all docker images env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} DOCKER_ORG: ${{ env.DOCKER_ORG }} run: docker-compose -f tests/build.yml build - name: Save all docker images @@ -89,26 +119,49 @@ jobs: - build steps: - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 - name: Extract branch name shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for branch testing + #For branch TESTING, we set the image tag to PR-xxxx + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing if: ${{ env.BRANCH == 'testing' }} shell: bash env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: - MAILU_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging + if: ${{ env.BRANCH == 'staging' }} + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for master + if: ${{ env.BRANCH == 'master' }} + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -128,7 +181,7 @@ jobs: run: python tests/compose/test.py core 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} DOCKER_ORG: ${{ env.DOCKER_ORG }} test-fetchmail: @@ -138,26 +191,49 @@ jobs: - build steps: - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 - name: Extract branch name shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for branch testing + #For branch TESTING, we set the image tag to PR-xxxx + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing if: ${{ env.BRANCH == 'testing' }} shell: bash env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: - MAILU_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging + if: ${{ env.BRANCH == 'staging' }} + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for master + if: ${{ env.BRANCH == 'master' }} + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -177,7 +253,7 @@ jobs: run: python tests/compose/test.py fetchmail 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} DOCKER_ORG: ${{ env.DOCKER_ORG }} test-filters: @@ -187,26 +263,49 @@ jobs: - build steps: - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 - name: Extract branch name shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for branch testing + #For branch TESTING, we set the image tag to PR-xxxx + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing if: ${{ env.BRANCH == 'testing' }} shell: bash env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: - MAILU_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging + if: ${{ env.BRANCH == 'staging' }} + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for master + if: ${{ env.BRANCH == 'master' }} + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -226,7 +325,7 @@ jobs: run: python tests/compose/test.py filters 3 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} DOCKER_ORG: ${{ env.DOCKER_ORG }} test-rainloop: @@ -236,26 +335,49 @@ jobs: - build steps: - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 - name: Extract branch name shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for branch testing + #For branch TESTING, we set the image tag to PR-xxxx + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing if: ${{ env.BRANCH == 'testing' }} shell: bash env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: - MAILU_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging + if: ${{ env.BRANCH == 'staging' }} + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for master + if: ${{ env.BRANCH == 'master' }} + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -275,7 +397,7 @@ jobs: run: python tests/compose/test.py rainloop 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} DOCKER_ORG: ${{ env.DOCKER_ORG }} test-roundcube: @@ -285,26 +407,49 @@ jobs: - build steps: - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 - name: Extract branch name shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for branch testing + #For branch TESTING, we set the image tag to PR-xxxx + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing if: ${{ env.BRANCH == 'testing' }} shell: bash env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: - MAILU_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging + if: ${{ env.BRANCH == 'staging' }} + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for master + if: ${{ env.BRANCH == 'master' }} + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -324,7 +469,7 @@ jobs: run: python tests/compose/test.py roundcube 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} DOCKER_ORG: ${{ env.DOCKER_ORG }} test-webdav: @@ -334,26 +479,49 @@ jobs: - build steps: - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 - name: Extract branch name shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: Derive MAILU_VERSION for branch testing + #For branch TESTING, we set the image tag to PR-xxxx + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing if: ${{ env.BRANCH == 'testing' }} shell: bash env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: - MAILU_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging + if: ${{ env.BRANCH == 'staging' }} + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for master + if: ${{ env.BRANCH == 'master' }} + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -373,7 +541,7 @@ jobs: run: python tests/compose/test.py webdav 2 env: MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} DOCKER_ORG: ${{ env.DOCKER_ORG }} deploy: @@ -389,27 +557,49 @@ jobs: - test-webdav steps: - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 - name: Extract branch name shell: bash run: | echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV #For branch TESTING, we set the image tag to PR-xxxx - - name: Derive MAILU_VERSION for branch testing + - name: Derive MAILU_VERSION and PINNED_MAILU_VERSION for branch testing if: ${{ env.BRANCH == 'testing' }} shell: bash env: COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} run: | echo "MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG_TESTS }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=pr-${COMMIT_MESSAGE//[!0-9]/}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG_TESTS" >> $GITHUB_ENV - name: Derive MAILU_VERSION for other branches than testing if: ${{ env.BRANCH != 'testing' }} shell: bash env: - MAILU_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: | - echo "MAILU_VERSION=${{ env.MAILU_BRANCH }}" >> $GITHUB_ENV - echo "DOCKER_ORG=${{ secrets.DOCKER_ORG }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for staging + if: ${{ env.BRANCH == 'staging' }} + shell: bash + run: | + echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for master + if: ${{ env.BRANCH == 'master' }} + shell: bash + env: + GITHUB_SHA: ${{ env.GITHUB_SHA }} + run: | + echo "PINNED_MAILU_VERSION=$GITHUB_SHA" >> $GITHUB_ENV - name: Create folder for storing images run: | sudo mkdir -p /images @@ -430,13 +620,49 @@ jobs: DOCKER_PW: ${{ secrets.Docker_Password }} DOCKER_ORG: ${{ env.DOCKER_ORG }} MAILU_VERSION: ${{ env.MAILU_VERSION }} - TRAVIS_BRANCH: ${{ env.BRANCH }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + BRANCH: ${{ env.BRANCH }} run: bash tests/deploy.sh + tag-release: + runs-on: ubuntu-latest + needs: + - deploy + steps: + - uses: actions/checkout@v2 + with: + # fetch-depth 0 is required to also retrieve all tags. + fetch-depth: 0 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: Derive MAILU_VERSION amd DOCKER_ORG + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' }} + shell: bash + env: + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: | + echo "MAILU_VERSION=${{ env.BRANCH }}" >> $GITHUB_ENV + echo "DOCKER_ORG=$DOCKER_ORG" >> $GITHUB_ENV + - name: Derive PINNED_MAILU_VERSION for normal release x.y + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' }} + shell: bash + run: | + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + - name: Tag and create release + if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' && env.PINNED_MAILU_VERSION != '' }} + uses: ncipollo/release-action@v1 + with: + bodyFile: "RELEASE_TEMPLATE.md" + commit: ${{ env.GITHUB_SHA }} + tag: ${{ env.PINNED_MAILU_VERSION }} + token: ${{ secrets.GITHUB_TOKEN }} + # This job is watched by bors. It only complets if building,testing and deploy worked. ci-success: name: CI-Done - #Returns true when none of the **previous** steps have failed or been canceled. + #Returns true when none of the **previous** steps have failed or have been canceled. if: ${{ success() }} needs: - deploy diff --git a/RELEASE_TEMPLATE.md b/RELEASE_TEMPLATE.md new file mode 100644 index 00000000..48178938 --- /dev/null +++ b/RELEASE_TEMPLATE.md @@ -0,0 +1,10 @@ +This is a new automatic release of Mailu. The new version can be seen in the tag name. +The main version X.Y (e.g. 1.8) will always reflect the latest version of the branch. To update your Mailu installation simply pull the latest images `docker-compose pull && docker-compose up -d`. +The pinned version X.Y.Z (e.g. 1.8.1) is not updated. It is pinned to the commit that was used for creating this release. You can use a pinned version to make sure your Mailu installation is not suddenly updated when recreating containers. The pinned version allows the user to manually update. It also allows to go back to a previous pinned version. + +To check what was changed: +- Go to https://github.com/Mailu/Mailu/tree/master/towncrier/newsfragments +- Change the branch to the tag of this release. +- Read the news fragment files to check what was changed. + +The release notes of the original release can be accessed via menu item 'Release notes' on [mailu.io](https://mailu.io/). \ No newline at end of file diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 1958ae61..ac8ae9fc 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -21,10 +21,14 @@ RUN set -eu \ # Actual application FROM $DISTRO +ARG VERSION COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN set -eu \ && apk add --no-cache python3 py3-pip py3-wheel git bash tzdata \ diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 7a2dbdc1..791b6c04 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -11,9 +11,12 @@ RUN git clone https://github.com/grosjo/fts-xapian.git \ && make install FROM $DISTRO - +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip git bash py3-multidict py3-yarl tzdata \ diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index d8899144..e5d30073 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -1,8 +1,12 @@ ARG DISTRO=alpine:3.14.2 FROM $DISTRO +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip git bash py3-multidict \ diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 9f0cc701..5c6238b3 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,8 +1,13 @@ ARG DISTRO=alpine:3.14.2 + FROM $DISTRO +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip git bash py3-multidict py3-yarl tzdata \ diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 33174c1a..d1172ffc 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -1,8 +1,11 @@ ARG DISTRO=alpine:3.14.2 FROM $DISTRO - +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip git bash py3-multidict tzdata \ diff --git a/docs/Dockerfile b/docs/Dockerfile index 289697da..b3d1af01 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -19,7 +19,11 @@ RUN apk add --no-cache --virtual .build-deps \ FROM nginx:1.21-alpine ARG version=master +ARG pinned_version=master ENV VERSION=$version +ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $pinned_version >> /version COPY ./nginx.conf /etc/nginx/conf.d/default.conf COPY --from=build /build/$VERSION /build/$VERSION diff --git a/optional/clamav/Dockerfile b/optional/clamav/Dockerfile index e17d2d70..c68b8888 100644 --- a/optional/clamav/Dockerfile +++ b/optional/clamav/Dockerfile @@ -1,8 +1,12 @@ ARG DISTRO=alpine:3.14.2 FROM $DISTRO +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip bash tzdata \ diff --git a/optional/fetchmail/Dockerfile b/optional/fetchmail/Dockerfile index 068a5dce..3c670bca 100644 --- a/optional/fetchmail/Dockerfile +++ b/optional/fetchmail/Dockerfile @@ -1,8 +1,12 @@ ARG DISTRO=alpine:3.14.2 FROM $DISTRO +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip bash tzdata \ diff --git a/optional/postgresql/Dockerfile b/optional/postgresql/Dockerfile index 3ddbb40a..14d879f3 100644 --- a/optional/postgresql/Dockerfile +++ b/optional/postgresql/Dockerfile @@ -1,8 +1,12 @@ ARG DISTRO=alpine:3.14.2 FROM $DISTRO +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip bash py3-multidict tzdata \ diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index 4eb8d5a7..2eee6915 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -1,8 +1,12 @@ ARG DISTRO=alpine:3.14.2 FROM $DISTRO +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip bash tzdata \ diff --git a/optional/traefik-certdumper/Dockerfile b/optional/traefik-certdumper/Dockerfile index 506c09c2..1aecc48e 100644 --- a/optional/traefik-certdumper/Dockerfile +++ b/optional/traefik-certdumper/Dockerfile @@ -1,7 +1,11 @@ FROM ldez/traefik-certs-dumper +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + RUN apk --no-cache add inotify-tools util-linux bash tzdata COPY run.sh / diff --git a/optional/unbound/Dockerfile b/optional/unbound/Dockerfile index bf5d4840..501d562a 100644 --- a/optional/unbound/Dockerfile +++ b/optional/unbound/Dockerfile @@ -1,8 +1,12 @@ ARG DISTRO=alpine:3.14.2 FROM $DISTRO +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip git bash py3-multidict tzdata \ diff --git a/setup/Dockerfile b/setup/Dockerfile index e0f685ee..0635566b 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -1,5 +1,9 @@ ARG DISTRO=alpine:3.14.2 FROM $DISTRO +ARG VERSION +ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version RUN mkdir -p /app WORKDIR /app diff --git a/tests/build.yml b/tests/build.yml index eda5550c..a61277ba 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -3,69 +3,114 @@ version: '3' services: front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-local} - build: ../core/nginx + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} + build: + context: ../core/nginx + args: + VERSION: ${PINNED_MAILU_VERSION:-local} resolver: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}unbound:${MAILU_VERSION:-local} - build: ../optional/unbound + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}unbound:${PINNED_MAILU_VERSION:-local} + build: + context: ../optional/unbound + args: + VERSION: ${PINNED_MAILU_VERSION:-local} imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-local} - build: ../core/dovecot + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} + build: + context: ../core/dovecot + args: + VERSION: ${PINNED_MAILU_VERSION:-local} smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-local} - build: ../core/postfix + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} + build: + context: ../core/postfix + args: + VERSION: ${PINNED_MAILU_VERSION:-local} antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-local} - build: ../core/rspamd + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} + build: + context: ../core/rspamd + args: + VERSION: ${PINNED_MAILU_VERSION:-local} antivirus: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}clamav:${MAILU_VERSION:-local} - build: ../optional/clamav + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}clamav:${PINNED_MAILU_VERSION:-local} + build: + context: ../optional/clamav + args: + VERSION: ${PINNED_MAILU_VERSION:-local} webdav: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}radicale:${MAILU_VERSION:-local} - build: ../optional/radicale + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}radicale:${PINNED_MAILU_VERSION:-local} + build: + context: ../optional/radicale + args: + VERSION: ${PINNED_MAILU_VERSION:-local} traefik-certdumper: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}traefik-certdumper:${MAILU_VERSION:-local} - build: ../optional/traefik-certdumper + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}traefik-certdumper:${PINNED_MAILU_VERSION:-local} + build: + context: ../optional/traefik-certdumper + args: + VERSION: ${PINNED_MAILU_VERSION:-local} admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-local} - build: ../core/admin + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} + build: + context: ../core/admin + args: + VERSION: ${PINNED_MAILU_VERSION:-local} postgresql: - image: ${DOCKER_ORG:-mailu}/postgresql:${MAILU_VERSION:-local} - build: ../optional/postgresql + image: ${DOCKER_ORG:-mailu}/postgresql:${PINNED_MAILU_VERSION:-local} + build: + context: ../optional/postgresql + args: + VERSION: ${PINNED_MAILU_VERSION:-local} roundcube: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}roundcube:${MAILU_VERSION:-local} - build: ../webmails/roundcube + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}roundcube:${PINNED_MAILU_VERSION:-local} + build: + context: ../webmails/roundcube + args: + VERSION: ${PINNED_MAILU_VERSION:-local} rainloop: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rainloop:${MAILU_VERSION:-local} - build: ../webmails/rainloop + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rainloop:${PINNED_MAILU_VERSION:-local} + build: + context: ../webmails/rainloop + args: + VERSION: ${PINNED_MAILU_VERSION:-local} fetchmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${MAILU_VERSION:-local} - build: ../optional/fetchmail + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${PINNED_MAILU_VERSION:-local} + build: + context: ../optional/fetchmail + args: + VERSION: ${PINNED_MAILU_VERSION:-local} none: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}none:${MAILU_VERSION:-local} - build: ../core/none + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}none:${PINNED_MAILU_VERSION:-local} + build: + context: ../core/none + args: + VERSION: ${PINNED_MAILU_VERSION:-local} docs: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}docs:${MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}docs:${PINNED_MAILU_VERSION:-local} build: context: ../docs args: version: ${MAILU_VERSION:-local} + pinned_version: ${PINNED_MAILU_VERSION:-local} setup: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}setup:${MAILU_VERSION:-local} - build: ../setup - + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}setup:${PINNED_MAILU_VERSION:-local} + build: + context: ../setup + args: + VERSION: ${PINNED_MAILU_VERSION:-local} diff --git a/tests/deploy.sh b/tests/deploy.sh index abb37b6b..6913e404 100755 --- a/tests/deploy.sh +++ b/tests/deploy.sh @@ -1,7 +1,51 @@ #!/bin/bash # Skip deploy for staging branch -[ "$TRAVIS_BRANCH" = "staging" ] && exit 0 +[ "$BRANCH" = "staging" ] && exit 0 docker login -u $DOCKER_UN -p $DOCKER_PW + +if [ "$BRANCH" = "testing" ] +then + docker-compose -f tests/build.yml push + exit 0 +fi + +#Deploy for main releases +#Images are built with tag PINNED_MAILU_VERSION (x.y.z). +#We are tagging them as well with MAILU_VERSION (x.y) +#After that, both tags are pushed to the docker repository. +if [ "$PINNED_MAILU_VERSION" != "" ] && [ "$BRANCH" != "master" ] +then + images=$(docker-compose -f tests/build.yml config | grep 'image: ' | awk -F ':' '{ print $2 }') + for image in $images + do + docker tag "${image}":"${PINNED_MAILU_VERSION}" "${image}":${MAILU_VERSION} + done +#Push PINNED_MAILU_VERSION images + docker-compose -f tests/build.yml push +#Push MAILU_VERSION images + PINNED_MAILU_VERSION=$MAILU_VERSION + docker-compose -f tests/build.yml push + exit 0 +fi + +#Deploy for master. For master we only publish images with tag master +#Images are built with tag PINNED_MAILU_VERSION (commit hash). +#We are tagging them as well with MAILU_VERSION (master) +#Then we publish the images with tag master +if [ "$PINNED_MAILU_VERSION" != "" ] && [ "$BRANCH" == "master" ] +then + images=$(docker-compose -f tests/build.yml config | grep 'image: ' | awk -F ':' '{ print $2 }') + for image in $images + do + docker tag "${image}":"${PINNED_MAILU_VERSION}" "${image}":${MAILU_VERSION} + done +#Push MAILU_VERSION images + PINNED_MAILU_VERSION=$MAILU_VERSION + docker-compose -f tests/build.yml push + exit 0 +fi + +#Fallback in case $PINNED_MAILU_VERSION is empty. This should never execute. docker-compose -f tests/build.yml push diff --git a/towncrier/newsfragments/1182.feature b/towncrier/newsfragments/1182.feature new file mode 100644 index 00000000..51ff61cb --- /dev/null +++ b/towncrier/newsfragments/1182.feature @@ -0,0 +1,10 @@ +Use semantic versioning for building releases. +- Add versioning (tagging) for branch x.y (1.8). E.g. 1.8.0, 1.8.1 etc. + - docker repo will contain x.y (latest) and x.y.z (pinned version) images. + - The X.Y.Z tag is incremented automatically. E.g. if 1.8.0 already exists, then the next merge on 1.8 will result in the new tag 1.8.1 being used. +- Make the version available in the image. + - For X.Y and X.Y.Z write the version (X.Y.Z) into /version on the image and add a label with version=X.Y.Z + - This means that the latest X.Y image shows the pinned version (X.Y.Z e.g. 1.8.1) it was based on. Via the tag X.Y.Z you can see the commit hash that triggered the built. + - For master write the commit hash into /version on the image and add a label with version={commit hash} +- Automatic releases. For x.y triggered builts (e.g. merge on 1.9) do a new github release for the pinned x.y.z (e.g. 1.9.2). + - Release shows a static message (see RELEASE_TEMPLATE.md) that explains how to reach the newsfragments folder and change the branch to the tag (x.y.z) mentioned in the release. Now you can get the changelog by reading all newsfragment files in this folder. \ No newline at end of file diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile index 02910e39..8039f69a 100644 --- a/webmails/rainloop/Dockerfile +++ b/webmails/rainloop/Dockerfile @@ -2,10 +2,14 @@ ARG ARCH="" # NOTE: only add file if building for arm FROM ${ARCH}alpine:3.14 +ARG VERSION ONBUILD COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/qemu-arm-static ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + # Shared later between dovecot postfix nginx rspamd rainloop and roundloop RUN apk add --no-cache \ python3 py3-pip tzdata \ diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index 4ad89022..5c411f84 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -7,9 +7,12 @@ ONBUILD COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/ FROM ${ARCH}php:7.4-apache as build_other FROM build_${QEMU} - +ARG VERSION ENV TZ Etc/UTC +LABEL version=$VERSION +RUN echo $VERSION >> /version + #Shared layer between rainloop and roundcube RUN apt-get update && apt-get install -y \ python3 curl python3-pip git python3-multidict tzdata \ From f7677543c69ebe7389a2f7e6e593427562eb45f3 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 18 Nov 2021 17:21:56 +0000 Subject: [PATCH 08/25] Process code review remarks - Moved run to bottom of Dockerfile to allow using unmodified / cached states. - Simplified bash code in deploy.sh. - Improved the large bash one-liner in CI.yml. It could not handle >9 for 1.x. --- .github/workflows/CI.yml | 22 +++++++++++----------- core/admin/Dockerfile | 2 +- core/dovecot/Dockerfile | 2 +- core/nginx/Dockerfile | 2 +- core/postfix/Dockerfile | 2 +- core/rspamd/Dockerfile | 2 +- docs/Dockerfile | 2 +- optional/clamav/Dockerfile | 2 +- optional/fetchmail/Dockerfile | 4 ++-- optional/postgresql/Dockerfile | 2 +- optional/radicale/Dockerfile | 2 +- optional/traefik-certdumper/Dockerfile | 2 +- optional/unbound/Dockerfile | 2 +- setup/Dockerfile | 3 ++- tests/deploy.sh | 4 ++-- webmails/rainloop/Dockerfile | 2 +- webmails/roundcube/Dockerfile | 2 +- 17 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2a0ff932..8cc174f7 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -27,8 +27,8 @@ on: # Code block that is used as one liner. ##!/bin/bash #version=$( git tag --list "{{ env.MAILU_VERSION }}.*" | tail -1 ) -#root_version=${version:0:3} -#patch_version=${version:4:4} +#root_version=${version%.*} +#patch_version=${version##*.} #if [ "$patch_version" == "" ] #then # pinned_version={{ env.MAILU_VERSION }}.0 @@ -73,7 +73,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION for staging if: ${{ env.BRANCH == 'staging' }} shell: bash @@ -149,7 +149,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION for staging if: ${{ env.BRANCH == 'staging' }} shell: bash @@ -221,7 +221,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION for staging if: ${{ env.BRANCH == 'staging' }} shell: bash @@ -293,7 +293,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION for staging if: ${{ env.BRANCH == 'staging' }} shell: bash @@ -365,7 +365,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION for staging if: ${{ env.BRANCH == 'staging' }} shell: bash @@ -437,7 +437,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION for staging if: ${{ env.BRANCH == 'staging' }} shell: bash @@ -509,7 +509,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION for staging if: ${{ env.BRANCH == 'staging' }} shell: bash @@ -587,7 +587,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Derive PINNED_MAILU_VERSION for staging if: ${{ env.BRANCH == 'staging' }} shell: bash @@ -649,7 +649,7 @@ jobs: if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' }} shell: bash run: | - version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version:0:3};patch_version=${version:4:4};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV + version=$( git tag --list "${{ env.MAILU_VERSION }}.*" | tail -1 );root_version=${version%.*};patch_version=${version##*.};if [ "$patch_version" == "" ]; then pinned_version=${{ env.MAILU_VERSION }}.0; else pinned_version=$root_version.$(expr $patch_version + 1); fi;echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV - name: Tag and create release if: ${{ env.BRANCH != 'testing' && env.BRANCH != 'staging' && env.BRANCH != 'master' && env.PINNED_MAILU_VERSION != '' }} uses: ncipollo/release-action@v1 diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index ac8ae9fc..fde7c877 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -27,7 +27,6 @@ COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/qemu-arm ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN set -eu \ @@ -60,3 +59,4 @@ ENV FLASK_APP mailu CMD /start.py HEALTHCHECK CMD curl -f -L http://localhost/sso/login?next=ui.index || exit 1 +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 791b6c04..16f35490 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -15,7 +15,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -44,3 +43,4 @@ VOLUME ["/mail"] CMD /start.py HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready." +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index e5d30073..744de810 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -5,7 +5,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -32,3 +31,4 @@ VOLUME ["/overrides"] CMD /start.py HEALTHCHECK CMD curl -k -f -L http://localhost/health || exit 1 +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 5c6238b3..deedd9b8 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -6,7 +6,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -35,3 +34,4 @@ VOLUME ["/queue"] CMD /start.py HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index d1172ffc..59e6fefa 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -4,7 +4,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -29,3 +28,4 @@ VOLUME ["/var/lib/rspamd"] CMD /start.py HEALTHCHECK --start-period=350s CMD curl -f -L http://localhost:11334/ || exit 1 +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/docs/Dockerfile b/docs/Dockerfile index b3d1af01..18ca8333 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -23,7 +23,6 @@ ARG pinned_version=master ENV VERSION=$version ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $pinned_version >> /version COPY ./nginx.conf /etc/nginx/conf.d/default.conf COPY --from=build /build/$VERSION /build/$VERSION @@ -31,3 +30,4 @@ COPY --from=build /build/$VERSION /build/$VERSION EXPOSE 80/tcp CMD nginx -g "daemon off;" +RUN echo $pinned_version >> /version \ No newline at end of file diff --git a/optional/clamav/Dockerfile b/optional/clamav/Dockerfile index c68b8888..97a47d30 100644 --- a/optional/clamav/Dockerfile +++ b/optional/clamav/Dockerfile @@ -5,7 +5,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -24,3 +23,4 @@ VOLUME ["/data"] CMD /start.py HEALTHCHECK --start-period=350s CMD /health.sh +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/fetchmail/Dockerfile b/optional/fetchmail/Dockerfile index 3c670bca..81ccc2d1 100644 --- a/optional/fetchmail/Dockerfile +++ b/optional/fetchmail/Dockerfile @@ -5,7 +5,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -20,4 +19,5 @@ RUN mkdir -p /data COPY fetchmail.py /fetchmail.py -CMD ["/fetchmail.py"] \ No newline at end of file +CMD ["/fetchmail.py"] +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/postgresql/Dockerfile b/optional/postgresql/Dockerfile index 14d879f3..e66096ec 100644 --- a/optional/postgresql/Dockerfile +++ b/optional/postgresql/Dockerfile @@ -5,7 +5,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -41,3 +40,4 @@ EXPOSE 5432 CMD /start.py HEALTHCHECK CMD psql -h 127.0.0.1 -d postgres -U health -c "select 1 as ok;" || exit 1 +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index 2eee6915..93796eb1 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -5,7 +5,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -25,3 +24,4 @@ VOLUME ["/data"] CMD radicale -S -C /radicale.conf HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/traefik-certdumper/Dockerfile b/optional/traefik-certdumper/Dockerfile index 1aecc48e..829655f0 100644 --- a/optional/traefik-certdumper/Dockerfile +++ b/optional/traefik-certdumper/Dockerfile @@ -4,7 +4,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version RUN apk --no-cache add inotify-tools util-linux bash tzdata @@ -14,3 +13,4 @@ VOLUME ["/traefik"] VOLUME ["/output"] ENTRYPOINT ["/run.sh"] +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/unbound/Dockerfile b/optional/unbound/Dockerfile index 501d562a..3a86cc6c 100644 --- a/optional/unbound/Dockerfile +++ b/optional/unbound/Dockerfile @@ -5,7 +5,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # python3 shared with most images RUN apk add --no-cache \ @@ -31,3 +30,4 @@ EXPOSE 53/udp 53/tcp CMD /start.py HEALTHCHECK CMD dig @127.0.0.1 || exit 1 +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/setup/Dockerfile b/setup/Dockerfile index 0635566b..baa35da9 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -3,7 +3,7 @@ FROM $DISTRO ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version + RUN mkdir -p /app WORKDIR /app @@ -21,3 +21,4 @@ COPY static ./static EXPOSE 80/tcp CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/tests/deploy.sh b/tests/deploy.sh index 6913e404..e66310af 100755 --- a/tests/deploy.sh +++ b/tests/deploy.sh @@ -17,7 +17,7 @@ fi #After that, both tags are pushed to the docker repository. if [ "$PINNED_MAILU_VERSION" != "" ] && [ "$BRANCH" != "master" ] then - images=$(docker-compose -f tests/build.yml config | grep 'image: ' | awk -F ':' '{ print $2 }') + images=$(docker-compose -f Mailu/tests/build.yml config | awk -F ':' '/image:/{ print $2 }') for image in $images do docker tag "${image}":"${PINNED_MAILU_VERSION}" "${image}":${MAILU_VERSION} @@ -36,7 +36,7 @@ fi #Then we publish the images with tag master if [ "$PINNED_MAILU_VERSION" != "" ] && [ "$BRANCH" == "master" ] then - images=$(docker-compose -f tests/build.yml config | grep 'image: ' | awk -F ':' '{ print $2 }') + images=$(docker-compose -f Mailu/tests/build.yml config | awk -F ':' '/image:/{ print $2 }') for image in $images do docker tag "${image}":"${PINNED_MAILU_VERSION}" "${image}":${MAILU_VERSION} diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile index 8039f69a..0938056c 100644 --- a/webmails/rainloop/Dockerfile +++ b/webmails/rainloop/Dockerfile @@ -8,7 +8,6 @@ ONBUILD COPY --from=balenalib/rpi-alpine:3.14 /usr/bin/qemu-arm-static /usr/bin/ ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version # Shared later between dovecot postfix nginx rspamd rainloop and roundloop RUN apk add --no-cache \ @@ -75,3 +74,4 @@ VOLUME ["/data"] CMD php-fpm7 && /start.py HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1 +RUN echo $VERSION >> /version \ No newline at end of file diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index 5c411f84..f28f316c 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -11,7 +11,6 @@ ARG VERSION ENV TZ Etc/UTC LABEL version=$VERSION -RUN echo $VERSION >> /version #Shared layer between rainloop and roundcube RUN apt-get update && apt-get install -y \ @@ -62,3 +61,4 @@ VOLUME ["/data"] CMD /start.py HEALTHCHECK CMD curl -f -L -H 'User-Agent: health' http://localhost/ || exit 1 +RUN echo $VERSION >> /version \ No newline at end of file From f247520fe5d0a33afd42dccb8b1b8d20c43c3f8c Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 19 Nov 2021 07:35:08 +0000 Subject: [PATCH 09/25] Forgot to update tests to use PINNED_MAILU_VERSION as tag. --- tests/compose/core/docker-compose.yml | 10 +++++----- tests/compose/fetchmail/docker-compose.yml | 12 ++++++------ tests/compose/filters/docker-compose.yml | 12 ++++++------ tests/compose/rainloop/docker-compose.yml | 12 ++++++------ tests/compose/roundcube/docker-compose.yml | 12 ++++++------ tests/compose/webdav/docker-compose.yml | 12 ++++++------ 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/tests/compose/core/docker-compose.yml b/tests/compose/core/docker-compose.yml index f64c82b0..195d8a59 100644 --- a/tests/compose/core/docker-compose.yml +++ b/tests/compose/core/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -44,7 +44,7 @@ services: - redis imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -54,7 +54,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -63,7 +63,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/compose/fetchmail/docker-compose.yml b/tests/compose/fetchmail/docker-compose.yml index 49c292e9..d31479d4 100644 --- a/tests/compose/fetchmail/docker-compose.yml +++ b/tests/compose/fetchmail/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -44,7 +44,7 @@ services: - redis imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -54,7 +54,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -63,7 +63,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -77,7 +77,7 @@ services: fetchmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env diff --git a/tests/compose/filters/docker-compose.yml b/tests/compose/filters/docker-compose.yml index 94b97399..381d3683 100644 --- a/tests/compose/filters/docker-compose.yml +++ b/tests/compose/filters/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -44,7 +44,7 @@ services: - redis imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -54,7 +54,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -63,7 +63,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -75,7 +75,7 @@ services: # Optional services antivirus: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}clamav:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}clamav:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/compose/rainloop/docker-compose.yml b/tests/compose/rainloop/docker-compose.yml index 98425d7a..62d5890f 100644 --- a/tests/compose/rainloop/docker-compose.yml +++ b/tests/compose/rainloop/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -44,7 +44,7 @@ services: - redis imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -54,7 +54,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -63,7 +63,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -79,7 +79,7 @@ services: # Webmail webmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rainloop:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rainloop:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/compose/roundcube/docker-compose.yml b/tests/compose/roundcube/docker-compose.yml index b2f415df..0bb54e8c 100644 --- a/tests/compose/roundcube/docker-compose.yml +++ b/tests/compose/roundcube/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -44,7 +44,7 @@ services: - redis imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -54,7 +54,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -63,7 +63,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -79,7 +79,7 @@ services: # Webmail webmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}roundcube:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}roundcube:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: diff --git a/tests/compose/webdav/docker-compose.yml b/tests/compose/webdav/docker-compose.yml index adc094a8..a597b2d2 100644 --- a/tests/compose/webdav/docker-compose.yml +++ b/tests/compose/webdav/docker-compose.yml @@ -15,7 +15,7 @@ services: # Core services front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env logging: @@ -34,7 +34,7 @@ services: - "/mailu/certs:/certs" admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -44,7 +44,7 @@ services: - redis imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -54,7 +54,7 @@ services: - front smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -63,7 +63,7 @@ services: - front antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: @@ -76,7 +76,7 @@ services: # Optional services webdav: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}radicale:${MAILU_VERSION:-master} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}radicale:${PINNED_MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: From 602accfba7d8657e76ad3f8d49199702542501f7 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 23 Nov 2021 10:17:51 +0100 Subject: [PATCH 10/25] fixed ipv6 access-control --- optional/unbound/unbound.conf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/optional/unbound/unbound.conf b/optional/unbound/unbound.conf index df0c76ff..3cb6b3f4 100644 --- a/optional/unbound/unbound.conf +++ b/optional/unbound/unbound.conf @@ -1,7 +1,7 @@ server: verbosity: 1 interface: 0.0.0.0 - {{ 'interface: ::0' if SUBNET6 }} +{{- ' interface: ::0' if SUBNET6 }} logfile: "" do-ip4: yes do-ip6: {{ 'yes' if SUBNET6 else 'no' }} @@ -9,7 +9,9 @@ server: do-tcp: yes do-daemonize: no access-control: {{ SUBNET }} allow - {{ 'access-control: {{ SUBNET6 }} allow' if SUBNET6 }} +{%- if SUBNET6 %} + access-control: {{ SUBNET6 }} allow +{%- endif %} directory: "/etc/unbound" username: unbound auto-trust-anchor-file: trusted-key.key From 5c52f08f41ccdf61d08d26c485b5e8bf5c686de2 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 23 Nov 2021 16:13:31 +0000 Subject: [PATCH 11/25] Added documentation for how to switch the database back-end used by Mailu. Added documentation for migrating from the deprecated Mailu PostgreSQL image to a different PostgreSQL database. --- docs/cli.rst | 2 + docs/configuration.rst | 6 +- docs/database.rst | 243 +++++++++++++++++++++++-------- towncrier/newsfragments/1037.doc | 2 + 4 files changed, 192 insertions(+), 61 deletions(-) create mode 100644 towncrier/newsfragments/1037.doc diff --git a/docs/cli.rst b/docs/cli.rst index 957e47e4..b407c6a3 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -121,6 +121,8 @@ additional fields: * wildcard +.. _config-export: + config-export ------------- diff --git a/docs/configuration.rst b/docs/configuration.rst index 8f363a63..f7bece71 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -124,12 +124,12 @@ Full-text search is enabled for IMAP is enabled by default. This feature can be Web settings ------------ -- ``WEB_ADMIN`` contains the path to the main admin interface +- ``WEB_ADMIN`` contains the path to the main admin interface - ``WEB_WEBMAIL`` contains the path to the Web email client. - ``WEBROOT_REDIRECT`` redirects all non-found queries to the set path. - An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables classic behavior of a 404 result when not found. + An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables classic behavior of a 404 result when not found. Alternatively, ``WEBROOT_REDIRECT`` can be set to ``none`` if you are using an Nginx override for ``location /``. All three options need a leading slash (``/``) to work. @@ -239,6 +239,8 @@ resolved. This can be used to rely on DNS based service discovery with changing When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to resolve the hostnames. +.. _db_settings: + Database settings ----------------- diff --git a/docs/database.rst b/docs/database.rst index 0f8318d5..0af26f4b 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -1,47 +1,84 @@ Changing the database back-end ============================== -By default Mailu uses a SQLite database. Recently, we have changed the internals of Mailu -to enable the support of alternative database solutions as postgresql and mysql/mariadb. -This functionality should still be considered experimental! +By default Mailu uses a SQLite database. We have changed the internals of Mailu +to enable the support of alternative database solutions as PostgreSQL and MySQL/MariaDB. -Mailu Postgresql ----------------- -Mailu optionally comes with a pre-configured Postgresql image, which as of 1.8, is deprecated -and will be removed in 1.9. -This images has the following features: +Migrating to a different database back-end +------------------------------------------ -- Automatic creation of users, db, extensions and password; -- TCP connections are only allowed from the mailu `SUBNET`; -- Automatic minutely *wal archiving* and weekly `pg_basebackup`; -- Automatic cleaning of *wal archives* and *base backups*; - Two versions always remain available; -- When `/data` is empty and backups are present, the backups are restored automatically; - Useful in swarm environments, since the /data directory should not be on any network - filesystem (performance). +From Mailu 1.9, Mailu has a :ref:`cli command (link) ` for exporting and importing the complete Mailu configuration. +Using this tool it is very easy to switch what database back-end is used for Mailu. +Unfortunately roundcube does not have a tool for exporting/importing roundcube configuration. +This means it is not possible to switch the database back-end used by roundcube using out of box tools. -To make use of this functionality, just select `postgresql` as database flavor. -Don't select the usage of an external database. The ``docker-compose.yml`` and ``mailu.env`` -will pull in ``mailu/postgresql``. This image and ``mailu/admin`` contain all the scripts -to automatically setup the database. +To switch to a different database back-end: -After bring up the service, it might be useful to check the logs with: +1. Run config-export to export the configuration. E.g. `docker-compose exec admin flask mailu config-export --secrets --output mail-config.yml` +2. Set up your new database server. Refer to the subsequent sections for tips for creating the database. +3. Modify the database settings (DB_*) in mailu.env. Refer to the :ref:`configuration guide (link) ` for the exact settings. +4. Start your Mailu deployment. +5. Run config-import to import the configuration. E.g. `docker exec -i $(docker-compose ps -q admin) flask mailu config-import -v < mail-config.yml` -.. code-block:: bash +Mailu has now been switched to the new database back-end. The Mailu configuration has also been migrated. - docker-compose logs -f admin database +.. note:: + The setup configuration wizard (setup.mailu.io) only supports creating config files for the same database back-end. When creating new config files, select the desired database flavour in the setup and enter dummy values for roundcube. + In the generated mailu.env file, configure all ROUNDCUBE_DB_* environment variables to the old values. For SQLite you can remove all the ROUNDCUBE_DB_* values. -External Postgresql + +External MySQL/MariaDB +---------------------- + +It is also possible to use a MySQL/MariaDB database server, hosted elsewhere. +In this case you'll have to take to create an empty database for Mailu, corresponding user, +password and sufficient privileges on the database to ``CREATE TABLE``, ``DROP`` etc. +Usually making the user owner of the database would be the easiest thing to do. + +The following commands can serve as an example on how to set up MySQL/MariaDB for Mailu usage. +Adjust this to your own liking. + +.. code-block:: sql + + mysql> CREATE DATABASE mailu; + mysql> CREATE USER 'mailu'@'%' IDENTIFIED BY 'my-strong-password-here'; + mysql> GRANT ALL PRIVILEGES ON mailu.* TO 'mailu'@'%'; + mysql> FLUSH PRIVILEGES; + +Note that if you get any errors related to ``caching_sha2_password`` it can be solved by changing the encryption +of the password to ``mysql_native_password`` instead of the latest authentication plugin ``caching_sha2_password``. + +.. code-block:: sql + + mysql> SELECT host, user, plugin FROM mysql.user; + + +-----------+-------+-----------------------+ + | host | user | plugin | + +-----------+-------+-----------------------+ + | % | mailu | caching_sha2_password | + +-----------+-------+-----------------------+ + + mysql> update mysql.user set plugin = 'mysql_native_password' where user = 'mailu'; + mysql> SELECT host, user, plugin FROM mysql.user; + + +------+-------+-----------------------+ + | host | user | plugin | + +------+-------+-----------------------+ + | % | mailu | mysql_native_password | + +------+-------+-----------------------+ + + +External PostgreSQL ------------------- -It is also possible to use a Postgresql database server, hosted elsewhere. +It is also possible to use a PostgreSQL database server, hosted elsewhere. In this case you'll have to take to create an empty database for Mailu, corresponding user, password and sufficient privileges on the database to ``CREATE TABLE``, ``DROP`` etc. Usually making the user owner of the database would be the easiest thing to do. Don't forget to set ``pg_hba.conf`` accordingly. -The following commands can serve as an example on how to set up postgresql for Mailu usage. +The following commands can serve as an example on how to set up PostgreSQL for Mailu usage. Adjust this to your own liking. .. code-block:: bash @@ -72,42 +109,130 @@ In ``pg_hba.conf`` there should be a line like this: Note that this example is the bare-minimum to get Mailu working. It goes without saying that the database admin will have to setup his own means of backups and TLS encrypted connections. -External MySQL/Mariadb ----------------------- +Nowadays it is recommended to use the official PostgreSQL image from the PostgreSQL community. The repository is located `here `_. -It is also possible to use a mysql/mariadb database server, hosted elsewhere. -In this case you'll have to take to create an empty database for Mailu, corresponding user, -password and sufficient privileges on the database to ``CREATE TABLE``, ``DROP`` etc. -Usually making the user owner of the database would be the easiest thing to do. +.. _migrate_mailu_postgresql: -The following commands can serve as an example on how to set up mysql/mariadb for Mailu usage. -Adjust this to your own liking. +Mailu PostgreSQL +---------------- -.. code-block:: sql +Mailu optionally came with a pre-configured PostgreSQL image which was deprecated in Mailu 1.8. +Since Mailu 1.9 it is removed from Mailu. The following section describes how to move to a different PostgreSQL image for novice administrators. The official PostgreSQL image (Postgres) will be used. - mysql> CREATE DATABASE mailu; - mysql> CREATE USER 'mailu'@'%' IDENTIFIED BY 'my-strong-password-here'; - mysql> GRANT ALL PRIVILEGES ON mailu.* TO 'mailu'@'%'; - mysql> FLUSH PRIVILEGES; - -Note that if you get any errors related to ``caching_sha2_password`` it can be solved by changing the encryption -of the password to ``mysql_native_password`` instead of the latest authentication plugin ``caching_sha2_password``. +A Mailu deployment with the Mailu PostgreSQL image, only used PostgreSQL for the Admin container (Web administration interface). Roundcube used SQLite as database back-end. +Mailu uses the following configuration for connecting to the database: -.. code-block:: sql +- Database host: 'database' +- Database name: 'mailu' +- Database user: 'mailu' +- Database password: See DB_PW in mailu.env. - mysql> SELECT host, user, plugin FROM mysql.user; - - +-----------+-------+-----------------------+ - | host | user | plugin | - +-----------+-------+-----------------------+ - | % | mailu | caching_sha2_password | - +-----------+-------+-----------------------+ - - mysql> update mysql.user set plugin = 'mysql_native_password' where user = 'mailu'; - mysql> SELECT host, user, plugin FROM mysql.user; - - +------+-------+-----------------------+ - | host | user | plugin | - +------+-------+-----------------------+ - | % | mailu | mysql_native_password | - +------+-------+-----------------------+ +.. note:: + + The following instructions assume that + - project mailu is used. (-p mailu). If a different project (prefix) is used, then a different project can be specified. + - the data folder is /mailu. Change this to a different value in case Mailu makes use of a different data folder. + - All commands must be executed as root. On Debian/Ubuntu the sudo command is used to execute commands as root. + +Prepare the environment. Mailu must not be in use. Only the database container. + +1. Open a terminal. +2. `cd /mailu` +3. `docker-compose -p mailu down` +4. `docker-compose -p mailu up -d database` + +Create the dump SQL file for recreating the database. + +1. `docker-compose -p mailu exec database /bin/bash` +2. `pg_dump -h database -p 5432 -U mailu > /backup/backup_db.sql` +3. Enter the password. See the value of DB_PW in mailu.env. +4. `exit` +5. The dump is saved to /mailu/data/psql_backup/backup_db.sql. +6. `docker-compose -p mailu down` + +Prepare the new PostgreSQL deployment. + +1. `mkdir -p /mailu/data/external_psql/pgdata` +2. Create the file docker-compose-postgresql.yml with the following contents: + +.. code-block:: docker + + version: '3.1' + services: + database: + image: postgres:13 + restart: always + environment: + - POSTGRES_USER=mailu + - POSTGRES_PASSWORD=DB_PW from mailu.env file + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - "/mailu/data/external_psql/pgdata:/var/lib/postgresql/data/pgdata" + - "/mailu/data/psql_backup:/dump" + + +3. `docker-compose -f docker-compose-postgresql.yml up -d` +4. `docker-compose -f docker-compose-postgresql.yml exec database /bin/bash` +5. `cat /dump/backup_db.sql | psql -h localhost -p 5432 -U mailu` +6. `exit` +7. `docker-compose -f docker-compose-postgresql.yml down` +8. Remove the file docker-compose-postgresql.yml. + +The new PostgreSQL deployment has the dump loaded now. Now it is time to modify Mailu to use the official PostgreSQL docker image. + +1. Edit docker-compose.yml and change: + +.. code-block:: docker + + database: + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postgresql:${MAILU_VERSION:-master} + restart: always + env_file: mailu.env + volumes: + - "/mailu_db/data/psql_db:/data" + - "/mailu_db/data/psql_backup:/backup" + +to + +.. code-block:: docker + + database: + image: postgres:13 + restart: always + environment: + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - "/mailu/data/external_psql/pgdata:/var/lib/postgresql/data/pgdata" + + +2. Edit mailu.env and append the following after the block + +.. code-block:: docker + + ################################### + # Database settings + ################################### + + +.. code-block:: docker + + DB_HOST=database + DB_PORT=5432 + DB_USER=mailu + DB_NAME=mailu + +Mailu is now configured to use the official PostgreSQL docker image. Bring your new deployment online + +1. `docker-compose -p mailu up -d` + +Optionally you can remove left-over files which were used by the old database: + +- /mailu/data/psql_backup (old database backup files +- /mailu/data/psql_db (old database files) + +.. note:: + The setup configuration wizard (setup.mailu.io) only supports creating config files for the same database back-end. When creating new config files, select PostgreSQL in the setup and enter dummy values for roundcube. + In the generated mailu.env file, remove all ROUNDCUBE_DB_* environment variables. + Now Admin will use PostgreSQL and roundcube will keep using Roundcube. + + Roundcube does not offer a migration tool for moving from SQLite to PostgreSQL. diff --git a/towncrier/newsfragments/1037.doc b/towncrier/newsfragments/1037.doc new file mode 100644 index 00000000..532ceaa6 --- /dev/null +++ b/towncrier/newsfragments/1037.doc @@ -0,0 +1,2 @@ +Added documentation for how to switch the database back-end used by Mailu. +Added documentation for migrating from the deprecated Mailu PostgreSQL image to a different PostgreSQL database. \ No newline at end of file From 6cb8f101d96eb31d162ebb683e0517b643b32a79 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 23 Nov 2021 18:20:35 +0000 Subject: [PATCH 12/25] Update stale bot with clearer message why an issue is marked stale. --- .github/stale.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/stale.yml b/.github/stale.yml index 2f644e76..20e1b88d 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -24,7 +24,7 @@ exemptLabels: - type/security # Set to true to ignore issues in a project (defaults to false) -exemptProjects: false +exemptProjects: true # Set to true to ignore issues in a milestone (defaults to false) exemptMilestones: true @@ -36,10 +36,14 @@ exemptAssignees: true staleLabel: status/response_needed # Comment to post when marking as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. +markComment: | + Issues not for bugs, enhancement requests or discussion go stale after 21 days of inactivity. This issue will be automatically closed after 14 days. + For all metrics refer to the [stale.yml file](https://github.com/Mailu/Mailu/blob/master/.github/stale.yml). + Github issues are not meant for user support. For **user-support questions**, reach out on the [matrix support channel](https://matrix.to/#/#mailu:tedomum.net). + + Mark the issue as fresh by simply adding a comment to the issue. + If this issue is safe to close, please do so now. + # Comment to post when removing the stale label. # unmarkComment: > @@ -47,9 +51,8 @@ markComment: > # Comment to post when closing a stale Issue or Pull Request. closeComment: > - This issue has not seen activity since as it has become stale. It will now be - automatically closed. Please note that this is an automatic action, and not - meant in any offensive way. + This issue has not seen activity since as it has become stale. + Stale issues are automatically closed after 14 days. # Limit the number of actions per hour, from 1-30. Default is 30 limitPerRun: 30 From 3afaeecfbb47a5fb00918a02f14f5f635544c033 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 23 Nov 2021 21:31:06 +0000 Subject: [PATCH 13/25] Further clarify memory requirements and create newsfragment. --- docs/compose/requirements.rst | 16 ++++++++++++++-- towncrier/newsfragments/470.doc | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/470.doc diff --git a/docs/compose/requirements.rst b/docs/compose/requirements.rst index 1db200b0..965e24d7 100644 --- a/docs/compose/requirements.rst +++ b/docs/compose/requirements.rst @@ -5,8 +5,20 @@ Hardware considerations ----------------------- You should make sure that your hardware (virtual or physical) is compatible with -the latest Linux kernel. Also, you should have at least 3GB of total memory and -1GB of free memory/swap when running Mailu. +the latest Linux kernel. The minimal required memory and swap are: + +* When using antivirus (clamav): + + * 3GB of memory + + * 1GB of swap + +* When not using antivirus (clamav): + + * 1GB of memory + + * 1GB of swap + Pick a distribution ------------------- diff --git a/towncrier/newsfragments/470.doc b/towncrier/newsfragments/470.doc new file mode 100644 index 00000000..e53ea233 --- /dev/null +++ b/towncrier/newsfragments/470.doc @@ -0,0 +1,4 @@ +Document hardware requirements when using clamav. +Clamav requires **at least** 2GB of memory. +This 2Gb does not entail any other software running on the box. +So in total you require at least 3GB of memory and 1GB swap when antivirus is enabled. From 1925b2e0fb0f6844407e8b114e1cb67e7b7116be Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 24 Nov 2021 16:46:35 +0100 Subject: [PATCH 14/25] Upgrade rspamd --- core/rspamd/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 33174c1a..6ec211c4 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -1,4 +1,4 @@ -ARG DISTRO=alpine:3.14.2 +ARG DISTRO=alpine:3.15 FROM $DISTRO ENV TZ Etc/UTC From 89a7a8ac13f20b595d3dd21310494cbafa657013 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 25 Nov 2021 15:29:31 +0100 Subject: [PATCH 15/25] Fix score of RCVD_NO_TLS_LAST --- core/rspamd/conf/headers_group.conf | 7 +++++++ core/rspamd/conf/settings.conf | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 core/rspamd/conf/headers_group.conf delete mode 100644 core/rspamd/conf/settings.conf diff --git a/core/rspamd/conf/headers_group.conf b/core/rspamd/conf/headers_group.conf new file mode 100644 index 00000000..bae6f02b --- /dev/null +++ b/core/rspamd/conf/headers_group.conf @@ -0,0 +1,7 @@ +symbols = { + "RCVD_NO_TLS_LAST" { + # see https://github.com/Mailu/Mailu/issues/1705 + weight = 0.0; + description = "Last hop did not use encrypted transports"; + } +} diff --git a/core/rspamd/conf/settings.conf b/core/rspamd/conf/settings.conf deleted file mode 100644 index 01e9780f..00000000 --- a/core/rspamd/conf/settings.conf +++ /dev/null @@ -1,4 +0,0 @@ -apply { - # see https://github.com/Mailu/Mailu/issues/1705 - RCVD_NO_TLS_LAST = 0; -} From 33e8de5911e79773fa5baea95b98d3218f7b1a32 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 25 Nov 2021 20:50:17 +0000 Subject: [PATCH 16/25] Process code review comments in PR#2064. --- docs/database.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/database.rst b/docs/database.rst index 0af26f4b..41de2fde 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -2,7 +2,7 @@ Changing the database back-end ============================== By default Mailu uses a SQLite database. We have changed the internals of Mailu -to enable the support of alternative database solutions as PostgreSQL and MySQL/MariaDB. +to enable the support of alternative database solutions such as PostgreSQL and MySQL/MariaDB. Migrating to a different database back-end @@ -10,7 +10,7 @@ Migrating to a different database back-end From Mailu 1.9, Mailu has a :ref:`cli command (link) ` for exporting and importing the complete Mailu configuration. Using this tool it is very easy to switch what database back-end is used for Mailu. -Unfortunately roundcube does not have a tool for exporting/importing roundcube configuration. +Unfortunately roundcube does not have a tool for exporting/importing its configuration. This means it is not possible to switch the database back-end used by roundcube using out of box tools. To switch to a different database back-end: @@ -42,12 +42,10 @@ Adjust this to your own liking. .. code-block:: sql mysql> CREATE DATABASE mailu; - mysql> CREATE USER 'mailu'@'%' IDENTIFIED BY 'my-strong-password-here'; + mysql> CREATE USER `mailu`@`%` IDENTIFIED WITH mysql_native_password BY `my-strong-password-here`; mysql> GRANT ALL PRIVILEGES ON mailu.* TO 'mailu'@'%'; mysql> FLUSH PRIVILEGES; -Note that if you get any errors related to ``caching_sha2_password`` it can be solved by changing the encryption -of the password to ``mysql_native_password`` instead of the latest authentication plugin ``caching_sha2_password``. .. code-block:: sql From 8fe2d227f01c84527d7d09a2ed6b5eafa1b86b49 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 25 Nov 2021 20:52:28 +0000 Subject: [PATCH 17/25] Now the paragraph is really removed. --- docs/database.rst | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/docs/database.rst b/docs/database.rst index 41de2fde..d4199c73 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -47,26 +47,6 @@ Adjust this to your own liking. mysql> FLUSH PRIVILEGES; -.. code-block:: sql - - mysql> SELECT host, user, plugin FROM mysql.user; - - +-----------+-------+-----------------------+ - | host | user | plugin | - +-----------+-------+-----------------------+ - | % | mailu | caching_sha2_password | - +-----------+-------+-----------------------+ - - mysql> update mysql.user set plugin = 'mysql_native_password' where user = 'mailu'; - mysql> SELECT host, user, plugin FROM mysql.user; - - +------+-------+-----------------------+ - | host | user | plugin | - +------+-------+-----------------------+ - | % | mailu | mysql_native_password | - +------+-------+-----------------------+ - - External PostgreSQL ------------------- From 4fffdd95e96de1b3b366af2004e64ea02ea58157 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 5 Dec 2021 14:37:11 +0100 Subject: [PATCH 18/25] Reduce logging level --- core/admin/mailu/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 024c487f..755be013 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -66,10 +66,10 @@ def has_dane_record(domain, timeout=10): return app.config['DEFER_ON_TLS_ERROR'] except dns.exception.Timeout: app.logger.warn(f'Timeout while resolving the TLSA record for {domain} ({timeout}s).') - except dns.resolver.NXDOMAIN: + except (dns.resolver.NXDOMAIN, dns.name.EmptyLabel): pass # this is expected, not TLSA record is fine except Exception as e: - app.logger.error(f'Error while looking up the TLSA record for {domain} {e}') + app.logger.info(f'Error while looking up the TLSA record for {domain} {e}') pass # Rate limiter From 04bbd9f5153c90a59bce480bf2022881c40f39ec Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sun, 5 Dec 2021 16:10:55 +0000 Subject: [PATCH 19/25] Fix folder path twice in deploy.sh. --- tests/deploy.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/deploy.sh b/tests/deploy.sh index e66310af..db843b9c 100755 --- a/tests/deploy.sh +++ b/tests/deploy.sh @@ -17,7 +17,7 @@ fi #After that, both tags are pushed to the docker repository. if [ "$PINNED_MAILU_VERSION" != "" ] && [ "$BRANCH" != "master" ] then - images=$(docker-compose -f Mailu/tests/build.yml config | awk -F ':' '/image:/{ print $2 }') + images=$(docker-compose -f tests/build.yml config | awk -F ':' '/image:/{ print $2 }') for image in $images do docker tag "${image}":"${PINNED_MAILU_VERSION}" "${image}":${MAILU_VERSION} @@ -36,7 +36,7 @@ fi #Then we publish the images with tag master if [ "$PINNED_MAILU_VERSION" != "" ] && [ "$BRANCH" == "master" ] then - images=$(docker-compose -f Mailu/tests/build.yml config | awk -F ':' '/image:/{ print $2 }') + images=$(docker-compose -f tests/build.yml config | awk -F ':' '/image:/{ print $2 }') for image in $images do docker tag "${image}":"${PINNED_MAILU_VERSION}" "${image}":${MAILU_VERSION} From 15e64e8e500ab0372bd42a776e025c513d4886a8 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Mon, 6 Dec 2021 15:22:42 +0000 Subject: [PATCH 20/25] Add concurrency to ensure that only a single workflow can run for a branch. --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8cc174f7..51f569f8 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -12,6 +12,8 @@ on: # test branches, e.g. test-debian - test-* +concurrency: ci-${{ github.ref }} + ############################################### # REQUIRED secrets # DOCKER_UN: ${{ secrets.Docker_Login }} From 593e3ac5a4de54c2a536dcb8f784ea55711d5675 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 8 Dec 2021 19:13:28 +0100 Subject: [PATCH 21/25] fix DEFER_ON_TLS_ERROR --- core/postfix/start.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/postfix/start.py b/core/postfix/start.py index 4c291061..09574051 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -35,6 +35,7 @@ def is_valid_postconf_line(line): and not line == '' # Actual startup script +os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True' os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_MILTER", "antispam:11332") From f26fa8da84edffe83f5d1af0ac79dd508b183773 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 14 Dec 2021 11:26:33 +0000 Subject: [PATCH 22/25] Fix Webmail token check. Fix Auth-Port for Webmail. #2079 --- core/admin/mailu/internal/nginx.py | 6 +++--- core/nginx/conf/nginx.conf | 2 +- towncrier/newsfragments/2079.fix | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 towncrier/newsfragments/2079.fix diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 027db935..a9889969 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -27,12 +27,12 @@ STATUSES = { }), } -def check_credentials(user, password, ip, protocol=None): +def check_credentials(user, password, ip, protocol=None, auth_port=None): if not user or not user.enabled or (protocol == "imap" and not user.enable_imap) or (protocol == "pop3" and not user.enable_pop): return False is_ok = False # webmails - if len(password) == 64 and ip == app.config['WEBMAIL_ADDRESS']: + if len(password) == 64 and auth_port == '10143': if user.verify_temp_token(password): is_ok = True # All tokens are 32 characters hex lowercase @@ -100,7 +100,7 @@ def handle_authentication(headers): app.logger.warn(f'Invalid user {user_email!r}: {exc}') else: ip = urllib.parse.unquote(headers["Client-Ip"]) - if check_credentials(user, password, ip, protocol): + if check_credentials(user, password, ip, protocol, headers["Auth-Port"]): server, port = get_server(headers["Auth-Protocol"], True) return { "Auth-Status": "OK", diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 71cbf9ee..62e5c54b 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -277,7 +277,7 @@ mail { listen 10143; protocol imap; smtp_auth plain; - auth_http_header Auth-Port 10043; + auth_http_header Auth-Port 10143; } # SMTP is always enabled, to avoid losing emails when TLS is failing diff --git a/towncrier/newsfragments/2079.fix b/towncrier/newsfragments/2079.fix new file mode 100644 index 00000000..82350ff6 --- /dev/null +++ b/towncrier/newsfragments/2079.fix @@ -0,0 +1,2 @@ +#2079 Webmail token check does not work if WEBMAIL_ADDRESS is set to a hostname. +#2081 Fix typo in nginx config for webmail port (10043 to 10143) \ No newline at end of file From d76773b1dff95aafe89625265d62618b160b9972 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 14 Dec 2021 14:52:15 +0000 Subject: [PATCH 23/25] Also check the SMTP port for webmail/token --- core/admin/mailu/internal/nginx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index a9889969..9271df8e 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -32,7 +32,7 @@ def check_credentials(user, password, ip, protocol=None, auth_port=None): return False is_ok = False # webmails - if len(password) == 64 and auth_port == '10143': + if len(password) == 64 and auth_port in ['10143', '10025']: if user.verify_temp_token(password): is_ok = True # All tokens are 32 characters hex lowercase From 5bedcc1cb1a0e2e3922136102b730f3681577486 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 14 Dec 2021 15:10:28 +0000 Subject: [PATCH 24/25] Fix #2078 --- core/nginx/conf/nginx.conf | 7 ++++++- towncrier/newsfragments/2078.fix | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/2078.fix diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 71cbf9ee..c9e21fb3 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -135,7 +135,7 @@ http { # Actual logic {% if ADMIN == 'true' or WEBMAIL != 'none' %} - location ~ ^/(sso|static) { + location ~ ^/(sso|static)/ { include /etc/nginx/proxy.conf; proxy_pass http://$admin; } @@ -165,7 +165,12 @@ http { proxy_pass http://$webmail; } + {% if WEB_WEBMAIL == '/' %} + location /sso.php { + {% endif %} + {% if WEB_WEBMAIL != '/' %} location {{ WEB_WEBMAIL }}/sso.php { + {% endif %} {% if WEB_WEBMAIL != '/' %} rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent; rewrite ^{{ WEB_WEBMAIL }}/(.*) /$1 break; diff --git a/towncrier/newsfragments/2078.fix b/towncrier/newsfragments/2078.fix new file mode 100644 index 00000000..2a05e5f4 --- /dev/null +++ b/towncrier/newsfragments/2078.fix @@ -0,0 +1 @@ +SSO login page to webmail did not work if WEB_WEBMAIL=/ was set. \ No newline at end of file From d7a8235b892dfa1408e5004b689bee5f23c58a1f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 15 Dec 2021 10:53:47 +0100 Subject: [PATCH 25/25] Simplify --- core/nginx/conf/nginx.conf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index c9e21fb3..2f5f3496 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -167,8 +167,7 @@ http { {% if WEB_WEBMAIL == '/' %} location /sso.php { - {% endif %} - {% if WEB_WEBMAIL != '/' %} + {% else %} location {{ WEB_WEBMAIL }}/sso.php { {% endif %} {% if WEB_WEBMAIL != '/' %}