From 58235bcc44e9a7449229ea0d851e35846a5ce176 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 08:25:15 +0000 Subject: [PATCH 1/2] Switch to github actions for CI/CD --- .github/worfklows/CI.yml | 276 +++++++++++++++++++++++ .travis.yml | 56 ----- bors.toml | 3 +- tests/compose/filters/00_create_users.sh | 5 + 4 files changed, 283 insertions(+), 57 deletions(-) create mode 100644 .github/worfklows/CI.yml delete mode 100644 .travis.yml create mode 100755 tests/compose/filters/00_create_users.sh diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml new file mode 100644 index 00000000..3dcaf096 --- /dev/null +++ b/.github/worfklows/CI.yml @@ -0,0 +1,276 @@ + + +name: CI +on: + push: + branches: + - staging + - testing + - '1.5' + - '1.6' + - '1.7' + - '1.8' + - master + # version tags, e.g. 1.7.1 + - '[1-9].[0-9].[0-9]' + # pre-releases, e.g. 1.8-pre1 + - 1.8-pre[0-9] + # test branches, e.g. test-debian + - test-* + +############################################### +# REQUIRED secrets +# DOCKER_UN: ${{ secrets.Docker_Login }} +# Username of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_PW: ${{ secrets.Docker_Password }} +# Password of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_ORG: ${{ secrets.DOCKER_ORG }} +# The docker repository where the images are pushed to. +# +# Add the above secrets to your github repo to determine where the images will be pushed. +################################################ + +jobs: + build: + name: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - name: check docker-compose version + run: docker-compose -v + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + # In this step, this action saves a list of existing images, + # the cache is created without them in the post run. + # It also restores the cache if it exists. + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: build all docker images + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: docker-compose -f tests/build.yml build + +#NOTE: It appears the filter test depends on the core test. The filter test requires an email user +#that is created by the core test. + core-test: + name: core test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test core suite + run: python tests/compose/test.py core 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + filter-test: + name: filter test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: 'test clamvav' + run: python tests/compose/test.py filters 2 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + fetch-test: + name: fetch test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test fetch + run: python tests/compose/test.py fetchmail 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + rainloop-test: + name: rainloop test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test rainloop + run: python tests/compose/test.py rainloop 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + roundcube-test: + name: roundcube test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test roundcube + run: python tests/compose/test.py roundcube 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + webdav-test: + name: webdav test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test webdav + run: python tests/compose/test.py webdav 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + deploy: + name: deploy step + runs-on: ubuntu-latest + needs: + - build + - core-test + - filter-test + - fetch-test + - rainloop-test + - roundcube-test + - webdav-test + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + - name: build all docker images + run: docker-compose -f tests/build.yml build + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + - name: deploy built docker images + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: bash tests/deploy.sh + + # This job is watched by bors. It only complets if building,testing and deploy worked. + ci-success: + name: CI-Done + #Returns true when none of the **previous** steps have failed or been canceled. + if: ${{ success() }} + needs: + - deploy + runs-on: ubuntu-latest + steps: + - name: CI/CD succeeded. + run: exit 0 + + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f2a85630..00000000 --- a/.travis.yml +++ /dev/null @@ -1,56 +0,0 @@ -branches: - only: - - staging - - testing - - '1.5' - - '1.6' - - '1.7' - - '1.8' - - master - # version tags, e.g. 1.7.1 - - /^1\.[5678]\.\d+$/ - # pre-releases, e.g. 1.8-pre1 - - /^1\.8-pre\d+$/ - # test branches, e.g. test-debian - - /^test-[\w\-\.]+$/ - -sudo: required -services: docker -addons: - apt: - packages: - - docker-ce - -env: - - MAILU_VERSION=${TRAVIS_BRANCH////-} - -language: python -python: - - "3.6" -install: - - pip install -r tests/requirements.txt - - sudo curl -L https://github.com/docker/compose/releases/download/1.23.0-rc3/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose - - sudo chmod +x /usr/local/bin/docker-compose - -before_script: - - docker-compose -v - - echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - - docker-compose -f tests/build.yml build - - sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - -script: -# test.py, test name and timeout between start and tests. - - python tests/compose/test.py core 1 - - python tests/compose/test.py fetchmail 1 - - travis_wait python tests/compose/test.py filters 10 - - python tests/compose/test.py rainloop 1 - - python tests/compose/test.py roundcube 1 - - python tests/compose/test.py webdav 1 - -deploy: - provider: script - script: bash tests/deploy.sh - on: - all_branches: true - condition: -n $DOCKER_UN diff --git a/bors.toml b/bors.toml index 5279fe72..272a6047 100644 --- a/bors.toml +++ b/bors.toml @@ -1,3 +1,4 @@ status = [ - "continuous-integration/travis-ci/push" + "CI-Done" ] + diff --git a/tests/compose/filters/00_create_users.sh b/tests/compose/filters/00_create_users.sh new file mode 100755 index 00000000..3c581685 --- /dev/null +++ b/tests/compose/filters/00_create_users.sh @@ -0,0 +1,5 @@ +echo "Creating user required for next test ..." +# Should not fail and update the password; update mode +docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu admin admin mailu.io 'password' --mode=update || exit 1 +docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu user user mailu.io 'password' || exit 1 +echo "User created successfully" From 54dd4cf224b510f657f0b07696cebf9e8d89d7c2 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 19:16:56 +0000 Subject: [PATCH 2/2] Added new docker repo for test image. Adapted deploy script to use env var for test repo name. Modified travis references to github actions references in docs. Added changelog entry. --- .github/worfklows/CI.yml | 4 +++- docs/contributors/environment.rst | 14 +++++++------- docs/contributors/workflow.rst | 2 +- docs/faq.rst | 2 +- tests/deploy.sh | 2 +- towncrier/newsfragments/1828.misc | 1 + 6 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 towncrier/newsfragments/1828.misc diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml index 3dcaf096..421fec3c 100644 --- a/.github/worfklows/CI.yml +++ b/.github/worfklows/CI.yml @@ -26,7 +26,8 @@ on: # Password of docker login for pushing the images to repo $DOCKER_ORG # DOCKER_ORG: ${{ secrets.DOCKER_ORG }} # The docker repository where the images are pushed to. -# +# DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} +# The docker repository for test images. Only used for the branch TESTING (BORS try). # Add the above secrets to your github repo to determine where the images will be pushed. ################################################ @@ -256,6 +257,7 @@ jobs: DOCKER_UN: ${{ secrets.Docker_Login }} DOCKER_PW: ${{ secrets.Docker_Password }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index 26c04d0b..cef71c6c 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -178,9 +178,9 @@ In the case of a PR from a fellow team member, a single review is enough to initiate merging. In all other cases, two approving reviews are required. There is also a possibility to set the ``review/need2`` to require a second review. -After Travis successfully tests the PR and the required amount of reviews are acquired, +After the Github Action workflow successfully tests the PR and the required amount of reviews are acquired, Mergify will trigger with a ``bors r+`` command. Bors will batch any approved PR's, -merges them with master in a staging branch where Travis builds and tests the result. +merges them with master in a staging branch where the Github Action workflow builds and tests the result. After a successful test, the actual master gets fast-forwarded to that point. System requirements @@ -201,16 +201,16 @@ us on `Matrix`_. Test images ``````````` -All PR's automatically get build by Travis, controlled by `bors-ng`_. +All PR's automatically get build by a Github Action workflow, controlled by `bors-ng`_. Some primitive auto testing is done. The resulting images get uploaded to Docker hub, under the -tag name ``mailutest/:pr-``. +tag name ``mailuci/:pr-``. For example, to test PR #500 against master, reviewers can use: .. code-block:: bash - export DOCKER_ORG="mailutest" + export DOCKER_ORG="mailuci" export MAILU_VERSION="pr-500" docker-compose pull docker-compose up -d @@ -232,8 +232,8 @@ after Bors confirms a successful build. When bors try fails ``````````````````` -Sometimes Travis fails when another PR triggers a ``bors try`` command, -before Travis cloned the git repository. +Sometimes the Github Action workflow fails when another PR triggers a ``bors try`` command, +before the Github Action workflow cloned the git repository. Inspect the build log in the link provided by *bors-ng* to find out the cause. If you see something like the following error on top of the logs, feel free to write a comment with ``bors retry``. diff --git a/docs/contributors/workflow.rst b/docs/contributors/workflow.rst index 16dcef52..31ffd793 100644 --- a/docs/contributors/workflow.rst +++ b/docs/contributors/workflow.rst @@ -41,7 +41,7 @@ PR Workflow ----------- All pull requests have to be against the main ``master`` branch. -The PR gets build by Travis and some primitive auto-testing is done. +The PR gets build by a Github Action workflow and some primitive auto-testing is done. Test images get uploaded to a separate section in Docker hub. Reviewers will check the PR and test the resulting images. See the :ref:`testing` section for more info. diff --git a/docs/faq.rst b/docs/faq.rst index 9c4f1d75..c4cea444 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -61,7 +61,7 @@ have to prevent pushing out something quickly. We currently maintain a strict work flow: #. Someone writes a solution and sends a pull request; -#. We use Travis-CI for some very basic building and testing; +#. We use Github actions for some very basic building and testing; #. The pull request needs to be code-reviewed and tested by at least two members from the contributors team. diff --git a/tests/deploy.sh b/tests/deploy.sh index 21aec444..a836417b 100755 --- a/tests/deploy.sh +++ b/tests/deploy.sh @@ -5,7 +5,7 @@ # Retag in case of `bors try` if [ "$TRAVIS_BRANCH" = "testing" ]; then - export DOCKER_ORG="mailutest" + export DOCKER_ORG=$DOCKER_ORG_TESTS # Commit message is like "Try #99". # This sets the version tag to "pr-99" export MAILU_VERSION="pr-${TRAVIS_COMMIT_MESSAGE//[!0-9]/}" diff --git a/towncrier/newsfragments/1828.misc b/towncrier/newsfragments/1828.misc new file mode 100644 index 00000000..09da59ad --- /dev/null +++ b/towncrier/newsfragments/1828.misc @@ -0,0 +1 @@ +Switched from Travis to Github actions for CI/CD. Improved CI workflow to perform all tests in parallel.