From 86637f025969800521ad7bbd4c933c087c4b1e72 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 14:18:51 +0100 Subject: [PATCH 01/16] Make setup use the base image --- core/none/Dockerfile | 2 +- setup/Dockerfile | 26 +++++++++++--------------- setup/requirements.txt | 12 ------------ tests/build.hcl | 3 +++ 4 files changed, 15 insertions(+), 28 deletions(-) delete mode 100644 setup/requirements.txt diff --git a/core/none/Dockerfile b/core/none/Dockerfile index f06cc31c..d605cb07 100644 --- a/core/none/Dockerfile +++ b/core/none/Dockerfile @@ -10,5 +10,5 @@ RUN echo $VERSION >/version HEALTHCHECK CMD true -USER app +USER mailu CMD ["/bin/bash", "-c", "sleep infinity"] diff --git a/setup/Dockerfile b/setup/Dockerfile index 85e5f55b..7be8f1b0 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -1,24 +1,20 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION -ENV TZ Etc/UTC +# syntax=docker/dockerfile-upstream:1.4.3 + +# setup image +FROM base + +ARG VERSION=local LABEL version=$VERSION - -RUN mkdir -p /app -WORKDIR /app - -COPY requirements.txt requirements.txt -RUN apk add --no-cache curl python3 py3-pip \ - && pip3 install -r requirements.txt - -COPY server.py ./server.py -COPY main.py ./main.py COPY flavors /data/flavors COPY templates /data/templates COPY static ./static +COPY server.py ./server.py +COPY main.py ./main.py + +RUN echo $VERSION >> /version EXPOSE 80/tcp +HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost/ CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app -RUN echo $VERSION >> /version diff --git a/setup/requirements.txt b/setup/requirements.txt deleted file mode 100644 index b6f9f713..00000000 --- a/setup/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -Flask==1.0.2 -Flask-Bootstrap==3.3.7.1 -gunicorn==19.9.0 -redis==3.2.1 -Jinja2==3.0.3 -MarkupSafe==2.1.0 -Werkzeug==2.0.3 -click==8.0.3 -dominate==2.6.0 -itsdangerous==2.0.1 -redis==3.2.1 -visitor==0.1.3 diff --git a/tests/build.hcl b/tests/build.hcl index 34955270..0f6226c8 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -107,6 +107,9 @@ target "docs" { target "setup" { inherits = ["defaults"] context = "setup/" + contexts = { + base = "target:base" + } tags = tag("setup") } From e5a1a353dbeb8f884e84b0b205a13927d9e173be Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 14:19:22 +0100 Subject: [PATCH 02/16] Upgrade to alpine 3.16.3 This has PHP fixes and a new rspamd --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 6f31e21c..af99ccc3 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile-upstream:1.4.3 # base system image (intermediate) -ARG DISTRO=alpine:3.16.2 +ARG DISTRO=alpine:3.16.3 FROM $DISTRO as system ENV TZ=Etc/UTC LANG=C.UTF-8 From 21b9f76ebced4ad11e586216b9328b2b5b74c9a4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 14:34:55 +0100 Subject: [PATCH 03/16] setup doesn't need root --- setup/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup/Dockerfile b/setup/Dockerfile index 7be8f1b0..d6a5f083 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -12,9 +12,14 @@ COPY static ./static COPY server.py ./server.py COPY main.py ./main.py +RUN set -euxo pipefail \ + ; apk add --no-cache libcap \ + ; setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn + RUN echo $VERSION >> /version EXPOSE 80/tcp HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost/ +USER mailu CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app From 80559ecb71112d38c6f6ec38fc5eb8e1489c0439 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 14:43:31 +0100 Subject: [PATCH 04/16] optimize caching --- setup/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/Dockerfile b/setup/Dockerfile index d6a5f083..7e04931a 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -6,16 +6,16 @@ FROM base ARG VERSION=local LABEL version=$VERSION +RUN set -euxo pipefail \ + ; apk add --no-cache libcap \ + ; setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn + COPY flavors /data/flavors COPY templates /data/templates COPY static ./static COPY server.py ./server.py COPY main.py ./main.py -RUN set -euxo pipefail \ - ; apk add --no-cache libcap \ - ; setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn - RUN echo $VERSION >> /version EXPOSE 80/tcp From 42cd5bf2dc43b67551edeb0d7a171138b29f3dee Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 15:17:24 +0100 Subject: [PATCH 05/16] Move it to base since admin will also use it --- core/base/Dockerfile | 3 ++- setup/Dockerfile | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index af99ccc3..814a946a 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -13,7 +13,7 @@ ARG TARGETPLATFORM RUN set -euxo pipefail \ ; addgroup -Sg ${MAILU_GID} mailu \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ - ; apk add --no-cache bash ca-certificates curl python3 tzdata \ + ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \ ; machine="$(uname -m)" \ ; ! [[ "${TARGETPLATFORM}" != linux/arm/v7 && \( "${machine}" == x86_64 || "${machine}" == armv8* || "${machine}" == aarch64 \) ]] \ || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc @@ -72,6 +72,7 @@ RUN set -euxo pipefail \ FROM system COPY --from=build /app/venv/ /app/venv/ +RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn ENV VIRTUAL_ENV=/app/venv ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" diff --git a/setup/Dockerfile b/setup/Dockerfile index 7e04931a..a410871d 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -6,10 +6,6 @@ FROM base ARG VERSION=local LABEL version=$VERSION -RUN set -euxo pipefail \ - ; apk add --no-cache libcap \ - ; setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn - COPY flavors /data/flavors COPY templates /data/templates COPY static ./static From 699be6f9fa62ef538fdac0f09003b2c54838eda4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 16:03:37 +0100 Subject: [PATCH 06/16] Drop privs when running admin too --- core/admin/Dockerfile | 18 +++++++++--------- core/admin/start.py | 6 ++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 600c3e9f..b43e46b6 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -9,23 +9,23 @@ LABEL version=$VERSION RUN set -euxo pipefail \ ; apk add --no-cache libressl mariadb-connector-c postgresql-libs -COPY --from=assets /work/static/ ./mailu/static/ +EXPOSE 80/tcp +HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login + +VOLUME ["/data","/dkim"] + +ENV FLASK_APP=mailu + +COPY --from=assets /work/static/ ./mailu/static/ COPY audit.py / COPY start.py / - COPY migrations/ ./migrations/ - COPY mailu/ ./mailu/ + RUN set -euxo pipefail \ ; venv/bin/pybabel compile -d mailu/translations RUN echo $VERSION >/version -EXPOSE 80/tcp -HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index - -VOLUME ["/data","/dkim"] - -ENV FLASK_APP=mailu CMD /start.py diff --git a/core/admin/start.py b/core/admin/start.py index 3cb5c422..99b34a01 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -2,8 +2,14 @@ import os import logging as log +from pwd import getpwnam import sys +os.system("chown mailu:mailu -R /data /dkim") +mailu_id = getpwnam('mailu') +os.setgid(mailu_id.pw_gid) +os.setuid(mailu_id.pw_uid) + log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO")) os.system("flask mailu advertise") From e79d7fed550f2c546de2535e6ca4b993a66b1a44 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 16:21:52 +0100 Subject: [PATCH 07/16] Reduce the number of warnings on the CI --- .github/workflows/build_test_deploy.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index d1395bec..6512ccd5 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -126,7 +126,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -182,7 +182,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -244,7 +244,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -307,7 +307,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -370,7 +370,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Install python packages @@ -416,7 +416,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Push image to Docker @@ -461,7 +461,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Push image to Docker From 3b5b00d87d4a701661efaa3bea785b8f88e93796 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 16:37:17 +0100 Subject: [PATCH 08/16] towncrier --- towncrier/newsfragments/2539.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/2539.misc diff --git a/towncrier/newsfragments/2539.misc b/towncrier/newsfragments/2539.misc new file mode 100644 index 00000000..9dbf4e4d --- /dev/null +++ b/towncrier/newsfragments/2539.misc @@ -0,0 +1 @@ +Upgrade to Alpine 3.16.3; Make both setup and admin run without root privs. From e3b875aa6b46d674731c6677d32908c7a3c51410 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:09:00 +0100 Subject: [PATCH 09/16] Well, -i stands for --insecure --- core/rspamd/start.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 537d996d..0f351057 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -33,4 +33,6 @@ while True: log.warning("Admin is not up just yet, retrying in 1 second") # Run rspamd -os.execv("/usr/sbin/rspamd", ["rspamd", "-i", "-f"]) +os.system("mkdir -m 755 -p /run/rspamd") +os.system("chown rspamd:rspamd /run/rspamd") +os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"]) From 3cb87b6e49352e97a1a578b3b22ce5199eaee295 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:10:53 +0100 Subject: [PATCH 10/16] Update entry --- towncrier/newsfragments/2539.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/towncrier/newsfragments/2539.misc b/towncrier/newsfragments/2539.misc index 9dbf4e4d..0d9907ac 100644 --- a/towncrier/newsfragments/2539.misc +++ b/towncrier/newsfragments/2539.misc @@ -1 +1 @@ -Upgrade to Alpine 3.16.3; Make both setup and admin run without root privs. +Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. From 6137f93d23265d839cca09d0b5da1018751635e0 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:17:41 +0100 Subject: [PATCH 11/16] add a GTUBE test to check the antispam --- tests/compose/filters/02_email_antispam.sh | 6 ++++++ tests/compose/filters/gtube.txt | 1 + 2 files changed, 7 insertions(+) create mode 100755 tests/compose/filters/02_email_antispam.sh create mode 100644 tests/compose/filters/gtube.txt diff --git a/tests/compose/filters/02_email_antispam.sh b/tests/compose/filters/02_email_antispam.sh new file mode 100755 index 00000000..009a13a2 --- /dev/null +++ b/tests/compose/filters/02_email_antispam.sh @@ -0,0 +1,6 @@ +python3 tests/email_test.py message-virus "tests/compose/filters/gtube.txt" +if [ $? -eq 99 ]; then + exit 0 +else + exit 1 +fi diff --git a/tests/compose/filters/gtube.txt b/tests/compose/filters/gtube.txt new file mode 100644 index 00000000..728c7cc6 --- /dev/null +++ b/tests/compose/filters/gtube.txt @@ -0,0 +1 @@ +XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X From 1bfab1dbfaf7b139f27fc30e35cf22fb2592c74c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:32:39 +0100 Subject: [PATCH 12/16] Maybe fix the test? --- tests/compose/filters/02_email_antispam.sh | 3 ++- tests/compose/filters/gtube.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 tests/compose/filters/gtube.txt diff --git a/tests/compose/filters/02_email_antispam.sh b/tests/compose/filters/02_email_antispam.sh index 009a13a2..923f1475 100755 --- a/tests/compose/filters/02_email_antispam.sh +++ b/tests/compose/filters/02_email_antispam.sh @@ -1,4 +1,5 @@ -python3 tests/email_test.py message-virus "tests/compose/filters/gtube.txt" +# GTUBE should be blocked, see https://rspamd.com/doc/gtube_patterns.html +python3 tests/email_test.py "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" if [ $? -eq 99 ]; then exit 0 else diff --git a/tests/compose/filters/gtube.txt b/tests/compose/filters/gtube.txt deleted file mode 100644 index 728c7cc6..00000000 --- a/tests/compose/filters/gtube.txt +++ /dev/null @@ -1 +0,0 @@ -XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X From b28798c74fed5dafc5da7d8d1fa599fc1c1a5b51 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:46:04 +0100 Subject: [PATCH 13/16] doh --- tests/compose/filters/02_email_antispam.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compose/filters/02_email_antispam.sh b/tests/compose/filters/02_email_antispam.sh index 923f1475..ac2653a3 100755 --- a/tests/compose/filters/02_email_antispam.sh +++ b/tests/compose/filters/02_email_antispam.sh @@ -1,6 +1,6 @@ # GTUBE should be blocked, see https://rspamd.com/doc/gtube_patterns.html python3 tests/email_test.py "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" -if [ $? -eq 99 ]; then +if [ $? -eq 25 ]; then exit 0 else exit 1 From bdc085048d009154ea4bf04264fe982cdebe4fef Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 18 Nov 2022 10:40:42 +0100 Subject: [PATCH 14/16] Restore the Dockerfile like it was --- core/admin/Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index b43e46b6..600c3e9f 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -9,23 +9,23 @@ LABEL version=$VERSION RUN set -euxo pipefail \ ; apk add --no-cache libressl mariadb-connector-c postgresql-libs -EXPOSE 80/tcp - -HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login - -VOLUME ["/data","/dkim"] - -ENV FLASK_APP=mailu - COPY --from=assets /work/static/ ./mailu/static/ + COPY audit.py / COPY start.py / -COPY migrations/ ./migrations/ -COPY mailu/ ./mailu/ +COPY migrations/ ./migrations/ + +COPY mailu/ ./mailu/ RUN set -euxo pipefail \ ; venv/bin/pybabel compile -d mailu/translations RUN echo $VERSION >/version +EXPOSE 80/tcp +HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index + +VOLUME ["/data","/dkim"] + +ENV FLASK_APP=mailu CMD /start.py From 44c47586eaa6798809cac77f6967e57c5173d2ab Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 17:50:57 +0100 Subject: [PATCH 15/16] Fix potential permission problems --- core/admin/start.py | 4 +++- towncrier/newsfragments/2539.misc | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/admin/start.py b/core/admin/start.py index 99b34a01..d9fa6bef 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -5,7 +5,9 @@ import logging as log from pwd import getpwnam import sys -os.system("chown mailu:mailu -R /data /dkim") +os.system("chown mailu:mailu -R /dkim") +os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu") +os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown mailu:mailu") mailu_id = getpwnam('mailu') os.setgid(mailu_id.pw_gid) os.setuid(mailu_id.pw_uid) diff --git a/towncrier/newsfragments/2539.misc b/towncrier/newsfragments/2539.misc index 0d9907ac..10e3954e 100644 --- a/towncrier/newsfragments/2539.misc +++ b/towncrier/newsfragments/2539.misc @@ -1 +1 @@ -Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. +Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. Please ensure that your folder overrides/rspamd is owned by 1000:1000 From f994c8687ed35402d6a1a7750f2a61ce8cfb8451 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 18:12:11 +0100 Subject: [PATCH 16/16] doh --- core/admin/start.py | 1 - core/rspamd/start.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/start.py b/core/admin/start.py index d9fa6bef..e2163398 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -7,7 +7,6 @@ import sys os.system("chown mailu:mailu -R /dkim") os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu") -os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown mailu:mailu") mailu_id = getpwnam('mailu') os.setgid(mailu_id.pw_gid) os.setuid(mailu_id.pw_uid) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 0f351057..37de1df9 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -35,4 +35,5 @@ while True: # Run rspamd os.system("mkdir -m 755 -p /run/rspamd") os.system("chown rspamd:rspamd /run/rspamd") +os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown rspamd:rspamd") os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"])