From 295d7ea6755928e66422a3864adab16584760b20 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:08:19 +0200 Subject: [PATCH 01/44] Move assets to own Dockerfile --- core/admin/assets/Dockerfile | 14 ++++++++++++++ core/admin/assets/{ => content}/app.css | 0 core/admin/assets/{ => content}/app.js | 0 core/admin/assets/{ => content}/mailu.png | Bin core/admin/{ => assets/content}/package.json | 0 core/admin/assets/{ => content}/vendor.js | 0 core/admin/{ => assets/content}/webpack.config.js | 0 7 files changed, 14 insertions(+) create mode 100644 core/admin/assets/Dockerfile rename core/admin/assets/{ => content}/app.css (100%) rename core/admin/assets/{ => content}/app.js (100%) rename core/admin/assets/{ => content}/mailu.png (100%) rename core/admin/{ => assets/content}/package.json (100%) rename core/admin/assets/{ => content}/vendor.js (100%) rename core/admin/{ => assets/content}/webpack.config.js (100%) diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile new file mode 100644 index 00000000..d799fa4f --- /dev/null +++ b/core/admin/assets/Dockerfile @@ -0,0 +1,14 @@ +# syntax=docker/dockerfile-upstream:1.4.3 + +FROM node:16-alpine3.16 + +COPY content ./ +RUN set -euxo pipefail \ + && npm config set update-notifier false \ + && npm install --no-fund \ + && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ + && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ + cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ + done \ + && node_modules/.bin/webpack-cli --color + diff --git a/core/admin/assets/app.css b/core/admin/assets/content/app.css similarity index 100% rename from core/admin/assets/app.css rename to core/admin/assets/content/app.css diff --git a/core/admin/assets/app.js b/core/admin/assets/content/app.js similarity index 100% rename from core/admin/assets/app.js rename to core/admin/assets/content/app.js diff --git a/core/admin/assets/mailu.png b/core/admin/assets/content/mailu.png similarity index 100% rename from core/admin/assets/mailu.png rename to core/admin/assets/content/mailu.png diff --git a/core/admin/package.json b/core/admin/assets/content/package.json similarity index 100% rename from core/admin/package.json rename to core/admin/assets/content/package.json diff --git a/core/admin/assets/vendor.js b/core/admin/assets/content/vendor.js similarity index 100% rename from core/admin/assets/vendor.js rename to core/admin/assets/content/vendor.js diff --git a/core/admin/webpack.config.js b/core/admin/assets/content/webpack.config.js similarity index 100% rename from core/admin/webpack.config.js rename to core/admin/assets/content/webpack.config.js From 5e552bae69bd4ff72b4377a8ab31c7b52e4be21d Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:08:37 +0200 Subject: [PATCH 02/44] Add base image --- core/base/Dockerfile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 core/base/Dockerfile diff --git a/core/base/Dockerfile b/core/base/Dockerfile new file mode 100644 index 00000000..71ac6e31 --- /dev/null +++ b/core/base/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile-upstream:1.4.3 + +ARG DISTRO=alpine:3.14.5 +FROM $DISTRO + +ENV TZ Etc/UTC +ENV LANG C.UTF-8 + +# TODO: use intermediate image to build virtual env + +RUN set -euxo pipefail \ + && adduser -s /bin/bash -Dh /app -k /var/empty -u 1000 -g mailu app \ + && apk add --no-cache bash ca-certificates tzdata python3 py3-pip py3-wheel \ + && pip3 install --no-cache-dir --upgrade pip + +WORKDIR /app + +COPY libs libs/ + +# TODO: work in virtual env (see above) +# && python3 -m venv . \ +RUN set -euxo pipefail \ + && pip3 install --no-cache-dir -r libs/requirements.txt --only-binary=:all: \ + || ( apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ + && pip3 install --no-cache-dir -r libs/requirements.txt \ + && apk del --no-cache .build-deps ) + +# TODO: clean image (or use intermediate - see above) +# && bin/pip uninstall -y pip distribute setuptools wheel \ +# && rm -rf /tmp/* /root/.cache/pip From 9fe452e3d1d868b076a34888659ea701e136d192 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:09:07 +0200 Subject: [PATCH 03/44] Use base image when building core images --- core/admin/Dockerfile | 53 ++++++++++------------------------------- core/dovecot/Dockerfile | 34 ++++++++------------------ core/nginx/Dockerfile | 43 ++++++++++++++++----------------- core/none/Dockerfile | 14 ++++++++--- core/postfix/Dockerfile | 41 ++++++++++--------------------- core/rspamd/Dockerfile | 29 ++++++++-------------- tests/build.hcl | 34 +++++++++++++++++++++++++- 7 files changed, 111 insertions(+), 137 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index e4d870e0..c3731fb2 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -1,61 +1,34 @@ -# First stage to build assets -ARG DISTRO=alpine:3.14.5 +# syntax=docker/dockerfile-upstream:1.4.3 -FROM node:16-alpine3.16 as assets - -COPY package.json ./ -RUN set -eu \ - && npm config set update-notifier false \ - && npm install --no-fund - -COPY webpack.config.js ./ -COPY assets ./assets -RUN set -eu \ - && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ - && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ - cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ - done \ - && node_modules/.bin/webpack-cli --color - - -# Actual application -FROM $DISTRO -ARG VERSION - -ENV TZ Etc/UTC +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN set -eu \ - && apk add --no-cache python3 py3-pip py3-wheel git bash tzdata \ - && pip3 install --upgrade pip - -RUN mkdir -p /app -WORKDIR /app - COPY requirements-prod.txt requirements.txt -RUN set -eu \ +RUN set -euxo pipefail \ && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ && pip install --no-cache-dir -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ || ( apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ - && pip install --upgrade pip \ && pip install -r requirements.txt \ && apk del --no-cache build-dep ) -COPY --from=assets static ./mailu/static COPY mailu ./mailu +RUN pybabel compile -d mailu/translations + COPY migrations ./migrations + COPY start.py /start.py COPY audit.py /audit.py -RUN pybabel compile -d mailu/translations +COPY --from=assets static ./mailu/static + +RUN echo $VERSION >> /version EXPOSE 80/tcp +HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index + VOLUME ["/data","/dkim"] + ENV FLASK_APP mailu - CMD /start.py - -HEALTHCHECK CMD curl -f -L http://localhost/sso/login?next=ui.index || exit 1 -RUN echo $VERSION >> /version diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index e44b69f7..2d74e59b 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -1,36 +1,22 @@ -ARG DISTRO=alpine:3.14.5 +# syntax=docker/dockerfile-upstream:1.4.3 + +FROM base -FROM $DISTRO ARG VERSION -ENV TZ Etc/UTC - LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict py3-yarl tzdata \ - && pip3 install --upgrade pip - -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 - -# Shared layer between dovecot and postfix -RUN apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ - && pip3 install "podop>0.2.5" \ - && apk del .build-deps - -# Image specific layers under this line -RUN apk add --no-cache \ - dovecot dovecot-lmtpd dovecot-pop3d dovecot-submissiond dovecot-pigeonhole-plugin rspamd-client xapian-core dovecot-fts-xapian \ - && mkdir /var/lib/dovecot +RUN set -euxo pipefail \ + && apk add --no-cache dovecot dovecot-lmtpd dovecot-pop3d dovecot-submissiond dovecot-pigeonhole-plugin rspamd-client xapian-core dovecot-fts-xapian \ + && mkdir /var/lib/dovecot COPY conf /conf COPY start.py /start.py +RUN echo $VERSION >> /version + EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp +HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready." + VOLUME ["/mail"] CMD /start.py - -HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready." -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 07021429..2a34403f 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -1,34 +1,33 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO +# syntax=docker/dockerfile-upstream:1.4.3 + +FROM base as static + +COPY static /static + +RUN set -euxo pipefail \ + && gzip -k9 /static/*.ico /static/*.txt \ + && chmod a+rX-w -R /static + + +FROM base + ARG VERSION - -ENV TZ Etc/UTC - LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict \ - && pip3 install --upgrade pip - -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 - # Image specific layers under this line -RUN apk add --no-cache certbot nginx nginx-mod-mail openssl curl tzdata \ - && pip3 install watchdog +RUN set -euxo pipefail \ + && apk add --no-cache certbot nginx nginx-mod-mail openssl curl \ + && pip3 install --no-cache-dir watchdog COPY conf /conf -COPY static /static +COPY --from=static /static /static COPY *.py / -RUN gzip -k9 /static/*.ico /static/*.txt; chmod a+rX -R /static +RUN echo $VERSION >> /version EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp -VOLUME ["/certs"] -VOLUME ["/overrides"] +HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://localhost/health + +VOLUME ["/certs", "/overrides"] CMD /start.py - -HEALTHCHECK CMD curl -k -f -L http://localhost/health || exit 1 -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/core/none/Dockerfile b/core/none/Dockerfile index 06d351a9..058b18c5 100644 --- a/core/none/Dockerfile +++ b/core/none/Dockerfile @@ -1,6 +1,14 @@ +# syntax=docker/dockerfile-upstream:1.4.3 # This is an idle image to dynamically replace any component if disabled. -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO +FROM base -CMD sleep 1000000d +ARG VERSION=local +LABEL version=$VERSION + +RUN echo $VERSION >> /version + +HEALTHCHECK CMD true + +USER app +CMD ["/bin/bash", "-c", "sleep infinity"] diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 373e5c8d..66adbde3 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,40 +1,25 @@ -ARG DISTRO=alpine:3.14.5 +# syntax=docker/dockerfile-upstream:1.4.3 -FROM $DISTRO -ARG VERSION - -ENV TZ Etc/UTC +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict py3-yarl tzdata \ - && pip3 install --upgrade pip - -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 - -# Shared layer between dovecot and postfix -RUN apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ - && pip3 install "podop>0.2.5" \ - && apk del .build-deps - -# Image specific layers under this line -# Building pycares from source requires py3-wheel and libffi-dev packages -RUN pip install --no-cache-dir --only-binary=:all: postfix-mta-sts-resolver==1.0.1 || (apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev py3-wheel libffi-dev \ - && pip3 install postfix-mta-sts-resolver==1.0.1 \ - && apk del .build-deps ) - -RUN apk add --no-cache postfix postfix-pcre cyrus-sasl-login rsyslog logrotate +RUN set -euxo pipefail \ + && apk add --no-cache postfix postfix-pcre cyrus-sasl-login rsyslog logrotate \ + && pip install --no-cache-dir --only-binary=:all: postfix-mta-sts-resolver==1.0.1 \ + || ( apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev py3-wheel libffi-dev \ + && pip3 install postfix-mta-sts-resolver==1.0.1 \ + && apk del .build-deps ) COPY conf /conf COPY start.py /start.py +RUN echo $VERSION >> /version + EXPOSE 25/tcp 10025/tcp +HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" + VOLUME ["/queue"] CMD /start.py - -HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" -RUN echo $VERSION >> /version diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 7b89aed0..5ee922e5 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -1,31 +1,22 @@ -ARG DISTRO=alpine:3.15 -FROM $DISTRO -ARG VERSION -ENV TZ Etc/UTC +# syntax=docker/dockerfile-upstream:1.4.3 +FROM base + +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict tzdata \ - && pip3 install --upgrade pip - -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 - -# Image specific layers under this line -RUN apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl - -RUN mkdir /run/rspamd +RUN set -euxo pipefail \ + && apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl \ + && mkdir /run/rspamd COPY conf/ /conf COPY start.py /start.py +RUN echo $VERSION >> /version + EXPOSE 11332/tcp 11334/tcp 11335/tcp +HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost:11334/ VOLUME ["/var/lib/rspamd"] CMD /start.py - -HEALTHCHECK --start-period=350s CMD curl -f -L http://localhost:11334/ || exit 1 -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/tests/build.hcl b/tests/build.hcl index 3df00b51..c32da8d5 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -78,6 +78,19 @@ function "tag" { # docker buildx bake -f tests\build.hcl docs #----------------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------------------- +# Base images +# ----------------------------------------------------------------------------------------- +target "base" { + inherits = ["defaults"] + context="core/base" +} + +target "assets" { + inherits = ["defaults"] + context="core/admin/assets" +} + # ----------------------------------------------------------------------------------------- # Documentation and setup images # ----------------------------------------------------------------------------------------- @@ -103,36 +116,55 @@ target "setup" { target "none" { inherits = ["defaults"] context="core/none" + contexts= { + base = "target:base" + } tags = tag("none") } target "admin" { inherits = ["defaults"] context="core/admin" + contexts= { + base = "target:base" + assets = "target:assets" + } tags = tag("admin") } target "antispam" { inherits = ["defaults"] context="core/rspamd" + contexts= { + base = "target:base" + } tags = tag("rspamd") } target "front" { inherits = ["defaults"] context="core/nginx" + contexts= { + base = "target:base" + } tags = tag("nginx") } target "imap" { inherits = ["defaults"] context="core/dovecot" + contexts= { + base = "target:base" + } tags = tag("dovecot") } target "smtp" { inherits = ["defaults"] context="core/postfix" + contexts= { + base = "target:base" + } tags = tag("postfix") } @@ -182,4 +214,4 @@ target "webdav" { inherits = ["defaults"] context="optional/radicale" tags = tag("radicale") -} \ No newline at end of file +} From b501498401fc4f09c79f6faa13b4c93858f467a2 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:09:31 +0200 Subject: [PATCH 04/44] Update .gitignore file --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8734dcdb..f5e9f8ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -*.pyc -*.mo +**/*.pyc +**/*.mo __pycache__ pip-selfcheck.json /core/admin/lib* From b1b0aeb69dc8d87dfce527f6ab67582683c73b1d Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Sun, 22 Jul 2018 20:06:30 +0200 Subject: [PATCH 05/44] Initial commit --- core/base/libs/podop/CONTRIBUTING.md | 7 ++ core/base/libs/podop/LICENSE.md | 25 ++++++ core/base/libs/podop/README.md | 112 ++++++++++++++++++++++++ core/base/libs/podop/podop/__init__.py | 44 ++++++++++ core/base/libs/podop/podop/dovecot.py | 95 ++++++++++++++++++++ core/base/libs/podop/podop/postfix.py | 115 +++++++++++++++++++++++++ core/base/libs/podop/podop/table.py | 26 ++++++ core/base/libs/podop/scripts/podop | 25 ++++++ core/base/libs/podop/setup.py | 14 +++ 9 files changed, 463 insertions(+) create mode 100644 core/base/libs/podop/CONTRIBUTING.md create mode 100644 core/base/libs/podop/LICENSE.md create mode 100644 core/base/libs/podop/README.md create mode 100644 core/base/libs/podop/podop/__init__.py create mode 100644 core/base/libs/podop/podop/dovecot.py create mode 100644 core/base/libs/podop/podop/postfix.py create mode 100644 core/base/libs/podop/podop/table.py create mode 100755 core/base/libs/podop/scripts/podop create mode 100644 core/base/libs/podop/setup.py diff --git a/core/base/libs/podop/CONTRIBUTING.md b/core/base/libs/podop/CONTRIBUTING.md new file mode 100644 index 00000000..6a09c85d --- /dev/null +++ b/core/base/libs/podop/CONTRIBUTING.md @@ -0,0 +1,7 @@ +This project is open source, and your contributions are all welcome. There are mostly three different ways one can contribute to the project: + +1. use Podop, either on test or on production servers, and report meaningful bugs when you find some; +2. write and publish, or contribute to mail distributions based on Podop, like Mailu; +2. contribute code and/or configuration to the repository (see [the development guidelines](https://mailu.io/contributors/guide.html) for details); + +Either way, keep in mind that the code you write must be licensed under the same conditions as the project itself. Additionally, all contributors are considered equal co-authors of the project. diff --git a/core/base/libs/podop/LICENSE.md b/core/base/libs/podop/LICENSE.md new file mode 100644 index 00000000..8aa0da5d --- /dev/null +++ b/core/base/libs/podop/LICENSE.md @@ -0,0 +1,25 @@ +MIT License + +Copyright (c) 2018 All Podop contributors at the date + +This software consists of voluntary contributions made by multiple individuals. +For exact contribution history, see the revision history available at +https://github.com/Mailu/podop.git + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/base/libs/podop/README.md b/core/base/libs/podop/README.md new file mode 100644 index 00000000..208b3ce5 --- /dev/null +++ b/core/base/libs/podop/README.md @@ -0,0 +1,112 @@ +Podop is a piece of middleware designed to run between Postfix or Dovecot +on one side, any Python implementation of a table lookup protocol on the +other side. + +It is thus able to forward Postfix maps and Dovecot dicts to the same +(or multiple) backends in order to write a single, more flexible backend +for a mail distribution. + +Examples +======== + +- Connect Postfix to a DNS lookup so that every domain that has a proper MX + record to your Postfix is actually accepted as a local domain +- Connect both Postfix and Dovecot to an HTTP microservice to run a high + availability microservice-based mail service +- Use a single database server running any Python-compatible API for both + your Postfix and Dovecot servers + +Configure Podop tables +====================== + +Podop tables are configured through CLI arguments when running the server. +You must provide a ``--name`` for the table, a ``--type`` for the table and +a ``--param`` that parametrizes the map. + +URL table +--------- + +The URL table will initiate an HTTP GET request for read access and an HTTP +POST request for write access to a table. The table is parametrized with +a template URL containing ``§`` (or ``{}``) for inserting the table key. + +``` +--name test --type url --param http://microservice/api/v1/map/tests/§ +``` + +GET requests should return ``200`` and a JSON-encoded object +that will be passed either to Postfix or Dovecot. They should return ``4XX`` +for access issues that will result in lookup miss, and ``5XX`` for backend +issues that will result in a temporary failure. + +POST requests will contain a JSON-encoded object in the request body, that +will be saved in the table. + +Postfix usage +============= + +In order to access Podop tables from Postfix, you should setup ``socketmap`` +Postfix maps. For instance, in order to access the ``test`` table on a Podop +socket at ``/tmp/podop.socket``, use the following setup: + +``` +virtual_alias_maps = socketmap:unix:/tmp/podop.socket:test +``` + +Multiple maps or identical maps can be configured for various usages. + +``` +virtual_alias_maps = socketmap:unix:/tmp/podop.socket:alias +virtual_mailbox_domains = socketmap:unix:/tmp/podop.socket:domain +virtual_mailbox_maps = socketmap:unix:/tmp/podop.socket:alias +``` + +In order to simplify the configuration, you can setup a shortcut. + +``` +podop = socketmap:unic:/tmp/podop.socket +virtual_alias_maps = ${podop}:alias +virtual_mailbox_domains = ${podop}:domain +virtual_mailbox_maps = ${podop}:alias +``` + +Dovecot usage +============= + +In order to access Podop tables from Dovecot, you should setup a ``proxy`` +Dovecot dictionary. For instance, in order to access the ``test`` table on +a Podop socket at ``/tmp/podop.socket``, use the following setup: + +``` +mail_attribute_dict = proxy:/tmp/podop.socket:test +``` + +Multiple maps or identical maps can be configured for various usages. + +``` +mail_attribute_dict = proxy:/tmp/podop.socket:meta + +passdb { + driver = dict + args = /etc/dovecot/auth.conf +} + +userdb { + driver = dict + args = /etc/dovecot/auth.conf +} + +# then in auth.conf +uri = proxy:/tmp/podop.socket:auth +iterate_disable = yes +default_pass_scheme = plain +password_key = passdb/%u +user_key = userdb/%u +``` + +Contributing +============ + +Podop is free software, open to suggestions and contributions. All +components are free software and compatible with the MIT license. All +the code is placed under the MIT license. diff --git a/core/base/libs/podop/podop/__init__.py b/core/base/libs/podop/podop/__init__.py new file mode 100644 index 00000000..8c2c4d8d --- /dev/null +++ b/core/base/libs/podop/podop/__init__.py @@ -0,0 +1,44 @@ +""" Podop is a *Po*stfix and *Do*vecot proxy + +It is able to proxify postfix maps and dovecot dicts to any table +""" + +import asyncio +import logging + +from podop import postfix, dovecot, table + + +SERVER_TYPES = dict( + postfix=postfix.SocketmapProtocol, + dovecot=dovecot.DictProtocol +) + +TABLE_TYPES = dict( + url=table.UrlTable +) + + +def run_server(server_type, socket, tables): + """ Run the server, given its type, socket path and table list + + The table list must be a list of tuples (name, type, param) + """ + # Prepare the maps + table_map = { + name: TABLE_TYPES[table_type](param) + for name, table_type, param in tables + } + # Run the main loop + logging.basicConfig(level=logging.DEBUG) + loop = asyncio.get_event_loop() + server = loop.run_until_complete(loop.create_unix_server( + SERVER_TYPES[server_type].factory(table_map), socket + )) + try: + loop.run_forever() + except KeyboardInterrupt: + pass + server.close() + loop.run_until_complete(server.wait_closed()) + loop.close() diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py new file mode 100644 index 00000000..bbf134cc --- /dev/null +++ b/core/base/libs/podop/podop/dovecot.py @@ -0,0 +1,95 @@ +""" Dovecot dict proxy implementation +""" + +import asyncio +import logging + + +class DictProtocol(asyncio.Protocol): + """ Protocol to answer Dovecot dict requests, as implemented in Dict proxy. + + There is very little documentation about the protocol, most of it was + reverse-engineered from : + + https://github.com/dovecot/core/blob/master/src/dict/dict-connection.c + https://github.com/dovecot/core/blob/master/src/dict/dict-commands.c + https://github.com/dovecot/core/blob/master/src/lib-dict/dict-client.h + """ + + DATA_TYPES = {0: str, 1: int} + + def __init__(self, table_map): + self.table_map = table_map + self.major_version = None + self.minor_version = None + self.dict = None + super(DictProtocol, self).__init__() + + def connection_made(self, transport): + logging.info('Connect {}'.format(transport.get_extra_info('peername'))) + self.transport = transport + + def data_received(self, data): + logging.debug("Received {}".format(data)) + results = [] + for line in data.split(b"\n"): + logging.debug("Line {}".format(line)) + if len(line) < 2: + continue + command = DictProtocol.COMMANDS.get(line[0]) + if command is None: + logging.warning('Unknown command {}'.format(line[0])) + return self.transport.abort() + args = line[1:].strip().split(b"\t") + try: + future = command(self, *args) + if future: + results.append(future) + except Exception: + logging.exception("Error when processing request") + return self.transport.abort() + logging.debug("Results {}".format(results)) + return asyncio.gather(*results) + + def process_hello(self, major, minor, value_type, user, dict_name): + self.major, self.minor = int(major), int(minor) + logging.debug('Client version {}.{}'.format(self.major, self.minor)) + assert self.major == 2 + self.value_type = DictProtocol.DATA_TYPES[int(value_type)] + self.user = user + self.dict = self.table_map[dict_name.decode("ascii")] + logging.debug("Value type {}, user {}, dict {}".format( + self.value_type, self.user, dict_name)) + + async def process_lookup(self, key): + logging.debug("Looking up {}".format(key)) + result = await self.dict.get(key.decode("utf8")) + if result is not None: + if type(result) is str: + response = result.encode("utf8") + elif type(result) is bytes: + response = result + else: + response = json.dumps(result).encode("ascii") + return self.reply(b"O", response) + else: + return self.reply(b"N") + + def reply(self, command, *args): + logging.debug("Replying {} with {}".format(command, args)) + self.transport.write(command) + self.transport.write(b"\t".join( + arg.replace(b"\t", b"\t\t") for arg in args + )) + self.transport.write(b"\n") + + @classmethod + def factory(cls, table_map): + """ Provide a protocol factory for a given map instance. + """ + return lambda: cls(table_map) + + COMMANDS = { + ord("H"): process_hello, + ord("L"): process_lookup + } diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py new file mode 100644 index 00000000..122cf962 --- /dev/null +++ b/core/base/libs/podop/podop/postfix.py @@ -0,0 +1,115 @@ +""" Postfix map proxy implementation +""" + +import asyncio +import logging + + +class NetstringProtocol(asyncio.Protocol): + """ Netstring asyncio protocol implementation. + + For protocol details, see https://cr.yp.to/proto/netstrings.txt + """ + + # Length of the smallest allocated buffer, larger buffers will be + # allocated dynamically + BASE_BUFFER = 1024 + + # Maximum length of a buffer, will crash when exceeded + MAX_BUFFER = 65535 + + def __init__(self): + super(NetstringProtocol, self).__init__() + self.init_buffer() + + def init_buffer(self): + self.len = None # None when waiting for a length to be sent) + self.separator = -1 # -1 when not yet detected (str.find) + self.index = 0 # relative to the buffer + self.buffer = bytearray(NetstringProtocol.BASE_BUFFER) + + def data_received(self, data): + # Manage the buffer + missing = len(data) - len(self.buffer) + self.index + if missing > 0: + if len(self.buffer) + missing > NetstringProtocol.MAX_BUFFER: + raise IOError("Not enough space when decoding netstring") + self.buffer.append(bytearray(missing + 1)) + new_index = self.index + len(data) + self.buffer[self.index:new_index] = data + self.index = new_index + # Try to detect a length at the beginning of the string + if self.len is None: + self.separator = self.buffer.find(0x3a) + if self.separator != -1 and self.buffer[:self.separator].isdigit(): + self.len = int(self.buffer[:self.separator], 10) + # Then get the complete string + if self.len is not None: + if self.index - self.separator == self.len + 2: + string = self.buffer[self.separator + 1:self.index - 1] + self.init_buffer() + self.string_received(string) + + def string_received(self, string): + pass + + def send_string(self, string): + """ Send a netstring + """ + self.transport.write(str(len(string)).encode('ascii')) + self.transport.write(b':') + self.transport.write(string) + self.transport.write(b',') + + +class SocketmapProtocol(NetstringProtocol): + """ Protocol to answer Postfix socketmap and proxify lookups to + an outside object. + + See http://www.postfix.org/socketmap_table.5.html for details on the + protocol. + + A table map must be provided as a dictionary to lookup tables. + """ + + def __init__(self, table_map): + self.table_map = table_map + super(SocketmapProtocol, self).__init__() + + def connection_made(self, transport): + logging.info('Connect {}'.format(transport.get_extra_info('peername'))) + self.transport = transport + + def string_received(self, string): + space = string.find(0x20) + if space != -1: + name = string[:space].decode('ascii') + key = string[space+1:].decode('utf8') + return asyncio.async(self.process_request(name, key)) + + def send_string(self, string): + logging.debug("Send {}".format(string)) + super(SocketmapProtocol, self).send_string(string) + + async def process_request(self, name, key): + """ Process a request by querying the provided map. + """ + logging.debug("Request {}/{}".format(name, key)) + try: + table = self.table_map.get(name) + except KeyError: + return self.send_string(b'TEMP no such map') + try: + result = await table.get(key) + return self.send_string(b'OK ' + str(result).encode('utf8')) + except KeyError: + return self.send_string(b'NOTFOUND ') + except Exception: + logging.exception("Error when processing request") + return self.send_string(b'TEMP unknown error') + + @classmethod + def factory(cls, table_map): + """ Provide a protocol factory for a given map instance. + """ + return lambda: cls(table_map) diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py new file mode 100644 index 00000000..f3b8cc1e --- /dev/null +++ b/core/base/libs/podop/podop/table.py @@ -0,0 +1,26 @@ +""" Table lookup backends for podop +""" + +import aiohttp +import logging + + +class UrlTable(object): + """ Resolve an entry by querying a parametrized GET URL. + """ + + def __init__(self, url_pattern): + """ url_pattern must contain a format ``{}`` so the key is injected in + the url before the query, the ``§`` character will be replaced with + ``{}`` for easier setup. + """ + self.url_pattern = url_pattern.replace('§', '{}') + + async def get(self, key): + logging.debug("Getting {} from url table".format(key)) + async with aiohttp.ClientSession() as session: + async with session.get(self.url_pattern.format(key)) as request: + if request.status == 200: + result = await request.json() + logging.debug("Got {} from url table".format(result)) + return result diff --git a/core/base/libs/podop/scripts/podop b/core/base/libs/podop/scripts/podop new file mode 100755 index 00000000..b22c830d --- /dev/null +++ b/core/base/libs/podop/scripts/podop @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import argparse + +from podop import run_server, SERVER_TYPES, TABLE_TYPES + + +def main(): + """ Run a podop server based on CLI arguments + """ + parser = argparse.ArgumentParser("Postfix and Dovecot proxy") + parser.add_argument("--socket", help="path to the socket", required=True) + parser.add_argument("--mode", choices=SERVER_TYPES.keys(), required=True) + parser.add_argument("--name", help="name of the table", action="append") + parser.add_argument("--type", choices=TABLE_TYPES.keys(), action="append") + parser.add_argument("--param", help="table parameter", action="append") + args = parser.parse_args() + run_server( + args.mode, args.socket, + zip(args.name, args.type, args.param) if args.name else [] + ) + + +if __name__ == "__main__": + main() diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py new file mode 100644 index 00000000..4d1e2ea3 --- /dev/null +++ b/core/base/libs/podop/setup.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +from distutils.core import setup + +setup( + name="Podop", + version="0.1", + description="Postfix and Dovecot proxy", + author="Pierre Jaury", + author_email="pierre@jaury.eu", + url="https://github.com/mailu/podop.git", + packages=["podop"], + scripts=["scripts/podop"] +) From eb6b1866f15bb2745b5fb36138c4a1e656ebaf7f Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Wed, 25 Jul 2018 20:49:16 +0200 Subject: [PATCH 06/44] Specify dependencies in the setup script --- core/base/libs/podop/setup.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index 4d1e2ea3..ac995168 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -2,13 +2,21 @@ from distutils.core import setup +with open("README.md", "r") as fh: + long_description = fh.read() + setup( - name="Podop", - version="0.1", + name="podop", + version="0.1.1", description="Postfix and Dovecot proxy", + long_description=long_description, + long_description_content_type="text/markdown", author="Pierre Jaury", author_email="pierre@jaury.eu", url="https://github.com/mailu/podop.git", packages=["podop"], - scripts=["scripts/podop"] + scripts=["scripts/podop"], + install_requires=[ + "aiohttp" + ] ) From c5fa0280a0e6312d7f734939fb4179fad21158b7 Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 20:48:04 +0200 Subject: [PATCH 07/44] Add support for dovecot dict_set operations --- core/base/libs/podop/podop/__init__.py | 6 ++- core/base/libs/podop/podop/dovecot.py | 69 ++++++++++++++++++++++---- core/base/libs/podop/podop/postfix.py | 8 +-- core/base/libs/podop/podop/table.py | 25 ++++++++-- core/base/libs/podop/scripts/podop | 20 +++++--- core/base/libs/podop/setup.py | 2 +- 6 files changed, 105 insertions(+), 25 deletions(-) diff --git a/core/base/libs/podop/podop/__init__.py b/core/base/libs/podop/podop/__init__.py index 8c2c4d8d..64ab3b67 100644 --- a/core/base/libs/podop/podop/__init__.py +++ b/core/base/libs/podop/podop/__init__.py @@ -5,6 +5,7 @@ It is able to proxify postfix maps and dovecot dicts to any table import asyncio import logging +import sys from podop import postfix, dovecot, table @@ -19,7 +20,7 @@ TABLE_TYPES = dict( ) -def run_server(server_type, socket, tables): +def run_server(verbosity, server_type, socket, tables): """ Run the server, given its type, socket path and table list The table list must be a list of tuples (name, type, param) @@ -30,7 +31,8 @@ def run_server(server_type, socket, tables): for name, table_type, param in tables } # Run the main loop - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(stream=sys.stderr, level=max(3 - verbosity, 0) * 10, + format='%(name)s (%(levelname)s): %(message)s') loop = asyncio.get_event_loop() server = loop.run_until_complete(loop.create_unix_server( SERVER_TYPES[server_type].factory(table_map), socket diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index bbf134cc..96eea3b8 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -3,11 +3,15 @@ import asyncio import logging +import json class DictProtocol(asyncio.Protocol): """ Protocol to answer Dovecot dict requests, as implemented in Dict proxy. + Only a subset of operations is handled properly by this proxy: hello, + lookup and transaction-based set. + There is very little documentation about the protocol, most of it was reverse-engineered from : @@ -20,9 +24,15 @@ class DictProtocol(asyncio.Protocol): def __init__(self, table_map): self.table_map = table_map + # Minor and major versions are not properly checked yet, but stored + # anyway self.major_version = None self.minor_version = None + # Every connection starts with specifying which table is used, dovecot + # tables are called dicts self.dict = None + # Dictionary of active transaction lists per transaction id + self.transactions = {} super(DictProtocol, self).__init__() def connection_made(self, transport): @@ -32,14 +42,17 @@ class DictProtocol(asyncio.Protocol): def data_received(self, data): logging.debug("Received {}".format(data)) results = [] + # Every command is separated by "\n" for line in data.split(b"\n"): - logging.debug("Line {}".format(line)) + # A command must at list have a type and one argument if len(line) < 2: continue + # The command function will handle the command itself command = DictProtocol.COMMANDS.get(line[0]) if command is None: logging.warning('Unknown command {}'.format(line[0])) return self.transport.abort() + # Args are separated by "\t" args = line[1:].strip().split(b"\t") try: future = command(self, *args) @@ -48,22 +61,30 @@ class DictProtocol(asyncio.Protocol): except Exception: logging.exception("Error when processing request") return self.transport.abort() - logging.debug("Results {}".format(results)) + # For asyncio consistency, wait for all results to fire before + # actually returning control return asyncio.gather(*results) def process_hello(self, major, minor, value_type, user, dict_name): + """ Process a dict protocol hello message + """ self.major, self.minor = int(major), int(minor) - logging.debug('Client version {}.{}'.format(self.major, self.minor)) - assert self.major == 2 self.value_type = DictProtocol.DATA_TYPES[int(value_type)] - self.user = user + self.user = user.decode("utf8") self.dict = self.table_map[dict_name.decode("ascii")] - logging.debug("Value type {}, user {}, dict {}".format( - self.value_type, self.user, dict_name)) + logging.debug("Client {}.{} type {}, user {}, dict {}".format( + self.major, self.minor, self.value_type, self.user, dict_name)) async def process_lookup(self, key): + """ Process a dict lookup message + """ logging.debug("Looking up {}".format(key)) - result = await self.dict.get(key.decode("utf8")) + # Priv and shared keys are handled slighlty differently + key_type, key = key.decode("utf8").split("/", 1) + result = await self.dict.get( + key, ns=(self.user if key_type == "priv" else None) + ) + # Handle various response types if result is not None: if type(result) is str: response = result.encode("utf8") @@ -75,6 +96,33 @@ class DictProtocol(asyncio.Protocol): else: return self.reply(b"N") + def process_begin(self, transaction_id): + """ Process a dict begin message + """ + self.transactions[transaction_id] = {} + + def process_set(self, transaction_id, key, value): + """ Process a dict set message + """ + # Nothing is actually set until everything is commited + self.transactions[transaction_id][key] = value + + async def process_commit(self, transaction_id): + """ Process a dict commit message + """ + # Actually handle all set operations from the transaction store + results = [] + for key, value in self.transactions[transaction_id].items(): + logging.debug("Storing {}={}".format(key, value)) + key_type, key = key.decode("utf8").split("/", 1) + result = await self.dict.set( + key, json.loads(value), + ns=(self.user if key_type == "priv" else None) + ) + # Remove stored transaction + del self.transactions[transaction_id] + return self.reply(b"O", transaction_id) + def reply(self, command, *args): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) @@ -91,5 +139,8 @@ class DictProtocol(asyncio.Protocol): COMMANDS = { ord("H"): process_hello, - ord("L"): process_lookup + ord("L"): process_lookup, + ord("B"): process_begin, + ord("C"): process_commit, + ord("S"): process_set } diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py index 122cf962..b0395f35 100644 --- a/core/base/libs/podop/podop/postfix.py +++ b/core/base/libs/podop/podop/postfix.py @@ -51,6 +51,8 @@ class NetstringProtocol(asyncio.Protocol): self.string_received(string) def string_received(self, string): + """ A new netstring was received + """ pass def send_string(self, string): @@ -81,16 +83,14 @@ class SocketmapProtocol(NetstringProtocol): self.transport = transport def string_received(self, string): + # The postfix format contains a space for separating the map name and + # the key space = string.find(0x20) if space != -1: name = string[:space].decode('ascii') key = string[space+1:].decode('utf8') return asyncio.async(self.process_request(name, key)) - def send_string(self, string): - logging.debug("Send {}".format(string)) - super(SocketmapProtocol, self).send_string(string) - async def process_request(self, name, key): """ Process a request by querying the provided map. """ diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py index f3b8cc1e..d30ff9fb 100644 --- a/core/base/libs/podop/podop/table.py +++ b/core/base/libs/podop/podop/table.py @@ -16,11 +16,30 @@ class UrlTable(object): """ self.url_pattern = url_pattern.replace('§', '{}') - async def get(self, key): - logging.debug("Getting {} from url table".format(key)) + async def get(self, key, ns=None): + """ Get the given key in the provided namespace + """ + if ns is not None: + key += "/" + ns async with aiohttp.ClientSession() as session: async with session.get(self.url_pattern.format(key)) as request: if request.status == 200: result = await request.json() - logging.debug("Got {} from url table".format(result)) + return result + + async def set(self, key, value, ns=None): + """ Set a value for the given key in the provided namespace + """ + if ns is not None: + key += "/" + ns + async with aiohttp.ClientSession() as session: + await session.post(self.url_pattern.format(key), json=value) + + async def iter(self, cat): + """ Iterate the given key (experimental) + """ + async with aiohttp.ClientSession() as session: + async with session.get(self.url_pattern.format(cat)) as request: + if request.status == 200: + result = await request.json() return result diff --git a/core/base/libs/podop/scripts/podop b/core/base/libs/podop/scripts/podop index b22c830d..f61c9e21 100755 --- a/core/base/libs/podop/scripts/podop +++ b/core/base/libs/podop/scripts/podop @@ -9,14 +9,22 @@ def main(): """ Run a podop server based on CLI arguments """ parser = argparse.ArgumentParser("Postfix and Dovecot proxy") - parser.add_argument("--socket", help="path to the socket", required=True) - parser.add_argument("--mode", choices=SERVER_TYPES.keys(), required=True) - parser.add_argument("--name", help="name of the table", action="append") - parser.add_argument("--type", choices=TABLE_TYPES.keys(), action="append") - parser.add_argument("--param", help="table parameter", action="append") + parser.add_argument("--socket", required=True, + help="path to the listening unix socket") + parser.add_argument("--mode", choices=SERVER_TYPES.keys(), required=True, + help="select which server will connect to Podop") + parser.add_argument("--name", action="append", + help="name of each configured table") + parser.add_argument("--type", choices=TABLE_TYPES.keys(), action="append", + help="type of each configured table") + parser.add_argument("--param", action="append", + help="mandatory param for each table configured") + parser.add_argument("-v", "--verbose", dest="verbosity", + action="count", default=0, + help="increases log verbosity for each occurence.") args = parser.parse_args() run_server( - args.mode, args.socket, + args.verbosity, args.mode, args.socket, zip(args.name, args.type, args.param) if args.name else [] ) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index ac995168..4951196d 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.1.1", + version="0.2", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From d640da8787dd40fa7a4d59da3dca4e2c8deff7f9 Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:10:07 +0200 Subject: [PATCH 08/44] Include package data in the package --- core/base/libs/podop/MANIFEST.in | 2 ++ core/base/libs/podop/setup.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 core/base/libs/podop/MANIFEST.in diff --git a/core/base/libs/podop/MANIFEST.in b/core/base/libs/podop/MANIFEST.in new file mode 100644 index 00000000..c28ab72d --- /dev/null +++ b/core/base/libs/podop/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include LICENSE.md diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index 4951196d..2d62fd78 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.2", + version="0.2.1", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", @@ -15,6 +15,7 @@ setup( author_email="pierre@jaury.eu", url="https://github.com/mailu/podop.git", packages=["podop"], + include_package_data=True, scripts=["scripts/podop"], install_requires=[ "aiohttp" From 81d171f978d67b03db78af92aa0044f1cbbd2a59 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 17:44:55 +0200 Subject: [PATCH 09/44] Add some debug logging to the table class --- core/base/libs/podop/podop/table.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py index d30ff9fb..ab53179e 100644 --- a/core/base/libs/podop/podop/table.py +++ b/core/base/libs/podop/podop/table.py @@ -19,17 +19,20 @@ class UrlTable(object): async def get(self, key, ns=None): """ Get the given key in the provided namespace """ + logging.debug("Table get {}".format(key)) if ns is not None: key += "/" + ns async with aiohttp.ClientSession() as session: async with session.get(self.url_pattern.format(key)) as request: if request.status == 200: result = await request.json() + logging.debug("Table get {} is {}".format(key, result)) return result async def set(self, key, value, ns=None): """ Set a value for the given key in the provided namespace """ + logging.debug("Table set {} to {}".format(key, value)) if ns is not None: key += "/" + ns async with aiohttp.ClientSession() as session: @@ -38,6 +41,7 @@ class UrlTable(object): async def iter(self, cat): """ Iterate the given key (experimental) """ + logging.debug("Table iter {}".format(cat)) async with aiohttp.ClientSession() as session: async with session.get(self.url_pattern.format(cat)) as request: if request.status == 200: From d2b98ae323e7695e61c409c831b50b01540de945 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 17:45:27 +0200 Subject: [PATCH 10/44] Update to 0.2.2 --- core/base/libs/podop/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index 2d62fd78..82c998bc 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.2.1", + version="0.2.2", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From 814bb1f36db0ea331644eee32ae83fb4f2e99b7f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 20:56:26 +0200 Subject: [PATCH 11/44] Properly miss when the web api returns 404 --- core/base/libs/podop/podop/dovecot.py | 11 +++++------ core/base/libs/podop/podop/postfix.py | 2 ++ core/base/libs/podop/podop/table.py | 4 ++++ core/base/libs/podop/setup.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 96eea3b8..64074e38 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -81,11 +81,10 @@ class DictProtocol(asyncio.Protocol): logging.debug("Looking up {}".format(key)) # Priv and shared keys are handled slighlty differently key_type, key = key.decode("utf8").split("/", 1) - result = await self.dict.get( - key, ns=(self.user if key_type == "priv" else None) - ) - # Handle various response types - if result is not None: + try: + result = await self.dict.get( + key, ns=(self.user if key_type == "priv" else None) + ) if type(result) is str: response = result.encode("utf8") elif type(result) is bytes: @@ -93,7 +92,7 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") return self.reply(b"O", response) - else: + except KeyError: return self.reply(b"N") def process_begin(self, transaction_id): diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py index b0395f35..84c7b08d 100644 --- a/core/base/libs/podop/podop/postfix.py +++ b/core/base/libs/podop/podop/postfix.py @@ -58,6 +58,7 @@ class NetstringProtocol(asyncio.Protocol): def send_string(self, string): """ Send a netstring """ + logging.debug("Replying {}".format(string)) self.transport.write(str(len(string)).encode('ascii')) self.transport.write(b':') self.transport.write(string) @@ -85,6 +86,7 @@ class SocketmapProtocol(NetstringProtocol): def string_received(self, string): # The postfix format contains a space for separating the map name and # the key + logging.debug("Received {}".format(string)) space = string.find(0x20) if space != -1: name = string[:space].decode('ascii') diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py index ab53179e..3869cafc 100644 --- a/core/base/libs/podop/podop/table.py +++ b/core/base/libs/podop/podop/table.py @@ -28,6 +28,10 @@ class UrlTable(object): result = await request.json() logging.debug("Table get {} is {}".format(key, result)) return result + elif request.status == 404: + raise KeyError() + else: + raise Exception(request.status) async def set(self, key, value, ns=None): """ Set a value for the given key in the provided namespace diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index 82c998bc..d681ae53 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.2.2", + version="0.2.3", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From 23e5aa2e051b03ec2f6948f0cb4cbcd5f2034f5e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 27 Sep 2018 13:48:28 +0200 Subject: [PATCH 12/44] Escape strings properly in the Dovecot dict dialect --- core/base/libs/podop/podop/dovecot.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 64074e38..68bfa172 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -125,9 +125,7 @@ class DictProtocol(asyncio.Protocol): def reply(self, command, *args): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) - self.transport.write(b"\t".join( - arg.replace(b"\t", b"\t\t") for arg in args - )) + self.transport.write(b"\t".join(map(tabescape, args))) self.transport.write(b"\n") @classmethod @@ -143,3 +141,25 @@ class DictProtocol(asyncio.Protocol): ord("C"): process_commit, ord("S"): process_set } + + +def tabescape(unescaped): + """ Escape a string using the specific Dovecot tabescape + See: https://github.com/dovecot/core/blob/master/src/lib/strescape.c + """ + return unescaped.replace(b"\x01", b"\x011")\ + .replace(b"\x00", b"\x010")\ + .replace(b"\t", b"\x01t")\ + .replace(b"\n", b"\x01n")\ + .replace(b"\r", b"\x01r") + + +def tabunescape(escaped): + """ Unescape a string using the specific Dovecot tabescape + See: https://github.com/dovecot/core/blob/master/src/lib/strescape.c + """ + return escaped.replace(b"\x01r", b"\r")\ + .replace(b"\x01n", b"\n")\ + .replace(b"\x01t", b"\t")\ + .replace(b"\x010", b"\x00")\ + .replace(b"\x011", b"\x01") From 080e76f972e6e4b1d2ff7497b9a64679bf7029d5 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 7 Oct 2018 13:30:48 +0200 Subject: [PATCH 13/44] Merge pull request #1 from rakshith-ravi/patch-1 Fixed a small typo --- core/base/libs/podop/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/README.md b/core/base/libs/podop/README.md index 208b3ce5..cd4a08a4 100644 --- a/core/base/libs/podop/README.md +++ b/core/base/libs/podop/README.md @@ -64,7 +64,7 @@ virtual_mailbox_maps = socketmap:unix:/tmp/podop.socket:alias In order to simplify the configuration, you can setup a shortcut. ``` -podop = socketmap:unic:/tmp/podop.socket +podop = socketmap:unix:/tmp/podop.socket virtual_alias_maps = ${podop}:alias virtual_mailbox_domains = ${podop}:domain virtual_mailbox_maps = ${podop}:alias From 6fadd39aeafcc5ff0a53aa43886444df4a631ae8 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 6 Jan 2019 13:21:00 +0100 Subject: [PATCH 14/44] Merge pull request #3 from Nebukadneza/add_key_url_quoting URL-Quote the key in HTTP requests --- core/base/libs/podop/podop/table.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py index 3869cafc..0f1ea680 100644 --- a/core/base/libs/podop/podop/table.py +++ b/core/base/libs/podop/podop/table.py @@ -3,7 +3,7 @@ import aiohttp import logging - +from urllib.parse import quote class UrlTable(object): """ Resolve an entry by querying a parametrized GET URL. @@ -23,7 +23,8 @@ class UrlTable(object): if ns is not None: key += "/" + ns async with aiohttp.ClientSession() as session: - async with session.get(self.url_pattern.format(key)) as request: + quoted_key = quote(key) + async with session.get(self.url_pattern.format(quoted_key)) as request: if request.status == 200: result = await request.json() logging.debug("Table get {} is {}".format(key, result)) @@ -40,7 +41,8 @@ class UrlTable(object): if ns is not None: key += "/" + ns async with aiohttp.ClientSession() as session: - await session.post(self.url_pattern.format(key), json=value) + quoted_key = quote(key) + await session.post(self.url_pattern.format(quoted_key), json=value) async def iter(self, cat): """ Iterate the given key (experimental) From e2979f9103c466156e964bd0ad9be7b7b8b7f5d7 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 25 Jun 2019 19:17:35 +0200 Subject: [PATCH 15/44] Merge pull request #6 from Nebukadneza/fix_py37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don’t use deprecated now-keyword "async" --- core/base/libs/podop/podop/postfix.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py index 84c7b08d..3140a3a8 100644 --- a/core/base/libs/podop/podop/postfix.py +++ b/core/base/libs/podop/podop/postfix.py @@ -4,7 +4,6 @@ import asyncio import logging - class NetstringProtocol(asyncio.Protocol): """ Netstring asyncio protocol implementation. @@ -91,7 +90,7 @@ class SocketmapProtocol(NetstringProtocol): if space != -1: name = string[:space].decode('ascii') key = string[space+1:].decode('utf8') - return asyncio.async(self.process_request(name, key)) + return asyncio.ensure_future(self.process_request(name, key)) async def process_request(self, name, key): """ Process a request by querying the provided map. From 3d0d831c76d842244cbd0de1a1a72b2ca6cb75cf Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 14 Jul 2019 13:30:01 +0200 Subject: [PATCH 16/44] Version 0.2.4 --- core/base/libs/podop/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index d681ae53..fda267ce 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.2.3", + version="0.2.4", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From dbec5f0a6cb8b2a727071ec6b81350a4b9b2bbf7 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 14 Jul 2019 13:33:57 +0200 Subject: [PATCH 17/44] Switch to setuptools and bump the version --- core/base/libs/podop/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index fda267ce..68c3e862 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -1,13 +1,13 @@ #!/usr/bin/env python -from distutils.core import setup +from setuptools import setup with open("README.md", "r") as fh: long_description = fh.read() setup( name="podop", - version="0.2.4", + version="0.2.5", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From ce9d886195cc08831111bc20436f29cca0cce2cb Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 25 Aug 2022 14:35:28 +0200 Subject: [PATCH 18/44] Merge pull request #10 from ghostwheel42/add_gitignore Add .gitignore file --- core/base/libs/podop/.gitignore | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 core/base/libs/podop/.gitignore diff --git a/core/base/libs/podop/.gitignore b/core/base/libs/podop/.gitignore new file mode 100644 index 00000000..9a7e75d9 --- /dev/null +++ b/core/base/libs/podop/.gitignore @@ -0,0 +1,20 @@ +.DS_Store +.idea +tmp + +*.bak +*~ +.*.swp + +__pycache__/ +*.pyc +*.pyo +*.egg-info/ + +.build +.env* +.venv + +*.code-workspace + +build/ From 0370b26f3e88886052dca3c73cc7beaa59d24ead Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 14:06:25 +0200 Subject: [PATCH 19/44] Initial commit --- core/base/libs/socrate/LICENSE.md | 21 ++++++++++ core/base/libs/socrate/MANIFEST.in | 2 + core/base/libs/socrate/README.md | 24 +++++++++++ core/base/libs/socrate/setup.py | 24 +++++++++++ core/base/libs/socrate/socrate/__init__.py | 0 core/base/libs/socrate/socrate/conf.py | 46 ++++++++++++++++++++++ core/base/libs/socrate/socrate/system.py | 20 ++++++++++ 7 files changed, 137 insertions(+) create mode 100644 core/base/libs/socrate/LICENSE.md create mode 100644 core/base/libs/socrate/MANIFEST.in create mode 100644 core/base/libs/socrate/README.md create mode 100644 core/base/libs/socrate/setup.py create mode 100644 core/base/libs/socrate/socrate/__init__.py create mode 100644 core/base/libs/socrate/socrate/conf.py create mode 100644 core/base/libs/socrate/socrate/system.py diff --git a/core/base/libs/socrate/LICENSE.md b/core/base/libs/socrate/LICENSE.md new file mode 100644 index 00000000..d360537d --- /dev/null +++ b/core/base/libs/socrate/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Mailu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/base/libs/socrate/MANIFEST.in b/core/base/libs/socrate/MANIFEST.in new file mode 100644 index 00000000..c28ab72d --- /dev/null +++ b/core/base/libs/socrate/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include LICENSE.md diff --git a/core/base/libs/socrate/README.md b/core/base/libs/socrate/README.md new file mode 100644 index 00000000..9b65a83b --- /dev/null +++ b/core/base/libs/socrate/README.md @@ -0,0 +1,24 @@ +Socrate is a simple Python module providing a set of utility functions for +Python daemon applications. + +The scope includes: +- configuration utilities (configuration parsing, etc.) +- system utilities (access to DNS, stats, etc.) + +Setup +====== + +Socrate is available on Pypi, simpy run: + +``` +pip install socrate +``` + + +Contributing +============ + +Podop is free software, open to suggestions and contributions. All +components are free software and compatible with the MIT license. All +the code is placed under the MIT license. + diff --git a/core/base/libs/socrate/setup.py b/core/base/libs/socrate/setup.py new file mode 100644 index 00000000..827fc6e0 --- /dev/null +++ b/core/base/libs/socrate/setup.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +from distutils.core import setup + +with open("README.md", "r") as fh: + long_description = fh.read() + +setup( + name="socrate", + version="0.1", + description="Socrate daemon utilities", + long_description=long_description, + long_description_content_type="text/markdown", + author="Pierre Jaury", + author_email="pierre@jaury.eu", + url="https://github.com/mailu/socrate.git", + packages=["socrate"], + include_package_data=True, + install_requires=[ + "jinja2", + "importlib", + "tenacity" + ] +) diff --git a/core/base/libs/socrate/socrate/__init__.py b/core/base/libs/socrate/socrate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/base/libs/socrate/socrate/conf.py b/core/base/libs/socrate/socrate/conf.py new file mode 100644 index 00000000..5ccde54d --- /dev/null +++ b/core/base/libs/socrate/socrate/conf.py @@ -0,0 +1,46 @@ +import jinja2 +import importlib + + +def jinja(source, environ, destination=None): + """ Render a Jinja configuration file + """ + with open(source, "r") as template: + result = jinja2.Template(template.read()).render(environ) + if destination is not None: + with open(destination, "w") as handle: + handle.write(result) + return result + + +def merge(*objects): + """ Merge simple python objects, which only consist of + strings, integers, bools, lists and dicts + """ + mode = type(objects[0]) + if not all(type(obj) is mode for obj in objects): + raise ValueError("Cannot merge mixed typed objects") + if len(objects) == 1: + return objects[0] + elif mode is dict: + return { + key: merge(*[obj[key] for obj in objects if key in obj]) + for obj in objects for key in obj.keys() + } + elif mode is list: + return sum(objects) + else: + raise ValueError("Cannot merge objects of type {}: {}".format( + mode, objects)) + + +def resolve_function(function, cache={}): + """ Resolve a fully qualified function name in Python, and caches + the result + """ + if function not in cache: + module, name = function.rsplit(".", 1) + cache[function] = getattr(importlib.import_module(module), name) + return cache[function] + + diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py new file mode 100644 index 00000000..64a87d15 --- /dev/null +++ b/core/base/libs/socrate/socrate/system.py @@ -0,0 +1,20 @@ +import socket +import tenacity + + +@retry(stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5)) +def resolve_hostname(hostname): + """ This function uses system DNS to resolve a hostname. + It is capable of retrying in case the host is not immediately available + """ + return socket.gethostbyname(hostname) + + +def resolve_address(address): + """ This function is identical to ``resolve_host`` but also supports + resolving an address, i.e. including a port. + """ + hostname, *rest = address.resplit(":", 1) + ip_address = resolve_hostname(hostname) + return ip_address + "".join(":" + port for port in rest) From 351b05b92d630a6778bb23fd331ac693e71d2d77 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 14:57:18 +0200 Subject: [PATCH 20/44] Allow jinja to load from file path or handle --- core/base/libs/socrate/socrate/conf.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/core/base/libs/socrate/socrate/conf.py b/core/base/libs/socrate/socrate/conf.py index 5ccde54d..d792879c 100644 --- a/core/base/libs/socrate/socrate/conf.py +++ b/core/base/libs/socrate/socrate/conf.py @@ -3,13 +3,22 @@ import importlib def jinja(source, environ, destination=None): - """ Render a Jinja configuration file + """ Render a Jinja configuration file, supports file handle or path """ - with open(source, "r") as template: - result = jinja2.Template(template.read()).render(environ) + close_source = close_destination = False + if type(source) is str: + source = open(source, "r") + close_source = True + if type(destination) is str: + destination = open(destination, "w") + close_destination = True + result = jinja2.Template(source.read()).render(environ) + if close_source: + source.close() if destination is not None: - with open(destination, "w") as handle: - handle.write(result) + destination.write(result) + if close_destination: + destination.close() return result @@ -28,7 +37,7 @@ def merge(*objects): for obj in objects for key in obj.keys() } elif mode is list: - return sum(objects) + return sum(objects, []) else: raise ValueError("Cannot merge objects of type {}: {}".format( mode, objects)) From 74a3e87de3f970e734cf8c9c1b9422744b6f1b0b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 14:57:38 +0200 Subject: [PATCH 21/44] Fix a couple syntax typos --- core/base/libs/socrate/socrate/system.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 64a87d15..b3b95012 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -2,8 +2,8 @@ import socket import tenacity -@retry(stop=tenacity.stop_after_attempt(100), - wait=tenacity.wait_random(min=2, max=5)) +@tenacity.retry(stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5)) def resolve_hostname(hostname): """ This function uses system DNS to resolve a hostname. It is capable of retrying in case the host is not immediately available @@ -15,6 +15,6 @@ def resolve_address(address): """ This function is identical to ``resolve_host`` but also supports resolving an address, i.e. including a port. """ - hostname, *rest = address.resplit(":", 1) + hostname, *rest = address.rsplit(":", 1) ip_address = resolve_hostname(hostname) return ip_address + "".join(":" + port for port in rest) From ef344c62f6117e6ab891d0c733dfd67ca401581c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 14:57:45 +0200 Subject: [PATCH 22/44] Add automated tests --- core/base/libs/socrate/test.py | 80 ++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 core/base/libs/socrate/test.py diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py new file mode 100644 index 00000000..efdcb343 --- /dev/null +++ b/core/base/libs/socrate/test.py @@ -0,0 +1,80 @@ +import unittest +import io + +from socrate import conf, system + + +class TestConf(unittest.TestCase): + """ Test configuration functions + """ + + MERGE_EXPECTATIONS = [ + ({"a": "1", "b": "2", "c": "3", "d": "4"}, + {"a": "1", "b": "2"}, + {"c": "3", "d": "4"}), + + ({"a": [1, 2, 3, 4, 5], "b": "4"}, + {"a": [1, 2, 3], "b": "4"}, + {"a": [4, 5]}), + + ({"a": {"x": "1", "y": "2", "z": 3}, "b": 4, "c": "5"}, + {"a": {"x": "1", "y": "2"}, "b": 4}, + {"a": {"z": 3}, "c": "5"}) + ] + + def test_jinja(self): + template = "Test {{ variable }}" + environ = {"variable": "ok"} + self.assertEqual( + conf.jinja(io.StringIO(template), environ), + "Test ok" + ) + result = io.StringIO() + conf.jinja(io.StringIO(template), environ, result) + self.assertEqual( + result.getvalue(), + "Test ok" + ) + + def test_merge(self): + for result, *parts in TestConf.MERGE_EXPECTATIONS: + self.assertEqual(result, conf.merge(*parts)) + + def test_merge_failure(self): + with self.assertRaises(ValueError): + conf.merge({"a": 1}, {"a": 2}) + with self.assertRaises(ValueError): + conf.merge(1, "a") + + def test_resolve(self): + self.assertEqual( + conf.resolve_function("unittest.TestCase"), + unittest.TestCase + ) + self.assertEqual( + conf.resolve_function("unittest.util.strclass"), + unittest.util.strclass + ) + + def test_resolve_failure(self): + with self.assertRaises(AttributeError): + conf.resolve_function("unittest.inexistant") + with self.assertRaises(ModuleNotFoundError): + conf.resolve_function("inexistant.function") + + +class TestSystem(unittest.TestCase): + """ Test the system functions + """ + + def test_resolve_hostname(self): + self.assertEqual( + system.resolve_hostname("one.one.one.one"), + "1.1.1.1" + ) + + def test_resolve_address(self): + self.assertEqual( + system.resolve_address("one.one.one.one:80"), + "1.1.1.1:80" + ) From 7f6d51904bf546473c58ec3bfb90d19d5ac16ead Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 15:09:49 +0200 Subject: [PATCH 23/44] Remove wrong dependency to importlib --- core/base/libs/socrate/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/socrate/setup.py b/core/base/libs/socrate/setup.py index 827fc6e0..da1c9520 100644 --- a/core/base/libs/socrate/setup.py +++ b/core/base/libs/socrate/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import setuptools from distutils.core import setup with open("README.md", "r") as fh: @@ -7,7 +8,7 @@ with open("README.md", "r") as fh: setup( name="socrate", - version="0.1", + version="0.1.1", description="Socrate daemon utilities", long_description=long_description, long_description_content_type="text/markdown", @@ -18,7 +19,6 @@ setup( include_package_data=True, install_requires=[ "jinja2", - "importlib", "tenacity" ] ) From b198fde75692387daebddbe4235aa811b0d81697 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 25 Aug 2019 11:51:18 +0200 Subject: [PATCH 24/44] Merge pull request #3 from micw/fix-random-failures Change test hostnames for stable test results --- core/base/libs/socrate/.gitignore | 1 + core/base/libs/socrate/test.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 core/base/libs/socrate/.gitignore diff --git a/core/base/libs/socrate/.gitignore b/core/base/libs/socrate/.gitignore new file mode 100644 index 00000000..c18dd8d8 --- /dev/null +++ b/core/base/libs/socrate/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py index efdcb343..b9b36d5f 100644 --- a/core/base/libs/socrate/test.py +++ b/core/base/libs/socrate/test.py @@ -69,12 +69,12 @@ class TestSystem(unittest.TestCase): def test_resolve_hostname(self): self.assertEqual( - system.resolve_hostname("one.one.one.one"), - "1.1.1.1" + system.resolve_hostname("1.2.3.4.xip.io"), + "1.2.3.4" ) def test_resolve_address(self): self.assertEqual( - system.resolve_address("one.one.one.one:80"), - "1.1.1.1:80" + system.resolve_address("1.2.3.4.xip.io:80"), + "1.2.3.4:80" ) From 68d44201abc6b0813fe77032966b98778285fcdf Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 25 Aug 2019 14:06:31 +0200 Subject: [PATCH 25/44] Merge pull request #4 from micw/resolve-host-if-address-not-set Resolve host if address not set --- core/base/libs/socrate/socrate/system.py | 12 ++++++++++ core/base/libs/socrate/test.py | 28 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index b3b95012..99e11bbb 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -1,5 +1,6 @@ import socket import tenacity +from os import environ @tenacity.retry(stop=tenacity.stop_after_attempt(100), @@ -18,3 +19,14 @@ def resolve_address(address): hostname, *rest = address.rsplit(":", 1) ip_address = resolve_hostname(hostname) return ip_address + "".join(":" + port for port in rest) + + +def get_host_address_from_environment(name, default): + """ This function looks up an envionment variable ``{{ name }}_ADDRESS``. + If it's defined, it is returned unmodified. If it's undefined, an environment + variable ``HOST_{{ name }}`` is looked up and resolved to an ip address. + If this is also not defined, the default is resolved to an ip address. + """ + if "{}_ADDRESS".format(name) in environ: + return environ.get("{}_ADDRESS".format(name)) + return resolve_address(environ.get("HOST_{}".format(name), default)) diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py index b9b36d5f..6fc87bfa 100644 --- a/core/base/libs/socrate/test.py +++ b/core/base/libs/socrate/test.py @@ -1,5 +1,6 @@ import unittest import io +import os from socrate import conf, system @@ -78,3 +79,30 @@ class TestSystem(unittest.TestCase): system.resolve_address("1.2.3.4.xip.io:80"), "1.2.3.4:80" ) + + def test_get_host_address_from_environment(self): + if "TEST_ADDRESS" in os.environ: + del os.environ["TEST_ADDRESS"] + if "HOST_TEST" in os.environ: + del os.environ["HOST_TEST"] + # if nothing is set, the default must be resolved + self.assertEqual( + system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + "1.2.3.4:80" + ) + # if HOST is set, the HOST must be resolved + os.environ['HOST_TEST']="1.2.3.5.xip.io:80" + self.assertEqual( + system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + "1.2.3.5:80" + ) + # if ADDRESS is set, the ADDRESS must be returned unresolved + os.environ['TEST_ADDRESS']="1.2.3.6.xip.io:80" + self.assertEqual( + system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + "1.2.3.6.xip.io:80" + ) + + +if __name__ == "__main__": + unittest.main() From f63837b8e162188d8b6a95a07bc364a31891566f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 26 Aug 2019 22:44:55 +0200 Subject: [PATCH 26/44] Update to 0.2.0 --- core/base/libs/socrate/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/socrate/setup.py b/core/base/libs/socrate/setup.py index da1c9520..25598a87 100644 --- a/core/base/libs/socrate/setup.py +++ b/core/base/libs/socrate/setup.py @@ -8,7 +8,7 @@ with open("README.md", "r") as fh: setup( name="socrate", - version="0.1.1", + version="0.2.0", description="Socrate daemon utilities", long_description=long_description, long_description_content_type="text/markdown", From c0066abd0187ae5b406bf811ed01fcc7a8c1787b Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Thu, 25 Aug 2022 15:31:11 +0200 Subject: [PATCH 27/44] Merge pull request #6 from micw/log-failed-dns Add logging for failed DNS lookups --- core/base/libs/socrate/socrate/system.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 99e11bbb..a43aba0d 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -1,7 +1,7 @@ import socket import tenacity from os import environ - +import logging as log @tenacity.retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) @@ -9,7 +9,11 @@ def resolve_hostname(hostname): """ This function uses system DNS to resolve a hostname. It is capable of retrying in case the host is not immediately available """ - return socket.gethostbyname(hostname) + try: + return socket.gethostbyname(hostname) + except Exception as e: + log.warn("Unable to lookup '%s': %s",hostname,e) + raise e def resolve_address(address): From b711f930ef877f121e6e5424a70cab9c295cebe2 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Thu, 25 Aug 2022 15:31:50 +0200 Subject: [PATCH 28/44] Merge pull request #9 from vavanade/patch-1 fix docstring --- core/base/libs/socrate/socrate/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index a43aba0d..65aa9367 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -17,7 +17,7 @@ def resolve_hostname(hostname): def resolve_address(address): - """ This function is identical to ``resolve_host`` but also supports + """ This function is identical to ``resolve_hostname`` but also supports resolving an address, i.e. including a port. """ hostname, *rest = address.rsplit(":", 1) From 9f511faf6476caa1a26b0094504f01191828e28c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 25 Aug 2022 16:41:39 +0200 Subject: [PATCH 29/44] Merge pull request #8 from NeverBehave/master fix: resolve IPv6 container hostname --- core/base/libs/socrate/.gitignore | 21 ++++++++++++++++++++ core/base/libs/socrate/socrate/system.py | 4 +++- core/base/libs/socrate/test.py | 25 ++++++++++++++++-------- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/core/base/libs/socrate/.gitignore b/core/base/libs/socrate/.gitignore index c18dd8d8..9305b479 100644 --- a/core/base/libs/socrate/.gitignore +++ b/core/base/libs/socrate/.gitignore @@ -1 +1,22 @@ +.DS_Store +.idea +tmp + +*.bak +*~ +.*.swp + __pycache__/ +*.pyc +*.pyo +*.egg-info/ + +.build +.env* +.venv + +*.code-workspace + +venv/ +build/ +dist/ diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 65aa9367..d4e3802a 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -10,7 +10,7 @@ def resolve_hostname(hostname): It is capable of retrying in case the host is not immediately available """ try: - return socket.gethostbyname(hostname) + return sorted(socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE), key=lambda s:s[0])[0][4][0] except Exception as e: log.warn("Unable to lookup '%s': %s",hostname,e) raise e @@ -22,6 +22,8 @@ def resolve_address(address): """ hostname, *rest = address.rsplit(":", 1) ip_address = resolve_hostname(hostname) + if ":" in ip_address: + ip_address = "[{}]".format(ip_address) return ip_address + "".join(":" + port for port in rest) diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py index 6fc87bfa..f6088345 100644 --- a/core/base/libs/socrate/test.py +++ b/core/base/libs/socrate/test.py @@ -70,15 +70,24 @@ class TestSystem(unittest.TestCase): def test_resolve_hostname(self): self.assertEqual( - system.resolve_hostname("1.2.3.4.xip.io"), + system.resolve_hostname("1.2.3.4.sslip.io"), "1.2.3.4" ) + self.assertEqual( + system.resolve_hostname("2001-db8--f00.sslip.io"), + "2001:db8::f00" + ) + def test_resolve_address(self): self.assertEqual( - system.resolve_address("1.2.3.4.xip.io:80"), + system.resolve_address("1.2.3.4.sslip.io:80"), "1.2.3.4:80" ) + self.assertEqual( + system.resolve_address("2001-db8--f00.sslip.io:80"), + "[2001:db8::f00]:80" + ) def test_get_host_address_from_environment(self): if "TEST_ADDRESS" in os.environ: @@ -87,20 +96,20 @@ class TestSystem(unittest.TestCase): del os.environ["HOST_TEST"] # if nothing is set, the default must be resolved self.assertEqual( - system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"), "1.2.3.4:80" ) # if HOST is set, the HOST must be resolved - os.environ['HOST_TEST']="1.2.3.5.xip.io:80" + os.environ['HOST_TEST']="1.2.3.5.sslip.io:80" self.assertEqual( - system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"), "1.2.3.5:80" ) # if ADDRESS is set, the ADDRESS must be returned unresolved - os.environ['TEST_ADDRESS']="1.2.3.6.xip.io:80" + os.environ['TEST_ADDRESS']="1.2.3.6.sslip.io:80" self.assertEqual( - system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), - "1.2.3.6.xip.io:80" + system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"), + "1.2.3.6.sslip.io:80" ) From 8668b269cdb25f935adb57837b31afc4e32b0e23 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:15:21 +0200 Subject: [PATCH 30/44] Add requirements.txt for base image --- core/base/libs/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 core/base/libs/requirements.txt diff --git a/core/base/libs/requirements.txt b/core/base/libs/requirements.txt new file mode 100644 index 00000000..0e537e6a --- /dev/null +++ b/core/base/libs/requirements.txt @@ -0,0 +1,2 @@ +libs/podop/ +libs/socrate/ From 768c0cc1cebb7307d7c92b413d21560b1c3a1938 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 29 Sep 2022 11:28:44 +0200 Subject: [PATCH 31/44] Fix assets build process --- core/admin/assets/Dockerfile | 7 +++++-- core/admin/assets/content/{ => assets}/app.css | 0 core/admin/assets/content/{ => assets}/app.js | 0 core/admin/assets/content/{ => assets}/mailu.png | Bin core/admin/assets/content/{ => assets}/vendor.js | 0 5 files changed, 5 insertions(+), 2 deletions(-) rename core/admin/assets/content/{ => assets}/app.css (100%) rename core/admin/assets/content/{ => assets}/app.js (100%) rename core/admin/assets/content/{ => assets}/mailu.png (100%) rename core/admin/assets/content/{ => assets}/vendor.js (100%) diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile index d799fa4f..2fb6f9b6 100644 --- a/core/admin/assets/Dockerfile +++ b/core/admin/assets/Dockerfile @@ -2,10 +2,13 @@ FROM node:16-alpine3.16 -COPY content ./ +WORKDIR /work + +COPY content /work + RUN set -euxo pipefail \ && npm config set update-notifier false \ - && npm install --no-fund \ + && npm install --no-audit --no-fund \ && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ diff --git a/core/admin/assets/content/app.css b/core/admin/assets/content/assets/app.css similarity index 100% rename from core/admin/assets/content/app.css rename to core/admin/assets/content/assets/app.css diff --git a/core/admin/assets/content/app.js b/core/admin/assets/content/assets/app.js similarity index 100% rename from core/admin/assets/content/app.js rename to core/admin/assets/content/assets/app.js diff --git a/core/admin/assets/content/mailu.png b/core/admin/assets/content/assets/mailu.png similarity index 100% rename from core/admin/assets/content/mailu.png rename to core/admin/assets/content/assets/mailu.png diff --git a/core/admin/assets/content/vendor.js b/core/admin/assets/content/assets/vendor.js similarity index 100% rename from core/admin/assets/content/vendor.js rename to core/admin/assets/content/assets/vendor.js From 52dd09d45230d035add1d8b39d85826f0f7c306e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 29 Sep 2022 12:49:37 +0200 Subject: [PATCH 32/44] Fix assets build process #2 --- core/admin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index c3731fb2..c1c63a29 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -21,7 +21,7 @@ COPY migrations ./migrations COPY start.py /start.py COPY audit.py /audit.py -COPY --from=assets static ./mailu/static +COPY --from=assets /work/static ./mailu/static RUN echo $VERSION >> /version From a29f0668584d9d689e2f73ece8c4c56998f7d835 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 8 Oct 2022 15:55:40 +0200 Subject: [PATCH 33/44] Move even more python deps to base image --- core/admin/Dockerfile | 23 +++--- core/admin/assets/Dockerfile | 3 +- core/admin/audit.py | 2 +- core/admin/requirements-dev.txt | 28 +++++++ core/admin/requirements-prod.txt | 78 ------------------ core/admin/requirements.txt | 106 ++++++++++++++++++------- core/admin/start.py | 2 +- core/base/Dockerfile | 59 ++++++++++---- core/base/{libs => }/requirements.txt | 0 core/dovecot/Dockerfile | 11 +-- core/dovecot/requirements.txt | 2 + core/dovecot/start.py | 2 +- core/nginx/Dockerfile | 18 ++--- core/nginx/certwatcher.py | 2 +- core/nginx/config.py | 2 +- core/nginx/letsencrypt.py | 2 +- core/nginx/requirements.txt | 2 + core/nginx/start.py | 2 +- core/none/Dockerfile | 4 +- core/postfix/Dockerfile | 13 ++- core/postfix/requirements.txt | 3 + core/postfix/start.py | 2 +- core/rspamd/Dockerfile | 11 +-- core/rspamd/requirements.txt | 1 + core/rspamd/start.py | 2 +- optional/clamav/Dockerfile | 28 +++---- optional/clamav/health.sh | 8 -- optional/clamav/start.py | 2 +- optional/fetchmail/Dockerfile | 26 +++--- optional/fetchmail/fetchmail.py | 2 +- optional/fetchmail/requirements.txt | 1 + optional/radicale/Dockerfile | 26 +++--- optional/radicale/requirements.txt | 2 + optional/traefik-certdumper/Dockerfile | 12 ++- optional/unbound/Dockerfile | 38 ++++----- optional/unbound/start.py | 2 +- tests/build.hcl | 62 +++++++++------ 37 files changed, 308 insertions(+), 281 deletions(-) create mode 100644 core/admin/requirements-dev.txt delete mode 100644 core/admin/requirements-prod.txt rename core/base/{libs => }/requirements.txt (100%) create mode 100644 core/dovecot/requirements.txt create mode 100644 core/nginx/requirements.txt create mode 100644 core/postfix/requirements.txt create mode 100644 core/rspamd/requirements.txt delete mode 100755 optional/clamav/health.sh create mode 100644 optional/fetchmail/requirements.txt create mode 100644 optional/radicale/requirements.txt diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index c1c63a29..ca98662d 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -1,29 +1,26 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# admin image FROM base ARG VERSION=local LABEL version=$VERSION -COPY requirements-prod.txt requirements.txt RUN set -euxo pipefail \ - && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ - && pip install --no-cache-dir -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ - || ( apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ - && pip install -r requirements.txt \ - && apk del --no-cache build-dep ) + ; apk add --no-cache curl libressl mariadb-connector-c postgresql-libs -COPY mailu ./mailu -RUN pybabel compile -d mailu/translations +COPY mailu/ ./mailu/ +RUN set -euxo pipefail \ + ; venv/bin/pybabel compile -d mailu/translations -COPY migrations ./migrations +COPY migrations/ ./migrations/ -COPY start.py /start.py -COPY audit.py /audit.py +COPY audit.py / +COPY start.py / -COPY --from=assets /work/static ./mailu/static +COPY --from=assets /work/static/ ./mailu/static/ -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 80/tcp HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile index 2fb6f9b6..c8556f47 100644 --- a/core/admin/assets/Dockerfile +++ b/core/admin/assets/Dockerfile @@ -4,7 +4,7 @@ FROM node:16-alpine3.16 WORKDIR /work -COPY content /work +COPY content/ ./ RUN set -euxo pipefail \ && npm config set update-notifier false \ @@ -14,4 +14,3 @@ RUN set -euxo pipefail \ cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ done \ && node_modules/.bin/webpack-cli --color - diff --git a/core/admin/audit.py b/core/admin/audit.py index 60583f83..31ee9665 100755 --- a/core/admin/audit.py +++ b/core/admin/audit.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import sys import tabulate diff --git a/core/admin/requirements-dev.txt b/core/admin/requirements-dev.txt new file mode 100644 index 00000000..5bf57874 --- /dev/null +++ b/core/admin/requirements-dev.txt @@ -0,0 +1,28 @@ +Flask +Flask-Login +Flask-SQLAlchemy +Flask-bootstrap +Flask-Babel +Flask-migrate +Flask-script +Flask-wtf +Flask-debugtoolbar +limits +redis +WTForms-Components +socrate +passlib +gunicorn +tabulate +PyYAML +PyOpenSSL +Pygments +dnspython +tenacity +mysql-connector-python +idna +srslib +marshmallow +flask-marshmallow +marshmallow-sqlalchemy +xmltodict diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt deleted file mode 100644 index 297c6902..00000000 --- a/core/admin/requirements-prod.txt +++ /dev/null @@ -1,78 +0,0 @@ -alembic==1.7.4 -appdirs==1.4.4 -Babel==2.9.1 -bcrypt==3.2.0 -blinker==1.4 -CacheControl==0.12.9 -certifi==2021.10.8 -# cffi==1.15.0 -chardet==4.0.0 -click==8.0.3 -colorama==0.4.4 -contextlib2==21.6.0 -cryptography==35.0.0 -decorator==5.1.0 -# distlib==0.3.1 -# distro==1.5.0 -dnspython==2.1.0 -dominate==2.6.0 -email-validator==1.1.3 -Flask==2.0.2 -Flask-Babel==2.0.0 -Flask-Bootstrap==3.3.7.1 -Flask-DebugToolbar==0.11.0 -Flask-Limiter==1.4 -Flask-Login==0.5.0 -flask-marshmallow==0.14.0 -Flask-Migrate==3.1.0 -Flask-Script==2.0.6 -Flask-SQLAlchemy==2.5.1 -Flask-WTF==0.15.1 -greenlet==1.1.2 -gunicorn==20.1.0 -html5lib==1.1 -idna==3.3 -infinity==1.5 -intervals==0.9.2 -itsdangerous==2.0.1 -Jinja2==3.0.2 -limits==1.5.1 -lockfile==0.12.2 -Mako==1.1.5 -MarkupSafe==2.0.1 -marshmallow==3.14.0 -marshmallow-sqlalchemy==0.26.1 -msgpack==1.0.2 -# mysqlclient==2.0.3 -mysql-connector-python==8.0.25 -ordered-set==4.0.2 -# packaging==20.9 -passlib==1.7.4 -# pep517==0.10.0 -progress==1.6 -#psycopg2==2.9.1 -psycopg2-binary==2.9.3 -pycparser==2.20 -Pygments==2.10.0 -pyOpenSSL==21.0.0 -pyparsing==3.0.4 -pytz==2021.3 -PyYAML==6.0 -redis==3.5.3 -requests==2.26.0 -retrying==1.3.3 -# six==1.15.0 -socrate==0.2.0 -SQLAlchemy==1.4.26 -srslib==0.1.4 -tabulate==0.8.9 -tenacity==8.0.1 -toml==0.10.2 -urllib3==1.26.7 -validators==0.18.2 -visitor==0.1.3 -webencodings==0.5.1 -Werkzeug==2.0.2 -WTForms==2.3.3 -WTForms-Components==0.10.5 -xmltodict==0.12.0 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index 5bf57874..297c6902 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -1,28 +1,78 @@ -Flask -Flask-Login -Flask-SQLAlchemy -Flask-bootstrap -Flask-Babel -Flask-migrate -Flask-script -Flask-wtf -Flask-debugtoolbar -limits -redis -WTForms-Components -socrate -passlib -gunicorn -tabulate -PyYAML -PyOpenSSL -Pygments -dnspython -tenacity -mysql-connector-python -idna -srslib -marshmallow -flask-marshmallow -marshmallow-sqlalchemy -xmltodict +alembic==1.7.4 +appdirs==1.4.4 +Babel==2.9.1 +bcrypt==3.2.0 +blinker==1.4 +CacheControl==0.12.9 +certifi==2021.10.8 +# cffi==1.15.0 +chardet==4.0.0 +click==8.0.3 +colorama==0.4.4 +contextlib2==21.6.0 +cryptography==35.0.0 +decorator==5.1.0 +# distlib==0.3.1 +# distro==1.5.0 +dnspython==2.1.0 +dominate==2.6.0 +email-validator==1.1.3 +Flask==2.0.2 +Flask-Babel==2.0.0 +Flask-Bootstrap==3.3.7.1 +Flask-DebugToolbar==0.11.0 +Flask-Limiter==1.4 +Flask-Login==0.5.0 +flask-marshmallow==0.14.0 +Flask-Migrate==3.1.0 +Flask-Script==2.0.6 +Flask-SQLAlchemy==2.5.1 +Flask-WTF==0.15.1 +greenlet==1.1.2 +gunicorn==20.1.0 +html5lib==1.1 +idna==3.3 +infinity==1.5 +intervals==0.9.2 +itsdangerous==2.0.1 +Jinja2==3.0.2 +limits==1.5.1 +lockfile==0.12.2 +Mako==1.1.5 +MarkupSafe==2.0.1 +marshmallow==3.14.0 +marshmallow-sqlalchemy==0.26.1 +msgpack==1.0.2 +# mysqlclient==2.0.3 +mysql-connector-python==8.0.25 +ordered-set==4.0.2 +# packaging==20.9 +passlib==1.7.4 +# pep517==0.10.0 +progress==1.6 +#psycopg2==2.9.1 +psycopg2-binary==2.9.3 +pycparser==2.20 +Pygments==2.10.0 +pyOpenSSL==21.0.0 +pyparsing==3.0.4 +pytz==2021.3 +PyYAML==6.0 +redis==3.5.3 +requests==2.26.0 +retrying==1.3.3 +# six==1.15.0 +socrate==0.2.0 +SQLAlchemy==1.4.26 +srslib==0.1.4 +tabulate==0.8.9 +tenacity==8.0.1 +toml==0.10.2 +urllib3==1.26.7 +validators==0.18.2 +visitor==0.1.3 +webencodings==0.5.1 +Werkzeug==2.0.2 +WTForms==2.3.3 +WTForms-Components==0.10.5 +xmltodict==0.12.0 diff --git a/core/admin/start.py b/core/admin/start.py index 8bb1cef1..ac8bd526 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import logging as log diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 71ac6e31..cd590596 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -1,30 +1,57 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# base system image (intermediate) ARG DISTRO=alpine:3.14.5 -FROM $DISTRO +FROM $DISTRO as system ENV TZ Etc/UTC ENV LANG C.UTF-8 -# TODO: use intermediate image to build virtual env - RUN set -euxo pipefail \ - && adduser -s /bin/bash -Dh /app -k /var/empty -u 1000 -g mailu app \ - && apk add --no-cache bash ca-certificates tzdata python3 py3-pip py3-wheel \ - && pip3 install --no-cache-dir --upgrade pip + ; adduser -s /bin/bash -Dh /app -k /var/empty -u 1000 -g mailu app \ + ; apk add --no-cache bash ca-certificates python3 tzdata WORKDIR /app -COPY libs libs/ +CMD /bin/bash + + +# build virtual env (intermediate) +FROM system as build + +ENV VIRTUAL_ENV=/app/venv -# TODO: work in virtual env (see above) -# && python3 -m venv . \ RUN set -euxo pipefail \ - && pip3 install --no-cache-dir -r libs/requirements.txt --only-binary=:all: \ - || ( apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ - && pip3 install --no-cache-dir -r libs/requirements.txt \ - && apk del --no-cache .build-deps ) + ; apk add --no-cache py3-pip \ + ; python3 -m venv ${VIRTUAL_ENV} \ + ; venv/bin/pip install --no-cache-dir --upgrade --no-warn-script-location pip wheel -# TODO: clean image (or use intermediate - see above) -# && bin/pip uninstall -y pip distribute setuptools wheel \ -# && rm -rf /tmp/* /root/.cache/pip +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +COPY libs/ libs/ +COPY --from=core ./ core/ +COPY --from=optional ./ optional/ + +RUN set -euxo pipefail \ + ; grep -hEv '(podop|socrate)==' core/*/requirements.txt optional/*/requirements.txt \ + | sort -u >libs/requirements.txt \ +\ + ; venv/bin/pip install --no-cache-dir -r libs/requirements.txt \ + || ( \ + apk add --no-cache --virtual .build-deps \ + build-base cargo gcc libffi-dev libressl-dev mariadb-connector-c-dev \ + musl-dev postgresql-dev python3-dev \ + ; venv/bin/pip install --no-cache-dir -r libs/requirements.txt \ + ; apk del .build-deps \ + ) \ +\ + ; venv/bin/pip freeze > venv/requirements.txt + + +# base mailu image +FROM system + +COPY --from=build /app/venv/ /app/venv/ + +ENV VIRTUAL_ENV=/app/venv +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" diff --git a/core/base/libs/requirements.txt b/core/base/requirements.txt similarity index 100% rename from core/base/libs/requirements.txt rename to core/base/requirements.txt diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 2d74e59b..0796e587 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -1,18 +1,19 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# dovecot image FROM base ARG VERSION LABEL version=$VERSION RUN set -euxo pipefail \ - && apk add --no-cache dovecot dovecot-lmtpd dovecot-pop3d dovecot-submissiond dovecot-pigeonhole-plugin rspamd-client xapian-core dovecot-fts-xapian \ - && mkdir /var/lib/dovecot + ; apk add --no-cache dovecot dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond rspamd-client xapian-core \ + ; mkdir /var/lib/dovecot -COPY conf /conf -COPY start.py /start.py +COPY conf/ /conf/ +COPY start.py / -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready." diff --git a/core/dovecot/requirements.txt b/core/dovecot/requirements.txt new file mode 100644 index 00000000..16005f74 --- /dev/null +++ b/core/dovecot/requirements.txt @@ -0,0 +1,2 @@ +podop==0.2.5 +socrate==0.2.0 diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 03bdfa80..a8c85ebf 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import glob diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 2a34403f..cbb9cd7c 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -1,29 +1,29 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# build static assets (intermediate) FROM base as static -COPY static /static +COPY static/ /static/ RUN set -euxo pipefail \ - && gzip -k9 /static/*.ico /static/*.txt \ - && chmod a+rX-w -R /static + ; gzip -k9 /static/*.ico /static/*.txt \ + ; chmod a+rX-w -R /static +# nginx image FROM base ARG VERSION LABEL version=$VERSION -# Image specific layers under this line RUN set -euxo pipefail \ - && apk add --no-cache certbot nginx nginx-mod-mail openssl curl \ - && pip3 install --no-cache-dir watchdog + ; apk add --no-cache certbot curl nginx nginx-mod-mail openssl -COPY conf /conf -COPY --from=static /static /static +COPY conf/ /conf/ +COPY --from=static /static/ /static/ COPY *.py / -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://localhost/health diff --git a/core/nginx/certwatcher.py b/core/nginx/certwatcher.py index 96ccdd7c..e86fc9ec 100755 --- a/core/nginx/certwatcher.py +++ b/core/nginx/certwatcher.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 """ Certificate watcher which reloads nginx or reconfigures it, depending on what happens to externally supplied certificates. Only executed by start.py in case diff --git a/core/nginx/config.py b/core/nginx/config.py index e9c4b50e..7930ff12 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import logging as log diff --git a/core/nginx/letsencrypt.py b/core/nginx/letsencrypt.py index e636dac9..993e7f9f 100755 --- a/core/nginx/letsencrypt.py +++ b/core/nginx/letsencrypt.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import time diff --git a/core/nginx/requirements.txt b/core/nginx/requirements.txt new file mode 100644 index 00000000..c96c3bb8 --- /dev/null +++ b/core/nginx/requirements.txt @@ -0,0 +1,2 @@ +socrate==0.2.0 +watchdog==2.1.9 diff --git a/core/nginx/start.py b/core/nginx/start.py index 8673f148..07932211 100755 --- a/core/nginx/start.py +++ b/core/nginx/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import subprocess diff --git a/core/none/Dockerfile b/core/none/Dockerfile index 058b18c5..f06cc31c 100644 --- a/core/none/Dockerfile +++ b/core/none/Dockerfile @@ -1,12 +1,12 @@ # syntax=docker/dockerfile-upstream:1.4.3 -# This is an idle image to dynamically replace any component if disabled. +# idle image (to dynamically replace any disabled component) FROM base ARG VERSION=local LABEL version=$VERSION -RUN echo $VERSION >> /version +RUN echo $VERSION >/version HEALTHCHECK CMD true diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 66adbde3..dab4396c 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,21 +1,18 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# postfix image FROM base ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - && apk add --no-cache postfix postfix-pcre cyrus-sasl-login rsyslog logrotate \ - && pip install --no-cache-dir --only-binary=:all: postfix-mta-sts-resolver==1.0.1 \ - || ( apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev py3-wheel libffi-dev \ - && pip3 install postfix-mta-sts-resolver==1.0.1 \ - && apk del .build-deps ) + ; apk add --no-cache cyrus-sasl-login logrotate postfix postfix-pcre rsyslog -COPY conf /conf -COPY start.py /start.py +COPY conf/ /conf/ +COPY start.py / -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 25/tcp 10025/tcp HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" diff --git a/core/postfix/requirements.txt b/core/postfix/requirements.txt new file mode 100644 index 00000000..2d9b8135 --- /dev/null +++ b/core/postfix/requirements.txt @@ -0,0 +1,3 @@ +podop==0.2.5 +postfix-mta-sts-resolver==1.1.4 +socrate==0.2.0 diff --git a/core/postfix/start.py b/core/postfix/start.py index 4faf2e2d..b12d0b54 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import glob diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 5ee922e5..2ccb5307 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -1,18 +1,19 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# rspamd image FROM base ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - && apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl \ - && mkdir /run/rspamd + ; apk add --no-cache curl rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \ + ; mkdir /run/rspamd -COPY conf/ /conf -COPY start.py /start.py +COPY conf/ /conf/ +COPY start.py / -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 11332/tcp 11334/tcp 11335/tcp HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost:11334/ diff --git a/core/rspamd/requirements.txt b/core/rspamd/requirements.txt new file mode 100644 index 00000000..be4b0107 --- /dev/null +++ b/core/rspamd/requirements.txt @@ -0,0 +1 @@ +socrate==0.2.0 diff --git a/core/rspamd/start.py b/core/rspamd/start.py index fcb33a97..58ec89ca 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import glob diff --git a/optional/clamav/Dockerfile b/optional/clamav/Dockerfile index e0ed0cdc..9beded99 100644 --- a/optional/clamav/Dockerfile +++ b/optional/clamav/Dockerfile @@ -1,26 +1,22 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION +# syntax=docker/dockerfile-upstream:1.4.3 -ENV TZ Etc/UTC +# clamav image +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip bash tzdata \ - && pip3 install --upgrade pip -# Image specific layers under this line -RUN apk add --no-cache clamav rsyslog wget clamav-libunrar +RUN set -euxo pipefail \ + ; apk add --no-cache clamav clamav-libunrar rsyslog wget -COPY conf /etc/clamav -COPY start.py /start.py -COPY health.sh /health.sh +COPY conf/ /etc/clamav/ +COPY start.py / + +RUN echo $VERSION >/version EXPOSE 3310/tcp +HEALTHCHECK --start-period=350s CMD echo PING|nc localhost 3310|grep "PONG" + VOLUME ["/data"] CMD /start.py - -HEALTHCHECK --start-period=350s CMD /health.sh -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/clamav/health.sh b/optional/clamav/health.sh deleted file mode 100755 index c4c55044..00000000 --- a/optional/clamav/health.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -if [ "$(echo PING | nc localhost 3310)" = "PONG" ]; then - echo "ping successful" -else - echo "ping failed" - exit 1 -fi diff --git a/optional/clamav/start.py b/optional/clamav/start.py index 56e1bcfe..3d0c306d 100755 --- a/optional/clamav/start.py +++ b/optional/clamav/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import logging as log diff --git a/optional/fetchmail/Dockerfile b/optional/fetchmail/Dockerfile index 71075ced..12488213 100644 --- a/optional/fetchmail/Dockerfile +++ b/optional/fetchmail/Dockerfile @@ -1,23 +1,21 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION +# syntax=docker/dockerfile-upstream:1.4.3 -ENV TZ Etc/UTC +# fetchmail image +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip bash tzdata \ - && pip3 install --upgrade pip +RUN set -euxo pipefail \ + ; apk add --no-cache fetchmail openssl \ + ; mkdir -p /data -# Image specific layers under this line -RUN apk add --no-cache fetchmail ca-certificates openssl \ - && pip3 install requests +COPY fetchmail.py / -RUN mkdir -p /data +RUN echo $VERSION >/version -COPY fetchmail.py /fetchmail.py +HEALTHCHECK --start-period=350s CMD ["/bin/sh", "-c", "ps ax | grep [/]fetchmail.py"] + +VOLUME ["/var/lib/rspamd"] CMD ["/fetchmail.py"] -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 5459de59..32751ed7 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import time import os diff --git a/optional/fetchmail/requirements.txt b/optional/fetchmail/requirements.txt new file mode 100644 index 00000000..a8ed785e --- /dev/null +++ b/optional/fetchmail/requirements.txt @@ -0,0 +1 @@ +requests==2.26.0 diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index 30055a14..f9fd7598 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -1,27 +1,21 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION +# syntax=docker/dockerfile-upstream:1.4.3 -ENV TZ Etc/UTC +# webdav image +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip bash tzdata \ - && pip3 install --upgrade pip +RUN set -euxo pipefail \ + ; apk add --no-cache curl -# Image specific layers under this line -RUN apk add --no-cache curl \ - && pip3 install pytz radicale~=3.0 +COPY radicale.conf / - -COPY radicale.conf /radicale.conf +RUN echo $VERSION >/version EXPOSE 5232/tcp +HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 + VOLUME ["/data"] CMD radicale -S -C /radicale.conf - -HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 -RUN echo $VERSION >> /version diff --git a/optional/radicale/requirements.txt b/optional/radicale/requirements.txt new file mode 100644 index 00000000..fc61502c --- /dev/null +++ b/optional/radicale/requirements.txt @@ -0,0 +1,2 @@ +pytz==2021.3 +radicale~=3.0 diff --git a/optional/traefik-certdumper/Dockerfile b/optional/traefik-certdumper/Dockerfile index 829655f0..a94f32ba 100644 --- a/optional/traefik-certdumper/Dockerfile +++ b/optional/traefik-certdumper/Dockerfile @@ -1,16 +1,22 @@ +# syntax=docker/dockerfile-upstream:1.4.3 + +# cert dumper image FROM ldez/traefik-certs-dumper -ARG VERSION ENV TZ Etc/UTC +ENV LANG C.UTF-8 +ARG VERSION LABEL version=$VERSION -RUN apk --no-cache add inotify-tools util-linux bash tzdata +RUN set -euxo pipefail \ + ; apk add --no-cache bash inotify-tools tzdata util-linux COPY run.sh / +RUN echo $VERSION >/version + VOLUME ["/traefik"] VOLUME ["/output"] ENTRYPOINT ["/run.sh"] -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/unbound/Dockerfile b/optional/unbound/Dockerfile index 342ceebc..343326fe 100644 --- a/optional/unbound/Dockerfile +++ b/optional/unbound/Dockerfile @@ -1,33 +1,25 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION +# syntax=docker/dockerfile-upstream:1.4.3 -ENV TZ Etc/UTC +# resolver image +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict tzdata \ - && pip3 install --upgrade pip +RUN set -euxo pipefail \ + ; apk add --no-cache bind-tools curl unbound \ + ; curl -so /etc/unbound/root.hints https://www.internic.net/domain/named.cache \ + ; chown root:unbound /etc/unbound \ + ; chmod 775 /etc/unbound \ + ; apk del --no-cache curl \ + ; /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key || true -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 +COPY unbound.conf / +COPY start.py / -# Image specific layers under this line -RUN apk add --no-cache unbound curl bind-tools \ - && curl -o /etc/unbound/root.hints https://www.internic.net/domain/named.cache \ - && chown root:unbound /etc/unbound \ - && chmod 775 /etc/unbound \ - && apk del --no-cache curl \ - && /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key | true - -COPY start.py /start.py -COPY unbound.conf /unbound.conf +RUN echo $VERSION >/version EXPOSE 53/udp 53/tcp +HEALTHCHECK CMD dig @127.0.0.1 || exit 1 CMD /start.py - -HEALTHCHECK CMD dig @127.0.0.1 || exit 1 -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/unbound/start.py b/optional/unbound/start.py index 0e7d0fdc..f3a5bee7 100755 --- a/optional/unbound/start.py +++ b/optional/unbound/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import logging as log diff --git a/tests/build.hcl b/tests/build.hcl index c32da8d5..75e4f996 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -83,12 +83,16 @@ function "tag" { # ----------------------------------------------------------------------------------------- target "base" { inherits = ["defaults"] - context="core/base" + context = "core/base/" + contexts = { + core = "core/" + optional = "optional/" + } } target "assets" { inherits = ["defaults"] - context="core/admin/assets" + context = "core/admin/assets/" } # ----------------------------------------------------------------------------------------- @@ -96,7 +100,7 @@ target "assets" { # ----------------------------------------------------------------------------------------- target "docs" { inherits = ["defaults"] - context = "docs" + context = "docs/" tags = tag("docs") args = { version = "${MAILU_VERSION}" @@ -106,7 +110,7 @@ target "docs" { target "setup" { inherits = ["defaults"] - context="setup" + context = "setup/" tags = tag("setup") } @@ -115,8 +119,8 @@ target "setup" { # ----------------------------------------------------------------------------------------- target "none" { inherits = ["defaults"] - context="core/none" - contexts= { + context = "core/none/" + contexts = { base = "target:base" } tags = tag("none") @@ -124,8 +128,8 @@ target "none" { target "admin" { inherits = ["defaults"] - context="core/admin" - contexts= { + context = "core/admin/" + contexts = { base = "target:base" assets = "target:assets" } @@ -134,8 +138,8 @@ target "admin" { target "antispam" { inherits = ["defaults"] - context="core/rspamd" - contexts= { + context = "core/rspamd/" + contexts = { base = "target:base" } tags = tag("rspamd") @@ -143,8 +147,8 @@ target "antispam" { target "front" { inherits = ["defaults"] - context="core/nginx" - contexts= { + context = "core/nginx/" + contexts = { base = "target:base" } tags = tag("nginx") @@ -152,8 +156,8 @@ target "front" { target "imap" { inherits = ["defaults"] - context="core/dovecot" - contexts= { + context = "core/dovecot/" + contexts = { base = "target:base" } tags = tag("dovecot") @@ -161,8 +165,8 @@ target "imap" { target "smtp" { inherits = ["defaults"] - context="core/postfix" - contexts= { + context = "core/postfix/" + contexts = { base = "target:base" } tags = tag("postfix") @@ -173,13 +177,13 @@ target "smtp" { # ----------------------------------------------------------------------------------------- target "snappymail" { inherits = ["defaults"] - context="webmails/snappymail" + context = "webmails/snappymail/" tags = tag("snappymail") } target "roundcube" { inherits = ["defaults"] - context="webmails/roundcube" + context = "webmails/roundcube/" tags = tag("roundcube") } @@ -188,30 +192,42 @@ target "roundcube" { # ----------------------------------------------------------------------------------------- target "antivirus" { inherits = ["defaults"] - context="optional/clamav" + context = "optional/clamav/" + contexts = { + base = "target:base" + } tags = tag("clamav") } target "fetchmail" { inherits = ["defaults"] - context="optional/fetchmail" + context = "optional/fetchmail/" + contexts = { + base = "target:base" + } tags = tag("fetchmail") } target "resolver" { inherits = ["defaults"] - context="optional/unbound" + context = "optional/unbound/" + contexts = { + base = "target:base" + } tags = tag("unbound") } target "traefik-certdumper" { inherits = ["defaults"] - context="optional/traefik-certdumper" + context = "optional/traefik-certdumper/" tags = tag("traefik-certdumper") } target "webdav" { inherits = ["defaults"] - context="optional/radicale" + context = "optional/radicale/" + contexts = { + base = "target:base" + } tags = tag("radicale") } From 4c1071a4976182e98b275c156c8990843c57c353 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 14 Oct 2022 14:34:27 +0200 Subject: [PATCH 34/44] Move all requirements*.txt to base image --- core/admin/requirements-dev.txt | 28 ---------- core/base/Dockerfile | 32 ++++++----- core/base/requirements-build.txt | 3 ++ core/base/requirements-dev.txt | 53 +++++++++++++++++++ .../requirements-prod.txt} | 32 +++++++---- core/base/requirements.txt | 2 - core/dovecot/requirements.txt | 2 - core/nginx/requirements.txt | 2 - core/postfix/requirements.txt | 3 -- core/rspamd/requirements.txt | 1 - optional/fetchmail/requirements.txt | 1 - optional/radicale/requirements.txt | 2 - 12 files changed, 93 insertions(+), 68 deletions(-) delete mode 100644 core/admin/requirements-dev.txt create mode 100644 core/base/requirements-build.txt create mode 100644 core/base/requirements-dev.txt rename core/{admin/requirements.txt => base/requirements-prod.txt} (74%) delete mode 100644 core/base/requirements.txt delete mode 100644 core/dovecot/requirements.txt delete mode 100644 core/nginx/requirements.txt delete mode 100644 core/postfix/requirements.txt delete mode 100644 core/rspamd/requirements.txt delete mode 100644 optional/fetchmail/requirements.txt delete mode 100644 optional/radicale/requirements.txt diff --git a/core/admin/requirements-dev.txt b/core/admin/requirements-dev.txt deleted file mode 100644 index 5bf57874..00000000 --- a/core/admin/requirements-dev.txt +++ /dev/null @@ -1,28 +0,0 @@ -Flask -Flask-Login -Flask-SQLAlchemy -Flask-bootstrap -Flask-Babel -Flask-migrate -Flask-script -Flask-wtf -Flask-debugtoolbar -limits -redis -WTForms-Components -socrate -passlib -gunicorn -tabulate -PyYAML -PyOpenSSL -Pygments -dnspython -tenacity -mysql-connector-python -idna -srslib -marshmallow -flask-marshmallow -marshmallow-sqlalchemy -xmltodict diff --git a/core/base/Dockerfile b/core/base/Dockerfile index cd590596..52f0bd3a 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -4,11 +4,14 @@ ARG DISTRO=alpine:3.14.5 FROM $DISTRO as system -ENV TZ Etc/UTC -ENV LANG C.UTF-8 +ENV TZ=Etc/UTC LANG=C.UTF-8 + +ARG MAILU_UID=1000 +ARG MAILU_GID=1000 RUN set -euxo pipefail \ - ; adduser -s /bin/bash -Dh /app -k /var/empty -u 1000 -g mailu app \ + ; addgroup -Sg ${MAILU_GID} mailu \ + ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ ; apk add --no-cache bash ca-certificates python3 tzdata WORKDIR /app @@ -21,31 +24,26 @@ FROM system as build ENV VIRTUAL_ENV=/app/venv +COPY requirements-*.txt ./ + RUN set -euxo pipefail \ ; apk add --no-cache py3-pip \ ; python3 -m venv ${VIRTUAL_ENV} \ - ; venv/bin/pip install --no-cache-dir --upgrade --no-warn-script-location pip wheel + ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" COPY libs/ libs/ -COPY --from=core ./ core/ -COPY --from=optional ./ optional/ RUN set -euxo pipefail \ - ; grep -hEv '(podop|socrate)==' core/*/requirements.txt optional/*/requirements.txt \ - | sort -u >libs/requirements.txt \ -\ - ; venv/bin/pip install --no-cache-dir -r libs/requirements.txt \ + ; pip install --no-cache-dir -r requirements-prod.txt \ || ( \ apk add --no-cache --virtual .build-deps \ - build-base cargo gcc libffi-dev libressl-dev mariadb-connector-c-dev \ - musl-dev postgresql-dev python3-dev \ - ; venv/bin/pip install --no-cache-dir -r libs/requirements.txt \ - ; apk del .build-deps \ - ) \ -\ - ; venv/bin/pip freeze > venv/requirements.txt + build-base cargo gcc libffi-dev libressl-dev mariadb-connector-c-dev \ + musl-dev postgresql-dev python3-dev \ + ; pip install --no-cache-dir -r requirements-prod.txt \ + ; apk del .build-deps \ + ) # base mailu image diff --git a/core/base/requirements-build.txt b/core/base/requirements-build.txt new file mode 100644 index 00000000..2858969f --- /dev/null +++ b/core/base/requirements-build.txt @@ -0,0 +1,3 @@ +pip==22.2.2 +setuptools==65.4.1 +wheel==0.37.1 diff --git a/core/base/requirements-dev.txt b/core/base/requirements-dev.txt new file mode 100644 index 00000000..2fb4352f --- /dev/null +++ b/core/base/requirements-dev.txt @@ -0,0 +1,53 @@ +# core/base +libs/podop +libs/socrate + +# core/admin +alembic +Babel +click +dnspython +Flask +Flask-Babel +Flask-Bootstrap +Flask-DebugToolbar +Flask-Login +flask-marshmallow +Flask-Migrate +Flask-SQLAlchemy +Flask-WTF +gunicorn +idna +itsdangerous +limits +marshmallow +marshmallow-sqlalchemy +mysql-connector-python +passlib +psycopg2-binary +Pygments +pyOpenSSL +PyYAML +redis +SQLAlchemy +srslib +tabulate +tenacity +validators +Werkzeug +WTForms +WTForms-Components +xmltodict + +# core/nginx +watchdog + +# core/postfix +postfix-mta-sts-resolver + +# optional/fetchmail +requests + +# optional/radicale +radicale + diff --git a/core/admin/requirements.txt b/core/base/requirements-prod.txt similarity index 74% rename from core/admin/requirements.txt rename to core/base/requirements-prod.txt index 297c6902..fbbce5d5 100644 --- a/core/admin/requirements.txt +++ b/core/base/requirements-prod.txt @@ -1,19 +1,24 @@ +aiodns==3.0.0 +aiohttp==3.8.3 +aiosignal==1.2.0 alembic==1.7.4 appdirs==1.4.4 +async-timeout==4.0.2 +attrs==22.1.0 Babel==2.9.1 bcrypt==3.2.0 blinker==1.4 CacheControl==0.12.9 certifi==2021.10.8 -# cffi==1.15.0 +cffi==1.15.1 chardet==4.0.0 +charset-normalizer==2.0.12 click==8.0.3 colorama==0.4.4 contextlib2==21.6.0 cryptography==35.0.0 decorator==5.1.0 -# distlib==0.3.1 -# distro==1.5.0 +defusedxml==0.7.1 dnspython==2.1.0 dominate==2.6.0 email-validator==1.1.3 @@ -28,6 +33,7 @@ Flask-Migrate==3.1.0 Flask-Script==2.0.6 Flask-SQLAlchemy==2.5.1 Flask-WTF==0.15.1 +frozenlist==1.3.1 greenlet==1.1.2 gunicorn==20.1.0 html5lib==1.1 @@ -38,31 +44,34 @@ itsdangerous==2.0.1 Jinja2==3.0.2 limits==1.5.1 lockfile==0.12.2 -Mako==1.1.5 +Mako==1.2.3 MarkupSafe==2.0.1 marshmallow==3.14.0 marshmallow-sqlalchemy==0.26.1 msgpack==1.0.2 -# mysqlclient==2.0.3 +multidict==6.0.2 mysql-connector-python==8.0.25 ordered-set==4.0.2 -# packaging==20.9 passlib==1.7.4 -# pep517==0.10.0 +podop @ file:///app/libs/podop +postfix-mta-sts-resolver==1.0.1 progress==1.6 -#psycopg2==2.9.1 +protobuf==4.21.7 psycopg2-binary==2.9.3 +pycares==4.2.2 pycparser==2.20 Pygments==2.10.0 pyOpenSSL==21.0.0 pyparsing==3.0.4 +python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 +Radicale==3.1.8 redis==3.5.3 requests==2.26.0 retrying==1.3.3 -# six==1.15.0 -socrate==0.2.0 +six==1.16.0 +socrate @ file:///app/libs/socrate SQLAlchemy==1.4.26 srslib==0.1.4 tabulate==0.8.9 @@ -71,8 +80,11 @@ toml==0.10.2 urllib3==1.26.7 validators==0.18.2 visitor==0.1.3 +vobject==0.9.6.1 +watchdog==2.1.9 webencodings==0.5.1 Werkzeug==2.0.2 WTForms==2.3.3 WTForms-Components==0.10.5 xmltodict==0.12.0 +yarl==1.8.1 diff --git a/core/base/requirements.txt b/core/base/requirements.txt deleted file mode 100644 index 0e537e6a..00000000 --- a/core/base/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -libs/podop/ -libs/socrate/ diff --git a/core/dovecot/requirements.txt b/core/dovecot/requirements.txt deleted file mode 100644 index 16005f74..00000000 --- a/core/dovecot/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -podop==0.2.5 -socrate==0.2.0 diff --git a/core/nginx/requirements.txt b/core/nginx/requirements.txt deleted file mode 100644 index c96c3bb8..00000000 --- a/core/nginx/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -socrate==0.2.0 -watchdog==2.1.9 diff --git a/core/postfix/requirements.txt b/core/postfix/requirements.txt deleted file mode 100644 index 2d9b8135..00000000 --- a/core/postfix/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -podop==0.2.5 -postfix-mta-sts-resolver==1.1.4 -socrate==0.2.0 diff --git a/core/rspamd/requirements.txt b/core/rspamd/requirements.txt deleted file mode 100644 index be4b0107..00000000 --- a/core/rspamd/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -socrate==0.2.0 diff --git a/optional/fetchmail/requirements.txt b/optional/fetchmail/requirements.txt deleted file mode 100644 index a8ed785e..00000000 --- a/optional/fetchmail/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests==2.26.0 diff --git a/optional/radicale/requirements.txt b/optional/radicale/requirements.txt deleted file mode 100644 index fc61502c..00000000 --- a/optional/radicale/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytz==2021.3 -radicale~=3.0 From 146921f6192a2950c62c98262ecb48097c974bae Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 14 Oct 2022 14:34:58 +0200 Subject: [PATCH 35/44] Move curl to base image --- core/admin/Dockerfile | 2 +- core/base/Dockerfile | 2 +- core/nginx/Dockerfile | 2 +- core/rspamd/Dockerfile | 2 +- optional/radicale/Dockerfile | 3 --- optional/unbound/Dockerfile | 3 +-- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index ca98662d..02dcd968 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -7,7 +7,7 @@ ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache curl libressl mariadb-connector-c postgresql-libs + ; apk add --no-cache libressl mariadb-connector-c postgresql-libs COPY mailu/ ./mailu/ RUN set -euxo pipefail \ diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 52f0bd3a..ca87dd70 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -12,7 +12,7 @@ ARG MAILU_GID=1000 RUN set -euxo pipefail \ ; addgroup -Sg ${MAILU_GID} mailu \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ - ; apk add --no-cache bash ca-certificates python3 tzdata + ; apk add --no-cache bash ca-certificates curl python3 tzdata WORKDIR /app diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index cbb9cd7c..f271fc07 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -17,7 +17,7 @@ ARG VERSION LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache certbot curl nginx nginx-mod-mail openssl + ; apk add --no-cache certbot nginx nginx-mod-mail openssl COPY conf/ /conf/ COPY --from=static /static/ /static/ diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 2ccb5307..eca8e62b 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -7,7 +7,7 @@ ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache curl rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \ + ; apk add --no-cache rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \ ; mkdir /run/rspamd COPY conf/ /conf/ diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index f9fd7598..56606494 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -6,9 +6,6 @@ FROM base ARG VERSION=local LABEL version=$VERSION -RUN set -euxo pipefail \ - ; apk add --no-cache curl - COPY radicale.conf / RUN echo $VERSION >/version diff --git a/optional/unbound/Dockerfile b/optional/unbound/Dockerfile index 343326fe..831476ab 100644 --- a/optional/unbound/Dockerfile +++ b/optional/unbound/Dockerfile @@ -7,11 +7,10 @@ ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache bind-tools curl unbound \ + ; apk add --no-cache bind-tools unbound \ ; curl -so /etc/unbound/root.hints https://www.internic.net/domain/named.cache \ ; chown root:unbound /etc/unbound \ ; chmod 775 /etc/unbound \ - ; apk del --no-cache curl \ ; /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key || true COPY unbound.conf / From 5c31120895dd6d23e52fc9e33034afee14ade74e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 14 Oct 2022 14:37:40 +0200 Subject: [PATCH 36/44] Remove obsolete contexts from base image --- tests/build.hcl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/build.hcl b/tests/build.hcl index 75e4f996..453e6d40 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -84,10 +84,6 @@ function "tag" { target "base" { inherits = ["defaults"] context = "core/base/" - contexts = { - core = "core/" - optional = "optional/" - } } target "assets" { From 7441a420c4eea54a336eae47ecc8b0ad17a95f4d Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 14 Oct 2022 16:17:46 +0200 Subject: [PATCH 37/44] Fix and speed-up arm build. Allow chosing of prod/dev env. --- core/base/Dockerfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index ca87dd70..d00ead10 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -22,28 +22,32 @@ CMD /bin/bash # build virtual env (intermediate) FROM system as build +ARG MAILU_ENV=prod + ENV VIRTUAL_ENV=/app/venv -COPY requirements-*.txt ./ +COPY requirements-build.txt ./ RUN set -euxo pipefail \ ; apk add --no-cache py3-pip \ ; python3 -m venv ${VIRTUAL_ENV} \ - ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt + ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt \ + ; apk del py3-pip ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" +COPY requirements-${MAILU_ENV}.txt ./ COPY libs/ libs/ RUN set -euxo pipefail \ - ; pip install --no-cache-dir -r requirements-prod.txt \ + ; pip install -r requirements-${MAILU_ENV}.txt \ || ( \ apk add --no-cache --virtual .build-deps \ - build-base cargo gcc libffi-dev libressl-dev mariadb-connector-c-dev \ - musl-dev postgresql-dev python3-dev \ - ; pip install --no-cache-dir -r requirements-prod.txt \ + build-base gcc libffi-dev python3-dev \ + ; pip install -r requirements-${MAILU_ENV}.txt \ ; apk del .build-deps \ - ) + ) \ + ; rm -rf /root/.cache /tmp/*.pem # base mailu image From d9bf6875e12255b7870448fd08d9012b41671752 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 17 Oct 2022 16:20:52 +0200 Subject: [PATCH 38/44] Optimize build order for better caching --- core/admin/Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 02dcd968..600c3e9f 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -9,16 +9,16 @@ LABEL version=$VERSION RUN set -euxo pipefail \ ; apk add --no-cache libressl mariadb-connector-c postgresql-libs -COPY mailu/ ./mailu/ -RUN set -euxo pipefail \ - ; venv/bin/pybabel compile -d mailu/translations - -COPY migrations/ ./migrations/ +COPY --from=assets /work/static/ ./mailu/static/ COPY audit.py / COPY start.py / -COPY --from=assets /work/static/ ./mailu/static/ +COPY migrations/ ./migrations/ + +COPY mailu/ ./mailu/ +RUN set -euxo pipefail \ + ; venv/bin/pybabel compile -d mailu/translations RUN echo $VERSION >/version @@ -27,5 +27,5 @@ HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index VOLUME ["/data","/dkim"] -ENV FLASK_APP mailu +ENV FLASK_APP=mailu CMD /start.py From 92cb8c146bc76d74e0f5254625df41b685f9c875 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 19 Oct 2022 11:02:22 +0000 Subject: [PATCH 39/44] Refine build_test_deploy.yml: Build base image before the other images. Change cache key to make it is re-used for all builds. This is not dangerous. The docker build process can determine itself whether a cache can be safely re-used or not. --- .github/workflows/build_test_deploy.yml | 161 +++++++++++++++++------- 1 file changed, 115 insertions(+), 46 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 2d721656..3172e411 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -89,6 +89,57 @@ jobs: run: | echo ${{ steps.targets.outputs.matrix }} +## This job buils the base image. The base image is used by all other images. + build-base-image: + name: Build base image + needs: + - targets + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v3 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/cache/base + key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + restore-keys: | + mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - uses: crazy-max/ghaction-github-runtime@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} + - name: Build all docker images + env: + DOCKER_ORG: ${{ env.DOCKER_ORG }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + uses: docker/bake-action@v2 + with: + files: ${{env.HCL_FILE}} + targets: base + load: false + push: false + set: | + *.cache-from=type=local,src=${{ runner.temp }}/cache/base + *.cache-to=type=local,dest=${{ runner.temp }}/cache/base,mode=max + *.platform=${{ inputs.architecture }} + # This job builds all the images. The build cache is stored in the github actions cache. # In further jobs, this cache is used to quickly rebuild the images. build: @@ -96,6 +147,7 @@ jobs: if: inputs.architecture == 'linux/amd64' needs: - targets + - build-base-image strategy: fail-fast: false matrix: @@ -113,11 +165,18 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/cache/base + key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + restore-keys: | + mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/${{ matrix.target }} - key: ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }}-${{ github.run_id }} + key: ${{ matrix.target }}-${{ inputs.architecture }}-${{ github.run_id }} restore-keys: | ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }} - name: Set up QEMU @@ -143,6 +202,7 @@ jobs: push: false set: | *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }} + *.cache-from=type=local,src=${{ runner.temp }}/cache/base *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max *.platform=${{ inputs.architecture }} @@ -153,6 +213,7 @@ jobs: if: inputs.architecture != 'linux/amd64' needs: - targets + - build-base-image strategy: fail-fast: false matrix: @@ -170,13 +231,20 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/cache/base + key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + restore-keys: | + mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/${{ matrix.target }} - key: ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }}-${{ github.run_id }} + key: ${{ matrix.target }}-${{ inputs.architecture }}-${{ github.run_id }} restore-keys: | - ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }} + ${{ matrix.target }}-${{ inputs.architecture }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -200,6 +268,7 @@ jobs: push: false set: | *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }} + *.cache-from=type=local,src=${{ runner.temp }}/cache/base *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max *.platform=${{ inputs.architecture }} @@ -238,72 +307,72 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/docs - key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }} + key: docs-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image setup uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/setup - key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }} + key: setup-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image admin uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/admin - key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }} + key: admin-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antispam uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antispam - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }} + key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image front uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/front - key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }} + key: front-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image imap uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/imap - key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }} + key: imap-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image smtp uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/smtp - key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }} + key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image snappymail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/snappymail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }} + key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image roundcube uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/roundcube - key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }} + key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antivirus uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antivirus - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }} + key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image fetchmail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/fetchmail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }} + key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image resolver uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/resolver - key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }} + key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image traefik-certdumper uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/traefik-certdumper - key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }} + key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image webdav uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/webdav - key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }} + key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -370,76 +439,76 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs + - name: Configure /cache for image docs uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/docs - key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }} + key: docs-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image setup uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/setup - key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }} + key: setup-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image admin uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/admin - key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }} + key: admin-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antispam uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antispam - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }} + key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image front uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/front - key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }} + key: front-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image imap uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/imap - key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }} + key: imap-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image smtp uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/smtp - key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }} + key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image snappymail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/snappymail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }} + key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image roundcube uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/roundcube - key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }} + key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antivirus uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antivirus - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }} + key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image fetchmail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/fetchmail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }} + key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image resolver uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/resolver - key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }} + key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image traefik-certdumper uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/traefik-certdumper - key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }} + key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image webdav uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/webdav - key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }} + key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -496,72 +565,72 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/docs - key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }} + key: docs-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image setup uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/setup - key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }} + key: setup-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image admin uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/admin - key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }} + key: admin-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antispam uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antispam - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }} + key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image front uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/front - key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }} + key: front-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image imap uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/imap - key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }} + key: imap-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image smtp uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/smtp - key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }} + key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image snappymail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/snappymail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }} + key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image roundcube uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/roundcube - key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }} + key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antivirus uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antivirus - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }} + key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image fetchmail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/fetchmail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }} + key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image resolver uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/resolver - key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }} + key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image traefik-certdumper uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/traefik-certdumper - key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }} + key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image webdav uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/webdav - key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }} + key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 From f9ba0e688fa3089b98db4dc8944cad9ea11c605e Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 19 Oct 2022 11:25:42 +0000 Subject: [PATCH 40/44] Removed syntax error --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 3172e411..8aee5c5f 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -439,7 +439,7 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs + - name: Configure /cache for image docs uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/docs From 451738e32b026b7b3e93a3c732e99427ef61add0 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 19 Oct 2022 11:35:57 +0000 Subject: [PATCH 41/44] We want the function result. Not the function statement. --- .github/workflows/build_test_deploy.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 8aee5c5f..7b7c357a 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -111,9 +111,9 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} restore-keys: | - mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') + mailu-base-${{ inputs.architecture }}-${{hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -169,9 +169,9 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} restore-keys: | - mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') + mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder uses: actions/cache@v3 with: @@ -235,9 +235,9 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} restore-keys: | - mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') + mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder uses: actions/cache@v3 with: From 6ea2d84a3cd0bd5b460fa3ea2b9f9939800adadb Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 19 Oct 2022 14:55:22 +0000 Subject: [PATCH 42/44] Remove outdated wrong documentation --- docs/contributors/environment.rst | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index 47f9f1af..1b53afe6 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -312,35 +312,6 @@ The following must be done on every PR or after every new commit to an existing If git opens a editor for a commit message just save and exit as-is. If you have a merge conflict, see above and do the complete procedure from ``git fetch`` onward again. -Web administration ------------------- - -The administration Web interface requires a proper dev environment that can easily be setup using -``virtualenv`` (make sure you are using Python 3) : - -.. code-block:: bash - - cd core/admin - virtualenv . - source bin/activate - pip install -r requirements.txt - -You can then export the path to the development database (use four slashes for absolute path): - -.. code-block:: bash - - export SQLALCHEMY_DATABASE_URI=sqlite:///path/to/dev.db - -And finally run the server with debug enabled: - -.. code-block:: bash - - python run.py - -Any change to the files will automatically restart the Web server and reload the files. - -When using the development environment, a debugging toolbar is displayed on the right side -of the screen, that you can open to access query details, internal variables, etc. Documentation ------------- From 4be0cbf4da68f56759bf6a458d1889df57650047 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 28 Oct 2022 11:52:49 +0000 Subject: [PATCH 43/44] Switch workflow to ghcr.io - Build images & build cache are pushed to ghcr.io. - Tests will make use of the images pushed to ghcr.io. - Deploy step only copies images from ghcr.io to docker.io. - Resolves strange build errors tied to buildx+intermediate builds - Results in quicker build times. --- .github/workflows/arm.yml | 4 +- .github/workflows/build_test_deploy.yml | 436 +++++------------------- .github/workflows/x64.yml | 4 +- 3 files changed, 90 insertions(+), 354 deletions(-) diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 22d58cd5..7125737c 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -61,7 +61,7 @@ jobs: echo "RELEASE=true" >> $GITHUB_ENV echo "DEPLOY=true" >> $GITHUB_ENV echo "RELEASE=true" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging for master + - name: Derive PINNED_MAILU_VERSION for master if: env.BRANCH == 'master' shell: bash env: @@ -98,4 +98,4 @@ jobs: #else # pinned_version=$root_version.$(expr $patch_version + 1) #fi -#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV \ No newline at end of file +#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 7b7c357a..e8bb1a0a 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -58,15 +58,15 @@ on: required: true type: string deploy: - description: Deploy to docker hub. Happens for all branches but staging + description: Deploy to docker hub. Happens for all branches but staging. Use string true or false. default: true required: false - type: boolean + type: string release: - description: 'Tag and create the github release. Only happens for branch x.y (release branch)' + description: Tag and create the github release. Use string true or false. default: false required: false - type: boolean + type: string env: HCL_FILE: ./tests/build.hcl @@ -84,7 +84,7 @@ jobs: - name: Create matrix id: targets run: | - echo ::set-output name=matrix::$(docker buildx bake -f ${{env.HCL_FILE}} --print | jq -cr '.group.default.targets') + echo matrix=$(docker buildx bake -f ${{env.HCL_FILE}} --print | jq -cr '.group.default.targets') >> $GITHUB_OUTPUT - name: Show matrix run: | echo ${{ steps.targets.outputs.matrix }} @@ -107,26 +107,25 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} - restore-keys: | - mailu-base-${{ inputs.architecture }}-${{hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 + with: + string: ${{ github.repository_owner }} - name: Build all docker images env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} uses: docker/bake-action@v2 @@ -136,8 +135,8 @@ jobs: load: false push: false set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/base - *.cache-to=type=local,dest=${{ runner.temp }}/cache/base,mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }},mode=max *.platform=${{ inputs.architecture }} # This job builds all the images. The build cache is stored in the github actions cache. @@ -165,33 +164,25 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} - restore-keys: | - mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/${{ matrix.target }} - key: ${{ matrix.target }}-${{ inputs.architecture }}-${{ github.run_id }} - restore-keys: | - ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 + with: + string: ${{ github.repository_owner }} - name: Build all docker images env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} uses: docker/bake-action@v2 @@ -199,11 +190,11 @@ jobs: files: ${{env.HCL_FILE}} targets: ${{ matrix.target }} load: false - push: false + push: true set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }} - *.cache-from=type=local,src=${{ runner.temp }}/cache/base - *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} *.platform=${{ inputs.architecture }} # This job builds all the images. The build cache is stored in the github actions cache. @@ -231,33 +222,25 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} - restore-keys: | - mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/${{ matrix.target }} - key: ${{ matrix.target }}-${{ inputs.architecture }}-${{ github.run_id }} - restore-keys: | - ${{ matrix.target }}-${{ inputs.architecture }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 + with: + string: ${{ github.repository_owner }} - name: Build all docker images env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} uses: docker/bake-action@v2 @@ -267,9 +250,10 @@ jobs: load: false push: false set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }} - *.cache-from=type=local,src=${{ runner.temp }}/cache/base - *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max *.platform=${{ inputs.architecture }} # This job runs all the tests. @@ -303,112 +287,22 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/docs - key: docs-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image setup - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/setup - key: setup-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image admin - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/admin - key: admin-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antispam - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antispam - key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image front - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/front - key: front-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image imap - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/imap - key: imap-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image smtp - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/smtp - key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image snappymail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/snappymail - key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image roundcube - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/roundcube - key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antivirus - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antivirus - key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image fetchmail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/fetchmail - key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image resolver - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/resolver - key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image traefik-certdumper - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/traefik-certdumper - key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image webdav - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/webdav - key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} - - name: Build docker images for testing from cache - env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 with: - files: ${{env.HCL_FILE}} - load: true - push: false - set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/docs - *.cache-from=type=local,src=${{ runner.temp }}/cache/setup - *.cache-from=type=local,src=${{ runner.temp }}/cache/admin - *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam - *.cache-from=type=local,src=${{ runner.temp }}/cache/front - *.cache-from=type=local,src=${{ runner.temp }}/cache/imap - *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp - *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail - *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube - *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus - *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail - *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver - *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper - *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav - *.platform=${{ inputs.architecture }} + string: ${{ github.repository_owner }} - name: Install python packages run: python3 -m pip install -r tests/requirements.txt - name: Copy all certs @@ -416,12 +310,10 @@ jobs: - name: Test ${{ matrix.target }} run: python tests/compose/test.py ${{ matrix.target }} ${{ matrix.time }} env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} -# This job deploys the docker images to the docker repository. The build.hcl file contains logic that determines what tags are pushed. -# E.g. for master only the :master and :latest tags are pushed. deploy: name: Deploy images # Deploying is not required for staging @@ -430,6 +322,10 @@ jobs: needs: - build - tests + strategy: + fail-fast: false + matrix: + target: ["setup", "docs", "fetchmail", "roundcube", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx", "snappymail"] steps: - uses: actions/checkout@v3 - name: Retrieve global variables @@ -439,76 +335,6 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/docs - key: docs-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image setup - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/setup - key: setup-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image admin - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/admin - key: admin-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antispam - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antispam - key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image front - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/front - key: front-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image imap - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/imap - key: imap-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image smtp - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/smtp - key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image snappymail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/snappymail - key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image roundcube - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/roundcube - key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antivirus - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antivirus - key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image fetchmail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/fetchmail - key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image resolver - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/resolver - key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image traefik-certdumper - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/traefik-certdumper - key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image webdav - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/webdav - key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -519,31 +345,19 @@ jobs: with: username: ${{ secrets.Docker_Login }} password: ${{ secrets.Docker_Password }} - - name: Deploy images to docker hub. Build.hcl contains the logic for the tags that are pushed. - env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 with: - files: ${{env.HCL_FILE}} - push: true - set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/docs - *.cache-from=type=local,src=${{ runner.temp }}/cache/setup - *.cache-from=type=local,src=${{ runner.temp }}/cache/admin - *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam - *.cache-from=type=local,src=${{ runner.temp }}/cache/front - *.cache-from=type=local,src=${{ runner.temp }}/cache/imap - *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp - *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail - *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube - *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus - *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail - *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver - *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper - *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav - *.platform=${{ inputs.architecture }} + string: ${{ github.repository_owner }} + - name: Push image to Docker + shell: bash + run: | + if [ '${{ env.MAILU_VERSION }}' == 'master' ]; then pinned_mailu_version='master'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi; + docker buildx imagetools create \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} deploy-arm: name: Deploy images for arm @@ -552,6 +366,10 @@ jobs: runs-on: self-hosted needs: - build-arm + strategy: + fail-fast: false + matrix: + target: ["setup", "docs", "fetchmail", "roundcube", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx", "snappymail"] steps: - uses: actions/checkout@v3 - name: Retrieve global variables @@ -561,76 +379,6 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/docs - key: docs-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image setup - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/setup - key: setup-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image admin - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/admin - key: admin-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antispam - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antispam - key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image front - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/front - key: front-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image imap - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/imap - key: imap-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image smtp - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/smtp - key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image snappymail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/snappymail - key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image roundcube - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/roundcube - key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antivirus - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antivirus - key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image fetchmail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/fetchmail - key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image resolver - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/resolver - key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image traefik-certdumper - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/traefik-certdumper - key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image webdav - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/webdav - key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -641,31 +389,19 @@ jobs: with: username: ${{ secrets.Docker_Login }} password: ${{ secrets.Docker_Password }} - - name: Deploy images to docker hub. Build.hcl contains the logic for the tags that are pushed. - env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 with: - files: ${{env.HCL_FILE}} - push: true - set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/docs - *.cache-from=type=local,src=${{ runner.temp }}/cache/setup - *.cache-from=type=local,src=${{ runner.temp }}/cache/admin - *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam - *.cache-from=type=local,src=${{ runner.temp }}/cache/front - *.cache-from=type=local,src=${{ runner.temp }}/cache/imap - *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp - *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail - *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube - *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus - *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail - *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver - *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper - *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav - *.platform=${{ inputs.architecture }} + string: ${{ github.repository_owner }} + - name: Push image to Docker + shell: bash + run: | + if [ '${{ env.MAILU_VERSION }}' == 'master-arm' ]; then pinned_mailu_version='master-arm'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi; + docker buildx imagetools create \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} #This job creates a tagged release. A tag is created for the pinned version x.y.z. The GH release refers to this tag. tag-release: diff --git a/.github/workflows/x64.yml b/.github/workflows/x64.yml index e1691756..97b6beac 100644 --- a/.github/workflows/x64.yml +++ b/.github/workflows/x64.yml @@ -82,7 +82,7 @@ jobs: echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV echo "DEPLOY=false" >> $GITHUB_ENV echo "RELEASE=false" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging for master + - name: Derive PINNED_MAILU_VERSION for master if: env.BRANCH == 'master' shell: bash env: @@ -131,4 +131,4 @@ jobs: #else # pinned_version=$root_version.$(expr $patch_version + 1) #fi -#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV \ No newline at end of file +#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV From 024b0573b385f65c7751ed7109cc7d25e6abf878 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 28 Oct 2022 15:47:48 +0200 Subject: [PATCH 44/44] Update build reqs and fix armv7 build --- core/base/Dockerfile | 13 ++++++------- core/base/requirements-build.txt | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index d00ead10..2cade640 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -32,7 +32,7 @@ RUN set -euxo pipefail \ ; apk add --no-cache py3-pip \ ; python3 -m venv ${VIRTUAL_ENV} \ ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt \ - ; apk del py3-pip + ; apk del -r py3-pip ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" @@ -40,13 +40,12 @@ COPY requirements-${MAILU_ENV}.txt ./ COPY libs/ libs/ RUN set -euxo pipefail \ + ; machine="$(uname -m)"; deps="" \ + ; [[ "${machine}" == arm* || "${machine}" == aarch64 ]] && deps="${deps} build-base gcc libffi-dev python3-dev" \ + ; [[ "${machine}" == armv7* ]] && deps="${deps} cargo libressl-dev mariadb-connector-c-dev postgresql-dev" \ + ; [[ "${deps}" ]] && apk add --virtual .build-deps ${deps} \ ; pip install -r requirements-${MAILU_ENV}.txt \ - || ( \ - apk add --no-cache --virtual .build-deps \ - build-base gcc libffi-dev python3-dev \ - ; pip install -r requirements-${MAILU_ENV}.txt \ - ; apk del .build-deps \ - ) \ + ; apk -e info -q .build-deps && apk del -r .build-deps \ ; rm -rf /root/.cache /tmp/*.pem diff --git a/core/base/requirements-build.txt b/core/base/requirements-build.txt index 2858969f..406eb725 100644 --- a/core/base/requirements-build.txt +++ b/core/base/requirements-build.txt @@ -1,3 +1,3 @@ -pip==22.2.2 -setuptools==65.4.1 +pip==22.3 +setuptools==65.5.0 wheel==0.37.1