From 6c83d253123debfc86b0746da70f1d5f6ca2385c Mon Sep 17 00:00:00 2001 From: enginefeeder101 Date: Tue, 26 Apr 2022 21:54:25 +0200 Subject: [PATCH 001/274] Configurable default spam threshold used for new users This commit adds functionality to set a custom default spam threshold for new users. The environment variable ``DEFAULT_SPAM_THRESHOLD`` can be used for this purpose. When not set, it defaults back to 80%, as the default value was before If ``DEFAULT_SPAM_THRESHOLD`` is set to a value that Python cannot parse as an integer, a ValueError is thrown. There is no error handling for that case built-in. --- core/admin/mailu/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 08738600..6e277fe3 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -509,7 +509,8 @@ class User(Base, Email): displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean, nullable=False, default=True) spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True) - spam_threshold = db.Column(db.Integer, nullable=False, default=80) + spam_threshold = db.Column(db.Integer, nullable=False, + default=int(os.environ.get('DEFAULT_SPAM_THRESHOLD', 80))) # Flask-login attributes is_authenticated = True From 4da0ff1856ff8a39b308305ad9257710a39ffb46 Mon Sep 17 00:00:00 2001 From: enginefeeder101 Date: Tue, 26 Apr 2022 22:07:50 +0200 Subject: [PATCH 002/274] Documentation for configurable default spam threshold --- docs/antispam.rst | 2 ++ docs/configuration.rst | 4 ++++ setup/flavors/compose/mailu.env | 3 +++ towncrier/newsfragments/2328.feature | 1 + 4 files changed, 10 insertions(+) create mode 100644 towncrier/newsfragments/2328.feature diff --git a/docs/antispam.rst b/docs/antispam.rst index a95109e8..7971200c 100644 --- a/docs/antispam.rst +++ b/docs/antispam.rst @@ -43,6 +43,8 @@ Rspamd rejects non-compliant email messages and email messages that contain viru 66% (10/15) is less than 80%, so the email is classified as ham. This email message will go to the inbox folder. If the user wants email messages with a score of 10 (66%) to be classified as spam, then the user defined spam filter tolerance can be lowered to 65% in the administration web interface. + The default spam filter tolerance used for new users can be configured using the environment variable ``DEFAULT_SPAM_THRESHOLD``. See also: :ref:`_advanced_cfg`. + .. image:: assets/screenshots/SpamFiltering.png The location in the administration web interface where the spam filter and spam filter tolerance can be configured. diff --git a/docs/configuration.rst b/docs/configuration.rst index 198315c6..0df8633f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -181,6 +181,8 @@ An example: Depending on your particular deployment you most probably will want to change the default. +.. _advanced_cfg: + Advanced settings ----------------- @@ -209,6 +211,8 @@ The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usua .. _`TZ database name`: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +The ``DEFAULT_SPAM_THRESHOLD`` (default: 80) setting is the default spam tolerance used when creating a new user. + Antivirus settings ------------------ diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index ed7ecfdd..675b80ef 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -173,6 +173,9 @@ LOG_LEVEL=WARNING # Timezone for the Mailu containers. See this link for all possible values https://en.wikipedia.org/wiki/List_of_tz_database_time_zones TZ=Etc/UTC +# Default spam threshold used for new users +DEFAULT_SPAM_THRESHOLD=80 + ################################### # Database settings ################################### diff --git a/towncrier/newsfragments/2328.feature b/towncrier/newsfragments/2328.feature new file mode 100644 index 00000000..f0d6eea7 --- /dev/null +++ b/towncrier/newsfragments/2328.feature @@ -0,0 +1 @@ +Configurable default spam threshold used for new users From 82860d0f8013742ae3d1d8be9d17b78dc55c59b9 Mon Sep 17 00:00:00 2001 From: enginefeeder101 Date: Wed, 8 Jun 2022 17:13:38 +0200 Subject: [PATCH 003/274] Moved parsing environment variable to global application config dictionary Per requested changes added the ``DEFAULT_SPAM_THRESHOLD`` to the main application configuration dictionary in ``configuration.py`` and updated ``models.py`` accordingly. No error handling is added, as that was not required. --- core/admin/mailu/configuration.py | 2 ++ core/admin/mailu/models.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 912a0204..f1bb2283 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -75,6 +75,7 @@ DEFAULT_CONFIG = { 'SESSION_COOKIE_SECURE': True, 'CREDENTIAL_ROUNDS': 12, 'TZ': 'Etc/UTC', + 'DEFAULT_SPAM_THRESHOLD': 80, # Host settings 'HOST_IMAP': 'imap', 'HOST_LMTP': 'imap:2525', @@ -163,6 +164,7 @@ class ConfigManager: self.config['MESSAGE_RATELIMIT_EXEMPTION'] = set([s for s in self.config['MESSAGE_RATELIMIT_EXEMPTION'].lower().replace(' ', '').split(',') if s]) self.config['HOSTNAMES'] = ','.join(hostnames) self.config['HOSTNAME'] = hostnames[0] + self.config['DEFAULT_SPAM_THRESHOLD'] = int(self.config['DEFAULT_SPAM_THRESHOLD']) # update the app config app.config.update(self.config) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6e277fe3..3722003c 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -509,8 +509,7 @@ class User(Base, Email): displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean, nullable=False, default=True) spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True) - spam_threshold = db.Column(db.Integer, nullable=False, - default=int(os.environ.get('DEFAULT_SPAM_THRESHOLD', 80))) + spam_threshold = db.Column(db.Integer, nullable=False, default=app.config['DEFAULT_SPAM_THRESHOLD']) # Flask-login attributes is_authenticated = True From 81c9e01d2487e813cb86652667e15b1fcb172d64 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 19 Aug 2022 17:58:33 +0000 Subject: [PATCH 004/274] finishing touches for PR# 2328 Antispam.rst contained a syntax error. Move config description to common section which is more fitting. Fixed wrong assignment of default value for DEFAULT_SPAM_THRESHOLD in models.py. --- core/admin/mailu/models.py | 7 ++++++- docs/antispam.rst | 32 ++++++++++++++++---------------- docs/configuration.rst | 6 +++--- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 3722003c..1e703e1a 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -509,7 +509,12 @@ class User(Base, Email): displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean, nullable=False, default=True) spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True) - spam_threshold = db.Column(db.Integer, nullable=False, default=app.config['DEFAULT_SPAM_THRESHOLD']) + def default_spam_threshold(cls): + if app: + return app.config['DEFAULT_SPAM_THRESHOLD'] + else: + return 80 + spam_threshold = db.Column(db.Integer, nullable=False, default=default_spam_threshold) # Flask-login attributes is_authenticated = True diff --git a/docs/antispam.rst b/docs/antispam.rst index 7971200c..9293653b 100644 --- a/docs/antispam.rst +++ b/docs/antispam.rst @@ -21,29 +21,29 @@ Rspamd rejects non-compliant email messages and email messages that contain viru 1. When an email message is received or send by Postfix, it is scanned by Rspamd. If the email message receives a spam score between 6 and 15, a header is added to the email message with the spam score. This is an example of an email header for a spam score of 14:: X-Spamd-Bar: ++++++++++++++ - X-Spam-Level: ************** + X-Spam-Level: ************** Authentication-Results: test.mailu.io; dkim=pass header.d=example.com header.s=mailing header.b=ABCDE; dkim=pass header.d=example.com header.s=mailing header.b=ABCDE; dmarc=pass (policy=none) header.from=eventim.de; spf=pass (test.mailu.io: domain of return@example.com designates 11.22.33.44 as permitted sender) smtp.mailfrom=return@example.com - X-Spam: Yes - + X-Spam: Yes + 2. Dovecot is then responsible for classifying the email message to the Junk folder based on user preferences. It works as following: * In the administration web interface, under settings under Antispam 'Enable spam filter' must be ticked. If this option is disabled, then all email messages will automatically go to the inbox folder. Except for email messages with a score of 15 or higher, as these email messages are rejected by Rspamd. - + * In the administration web interface, under settings under Antispam, the user defined spam filter tolerance must be configured. The default value is 80%. The lower the spam filter tolerance, the more false positives (ham classified as spam). The user can change this setting to finetune when an email message is classified as spam. * Dovecot extracts the X-Spam-Level email header from the email message and converts the spam score (0 - 15) to a 0 - 100 percent scale. This spam score is compared with the user defined spam filter tolerance. If the spam score is lower than the user defined spam filter tolerance, then the email message is accepted. In logic: - + If is greater than , then move the email message to the spam folder and mark the email message as read. - + For example if the user defined spam filter tolerance is set to 80%(default) and the spam score of an email message is 10: - + 66% (10/15) is less than 80%, so the email is classified as ham. This email message will go to the inbox folder. If the user wants email messages with a score of 10 (66%) to be classified as spam, then the user defined spam filter tolerance can be lowered to 65% in the administration web interface. - - The default spam filter tolerance used for new users can be configured using the environment variable ``DEFAULT_SPAM_THRESHOLD``. See also: :ref:`_advanced_cfg`. + + The default spam filter tolerance used for new users can be configured using the environment variable ``DEFAULT_SPAM_THRESHOLD``. See also :ref:`common_cfg` in the configuration reference. .. image:: assets/screenshots/SpamFiltering.png @@ -62,7 +62,7 @@ If you already have an existing mailbox and want Mailu to learn them all as ham rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /mail/user\@example.com/.Ham_Learn/cur/ -This should learn every file located in the ``Ham_Learn`` folder from user@example.com +This should learn every file located in the ``Ham_Learn`` folder from user@example.com Likewise, to learn all messages within the folder ``Spam_Learn`` as spam messages : @@ -77,7 +77,7 @@ Likewise, to learn all messages within the folder ``Spam_Learn`` as spam message How can I block emails from a domain? ------------------------------------- -Via the multimap filter it is possible to block emails from a sender domain. See the `official rspamd documentation`_ for more information. A local blacklist that contains the domains to be blocked can be configured via the :ref:`Rspamd overrides folder `. +Via the multimap filter it is possible to block emails from a sender domain. See the `official rspamd documentation`_ for more information. A local blacklist that contains the domains to be blocked can be configured via the :ref:`Rspamd overrides folder `. The following steps have to be taken to configure an additional symbol (rule) that uses the multimap filter to block emails from sender domain. @@ -99,7 +99,7 @@ The following steps have to be taken to configure an additional symbol (rule) th } Note the "action = "reject";" line. This is a so-called pre-filter. No further filters/rules are processed when a pre-filter is used. If you omit this line, then the configured score will be added to the total score of the email message. Depending on the end-score after processing all rules, a verdict is made. To override this, you can add the action line. When this symbol (rule) is fired, then this action is immediately taken and no further processing occurs. You can use the following actions: - + * discard: drop an email message, but return success for sender (should be used merely in special cases) * reject: reject the email message. This enables the actual blocking of mails from the domain. @@ -112,9 +112,9 @@ The following steps have to be taken to configure an additional symbol (rule) th To move an email message to the Junk (Spam) folder, a score of 15 can be used in combination with the action "add header". The above example configuration will reject all emails send from domains that are listed in '/etc/rspamd/override.d/blacklist.inc'. - -2. In the Rspamd overrides folder create a map that contains the domains to be blocked. You can use # to add comments. + +2. In the Rspamd overrides folder create a map that contains the domains to be blocked. You can use # to add comments. Create the file /mailu/overrides/rspamd/blacklist.inc with the following contents: .. code-block:: bash @@ -130,12 +130,12 @@ The following steps have to be taken to configure an additional symbol (rule) th docker-compose scale antispam=0 docker-compose scale antispam=1 -4. (Optional) Check if the custom symbol is loaded. To access the Rspamd webgui, log in the Mailu administration web interface with a user that is an administrator and go to Antispam. In Rspamd webgui go to tab Symbols. Change the group drop-down box to local_bl. The following additional rule will be listed. +4. (Optional) Check if the custom symbol is loaded. To access the Rspamd webgui, log in the Mailu administration web interface with a user that is an administrator and go to Antispam. In Rspamd webgui go to tab Symbols. Change the group drop-down box to local_bl. The following additional rule will be listed. .. image:: assets/screenshots/RspamdSymbolBlacklist.png The symbol is only displayed if the symbol has no pre-filter (action= line) configured. Changes made in this screen are not saved to the configuration file. - + 5. Check if the map is available. In rspamd webgui to to configuration. A map is available with the path: /etc/rspamd/override.d/blacklist.inc Senders domain part is on the local blacklist diff --git a/docs/configuration.rst b/docs/configuration.rst index 0df8633f..3bbc42f0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -62,6 +62,8 @@ there is a good way to disable rate limiting altogether. The ``TLS_FLAVOR`` sets how Mailu handles TLS connections. Setting this value to ``notls`` will cause Mailu not to server any web content! More on :ref:`tls_flavor`. +The ``DEFAULT_SPAM_THRESHOLD`` (default: 80) is the default spam tolerance used when creating a new user. + Mail settings ------------- @@ -163,7 +165,7 @@ To have the account created automatically, you just need to define a few environ - ``INITIAL_ADMIN_DOMAIN``: the domain appendix: Most probably identical to the ``DOMAIN`` variable. - ``INITIAL_ADMIN_PW``: the admin password. - ``INITIAL_ADMIN_MODE``: use one of the options below for configuring how the admin account must be created: - + - ``create``: (default) creates a new admin account and raises an exception when it already exists. - ``ifmissing``: creates a new admin account when the admin account does not exist. - ``update``: creates a new admin account when it does not exist, or update the password of an existing admin account. @@ -211,8 +213,6 @@ The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usua .. _`TZ database name`: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones -The ``DEFAULT_SPAM_THRESHOLD`` (default: 80) setting is the default spam tolerance used when creating a new user. - Antivirus settings ------------------ From 102d96bc7d9226ec9e7d9f4e7412f3d95ab8abb1 Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Mon, 26 Sep 2022 09:48:29 +0200 Subject: [PATCH 005/274] Implement event lister to keep updated_at unchanged on quota_bytes_used updates --- core/admin/mailu/models.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 70a9528f..03434fee 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -25,6 +25,7 @@ from flask import current_app as app from sqlalchemy.ext import declarative from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.inspection import inspect +from sqlalchemy.orm.attributes import flag_modified from werkzeug.utils import cached_property from mailu import dkim, utils @@ -982,3 +983,13 @@ class MailuConfig: alias = MailuCollection(Alias) relay = MailuCollection(Relay) config = MailuCollection(Config) + + +@db.event.listens_for(User, 'before_update') +def receive_before_update(mapper, connection, target): + """ Mark updated_at as modified, but keep the old date when updating the quota_bytes_used + """ + insp = db.inspect(target) + quota_bytes_used_changed, _, _ = insp.attrs.quota_bytes_used.history + if quota_bytes_used_changed: + flag_modified(target, 'updated_at') From 10583f57dd3ce851e3c4af9a5e87a1e1887d2d5a Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Mon, 26 Sep 2022 09:53:10 +0200 Subject: [PATCH 006/274] Add newsfragment --- towncrier/newsfragments/1363.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/1363.bugfix diff --git a/towncrier/newsfragments/1363.bugfix b/towncrier/newsfragments/1363.bugfix new file mode 100644 index 00000000..edb198d7 --- /dev/null +++ b/towncrier/newsfragments/1363.bugfix @@ -0,0 +1 @@ +Do not update the updated_at field of the User model when quota_bytes_used is updated \ No newline at end of file From bda404182f874b53d7f021471bd30f2c7c128469 Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Fri, 7 Oct 2022 11:17:46 +0200 Subject: [PATCH 007/274] Replace before update listener with method in the Base class --- core/admin/mailu/internal/views/dovecot.py | 1 + core/admin/mailu/models.py | 14 ++++---------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 2f5f68a4..e949e2eb 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -33,6 +33,7 @@ def dovecot_quota(ns, user_email): user = models.User.query.get(user_email) or flask.abort(404) if ns == "storage": user.quota_bytes_used = flask.request.get_json() + user.flag_updated_at_as_modified() models.db.session.commit() return flask.jsonify(None) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 03434fee..edcdcf27 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -155,6 +155,10 @@ class Base(db.Model): self.__hashed = id(self) if primary is None else hash(primary) return self.__hashed + def flag_updated_at_as_modified(self): + """ Mark updated_at as modified, but keep the old date when updating the model""" + flag_modified(self, 'updated_at') + # Many-to-many association table for domain managers managers = db.Table('manager', Base.metadata, @@ -983,13 +987,3 @@ class MailuConfig: alias = MailuCollection(Alias) relay = MailuCollection(Relay) config = MailuCollection(Config) - - -@db.event.listens_for(User, 'before_update') -def receive_before_update(mapper, connection, target): - """ Mark updated_at as modified, but keep the old date when updating the quota_bytes_used - """ - insp = db.inspect(target) - quota_bytes_used_changed, _, _ = insp.attrs.quota_bytes_used.history - if quota_bytes_used_changed: - flag_modified(target, 'updated_at') From 295d7ea6755928e66422a3864adab16584760b20 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:08:19 +0200 Subject: [PATCH 008/274] Move assets to own Dockerfile --- core/admin/assets/Dockerfile | 14 ++++++++++++++ core/admin/assets/{ => content}/app.css | 0 core/admin/assets/{ => content}/app.js | 0 core/admin/assets/{ => content}/mailu.png | Bin core/admin/{ => assets/content}/package.json | 0 core/admin/assets/{ => content}/vendor.js | 0 core/admin/{ => assets/content}/webpack.config.js | 0 7 files changed, 14 insertions(+) create mode 100644 core/admin/assets/Dockerfile rename core/admin/assets/{ => content}/app.css (100%) rename core/admin/assets/{ => content}/app.js (100%) rename core/admin/assets/{ => content}/mailu.png (100%) rename core/admin/{ => assets/content}/package.json (100%) rename core/admin/assets/{ => content}/vendor.js (100%) rename core/admin/{ => assets/content}/webpack.config.js (100%) diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile new file mode 100644 index 00000000..d799fa4f --- /dev/null +++ b/core/admin/assets/Dockerfile @@ -0,0 +1,14 @@ +# syntax=docker/dockerfile-upstream:1.4.3 + +FROM node:16-alpine3.16 + +COPY content ./ +RUN set -euxo pipefail \ + && npm config set update-notifier false \ + && npm install --no-fund \ + && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ + && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ + cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ + done \ + && node_modules/.bin/webpack-cli --color + diff --git a/core/admin/assets/app.css b/core/admin/assets/content/app.css similarity index 100% rename from core/admin/assets/app.css rename to core/admin/assets/content/app.css diff --git a/core/admin/assets/app.js b/core/admin/assets/content/app.js similarity index 100% rename from core/admin/assets/app.js rename to core/admin/assets/content/app.js diff --git a/core/admin/assets/mailu.png b/core/admin/assets/content/mailu.png similarity index 100% rename from core/admin/assets/mailu.png rename to core/admin/assets/content/mailu.png diff --git a/core/admin/package.json b/core/admin/assets/content/package.json similarity index 100% rename from core/admin/package.json rename to core/admin/assets/content/package.json diff --git a/core/admin/assets/vendor.js b/core/admin/assets/content/vendor.js similarity index 100% rename from core/admin/assets/vendor.js rename to core/admin/assets/content/vendor.js diff --git a/core/admin/webpack.config.js b/core/admin/assets/content/webpack.config.js similarity index 100% rename from core/admin/webpack.config.js rename to core/admin/assets/content/webpack.config.js From 5e552bae69bd4ff72b4377a8ab31c7b52e4be21d Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:08:37 +0200 Subject: [PATCH 009/274] Add base image --- core/base/Dockerfile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 core/base/Dockerfile diff --git a/core/base/Dockerfile b/core/base/Dockerfile new file mode 100644 index 00000000..71ac6e31 --- /dev/null +++ b/core/base/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile-upstream:1.4.3 + +ARG DISTRO=alpine:3.14.5 +FROM $DISTRO + +ENV TZ Etc/UTC +ENV LANG C.UTF-8 + +# TODO: use intermediate image to build virtual env + +RUN set -euxo pipefail \ + && adduser -s /bin/bash -Dh /app -k /var/empty -u 1000 -g mailu app \ + && apk add --no-cache bash ca-certificates tzdata python3 py3-pip py3-wheel \ + && pip3 install --no-cache-dir --upgrade pip + +WORKDIR /app + +COPY libs libs/ + +# TODO: work in virtual env (see above) +# && python3 -m venv . \ +RUN set -euxo pipefail \ + && pip3 install --no-cache-dir -r libs/requirements.txt --only-binary=:all: \ + || ( apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ + && pip3 install --no-cache-dir -r libs/requirements.txt \ + && apk del --no-cache .build-deps ) + +# TODO: clean image (or use intermediate - see above) +# && bin/pip uninstall -y pip distribute setuptools wheel \ +# && rm -rf /tmp/* /root/.cache/pip From 9fe452e3d1d868b076a34888659ea701e136d192 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:09:07 +0200 Subject: [PATCH 010/274] Use base image when building core images --- core/admin/Dockerfile | 53 ++++++++++------------------------------- core/dovecot/Dockerfile | 34 ++++++++------------------ core/nginx/Dockerfile | 43 ++++++++++++++++----------------- core/none/Dockerfile | 14 ++++++++--- core/postfix/Dockerfile | 41 ++++++++++--------------------- core/rspamd/Dockerfile | 29 ++++++++-------------- tests/build.hcl | 34 +++++++++++++++++++++++++- 7 files changed, 111 insertions(+), 137 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index e4d870e0..c3731fb2 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -1,61 +1,34 @@ -# First stage to build assets -ARG DISTRO=alpine:3.14.5 +# syntax=docker/dockerfile-upstream:1.4.3 -FROM node:16-alpine3.16 as assets - -COPY package.json ./ -RUN set -eu \ - && npm config set update-notifier false \ - && npm install --no-fund - -COPY webpack.config.js ./ -COPY assets ./assets -RUN set -eu \ - && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ - && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ - cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ - done \ - && node_modules/.bin/webpack-cli --color - - -# Actual application -FROM $DISTRO -ARG VERSION - -ENV TZ Etc/UTC +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN set -eu \ - && apk add --no-cache python3 py3-pip py3-wheel git bash tzdata \ - && pip3 install --upgrade pip - -RUN mkdir -p /app -WORKDIR /app - COPY requirements-prod.txt requirements.txt -RUN set -eu \ +RUN set -euxo pipefail \ && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ && pip install --no-cache-dir -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ || ( apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ - && pip install --upgrade pip \ && pip install -r requirements.txt \ && apk del --no-cache build-dep ) -COPY --from=assets static ./mailu/static COPY mailu ./mailu +RUN pybabel compile -d mailu/translations + COPY migrations ./migrations + COPY start.py /start.py COPY audit.py /audit.py -RUN pybabel compile -d mailu/translations +COPY --from=assets static ./mailu/static + +RUN echo $VERSION >> /version EXPOSE 80/tcp +HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index + VOLUME ["/data","/dkim"] + ENV FLASK_APP mailu - CMD /start.py - -HEALTHCHECK CMD curl -f -L http://localhost/sso/login?next=ui.index || exit 1 -RUN echo $VERSION >> /version diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index e44b69f7..2d74e59b 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -1,36 +1,22 @@ -ARG DISTRO=alpine:3.14.5 +# syntax=docker/dockerfile-upstream:1.4.3 + +FROM base -FROM $DISTRO ARG VERSION -ENV TZ Etc/UTC - LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict py3-yarl tzdata \ - && pip3 install --upgrade pip - -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 - -# Shared layer between dovecot and postfix -RUN apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ - && pip3 install "podop>0.2.5" \ - && apk del .build-deps - -# Image specific layers under this line -RUN apk add --no-cache \ - dovecot dovecot-lmtpd dovecot-pop3d dovecot-submissiond dovecot-pigeonhole-plugin rspamd-client xapian-core dovecot-fts-xapian \ - && mkdir /var/lib/dovecot +RUN set -euxo pipefail \ + && apk add --no-cache dovecot dovecot-lmtpd dovecot-pop3d dovecot-submissiond dovecot-pigeonhole-plugin rspamd-client xapian-core dovecot-fts-xapian \ + && mkdir /var/lib/dovecot COPY conf /conf COPY start.py /start.py +RUN echo $VERSION >> /version + EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp +HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready." + VOLUME ["/mail"] CMD /start.py - -HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready." -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 07021429..2a34403f 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -1,34 +1,33 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO +# syntax=docker/dockerfile-upstream:1.4.3 + +FROM base as static + +COPY static /static + +RUN set -euxo pipefail \ + && gzip -k9 /static/*.ico /static/*.txt \ + && chmod a+rX-w -R /static + + +FROM base + ARG VERSION - -ENV TZ Etc/UTC - LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict \ - && pip3 install --upgrade pip - -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 - # Image specific layers under this line -RUN apk add --no-cache certbot nginx nginx-mod-mail openssl curl tzdata \ - && pip3 install watchdog +RUN set -euxo pipefail \ + && apk add --no-cache certbot nginx nginx-mod-mail openssl curl \ + && pip3 install --no-cache-dir watchdog COPY conf /conf -COPY static /static +COPY --from=static /static /static COPY *.py / -RUN gzip -k9 /static/*.ico /static/*.txt; chmod a+rX -R /static +RUN echo $VERSION >> /version EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp -VOLUME ["/certs"] -VOLUME ["/overrides"] +HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://localhost/health + +VOLUME ["/certs", "/overrides"] CMD /start.py - -HEALTHCHECK CMD curl -k -f -L http://localhost/health || exit 1 -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/core/none/Dockerfile b/core/none/Dockerfile index 06d351a9..058b18c5 100644 --- a/core/none/Dockerfile +++ b/core/none/Dockerfile @@ -1,6 +1,14 @@ +# syntax=docker/dockerfile-upstream:1.4.3 # This is an idle image to dynamically replace any component if disabled. -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO +FROM base -CMD sleep 1000000d +ARG VERSION=local +LABEL version=$VERSION + +RUN echo $VERSION >> /version + +HEALTHCHECK CMD true + +USER app +CMD ["/bin/bash", "-c", "sleep infinity"] diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 373e5c8d..66adbde3 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,40 +1,25 @@ -ARG DISTRO=alpine:3.14.5 +# syntax=docker/dockerfile-upstream:1.4.3 -FROM $DISTRO -ARG VERSION - -ENV TZ Etc/UTC +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict py3-yarl tzdata \ - && pip3 install --upgrade pip - -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 - -# Shared layer between dovecot and postfix -RUN apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ - && pip3 install "podop>0.2.5" \ - && apk del .build-deps - -# Image specific layers under this line -# Building pycares from source requires py3-wheel and libffi-dev packages -RUN pip install --no-cache-dir --only-binary=:all: postfix-mta-sts-resolver==1.0.1 || (apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev py3-wheel libffi-dev \ - && pip3 install postfix-mta-sts-resolver==1.0.1 \ - && apk del .build-deps ) - -RUN apk add --no-cache postfix postfix-pcre cyrus-sasl-login rsyslog logrotate +RUN set -euxo pipefail \ + && apk add --no-cache postfix postfix-pcre cyrus-sasl-login rsyslog logrotate \ + && pip install --no-cache-dir --only-binary=:all: postfix-mta-sts-resolver==1.0.1 \ + || ( apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev py3-wheel libffi-dev \ + && pip3 install postfix-mta-sts-resolver==1.0.1 \ + && apk del .build-deps ) COPY conf /conf COPY start.py /start.py +RUN echo $VERSION >> /version + EXPOSE 25/tcp 10025/tcp +HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" + VOLUME ["/queue"] CMD /start.py - -HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" -RUN echo $VERSION >> /version diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 7b89aed0..5ee922e5 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -1,31 +1,22 @@ -ARG DISTRO=alpine:3.15 -FROM $DISTRO -ARG VERSION -ENV TZ Etc/UTC +# syntax=docker/dockerfile-upstream:1.4.3 +FROM base + +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict tzdata \ - && pip3 install --upgrade pip - -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 - -# Image specific layers under this line -RUN apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl - -RUN mkdir /run/rspamd +RUN set -euxo pipefail \ + && apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl \ + && mkdir /run/rspamd COPY conf/ /conf COPY start.py /start.py +RUN echo $VERSION >> /version + EXPOSE 11332/tcp 11334/tcp 11335/tcp +HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost:11334/ VOLUME ["/var/lib/rspamd"] CMD /start.py - -HEALTHCHECK --start-period=350s CMD curl -f -L http://localhost:11334/ || exit 1 -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/tests/build.hcl b/tests/build.hcl index 3df00b51..c32da8d5 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -78,6 +78,19 @@ function "tag" { # docker buildx bake -f tests\build.hcl docs #----------------------------------------------------------------------------------------- +# ----------------------------------------------------------------------------------------- +# Base images +# ----------------------------------------------------------------------------------------- +target "base" { + inherits = ["defaults"] + context="core/base" +} + +target "assets" { + inherits = ["defaults"] + context="core/admin/assets" +} + # ----------------------------------------------------------------------------------------- # Documentation and setup images # ----------------------------------------------------------------------------------------- @@ -103,36 +116,55 @@ target "setup" { target "none" { inherits = ["defaults"] context="core/none" + contexts= { + base = "target:base" + } tags = tag("none") } target "admin" { inherits = ["defaults"] context="core/admin" + contexts= { + base = "target:base" + assets = "target:assets" + } tags = tag("admin") } target "antispam" { inherits = ["defaults"] context="core/rspamd" + contexts= { + base = "target:base" + } tags = tag("rspamd") } target "front" { inherits = ["defaults"] context="core/nginx" + contexts= { + base = "target:base" + } tags = tag("nginx") } target "imap" { inherits = ["defaults"] context="core/dovecot" + contexts= { + base = "target:base" + } tags = tag("dovecot") } target "smtp" { inherits = ["defaults"] context="core/postfix" + contexts= { + base = "target:base" + } tags = tag("postfix") } @@ -182,4 +214,4 @@ target "webdav" { inherits = ["defaults"] context="optional/radicale" tags = tag("radicale") -} \ No newline at end of file +} From b501498401fc4f09c79f6faa13b4c93858f467a2 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:09:31 +0200 Subject: [PATCH 011/274] Update .gitignore file --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8734dcdb..f5e9f8ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -*.pyc -*.mo +**/*.pyc +**/*.mo __pycache__ pip-selfcheck.json /core/admin/lib* From b1b0aeb69dc8d87dfce527f6ab67582683c73b1d Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Sun, 22 Jul 2018 20:06:30 +0200 Subject: [PATCH 012/274] Initial commit --- core/base/libs/podop/CONTRIBUTING.md | 7 ++ core/base/libs/podop/LICENSE.md | 25 ++++++ core/base/libs/podop/README.md | 112 ++++++++++++++++++++++++ core/base/libs/podop/podop/__init__.py | 44 ++++++++++ core/base/libs/podop/podop/dovecot.py | 95 ++++++++++++++++++++ core/base/libs/podop/podop/postfix.py | 115 +++++++++++++++++++++++++ core/base/libs/podop/podop/table.py | 26 ++++++ core/base/libs/podop/scripts/podop | 25 ++++++ core/base/libs/podop/setup.py | 14 +++ 9 files changed, 463 insertions(+) create mode 100644 core/base/libs/podop/CONTRIBUTING.md create mode 100644 core/base/libs/podop/LICENSE.md create mode 100644 core/base/libs/podop/README.md create mode 100644 core/base/libs/podop/podop/__init__.py create mode 100644 core/base/libs/podop/podop/dovecot.py create mode 100644 core/base/libs/podop/podop/postfix.py create mode 100644 core/base/libs/podop/podop/table.py create mode 100755 core/base/libs/podop/scripts/podop create mode 100644 core/base/libs/podop/setup.py diff --git a/core/base/libs/podop/CONTRIBUTING.md b/core/base/libs/podop/CONTRIBUTING.md new file mode 100644 index 00000000..6a09c85d --- /dev/null +++ b/core/base/libs/podop/CONTRIBUTING.md @@ -0,0 +1,7 @@ +This project is open source, and your contributions are all welcome. There are mostly three different ways one can contribute to the project: + +1. use Podop, either on test or on production servers, and report meaningful bugs when you find some; +2. write and publish, or contribute to mail distributions based on Podop, like Mailu; +2. contribute code and/or configuration to the repository (see [the development guidelines](https://mailu.io/contributors/guide.html) for details); + +Either way, keep in mind that the code you write must be licensed under the same conditions as the project itself. Additionally, all contributors are considered equal co-authors of the project. diff --git a/core/base/libs/podop/LICENSE.md b/core/base/libs/podop/LICENSE.md new file mode 100644 index 00000000..8aa0da5d --- /dev/null +++ b/core/base/libs/podop/LICENSE.md @@ -0,0 +1,25 @@ +MIT License + +Copyright (c) 2018 All Podop contributors at the date + +This software consists of voluntary contributions made by multiple individuals. +For exact contribution history, see the revision history available at +https://github.com/Mailu/podop.git + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/base/libs/podop/README.md b/core/base/libs/podop/README.md new file mode 100644 index 00000000..208b3ce5 --- /dev/null +++ b/core/base/libs/podop/README.md @@ -0,0 +1,112 @@ +Podop is a piece of middleware designed to run between Postfix or Dovecot +on one side, any Python implementation of a table lookup protocol on the +other side. + +It is thus able to forward Postfix maps and Dovecot dicts to the same +(or multiple) backends in order to write a single, more flexible backend +for a mail distribution. + +Examples +======== + +- Connect Postfix to a DNS lookup so that every domain that has a proper MX + record to your Postfix is actually accepted as a local domain +- Connect both Postfix and Dovecot to an HTTP microservice to run a high + availability microservice-based mail service +- Use a single database server running any Python-compatible API for both + your Postfix and Dovecot servers + +Configure Podop tables +====================== + +Podop tables are configured through CLI arguments when running the server. +You must provide a ``--name`` for the table, a ``--type`` for the table and +a ``--param`` that parametrizes the map. + +URL table +--------- + +The URL table will initiate an HTTP GET request for read access and an HTTP +POST request for write access to a table. The table is parametrized with +a template URL containing ``§`` (or ``{}``) for inserting the table key. + +``` +--name test --type url --param http://microservice/api/v1/map/tests/§ +``` + +GET requests should return ``200`` and a JSON-encoded object +that will be passed either to Postfix or Dovecot. They should return ``4XX`` +for access issues that will result in lookup miss, and ``5XX`` for backend +issues that will result in a temporary failure. + +POST requests will contain a JSON-encoded object in the request body, that +will be saved in the table. + +Postfix usage +============= + +In order to access Podop tables from Postfix, you should setup ``socketmap`` +Postfix maps. For instance, in order to access the ``test`` table on a Podop +socket at ``/tmp/podop.socket``, use the following setup: + +``` +virtual_alias_maps = socketmap:unix:/tmp/podop.socket:test +``` + +Multiple maps or identical maps can be configured for various usages. + +``` +virtual_alias_maps = socketmap:unix:/tmp/podop.socket:alias +virtual_mailbox_domains = socketmap:unix:/tmp/podop.socket:domain +virtual_mailbox_maps = socketmap:unix:/tmp/podop.socket:alias +``` + +In order to simplify the configuration, you can setup a shortcut. + +``` +podop = socketmap:unic:/tmp/podop.socket +virtual_alias_maps = ${podop}:alias +virtual_mailbox_domains = ${podop}:domain +virtual_mailbox_maps = ${podop}:alias +``` + +Dovecot usage +============= + +In order to access Podop tables from Dovecot, you should setup a ``proxy`` +Dovecot dictionary. For instance, in order to access the ``test`` table on +a Podop socket at ``/tmp/podop.socket``, use the following setup: + +``` +mail_attribute_dict = proxy:/tmp/podop.socket:test +``` + +Multiple maps or identical maps can be configured for various usages. + +``` +mail_attribute_dict = proxy:/tmp/podop.socket:meta + +passdb { + driver = dict + args = /etc/dovecot/auth.conf +} + +userdb { + driver = dict + args = /etc/dovecot/auth.conf +} + +# then in auth.conf +uri = proxy:/tmp/podop.socket:auth +iterate_disable = yes +default_pass_scheme = plain +password_key = passdb/%u +user_key = userdb/%u +``` + +Contributing +============ + +Podop is free software, open to suggestions and contributions. All +components are free software and compatible with the MIT license. All +the code is placed under the MIT license. diff --git a/core/base/libs/podop/podop/__init__.py b/core/base/libs/podop/podop/__init__.py new file mode 100644 index 00000000..8c2c4d8d --- /dev/null +++ b/core/base/libs/podop/podop/__init__.py @@ -0,0 +1,44 @@ +""" Podop is a *Po*stfix and *Do*vecot proxy + +It is able to proxify postfix maps and dovecot dicts to any table +""" + +import asyncio +import logging + +from podop import postfix, dovecot, table + + +SERVER_TYPES = dict( + postfix=postfix.SocketmapProtocol, + dovecot=dovecot.DictProtocol +) + +TABLE_TYPES = dict( + url=table.UrlTable +) + + +def run_server(server_type, socket, tables): + """ Run the server, given its type, socket path and table list + + The table list must be a list of tuples (name, type, param) + """ + # Prepare the maps + table_map = { + name: TABLE_TYPES[table_type](param) + for name, table_type, param in tables + } + # Run the main loop + logging.basicConfig(level=logging.DEBUG) + loop = asyncio.get_event_loop() + server = loop.run_until_complete(loop.create_unix_server( + SERVER_TYPES[server_type].factory(table_map), socket + )) + try: + loop.run_forever() + except KeyboardInterrupt: + pass + server.close() + loop.run_until_complete(server.wait_closed()) + loop.close() diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py new file mode 100644 index 00000000..bbf134cc --- /dev/null +++ b/core/base/libs/podop/podop/dovecot.py @@ -0,0 +1,95 @@ +""" Dovecot dict proxy implementation +""" + +import asyncio +import logging + + +class DictProtocol(asyncio.Protocol): + """ Protocol to answer Dovecot dict requests, as implemented in Dict proxy. + + There is very little documentation about the protocol, most of it was + reverse-engineered from : + + https://github.com/dovecot/core/blob/master/src/dict/dict-connection.c + https://github.com/dovecot/core/blob/master/src/dict/dict-commands.c + https://github.com/dovecot/core/blob/master/src/lib-dict/dict-client.h + """ + + DATA_TYPES = {0: str, 1: int} + + def __init__(self, table_map): + self.table_map = table_map + self.major_version = None + self.minor_version = None + self.dict = None + super(DictProtocol, self).__init__() + + def connection_made(self, transport): + logging.info('Connect {}'.format(transport.get_extra_info('peername'))) + self.transport = transport + + def data_received(self, data): + logging.debug("Received {}".format(data)) + results = [] + for line in data.split(b"\n"): + logging.debug("Line {}".format(line)) + if len(line) < 2: + continue + command = DictProtocol.COMMANDS.get(line[0]) + if command is None: + logging.warning('Unknown command {}'.format(line[0])) + return self.transport.abort() + args = line[1:].strip().split(b"\t") + try: + future = command(self, *args) + if future: + results.append(future) + except Exception: + logging.exception("Error when processing request") + return self.transport.abort() + logging.debug("Results {}".format(results)) + return asyncio.gather(*results) + + def process_hello(self, major, minor, value_type, user, dict_name): + self.major, self.minor = int(major), int(minor) + logging.debug('Client version {}.{}'.format(self.major, self.minor)) + assert self.major == 2 + self.value_type = DictProtocol.DATA_TYPES[int(value_type)] + self.user = user + self.dict = self.table_map[dict_name.decode("ascii")] + logging.debug("Value type {}, user {}, dict {}".format( + self.value_type, self.user, dict_name)) + + async def process_lookup(self, key): + logging.debug("Looking up {}".format(key)) + result = await self.dict.get(key.decode("utf8")) + if result is not None: + if type(result) is str: + response = result.encode("utf8") + elif type(result) is bytes: + response = result + else: + response = json.dumps(result).encode("ascii") + return self.reply(b"O", response) + else: + return self.reply(b"N") + + def reply(self, command, *args): + logging.debug("Replying {} with {}".format(command, args)) + self.transport.write(command) + self.transport.write(b"\t".join( + arg.replace(b"\t", b"\t\t") for arg in args + )) + self.transport.write(b"\n") + + @classmethod + def factory(cls, table_map): + """ Provide a protocol factory for a given map instance. + """ + return lambda: cls(table_map) + + COMMANDS = { + ord("H"): process_hello, + ord("L"): process_lookup + } diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py new file mode 100644 index 00000000..122cf962 --- /dev/null +++ b/core/base/libs/podop/podop/postfix.py @@ -0,0 +1,115 @@ +""" Postfix map proxy implementation +""" + +import asyncio +import logging + + +class NetstringProtocol(asyncio.Protocol): + """ Netstring asyncio protocol implementation. + + For protocol details, see https://cr.yp.to/proto/netstrings.txt + """ + + # Length of the smallest allocated buffer, larger buffers will be + # allocated dynamically + BASE_BUFFER = 1024 + + # Maximum length of a buffer, will crash when exceeded + MAX_BUFFER = 65535 + + def __init__(self): + super(NetstringProtocol, self).__init__() + self.init_buffer() + + def init_buffer(self): + self.len = None # None when waiting for a length to be sent) + self.separator = -1 # -1 when not yet detected (str.find) + self.index = 0 # relative to the buffer + self.buffer = bytearray(NetstringProtocol.BASE_BUFFER) + + def data_received(self, data): + # Manage the buffer + missing = len(data) - len(self.buffer) + self.index + if missing > 0: + if len(self.buffer) + missing > NetstringProtocol.MAX_BUFFER: + raise IOError("Not enough space when decoding netstring") + self.buffer.append(bytearray(missing + 1)) + new_index = self.index + len(data) + self.buffer[self.index:new_index] = data + self.index = new_index + # Try to detect a length at the beginning of the string + if self.len is None: + self.separator = self.buffer.find(0x3a) + if self.separator != -1 and self.buffer[:self.separator].isdigit(): + self.len = int(self.buffer[:self.separator], 10) + # Then get the complete string + if self.len is not None: + if self.index - self.separator == self.len + 2: + string = self.buffer[self.separator + 1:self.index - 1] + self.init_buffer() + self.string_received(string) + + def string_received(self, string): + pass + + def send_string(self, string): + """ Send a netstring + """ + self.transport.write(str(len(string)).encode('ascii')) + self.transport.write(b':') + self.transport.write(string) + self.transport.write(b',') + + +class SocketmapProtocol(NetstringProtocol): + """ Protocol to answer Postfix socketmap and proxify lookups to + an outside object. + + See http://www.postfix.org/socketmap_table.5.html for details on the + protocol. + + A table map must be provided as a dictionary to lookup tables. + """ + + def __init__(self, table_map): + self.table_map = table_map + super(SocketmapProtocol, self).__init__() + + def connection_made(self, transport): + logging.info('Connect {}'.format(transport.get_extra_info('peername'))) + self.transport = transport + + def string_received(self, string): + space = string.find(0x20) + if space != -1: + name = string[:space].decode('ascii') + key = string[space+1:].decode('utf8') + return asyncio.async(self.process_request(name, key)) + + def send_string(self, string): + logging.debug("Send {}".format(string)) + super(SocketmapProtocol, self).send_string(string) + + async def process_request(self, name, key): + """ Process a request by querying the provided map. + """ + logging.debug("Request {}/{}".format(name, key)) + try: + table = self.table_map.get(name) + except KeyError: + return self.send_string(b'TEMP no such map') + try: + result = await table.get(key) + return self.send_string(b'OK ' + str(result).encode('utf8')) + except KeyError: + return self.send_string(b'NOTFOUND ') + except Exception: + logging.exception("Error when processing request") + return self.send_string(b'TEMP unknown error') + + @classmethod + def factory(cls, table_map): + """ Provide a protocol factory for a given map instance. + """ + return lambda: cls(table_map) diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py new file mode 100644 index 00000000..f3b8cc1e --- /dev/null +++ b/core/base/libs/podop/podop/table.py @@ -0,0 +1,26 @@ +""" Table lookup backends for podop +""" + +import aiohttp +import logging + + +class UrlTable(object): + """ Resolve an entry by querying a parametrized GET URL. + """ + + def __init__(self, url_pattern): + """ url_pattern must contain a format ``{}`` so the key is injected in + the url before the query, the ``§`` character will be replaced with + ``{}`` for easier setup. + """ + self.url_pattern = url_pattern.replace('§', '{}') + + async def get(self, key): + logging.debug("Getting {} from url table".format(key)) + async with aiohttp.ClientSession() as session: + async with session.get(self.url_pattern.format(key)) as request: + if request.status == 200: + result = await request.json() + logging.debug("Got {} from url table".format(result)) + return result diff --git a/core/base/libs/podop/scripts/podop b/core/base/libs/podop/scripts/podop new file mode 100755 index 00000000..b22c830d --- /dev/null +++ b/core/base/libs/podop/scripts/podop @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import argparse + +from podop import run_server, SERVER_TYPES, TABLE_TYPES + + +def main(): + """ Run a podop server based on CLI arguments + """ + parser = argparse.ArgumentParser("Postfix and Dovecot proxy") + parser.add_argument("--socket", help="path to the socket", required=True) + parser.add_argument("--mode", choices=SERVER_TYPES.keys(), required=True) + parser.add_argument("--name", help="name of the table", action="append") + parser.add_argument("--type", choices=TABLE_TYPES.keys(), action="append") + parser.add_argument("--param", help="table parameter", action="append") + args = parser.parse_args() + run_server( + args.mode, args.socket, + zip(args.name, args.type, args.param) if args.name else [] + ) + + +if __name__ == "__main__": + main() diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py new file mode 100644 index 00000000..4d1e2ea3 --- /dev/null +++ b/core/base/libs/podop/setup.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +from distutils.core import setup + +setup( + name="Podop", + version="0.1", + description="Postfix and Dovecot proxy", + author="Pierre Jaury", + author_email="pierre@jaury.eu", + url="https://github.com/mailu/podop.git", + packages=["podop"], + scripts=["scripts/podop"] +) From eb6b1866f15bb2745b5fb36138c4a1e656ebaf7f Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Wed, 25 Jul 2018 20:49:16 +0200 Subject: [PATCH 013/274] Specify dependencies in the setup script --- core/base/libs/podop/setup.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index 4d1e2ea3..ac995168 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -2,13 +2,21 @@ from distutils.core import setup +with open("README.md", "r") as fh: + long_description = fh.read() + setup( - name="Podop", - version="0.1", + name="podop", + version="0.1.1", description="Postfix and Dovecot proxy", + long_description=long_description, + long_description_content_type="text/markdown", author="Pierre Jaury", author_email="pierre@jaury.eu", url="https://github.com/mailu/podop.git", packages=["podop"], - scripts=["scripts/podop"] + scripts=["scripts/podop"], + install_requires=[ + "aiohttp" + ] ) From c5fa0280a0e6312d7f734939fb4179fad21158b7 Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 20:48:04 +0200 Subject: [PATCH 014/274] Add support for dovecot dict_set operations --- core/base/libs/podop/podop/__init__.py | 6 ++- core/base/libs/podop/podop/dovecot.py | 69 ++++++++++++++++++++++---- core/base/libs/podop/podop/postfix.py | 8 +-- core/base/libs/podop/podop/table.py | 25 ++++++++-- core/base/libs/podop/scripts/podop | 20 +++++--- core/base/libs/podop/setup.py | 2 +- 6 files changed, 105 insertions(+), 25 deletions(-) diff --git a/core/base/libs/podop/podop/__init__.py b/core/base/libs/podop/podop/__init__.py index 8c2c4d8d..64ab3b67 100644 --- a/core/base/libs/podop/podop/__init__.py +++ b/core/base/libs/podop/podop/__init__.py @@ -5,6 +5,7 @@ It is able to proxify postfix maps and dovecot dicts to any table import asyncio import logging +import sys from podop import postfix, dovecot, table @@ -19,7 +20,7 @@ TABLE_TYPES = dict( ) -def run_server(server_type, socket, tables): +def run_server(verbosity, server_type, socket, tables): """ Run the server, given its type, socket path and table list The table list must be a list of tuples (name, type, param) @@ -30,7 +31,8 @@ def run_server(server_type, socket, tables): for name, table_type, param in tables } # Run the main loop - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(stream=sys.stderr, level=max(3 - verbosity, 0) * 10, + format='%(name)s (%(levelname)s): %(message)s') loop = asyncio.get_event_loop() server = loop.run_until_complete(loop.create_unix_server( SERVER_TYPES[server_type].factory(table_map), socket diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index bbf134cc..96eea3b8 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -3,11 +3,15 @@ import asyncio import logging +import json class DictProtocol(asyncio.Protocol): """ Protocol to answer Dovecot dict requests, as implemented in Dict proxy. + Only a subset of operations is handled properly by this proxy: hello, + lookup and transaction-based set. + There is very little documentation about the protocol, most of it was reverse-engineered from : @@ -20,9 +24,15 @@ class DictProtocol(asyncio.Protocol): def __init__(self, table_map): self.table_map = table_map + # Minor and major versions are not properly checked yet, but stored + # anyway self.major_version = None self.minor_version = None + # Every connection starts with specifying which table is used, dovecot + # tables are called dicts self.dict = None + # Dictionary of active transaction lists per transaction id + self.transactions = {} super(DictProtocol, self).__init__() def connection_made(self, transport): @@ -32,14 +42,17 @@ class DictProtocol(asyncio.Protocol): def data_received(self, data): logging.debug("Received {}".format(data)) results = [] + # Every command is separated by "\n" for line in data.split(b"\n"): - logging.debug("Line {}".format(line)) + # A command must at list have a type and one argument if len(line) < 2: continue + # The command function will handle the command itself command = DictProtocol.COMMANDS.get(line[0]) if command is None: logging.warning('Unknown command {}'.format(line[0])) return self.transport.abort() + # Args are separated by "\t" args = line[1:].strip().split(b"\t") try: future = command(self, *args) @@ -48,22 +61,30 @@ class DictProtocol(asyncio.Protocol): except Exception: logging.exception("Error when processing request") return self.transport.abort() - logging.debug("Results {}".format(results)) + # For asyncio consistency, wait for all results to fire before + # actually returning control return asyncio.gather(*results) def process_hello(self, major, minor, value_type, user, dict_name): + """ Process a dict protocol hello message + """ self.major, self.minor = int(major), int(minor) - logging.debug('Client version {}.{}'.format(self.major, self.minor)) - assert self.major == 2 self.value_type = DictProtocol.DATA_TYPES[int(value_type)] - self.user = user + self.user = user.decode("utf8") self.dict = self.table_map[dict_name.decode("ascii")] - logging.debug("Value type {}, user {}, dict {}".format( - self.value_type, self.user, dict_name)) + logging.debug("Client {}.{} type {}, user {}, dict {}".format( + self.major, self.minor, self.value_type, self.user, dict_name)) async def process_lookup(self, key): + """ Process a dict lookup message + """ logging.debug("Looking up {}".format(key)) - result = await self.dict.get(key.decode("utf8")) + # Priv and shared keys are handled slighlty differently + key_type, key = key.decode("utf8").split("/", 1) + result = await self.dict.get( + key, ns=(self.user if key_type == "priv" else None) + ) + # Handle various response types if result is not None: if type(result) is str: response = result.encode("utf8") @@ -75,6 +96,33 @@ class DictProtocol(asyncio.Protocol): else: return self.reply(b"N") + def process_begin(self, transaction_id): + """ Process a dict begin message + """ + self.transactions[transaction_id] = {} + + def process_set(self, transaction_id, key, value): + """ Process a dict set message + """ + # Nothing is actually set until everything is commited + self.transactions[transaction_id][key] = value + + async def process_commit(self, transaction_id): + """ Process a dict commit message + """ + # Actually handle all set operations from the transaction store + results = [] + for key, value in self.transactions[transaction_id].items(): + logging.debug("Storing {}={}".format(key, value)) + key_type, key = key.decode("utf8").split("/", 1) + result = await self.dict.set( + key, json.loads(value), + ns=(self.user if key_type == "priv" else None) + ) + # Remove stored transaction + del self.transactions[transaction_id] + return self.reply(b"O", transaction_id) + def reply(self, command, *args): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) @@ -91,5 +139,8 @@ class DictProtocol(asyncio.Protocol): COMMANDS = { ord("H"): process_hello, - ord("L"): process_lookup + ord("L"): process_lookup, + ord("B"): process_begin, + ord("C"): process_commit, + ord("S"): process_set } diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py index 122cf962..b0395f35 100644 --- a/core/base/libs/podop/podop/postfix.py +++ b/core/base/libs/podop/podop/postfix.py @@ -51,6 +51,8 @@ class NetstringProtocol(asyncio.Protocol): self.string_received(string) def string_received(self, string): + """ A new netstring was received + """ pass def send_string(self, string): @@ -81,16 +83,14 @@ class SocketmapProtocol(NetstringProtocol): self.transport = transport def string_received(self, string): + # The postfix format contains a space for separating the map name and + # the key space = string.find(0x20) if space != -1: name = string[:space].decode('ascii') key = string[space+1:].decode('utf8') return asyncio.async(self.process_request(name, key)) - def send_string(self, string): - logging.debug("Send {}".format(string)) - super(SocketmapProtocol, self).send_string(string) - async def process_request(self, name, key): """ Process a request by querying the provided map. """ diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py index f3b8cc1e..d30ff9fb 100644 --- a/core/base/libs/podop/podop/table.py +++ b/core/base/libs/podop/podop/table.py @@ -16,11 +16,30 @@ class UrlTable(object): """ self.url_pattern = url_pattern.replace('§', '{}') - async def get(self, key): - logging.debug("Getting {} from url table".format(key)) + async def get(self, key, ns=None): + """ Get the given key in the provided namespace + """ + if ns is not None: + key += "/" + ns async with aiohttp.ClientSession() as session: async with session.get(self.url_pattern.format(key)) as request: if request.status == 200: result = await request.json() - logging.debug("Got {} from url table".format(result)) + return result + + async def set(self, key, value, ns=None): + """ Set a value for the given key in the provided namespace + """ + if ns is not None: + key += "/" + ns + async with aiohttp.ClientSession() as session: + await session.post(self.url_pattern.format(key), json=value) + + async def iter(self, cat): + """ Iterate the given key (experimental) + """ + async with aiohttp.ClientSession() as session: + async with session.get(self.url_pattern.format(cat)) as request: + if request.status == 200: + result = await request.json() return result diff --git a/core/base/libs/podop/scripts/podop b/core/base/libs/podop/scripts/podop index b22c830d..f61c9e21 100755 --- a/core/base/libs/podop/scripts/podop +++ b/core/base/libs/podop/scripts/podop @@ -9,14 +9,22 @@ def main(): """ Run a podop server based on CLI arguments """ parser = argparse.ArgumentParser("Postfix and Dovecot proxy") - parser.add_argument("--socket", help="path to the socket", required=True) - parser.add_argument("--mode", choices=SERVER_TYPES.keys(), required=True) - parser.add_argument("--name", help="name of the table", action="append") - parser.add_argument("--type", choices=TABLE_TYPES.keys(), action="append") - parser.add_argument("--param", help="table parameter", action="append") + parser.add_argument("--socket", required=True, + help="path to the listening unix socket") + parser.add_argument("--mode", choices=SERVER_TYPES.keys(), required=True, + help="select which server will connect to Podop") + parser.add_argument("--name", action="append", + help="name of each configured table") + parser.add_argument("--type", choices=TABLE_TYPES.keys(), action="append", + help="type of each configured table") + parser.add_argument("--param", action="append", + help="mandatory param for each table configured") + parser.add_argument("-v", "--verbose", dest="verbosity", + action="count", default=0, + help="increases log verbosity for each occurence.") args = parser.parse_args() run_server( - args.mode, args.socket, + args.verbosity, args.mode, args.socket, zip(args.name, args.type, args.param) if args.name else [] ) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index ac995168..4951196d 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.1.1", + version="0.2", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From d640da8787dd40fa7a4d59da3dca4e2c8deff7f9 Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:10:07 +0200 Subject: [PATCH 015/274] Include package data in the package --- core/base/libs/podop/MANIFEST.in | 2 ++ core/base/libs/podop/setup.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 core/base/libs/podop/MANIFEST.in diff --git a/core/base/libs/podop/MANIFEST.in b/core/base/libs/podop/MANIFEST.in new file mode 100644 index 00000000..c28ab72d --- /dev/null +++ b/core/base/libs/podop/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include LICENSE.md diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index 4951196d..2d62fd78 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.2", + version="0.2.1", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", @@ -15,6 +15,7 @@ setup( author_email="pierre@jaury.eu", url="https://github.com/mailu/podop.git", packages=["podop"], + include_package_data=True, scripts=["scripts/podop"], install_requires=[ "aiohttp" From 81d171f978d67b03db78af92aa0044f1cbbd2a59 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 17:44:55 +0200 Subject: [PATCH 016/274] Add some debug logging to the table class --- core/base/libs/podop/podop/table.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py index d30ff9fb..ab53179e 100644 --- a/core/base/libs/podop/podop/table.py +++ b/core/base/libs/podop/podop/table.py @@ -19,17 +19,20 @@ class UrlTable(object): async def get(self, key, ns=None): """ Get the given key in the provided namespace """ + logging.debug("Table get {}".format(key)) if ns is not None: key += "/" + ns async with aiohttp.ClientSession() as session: async with session.get(self.url_pattern.format(key)) as request: if request.status == 200: result = await request.json() + logging.debug("Table get {} is {}".format(key, result)) return result async def set(self, key, value, ns=None): """ Set a value for the given key in the provided namespace """ + logging.debug("Table set {} to {}".format(key, value)) if ns is not None: key += "/" + ns async with aiohttp.ClientSession() as session: @@ -38,6 +41,7 @@ class UrlTable(object): async def iter(self, cat): """ Iterate the given key (experimental) """ + logging.debug("Table iter {}".format(cat)) async with aiohttp.ClientSession() as session: async with session.get(self.url_pattern.format(cat)) as request: if request.status == 200: From d2b98ae323e7695e61c409c831b50b01540de945 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 17:45:27 +0200 Subject: [PATCH 017/274] Update to 0.2.2 --- core/base/libs/podop/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index 2d62fd78..82c998bc 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.2.1", + version="0.2.2", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From 814bb1f36db0ea331644eee32ae83fb4f2e99b7f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 20:56:26 +0200 Subject: [PATCH 018/274] Properly miss when the web api returns 404 --- core/base/libs/podop/podop/dovecot.py | 11 +++++------ core/base/libs/podop/podop/postfix.py | 2 ++ core/base/libs/podop/podop/table.py | 4 ++++ core/base/libs/podop/setup.py | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 96eea3b8..64074e38 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -81,11 +81,10 @@ class DictProtocol(asyncio.Protocol): logging.debug("Looking up {}".format(key)) # Priv and shared keys are handled slighlty differently key_type, key = key.decode("utf8").split("/", 1) - result = await self.dict.get( - key, ns=(self.user if key_type == "priv" else None) - ) - # Handle various response types - if result is not None: + try: + result = await self.dict.get( + key, ns=(self.user if key_type == "priv" else None) + ) if type(result) is str: response = result.encode("utf8") elif type(result) is bytes: @@ -93,7 +92,7 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") return self.reply(b"O", response) - else: + except KeyError: return self.reply(b"N") def process_begin(self, transaction_id): diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py index b0395f35..84c7b08d 100644 --- a/core/base/libs/podop/podop/postfix.py +++ b/core/base/libs/podop/podop/postfix.py @@ -58,6 +58,7 @@ class NetstringProtocol(asyncio.Protocol): def send_string(self, string): """ Send a netstring """ + logging.debug("Replying {}".format(string)) self.transport.write(str(len(string)).encode('ascii')) self.transport.write(b':') self.transport.write(string) @@ -85,6 +86,7 @@ class SocketmapProtocol(NetstringProtocol): def string_received(self, string): # The postfix format contains a space for separating the map name and # the key + logging.debug("Received {}".format(string)) space = string.find(0x20) if space != -1: name = string[:space].decode('ascii') diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py index ab53179e..3869cafc 100644 --- a/core/base/libs/podop/podop/table.py +++ b/core/base/libs/podop/podop/table.py @@ -28,6 +28,10 @@ class UrlTable(object): result = await request.json() logging.debug("Table get {} is {}".format(key, result)) return result + elif request.status == 404: + raise KeyError() + else: + raise Exception(request.status) async def set(self, key, value, ns=None): """ Set a value for the given key in the provided namespace diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index 82c998bc..d681ae53 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.2.2", + version="0.2.3", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From 23e5aa2e051b03ec2f6948f0cb4cbcd5f2034f5e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 27 Sep 2018 13:48:28 +0200 Subject: [PATCH 019/274] Escape strings properly in the Dovecot dict dialect --- core/base/libs/podop/podop/dovecot.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 64074e38..68bfa172 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -125,9 +125,7 @@ class DictProtocol(asyncio.Protocol): def reply(self, command, *args): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) - self.transport.write(b"\t".join( - arg.replace(b"\t", b"\t\t") for arg in args - )) + self.transport.write(b"\t".join(map(tabescape, args))) self.transport.write(b"\n") @classmethod @@ -143,3 +141,25 @@ class DictProtocol(asyncio.Protocol): ord("C"): process_commit, ord("S"): process_set } + + +def tabescape(unescaped): + """ Escape a string using the specific Dovecot tabescape + See: https://github.com/dovecot/core/blob/master/src/lib/strescape.c + """ + return unescaped.replace(b"\x01", b"\x011")\ + .replace(b"\x00", b"\x010")\ + .replace(b"\t", b"\x01t")\ + .replace(b"\n", b"\x01n")\ + .replace(b"\r", b"\x01r") + + +def tabunescape(escaped): + """ Unescape a string using the specific Dovecot tabescape + See: https://github.com/dovecot/core/blob/master/src/lib/strescape.c + """ + return escaped.replace(b"\x01r", b"\r")\ + .replace(b"\x01n", b"\n")\ + .replace(b"\x01t", b"\t")\ + .replace(b"\x010", b"\x00")\ + .replace(b"\x011", b"\x01") From 080e76f972e6e4b1d2ff7497b9a64679bf7029d5 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 7 Oct 2018 13:30:48 +0200 Subject: [PATCH 020/274] Merge pull request #1 from rakshith-ravi/patch-1 Fixed a small typo --- core/base/libs/podop/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/README.md b/core/base/libs/podop/README.md index 208b3ce5..cd4a08a4 100644 --- a/core/base/libs/podop/README.md +++ b/core/base/libs/podop/README.md @@ -64,7 +64,7 @@ virtual_mailbox_maps = socketmap:unix:/tmp/podop.socket:alias In order to simplify the configuration, you can setup a shortcut. ``` -podop = socketmap:unic:/tmp/podop.socket +podop = socketmap:unix:/tmp/podop.socket virtual_alias_maps = ${podop}:alias virtual_mailbox_domains = ${podop}:domain virtual_mailbox_maps = ${podop}:alias From 6fadd39aeafcc5ff0a53aa43886444df4a631ae8 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 6 Jan 2019 13:21:00 +0100 Subject: [PATCH 021/274] Merge pull request #3 from Nebukadneza/add_key_url_quoting URL-Quote the key in HTTP requests --- core/base/libs/podop/podop/table.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py index 3869cafc..0f1ea680 100644 --- a/core/base/libs/podop/podop/table.py +++ b/core/base/libs/podop/podop/table.py @@ -3,7 +3,7 @@ import aiohttp import logging - +from urllib.parse import quote class UrlTable(object): """ Resolve an entry by querying a parametrized GET URL. @@ -23,7 +23,8 @@ class UrlTable(object): if ns is not None: key += "/" + ns async with aiohttp.ClientSession() as session: - async with session.get(self.url_pattern.format(key)) as request: + quoted_key = quote(key) + async with session.get(self.url_pattern.format(quoted_key)) as request: if request.status == 200: result = await request.json() logging.debug("Table get {} is {}".format(key, result)) @@ -40,7 +41,8 @@ class UrlTable(object): if ns is not None: key += "/" + ns async with aiohttp.ClientSession() as session: - await session.post(self.url_pattern.format(key), json=value) + quoted_key = quote(key) + await session.post(self.url_pattern.format(quoted_key), json=value) async def iter(self, cat): """ Iterate the given key (experimental) From e2979f9103c466156e964bd0ad9be7b7b8b7f5d7 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 25 Jun 2019 19:17:35 +0200 Subject: [PATCH 022/274] Merge pull request #6 from Nebukadneza/fix_py37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don’t use deprecated now-keyword "async" --- core/base/libs/podop/podop/postfix.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py index 84c7b08d..3140a3a8 100644 --- a/core/base/libs/podop/podop/postfix.py +++ b/core/base/libs/podop/podop/postfix.py @@ -4,7 +4,6 @@ import asyncio import logging - class NetstringProtocol(asyncio.Protocol): """ Netstring asyncio protocol implementation. @@ -91,7 +90,7 @@ class SocketmapProtocol(NetstringProtocol): if space != -1: name = string[:space].decode('ascii') key = string[space+1:].decode('utf8') - return asyncio.async(self.process_request(name, key)) + return asyncio.ensure_future(self.process_request(name, key)) async def process_request(self, name, key): """ Process a request by querying the provided map. From 3d0d831c76d842244cbd0de1a1a72b2ca6cb75cf Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 14 Jul 2019 13:30:01 +0200 Subject: [PATCH 023/274] Version 0.2.4 --- core/base/libs/podop/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index d681ae53..fda267ce 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -7,7 +7,7 @@ with open("README.md", "r") as fh: setup( name="podop", - version="0.2.3", + version="0.2.4", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From dbec5f0a6cb8b2a727071ec6b81350a4b9b2bbf7 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 14 Jul 2019 13:33:57 +0200 Subject: [PATCH 024/274] Switch to setuptools and bump the version --- core/base/libs/podop/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py index fda267ce..68c3e862 100644 --- a/core/base/libs/podop/setup.py +++ b/core/base/libs/podop/setup.py @@ -1,13 +1,13 @@ #!/usr/bin/env python -from distutils.core import setup +from setuptools import setup with open("README.md", "r") as fh: long_description = fh.read() setup( name="podop", - version="0.2.4", + version="0.2.5", description="Postfix and Dovecot proxy", long_description=long_description, long_description_content_type="text/markdown", From ce9d886195cc08831111bc20436f29cca0cce2cb Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 25 Aug 2022 14:35:28 +0200 Subject: [PATCH 025/274] Merge pull request #10 from ghostwheel42/add_gitignore Add .gitignore file --- core/base/libs/podop/.gitignore | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 core/base/libs/podop/.gitignore diff --git a/core/base/libs/podop/.gitignore b/core/base/libs/podop/.gitignore new file mode 100644 index 00000000..9a7e75d9 --- /dev/null +++ b/core/base/libs/podop/.gitignore @@ -0,0 +1,20 @@ +.DS_Store +.idea +tmp + +*.bak +*~ +.*.swp + +__pycache__/ +*.pyc +*.pyo +*.egg-info/ + +.build +.env* +.venv + +*.code-workspace + +build/ From 0370b26f3e88886052dca3c73cc7beaa59d24ead Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 14:06:25 +0200 Subject: [PATCH 026/274] Initial commit --- core/base/libs/socrate/LICENSE.md | 21 ++++++++++ core/base/libs/socrate/MANIFEST.in | 2 + core/base/libs/socrate/README.md | 24 +++++++++++ core/base/libs/socrate/setup.py | 24 +++++++++++ core/base/libs/socrate/socrate/__init__.py | 0 core/base/libs/socrate/socrate/conf.py | 46 ++++++++++++++++++++++ core/base/libs/socrate/socrate/system.py | 20 ++++++++++ 7 files changed, 137 insertions(+) create mode 100644 core/base/libs/socrate/LICENSE.md create mode 100644 core/base/libs/socrate/MANIFEST.in create mode 100644 core/base/libs/socrate/README.md create mode 100644 core/base/libs/socrate/setup.py create mode 100644 core/base/libs/socrate/socrate/__init__.py create mode 100644 core/base/libs/socrate/socrate/conf.py create mode 100644 core/base/libs/socrate/socrate/system.py diff --git a/core/base/libs/socrate/LICENSE.md b/core/base/libs/socrate/LICENSE.md new file mode 100644 index 00000000..d360537d --- /dev/null +++ b/core/base/libs/socrate/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Mailu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/base/libs/socrate/MANIFEST.in b/core/base/libs/socrate/MANIFEST.in new file mode 100644 index 00000000..c28ab72d --- /dev/null +++ b/core/base/libs/socrate/MANIFEST.in @@ -0,0 +1,2 @@ +include README.md +include LICENSE.md diff --git a/core/base/libs/socrate/README.md b/core/base/libs/socrate/README.md new file mode 100644 index 00000000..9b65a83b --- /dev/null +++ b/core/base/libs/socrate/README.md @@ -0,0 +1,24 @@ +Socrate is a simple Python module providing a set of utility functions for +Python daemon applications. + +The scope includes: +- configuration utilities (configuration parsing, etc.) +- system utilities (access to DNS, stats, etc.) + +Setup +====== + +Socrate is available on Pypi, simpy run: + +``` +pip install socrate +``` + + +Contributing +============ + +Podop is free software, open to suggestions and contributions. All +components are free software and compatible with the MIT license. All +the code is placed under the MIT license. + diff --git a/core/base/libs/socrate/setup.py b/core/base/libs/socrate/setup.py new file mode 100644 index 00000000..827fc6e0 --- /dev/null +++ b/core/base/libs/socrate/setup.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +from distutils.core import setup + +with open("README.md", "r") as fh: + long_description = fh.read() + +setup( + name="socrate", + version="0.1", + description="Socrate daemon utilities", + long_description=long_description, + long_description_content_type="text/markdown", + author="Pierre Jaury", + author_email="pierre@jaury.eu", + url="https://github.com/mailu/socrate.git", + packages=["socrate"], + include_package_data=True, + install_requires=[ + "jinja2", + "importlib", + "tenacity" + ] +) diff --git a/core/base/libs/socrate/socrate/__init__.py b/core/base/libs/socrate/socrate/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/core/base/libs/socrate/socrate/conf.py b/core/base/libs/socrate/socrate/conf.py new file mode 100644 index 00000000..5ccde54d --- /dev/null +++ b/core/base/libs/socrate/socrate/conf.py @@ -0,0 +1,46 @@ +import jinja2 +import importlib + + +def jinja(source, environ, destination=None): + """ Render a Jinja configuration file + """ + with open(source, "r") as template: + result = jinja2.Template(template.read()).render(environ) + if destination is not None: + with open(destination, "w") as handle: + handle.write(result) + return result + + +def merge(*objects): + """ Merge simple python objects, which only consist of + strings, integers, bools, lists and dicts + """ + mode = type(objects[0]) + if not all(type(obj) is mode for obj in objects): + raise ValueError("Cannot merge mixed typed objects") + if len(objects) == 1: + return objects[0] + elif mode is dict: + return { + key: merge(*[obj[key] for obj in objects if key in obj]) + for obj in objects for key in obj.keys() + } + elif mode is list: + return sum(objects) + else: + raise ValueError("Cannot merge objects of type {}: {}".format( + mode, objects)) + + +def resolve_function(function, cache={}): + """ Resolve a fully qualified function name in Python, and caches + the result + """ + if function not in cache: + module, name = function.rsplit(".", 1) + cache[function] = getattr(importlib.import_module(module), name) + return cache[function] + + diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py new file mode 100644 index 00000000..64a87d15 --- /dev/null +++ b/core/base/libs/socrate/socrate/system.py @@ -0,0 +1,20 @@ +import socket +import tenacity + + +@retry(stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5)) +def resolve_hostname(hostname): + """ This function uses system DNS to resolve a hostname. + It is capable of retrying in case the host is not immediately available + """ + return socket.gethostbyname(hostname) + + +def resolve_address(address): + """ This function is identical to ``resolve_host`` but also supports + resolving an address, i.e. including a port. + """ + hostname, *rest = address.resplit(":", 1) + ip_address = resolve_hostname(hostname) + return ip_address + "".join(":" + port for port in rest) From 351b05b92d630a6778bb23fd331ac693e71d2d77 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 14:57:18 +0200 Subject: [PATCH 027/274] Allow jinja to load from file path or handle --- core/base/libs/socrate/socrate/conf.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/core/base/libs/socrate/socrate/conf.py b/core/base/libs/socrate/socrate/conf.py index 5ccde54d..d792879c 100644 --- a/core/base/libs/socrate/socrate/conf.py +++ b/core/base/libs/socrate/socrate/conf.py @@ -3,13 +3,22 @@ import importlib def jinja(source, environ, destination=None): - """ Render a Jinja configuration file + """ Render a Jinja configuration file, supports file handle or path """ - with open(source, "r") as template: - result = jinja2.Template(template.read()).render(environ) + close_source = close_destination = False + if type(source) is str: + source = open(source, "r") + close_source = True + if type(destination) is str: + destination = open(destination, "w") + close_destination = True + result = jinja2.Template(source.read()).render(environ) + if close_source: + source.close() if destination is not None: - with open(destination, "w") as handle: - handle.write(result) + destination.write(result) + if close_destination: + destination.close() return result @@ -28,7 +37,7 @@ def merge(*objects): for obj in objects for key in obj.keys() } elif mode is list: - return sum(objects) + return sum(objects, []) else: raise ValueError("Cannot merge objects of type {}: {}".format( mode, objects)) From 74a3e87de3f970e734cf8c9c1b9422744b6f1b0b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 14:57:38 +0200 Subject: [PATCH 028/274] Fix a couple syntax typos --- core/base/libs/socrate/socrate/system.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 64a87d15..b3b95012 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -2,8 +2,8 @@ import socket import tenacity -@retry(stop=tenacity.stop_after_attempt(100), - wait=tenacity.wait_random(min=2, max=5)) +@tenacity.retry(stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5)) def resolve_hostname(hostname): """ This function uses system DNS to resolve a hostname. It is capable of retrying in case the host is not immediately available @@ -15,6 +15,6 @@ def resolve_address(address): """ This function is identical to ``resolve_host`` but also supports resolving an address, i.e. including a port. """ - hostname, *rest = address.resplit(":", 1) + hostname, *rest = address.rsplit(":", 1) ip_address = resolve_hostname(hostname) return ip_address + "".join(":" + port for port in rest) From ef344c62f6117e6ab891d0c733dfd67ca401581c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 14:57:45 +0200 Subject: [PATCH 029/274] Add automated tests --- core/base/libs/socrate/test.py | 80 ++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 core/base/libs/socrate/test.py diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py new file mode 100644 index 00000000..efdcb343 --- /dev/null +++ b/core/base/libs/socrate/test.py @@ -0,0 +1,80 @@ +import unittest +import io + +from socrate import conf, system + + +class TestConf(unittest.TestCase): + """ Test configuration functions + """ + + MERGE_EXPECTATIONS = [ + ({"a": "1", "b": "2", "c": "3", "d": "4"}, + {"a": "1", "b": "2"}, + {"c": "3", "d": "4"}), + + ({"a": [1, 2, 3, 4, 5], "b": "4"}, + {"a": [1, 2, 3], "b": "4"}, + {"a": [4, 5]}), + + ({"a": {"x": "1", "y": "2", "z": 3}, "b": 4, "c": "5"}, + {"a": {"x": "1", "y": "2"}, "b": 4}, + {"a": {"z": 3}, "c": "5"}) + ] + + def test_jinja(self): + template = "Test {{ variable }}" + environ = {"variable": "ok"} + self.assertEqual( + conf.jinja(io.StringIO(template), environ), + "Test ok" + ) + result = io.StringIO() + conf.jinja(io.StringIO(template), environ, result) + self.assertEqual( + result.getvalue(), + "Test ok" + ) + + def test_merge(self): + for result, *parts in TestConf.MERGE_EXPECTATIONS: + self.assertEqual(result, conf.merge(*parts)) + + def test_merge_failure(self): + with self.assertRaises(ValueError): + conf.merge({"a": 1}, {"a": 2}) + with self.assertRaises(ValueError): + conf.merge(1, "a") + + def test_resolve(self): + self.assertEqual( + conf.resolve_function("unittest.TestCase"), + unittest.TestCase + ) + self.assertEqual( + conf.resolve_function("unittest.util.strclass"), + unittest.util.strclass + ) + + def test_resolve_failure(self): + with self.assertRaises(AttributeError): + conf.resolve_function("unittest.inexistant") + with self.assertRaises(ModuleNotFoundError): + conf.resolve_function("inexistant.function") + + +class TestSystem(unittest.TestCase): + """ Test the system functions + """ + + def test_resolve_hostname(self): + self.assertEqual( + system.resolve_hostname("one.one.one.one"), + "1.1.1.1" + ) + + def test_resolve_address(self): + self.assertEqual( + system.resolve_address("one.one.one.one:80"), + "1.1.1.1:80" + ) From 7f6d51904bf546473c58ec3bfb90d19d5ac16ead Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 6 May 2019 15:09:49 +0200 Subject: [PATCH 030/274] Remove wrong dependency to importlib --- core/base/libs/socrate/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/socrate/setup.py b/core/base/libs/socrate/setup.py index 827fc6e0..da1c9520 100644 --- a/core/base/libs/socrate/setup.py +++ b/core/base/libs/socrate/setup.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import setuptools from distutils.core import setup with open("README.md", "r") as fh: @@ -7,7 +8,7 @@ with open("README.md", "r") as fh: setup( name="socrate", - version="0.1", + version="0.1.1", description="Socrate daemon utilities", long_description=long_description, long_description_content_type="text/markdown", @@ -18,7 +19,6 @@ setup( include_package_data=True, install_requires=[ "jinja2", - "importlib", "tenacity" ] ) From b198fde75692387daebddbe4235aa811b0d81697 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 25 Aug 2019 11:51:18 +0200 Subject: [PATCH 031/274] Merge pull request #3 from micw/fix-random-failures Change test hostnames for stable test results --- core/base/libs/socrate/.gitignore | 1 + core/base/libs/socrate/test.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 core/base/libs/socrate/.gitignore diff --git a/core/base/libs/socrate/.gitignore b/core/base/libs/socrate/.gitignore new file mode 100644 index 00000000..c18dd8d8 --- /dev/null +++ b/core/base/libs/socrate/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py index efdcb343..b9b36d5f 100644 --- a/core/base/libs/socrate/test.py +++ b/core/base/libs/socrate/test.py @@ -69,12 +69,12 @@ class TestSystem(unittest.TestCase): def test_resolve_hostname(self): self.assertEqual( - system.resolve_hostname("one.one.one.one"), - "1.1.1.1" + system.resolve_hostname("1.2.3.4.xip.io"), + "1.2.3.4" ) def test_resolve_address(self): self.assertEqual( - system.resolve_address("one.one.one.one:80"), - "1.1.1.1:80" + system.resolve_address("1.2.3.4.xip.io:80"), + "1.2.3.4:80" ) From 68d44201abc6b0813fe77032966b98778285fcdf Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 25 Aug 2019 14:06:31 +0200 Subject: [PATCH 032/274] Merge pull request #4 from micw/resolve-host-if-address-not-set Resolve host if address not set --- core/base/libs/socrate/socrate/system.py | 12 ++++++++++ core/base/libs/socrate/test.py | 28 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index b3b95012..99e11bbb 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -1,5 +1,6 @@ import socket import tenacity +from os import environ @tenacity.retry(stop=tenacity.stop_after_attempt(100), @@ -18,3 +19,14 @@ def resolve_address(address): hostname, *rest = address.rsplit(":", 1) ip_address = resolve_hostname(hostname) return ip_address + "".join(":" + port for port in rest) + + +def get_host_address_from_environment(name, default): + """ This function looks up an envionment variable ``{{ name }}_ADDRESS``. + If it's defined, it is returned unmodified. If it's undefined, an environment + variable ``HOST_{{ name }}`` is looked up and resolved to an ip address. + If this is also not defined, the default is resolved to an ip address. + """ + if "{}_ADDRESS".format(name) in environ: + return environ.get("{}_ADDRESS".format(name)) + return resolve_address(environ.get("HOST_{}".format(name), default)) diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py index b9b36d5f..6fc87bfa 100644 --- a/core/base/libs/socrate/test.py +++ b/core/base/libs/socrate/test.py @@ -1,5 +1,6 @@ import unittest import io +import os from socrate import conf, system @@ -78,3 +79,30 @@ class TestSystem(unittest.TestCase): system.resolve_address("1.2.3.4.xip.io:80"), "1.2.3.4:80" ) + + def test_get_host_address_from_environment(self): + if "TEST_ADDRESS" in os.environ: + del os.environ["TEST_ADDRESS"] + if "HOST_TEST" in os.environ: + del os.environ["HOST_TEST"] + # if nothing is set, the default must be resolved + self.assertEqual( + system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + "1.2.3.4:80" + ) + # if HOST is set, the HOST must be resolved + os.environ['HOST_TEST']="1.2.3.5.xip.io:80" + self.assertEqual( + system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + "1.2.3.5:80" + ) + # if ADDRESS is set, the ADDRESS must be returned unresolved + os.environ['TEST_ADDRESS']="1.2.3.6.xip.io:80" + self.assertEqual( + system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + "1.2.3.6.xip.io:80" + ) + + +if __name__ == "__main__": + unittest.main() From f63837b8e162188d8b6a95a07bc364a31891566f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 26 Aug 2019 22:44:55 +0200 Subject: [PATCH 033/274] Update to 0.2.0 --- core/base/libs/socrate/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/socrate/setup.py b/core/base/libs/socrate/setup.py index da1c9520..25598a87 100644 --- a/core/base/libs/socrate/setup.py +++ b/core/base/libs/socrate/setup.py @@ -8,7 +8,7 @@ with open("README.md", "r") as fh: setup( name="socrate", - version="0.1.1", + version="0.2.0", description="Socrate daemon utilities", long_description=long_description, long_description_content_type="text/markdown", From c0066abd0187ae5b406bf811ed01fcc7a8c1787b Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Thu, 25 Aug 2022 15:31:11 +0200 Subject: [PATCH 034/274] Merge pull request #6 from micw/log-failed-dns Add logging for failed DNS lookups --- core/base/libs/socrate/socrate/system.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 99e11bbb..a43aba0d 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -1,7 +1,7 @@ import socket import tenacity from os import environ - +import logging as log @tenacity.retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) @@ -9,7 +9,11 @@ def resolve_hostname(hostname): """ This function uses system DNS to resolve a hostname. It is capable of retrying in case the host is not immediately available """ - return socket.gethostbyname(hostname) + try: + return socket.gethostbyname(hostname) + except Exception as e: + log.warn("Unable to lookup '%s': %s",hostname,e) + raise e def resolve_address(address): From b711f930ef877f121e6e5424a70cab9c295cebe2 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Thu, 25 Aug 2022 15:31:50 +0200 Subject: [PATCH 035/274] Merge pull request #9 from vavanade/patch-1 fix docstring --- core/base/libs/socrate/socrate/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index a43aba0d..65aa9367 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -17,7 +17,7 @@ def resolve_hostname(hostname): def resolve_address(address): - """ This function is identical to ``resolve_host`` but also supports + """ This function is identical to ``resolve_hostname`` but also supports resolving an address, i.e. including a port. """ hostname, *rest = address.rsplit(":", 1) From 9f511faf6476caa1a26b0094504f01191828e28c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 25 Aug 2022 16:41:39 +0200 Subject: [PATCH 036/274] Merge pull request #8 from NeverBehave/master fix: resolve IPv6 container hostname --- core/base/libs/socrate/.gitignore | 21 ++++++++++++++++++++ core/base/libs/socrate/socrate/system.py | 4 +++- core/base/libs/socrate/test.py | 25 ++++++++++++++++-------- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/core/base/libs/socrate/.gitignore b/core/base/libs/socrate/.gitignore index c18dd8d8..9305b479 100644 --- a/core/base/libs/socrate/.gitignore +++ b/core/base/libs/socrate/.gitignore @@ -1 +1,22 @@ +.DS_Store +.idea +tmp + +*.bak +*~ +.*.swp + __pycache__/ +*.pyc +*.pyo +*.egg-info/ + +.build +.env* +.venv + +*.code-workspace + +venv/ +build/ +dist/ diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index 65aa9367..d4e3802a 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -10,7 +10,7 @@ def resolve_hostname(hostname): It is capable of retrying in case the host is not immediately available """ try: - return socket.gethostbyname(hostname) + return sorted(socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE), key=lambda s:s[0])[0][4][0] except Exception as e: log.warn("Unable to lookup '%s': %s",hostname,e) raise e @@ -22,6 +22,8 @@ def resolve_address(address): """ hostname, *rest = address.rsplit(":", 1) ip_address = resolve_hostname(hostname) + if ":" in ip_address: + ip_address = "[{}]".format(ip_address) return ip_address + "".join(":" + port for port in rest) diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py index 6fc87bfa..f6088345 100644 --- a/core/base/libs/socrate/test.py +++ b/core/base/libs/socrate/test.py @@ -70,15 +70,24 @@ class TestSystem(unittest.TestCase): def test_resolve_hostname(self): self.assertEqual( - system.resolve_hostname("1.2.3.4.xip.io"), + system.resolve_hostname("1.2.3.4.sslip.io"), "1.2.3.4" ) + self.assertEqual( + system.resolve_hostname("2001-db8--f00.sslip.io"), + "2001:db8::f00" + ) + def test_resolve_address(self): self.assertEqual( - system.resolve_address("1.2.3.4.xip.io:80"), + system.resolve_address("1.2.3.4.sslip.io:80"), "1.2.3.4:80" ) + self.assertEqual( + system.resolve_address("2001-db8--f00.sslip.io:80"), + "[2001:db8::f00]:80" + ) def test_get_host_address_from_environment(self): if "TEST_ADDRESS" in os.environ: @@ -87,20 +96,20 @@ class TestSystem(unittest.TestCase): del os.environ["HOST_TEST"] # if nothing is set, the default must be resolved self.assertEqual( - system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"), "1.2.3.4:80" ) # if HOST is set, the HOST must be resolved - os.environ['HOST_TEST']="1.2.3.5.xip.io:80" + os.environ['HOST_TEST']="1.2.3.5.sslip.io:80" self.assertEqual( - system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), + system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"), "1.2.3.5:80" ) # if ADDRESS is set, the ADDRESS must be returned unresolved - os.environ['TEST_ADDRESS']="1.2.3.6.xip.io:80" + os.environ['TEST_ADDRESS']="1.2.3.6.sslip.io:80" self.assertEqual( - system.get_host_address_from_environment("TEST", "1.2.3.4.xip.io:80"), - "1.2.3.6.xip.io:80" + system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"), + "1.2.3.6.sslip.io:80" ) From 8668b269cdb25f935adb57837b31afc4e32b0e23 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Sep 2022 22:15:21 +0200 Subject: [PATCH 037/274] Add requirements.txt for base image --- core/base/libs/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 core/base/libs/requirements.txt diff --git a/core/base/libs/requirements.txt b/core/base/libs/requirements.txt new file mode 100644 index 00000000..0e537e6a --- /dev/null +++ b/core/base/libs/requirements.txt @@ -0,0 +1,2 @@ +libs/podop/ +libs/socrate/ From 768c0cc1cebb7307d7c92b413d21560b1c3a1938 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 29 Sep 2022 11:28:44 +0200 Subject: [PATCH 038/274] Fix assets build process --- core/admin/assets/Dockerfile | 7 +++++-- core/admin/assets/content/{ => assets}/app.css | 0 core/admin/assets/content/{ => assets}/app.js | 0 core/admin/assets/content/{ => assets}/mailu.png | Bin core/admin/assets/content/{ => assets}/vendor.js | 0 5 files changed, 5 insertions(+), 2 deletions(-) rename core/admin/assets/content/{ => assets}/app.css (100%) rename core/admin/assets/content/{ => assets}/app.js (100%) rename core/admin/assets/content/{ => assets}/mailu.png (100%) rename core/admin/assets/content/{ => assets}/vendor.js (100%) diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile index d799fa4f..2fb6f9b6 100644 --- a/core/admin/assets/Dockerfile +++ b/core/admin/assets/Dockerfile @@ -2,10 +2,13 @@ FROM node:16-alpine3.16 -COPY content ./ +WORKDIR /work + +COPY content /work + RUN set -euxo pipefail \ && npm config set update-notifier false \ - && npm install --no-fund \ + && npm install --no-audit --no-fund \ && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ diff --git a/core/admin/assets/content/app.css b/core/admin/assets/content/assets/app.css similarity index 100% rename from core/admin/assets/content/app.css rename to core/admin/assets/content/assets/app.css diff --git a/core/admin/assets/content/app.js b/core/admin/assets/content/assets/app.js similarity index 100% rename from core/admin/assets/content/app.js rename to core/admin/assets/content/assets/app.js diff --git a/core/admin/assets/content/mailu.png b/core/admin/assets/content/assets/mailu.png similarity index 100% rename from core/admin/assets/content/mailu.png rename to core/admin/assets/content/assets/mailu.png diff --git a/core/admin/assets/content/vendor.js b/core/admin/assets/content/assets/vendor.js similarity index 100% rename from core/admin/assets/content/vendor.js rename to core/admin/assets/content/assets/vendor.js From 52dd09d45230d035add1d8b39d85826f0f7c306e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 29 Sep 2022 12:49:37 +0200 Subject: [PATCH 039/274] Fix assets build process #2 --- core/admin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index c3731fb2..c1c63a29 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -21,7 +21,7 @@ COPY migrations ./migrations COPY start.py /start.py COPY audit.py /audit.py -COPY --from=assets static ./mailu/static +COPY --from=assets /work/static ./mailu/static RUN echo $VERSION >> /version From a29f0668584d9d689e2f73ece8c4c56998f7d835 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 8 Oct 2022 15:55:40 +0200 Subject: [PATCH 040/274] Move even more python deps to base image --- core/admin/Dockerfile | 23 +++--- core/admin/assets/Dockerfile | 3 +- core/admin/audit.py | 2 +- core/admin/requirements-dev.txt | 28 +++++++ core/admin/requirements-prod.txt | 78 ------------------ core/admin/requirements.txt | 106 ++++++++++++++++++------- core/admin/start.py | 2 +- core/base/Dockerfile | 59 ++++++++++---- core/base/{libs => }/requirements.txt | 0 core/dovecot/Dockerfile | 11 +-- core/dovecot/requirements.txt | 2 + core/dovecot/start.py | 2 +- core/nginx/Dockerfile | 18 ++--- core/nginx/certwatcher.py | 2 +- core/nginx/config.py | 2 +- core/nginx/letsencrypt.py | 2 +- core/nginx/requirements.txt | 2 + core/nginx/start.py | 2 +- core/none/Dockerfile | 4 +- core/postfix/Dockerfile | 13 ++- core/postfix/requirements.txt | 3 + core/postfix/start.py | 2 +- core/rspamd/Dockerfile | 11 +-- core/rspamd/requirements.txt | 1 + core/rspamd/start.py | 2 +- optional/clamav/Dockerfile | 28 +++---- optional/clamav/health.sh | 8 -- optional/clamav/start.py | 2 +- optional/fetchmail/Dockerfile | 26 +++--- optional/fetchmail/fetchmail.py | 2 +- optional/fetchmail/requirements.txt | 1 + optional/radicale/Dockerfile | 26 +++--- optional/radicale/requirements.txt | 2 + optional/traefik-certdumper/Dockerfile | 12 ++- optional/unbound/Dockerfile | 38 ++++----- optional/unbound/start.py | 2 +- tests/build.hcl | 62 +++++++++------ 37 files changed, 308 insertions(+), 281 deletions(-) create mode 100644 core/admin/requirements-dev.txt delete mode 100644 core/admin/requirements-prod.txt rename core/base/{libs => }/requirements.txt (100%) create mode 100644 core/dovecot/requirements.txt create mode 100644 core/nginx/requirements.txt create mode 100644 core/postfix/requirements.txt create mode 100644 core/rspamd/requirements.txt delete mode 100755 optional/clamav/health.sh create mode 100644 optional/fetchmail/requirements.txt create mode 100644 optional/radicale/requirements.txt diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index c1c63a29..ca98662d 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -1,29 +1,26 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# admin image FROM base ARG VERSION=local LABEL version=$VERSION -COPY requirements-prod.txt requirements.txt RUN set -euxo pipefail \ - && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ - && pip install --no-cache-dir -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ - || ( apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ - && pip install -r requirements.txt \ - && apk del --no-cache build-dep ) + ; apk add --no-cache curl libressl mariadb-connector-c postgresql-libs -COPY mailu ./mailu -RUN pybabel compile -d mailu/translations +COPY mailu/ ./mailu/ +RUN set -euxo pipefail \ + ; venv/bin/pybabel compile -d mailu/translations -COPY migrations ./migrations +COPY migrations/ ./migrations/ -COPY start.py /start.py -COPY audit.py /audit.py +COPY audit.py / +COPY start.py / -COPY --from=assets /work/static ./mailu/static +COPY --from=assets /work/static/ ./mailu/static/ -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 80/tcp HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile index 2fb6f9b6..c8556f47 100644 --- a/core/admin/assets/Dockerfile +++ b/core/admin/assets/Dockerfile @@ -4,7 +4,7 @@ FROM node:16-alpine3.16 WORKDIR /work -COPY content /work +COPY content/ ./ RUN set -euxo pipefail \ && npm config set update-notifier false \ @@ -14,4 +14,3 @@ RUN set -euxo pipefail \ cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ done \ && node_modules/.bin/webpack-cli --color - diff --git a/core/admin/audit.py b/core/admin/audit.py index 60583f83..31ee9665 100755 --- a/core/admin/audit.py +++ b/core/admin/audit.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import sys import tabulate diff --git a/core/admin/requirements-dev.txt b/core/admin/requirements-dev.txt new file mode 100644 index 00000000..5bf57874 --- /dev/null +++ b/core/admin/requirements-dev.txt @@ -0,0 +1,28 @@ +Flask +Flask-Login +Flask-SQLAlchemy +Flask-bootstrap +Flask-Babel +Flask-migrate +Flask-script +Flask-wtf +Flask-debugtoolbar +limits +redis +WTForms-Components +socrate +passlib +gunicorn +tabulate +PyYAML +PyOpenSSL +Pygments +dnspython +tenacity +mysql-connector-python +idna +srslib +marshmallow +flask-marshmallow +marshmallow-sqlalchemy +xmltodict diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt deleted file mode 100644 index 297c6902..00000000 --- a/core/admin/requirements-prod.txt +++ /dev/null @@ -1,78 +0,0 @@ -alembic==1.7.4 -appdirs==1.4.4 -Babel==2.9.1 -bcrypt==3.2.0 -blinker==1.4 -CacheControl==0.12.9 -certifi==2021.10.8 -# cffi==1.15.0 -chardet==4.0.0 -click==8.0.3 -colorama==0.4.4 -contextlib2==21.6.0 -cryptography==35.0.0 -decorator==5.1.0 -# distlib==0.3.1 -# distro==1.5.0 -dnspython==2.1.0 -dominate==2.6.0 -email-validator==1.1.3 -Flask==2.0.2 -Flask-Babel==2.0.0 -Flask-Bootstrap==3.3.7.1 -Flask-DebugToolbar==0.11.0 -Flask-Limiter==1.4 -Flask-Login==0.5.0 -flask-marshmallow==0.14.0 -Flask-Migrate==3.1.0 -Flask-Script==2.0.6 -Flask-SQLAlchemy==2.5.1 -Flask-WTF==0.15.1 -greenlet==1.1.2 -gunicorn==20.1.0 -html5lib==1.1 -idna==3.3 -infinity==1.5 -intervals==0.9.2 -itsdangerous==2.0.1 -Jinja2==3.0.2 -limits==1.5.1 -lockfile==0.12.2 -Mako==1.1.5 -MarkupSafe==2.0.1 -marshmallow==3.14.0 -marshmallow-sqlalchemy==0.26.1 -msgpack==1.0.2 -# mysqlclient==2.0.3 -mysql-connector-python==8.0.25 -ordered-set==4.0.2 -# packaging==20.9 -passlib==1.7.4 -# pep517==0.10.0 -progress==1.6 -#psycopg2==2.9.1 -psycopg2-binary==2.9.3 -pycparser==2.20 -Pygments==2.10.0 -pyOpenSSL==21.0.0 -pyparsing==3.0.4 -pytz==2021.3 -PyYAML==6.0 -redis==3.5.3 -requests==2.26.0 -retrying==1.3.3 -# six==1.15.0 -socrate==0.2.0 -SQLAlchemy==1.4.26 -srslib==0.1.4 -tabulate==0.8.9 -tenacity==8.0.1 -toml==0.10.2 -urllib3==1.26.7 -validators==0.18.2 -visitor==0.1.3 -webencodings==0.5.1 -Werkzeug==2.0.2 -WTForms==2.3.3 -WTForms-Components==0.10.5 -xmltodict==0.12.0 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index 5bf57874..297c6902 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -1,28 +1,78 @@ -Flask -Flask-Login -Flask-SQLAlchemy -Flask-bootstrap -Flask-Babel -Flask-migrate -Flask-script -Flask-wtf -Flask-debugtoolbar -limits -redis -WTForms-Components -socrate -passlib -gunicorn -tabulate -PyYAML -PyOpenSSL -Pygments -dnspython -tenacity -mysql-connector-python -idna -srslib -marshmallow -flask-marshmallow -marshmallow-sqlalchemy -xmltodict +alembic==1.7.4 +appdirs==1.4.4 +Babel==2.9.1 +bcrypt==3.2.0 +blinker==1.4 +CacheControl==0.12.9 +certifi==2021.10.8 +# cffi==1.15.0 +chardet==4.0.0 +click==8.0.3 +colorama==0.4.4 +contextlib2==21.6.0 +cryptography==35.0.0 +decorator==5.1.0 +# distlib==0.3.1 +# distro==1.5.0 +dnspython==2.1.0 +dominate==2.6.0 +email-validator==1.1.3 +Flask==2.0.2 +Flask-Babel==2.0.0 +Flask-Bootstrap==3.3.7.1 +Flask-DebugToolbar==0.11.0 +Flask-Limiter==1.4 +Flask-Login==0.5.0 +flask-marshmallow==0.14.0 +Flask-Migrate==3.1.0 +Flask-Script==2.0.6 +Flask-SQLAlchemy==2.5.1 +Flask-WTF==0.15.1 +greenlet==1.1.2 +gunicorn==20.1.0 +html5lib==1.1 +idna==3.3 +infinity==1.5 +intervals==0.9.2 +itsdangerous==2.0.1 +Jinja2==3.0.2 +limits==1.5.1 +lockfile==0.12.2 +Mako==1.1.5 +MarkupSafe==2.0.1 +marshmallow==3.14.0 +marshmallow-sqlalchemy==0.26.1 +msgpack==1.0.2 +# mysqlclient==2.0.3 +mysql-connector-python==8.0.25 +ordered-set==4.0.2 +# packaging==20.9 +passlib==1.7.4 +# pep517==0.10.0 +progress==1.6 +#psycopg2==2.9.1 +psycopg2-binary==2.9.3 +pycparser==2.20 +Pygments==2.10.0 +pyOpenSSL==21.0.0 +pyparsing==3.0.4 +pytz==2021.3 +PyYAML==6.0 +redis==3.5.3 +requests==2.26.0 +retrying==1.3.3 +# six==1.15.0 +socrate==0.2.0 +SQLAlchemy==1.4.26 +srslib==0.1.4 +tabulate==0.8.9 +tenacity==8.0.1 +toml==0.10.2 +urllib3==1.26.7 +validators==0.18.2 +visitor==0.1.3 +webencodings==0.5.1 +Werkzeug==2.0.2 +WTForms==2.3.3 +WTForms-Components==0.10.5 +xmltodict==0.12.0 diff --git a/core/admin/start.py b/core/admin/start.py index 8bb1cef1..ac8bd526 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import logging as log diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 71ac6e31..cd590596 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -1,30 +1,57 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# base system image (intermediate) ARG DISTRO=alpine:3.14.5 -FROM $DISTRO +FROM $DISTRO as system ENV TZ Etc/UTC ENV LANG C.UTF-8 -# TODO: use intermediate image to build virtual env - RUN set -euxo pipefail \ - && adduser -s /bin/bash -Dh /app -k /var/empty -u 1000 -g mailu app \ - && apk add --no-cache bash ca-certificates tzdata python3 py3-pip py3-wheel \ - && pip3 install --no-cache-dir --upgrade pip + ; adduser -s /bin/bash -Dh /app -k /var/empty -u 1000 -g mailu app \ + ; apk add --no-cache bash ca-certificates python3 tzdata WORKDIR /app -COPY libs libs/ +CMD /bin/bash + + +# build virtual env (intermediate) +FROM system as build + +ENV VIRTUAL_ENV=/app/venv -# TODO: work in virtual env (see above) -# && python3 -m venv . \ RUN set -euxo pipefail \ - && pip3 install --no-cache-dir -r libs/requirements.txt --only-binary=:all: \ - || ( apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ - && pip3 install --no-cache-dir -r libs/requirements.txt \ - && apk del --no-cache .build-deps ) + ; apk add --no-cache py3-pip \ + ; python3 -m venv ${VIRTUAL_ENV} \ + ; venv/bin/pip install --no-cache-dir --upgrade --no-warn-script-location pip wheel -# TODO: clean image (or use intermediate - see above) -# && bin/pip uninstall -y pip distribute setuptools wheel \ -# && rm -rf /tmp/* /root/.cache/pip +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" + +COPY libs/ libs/ +COPY --from=core ./ core/ +COPY --from=optional ./ optional/ + +RUN set -euxo pipefail \ + ; grep -hEv '(podop|socrate)==' core/*/requirements.txt optional/*/requirements.txt \ + | sort -u >libs/requirements.txt \ +\ + ; venv/bin/pip install --no-cache-dir -r libs/requirements.txt \ + || ( \ + apk add --no-cache --virtual .build-deps \ + build-base cargo gcc libffi-dev libressl-dev mariadb-connector-c-dev \ + musl-dev postgresql-dev python3-dev \ + ; venv/bin/pip install --no-cache-dir -r libs/requirements.txt \ + ; apk del .build-deps \ + ) \ +\ + ; venv/bin/pip freeze > venv/requirements.txt + + +# base mailu image +FROM system + +COPY --from=build /app/venv/ /app/venv/ + +ENV VIRTUAL_ENV=/app/venv +ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" diff --git a/core/base/libs/requirements.txt b/core/base/requirements.txt similarity index 100% rename from core/base/libs/requirements.txt rename to core/base/requirements.txt diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 2d74e59b..0796e587 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -1,18 +1,19 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# dovecot image FROM base ARG VERSION LABEL version=$VERSION RUN set -euxo pipefail \ - && apk add --no-cache dovecot dovecot-lmtpd dovecot-pop3d dovecot-submissiond dovecot-pigeonhole-plugin rspamd-client xapian-core dovecot-fts-xapian \ - && mkdir /var/lib/dovecot + ; apk add --no-cache dovecot dovecot-fts-xapian dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-pop3d dovecot-submissiond rspamd-client xapian-core \ + ; mkdir /var/lib/dovecot -COPY conf /conf -COPY start.py /start.py +COPY conf/ /conf/ +COPY start.py / -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready." diff --git a/core/dovecot/requirements.txt b/core/dovecot/requirements.txt new file mode 100644 index 00000000..16005f74 --- /dev/null +++ b/core/dovecot/requirements.txt @@ -0,0 +1,2 @@ +podop==0.2.5 +socrate==0.2.0 diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 03bdfa80..a8c85ebf 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import glob diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 2a34403f..cbb9cd7c 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -1,29 +1,29 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# build static assets (intermediate) FROM base as static -COPY static /static +COPY static/ /static/ RUN set -euxo pipefail \ - && gzip -k9 /static/*.ico /static/*.txt \ - && chmod a+rX-w -R /static + ; gzip -k9 /static/*.ico /static/*.txt \ + ; chmod a+rX-w -R /static +# nginx image FROM base ARG VERSION LABEL version=$VERSION -# Image specific layers under this line RUN set -euxo pipefail \ - && apk add --no-cache certbot nginx nginx-mod-mail openssl curl \ - && pip3 install --no-cache-dir watchdog + ; apk add --no-cache certbot curl nginx nginx-mod-mail openssl -COPY conf /conf -COPY --from=static /static /static +COPY conf/ /conf/ +COPY --from=static /static/ /static/ COPY *.py / -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://localhost/health diff --git a/core/nginx/certwatcher.py b/core/nginx/certwatcher.py index 96ccdd7c..e86fc9ec 100755 --- a/core/nginx/certwatcher.py +++ b/core/nginx/certwatcher.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 """ Certificate watcher which reloads nginx or reconfigures it, depending on what happens to externally supplied certificates. Only executed by start.py in case diff --git a/core/nginx/config.py b/core/nginx/config.py index e9c4b50e..7930ff12 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import logging as log diff --git a/core/nginx/letsencrypt.py b/core/nginx/letsencrypt.py index e636dac9..993e7f9f 100755 --- a/core/nginx/letsencrypt.py +++ b/core/nginx/letsencrypt.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import time diff --git a/core/nginx/requirements.txt b/core/nginx/requirements.txt new file mode 100644 index 00000000..c96c3bb8 --- /dev/null +++ b/core/nginx/requirements.txt @@ -0,0 +1,2 @@ +socrate==0.2.0 +watchdog==2.1.9 diff --git a/core/nginx/start.py b/core/nginx/start.py index 8673f148..07932211 100755 --- a/core/nginx/start.py +++ b/core/nginx/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import subprocess diff --git a/core/none/Dockerfile b/core/none/Dockerfile index 058b18c5..f06cc31c 100644 --- a/core/none/Dockerfile +++ b/core/none/Dockerfile @@ -1,12 +1,12 @@ # syntax=docker/dockerfile-upstream:1.4.3 -# This is an idle image to dynamically replace any component if disabled. +# idle image (to dynamically replace any disabled component) FROM base ARG VERSION=local LABEL version=$VERSION -RUN echo $VERSION >> /version +RUN echo $VERSION >/version HEALTHCHECK CMD true diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 66adbde3..dab4396c 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,21 +1,18 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# postfix image FROM base ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - && apk add --no-cache postfix postfix-pcre cyrus-sasl-login rsyslog logrotate \ - && pip install --no-cache-dir --only-binary=:all: postfix-mta-sts-resolver==1.0.1 \ - || ( apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev py3-wheel libffi-dev \ - && pip3 install postfix-mta-sts-resolver==1.0.1 \ - && apk del .build-deps ) + ; apk add --no-cache cyrus-sasl-login logrotate postfix postfix-pcre rsyslog -COPY conf /conf -COPY start.py /start.py +COPY conf/ /conf/ +COPY start.py / -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 25/tcp 10025/tcp HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" diff --git a/core/postfix/requirements.txt b/core/postfix/requirements.txt new file mode 100644 index 00000000..2d9b8135 --- /dev/null +++ b/core/postfix/requirements.txt @@ -0,0 +1,3 @@ +podop==0.2.5 +postfix-mta-sts-resolver==1.1.4 +socrate==0.2.0 diff --git a/core/postfix/start.py b/core/postfix/start.py index 4faf2e2d..b12d0b54 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import glob diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 5ee922e5..2ccb5307 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -1,18 +1,19 @@ # syntax=docker/dockerfile-upstream:1.4.3 +# rspamd image FROM base ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - && apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl \ - && mkdir /run/rspamd + ; apk add --no-cache curl rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \ + ; mkdir /run/rspamd -COPY conf/ /conf -COPY start.py /start.py +COPY conf/ /conf/ +COPY start.py / -RUN echo $VERSION >> /version +RUN echo $VERSION >/version EXPOSE 11332/tcp 11334/tcp 11335/tcp HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost:11334/ diff --git a/core/rspamd/requirements.txt b/core/rspamd/requirements.txt new file mode 100644 index 00000000..be4b0107 --- /dev/null +++ b/core/rspamd/requirements.txt @@ -0,0 +1 @@ +socrate==0.2.0 diff --git a/core/rspamd/start.py b/core/rspamd/start.py index fcb33a97..58ec89ca 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import glob diff --git a/optional/clamav/Dockerfile b/optional/clamav/Dockerfile index e0ed0cdc..9beded99 100644 --- a/optional/clamav/Dockerfile +++ b/optional/clamav/Dockerfile @@ -1,26 +1,22 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION +# syntax=docker/dockerfile-upstream:1.4.3 -ENV TZ Etc/UTC +# clamav image +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip bash tzdata \ - && pip3 install --upgrade pip -# Image specific layers under this line -RUN apk add --no-cache clamav rsyslog wget clamav-libunrar +RUN set -euxo pipefail \ + ; apk add --no-cache clamav clamav-libunrar rsyslog wget -COPY conf /etc/clamav -COPY start.py /start.py -COPY health.sh /health.sh +COPY conf/ /etc/clamav/ +COPY start.py / + +RUN echo $VERSION >/version EXPOSE 3310/tcp +HEALTHCHECK --start-period=350s CMD echo PING|nc localhost 3310|grep "PONG" + VOLUME ["/data"] CMD /start.py - -HEALTHCHECK --start-period=350s CMD /health.sh -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/clamav/health.sh b/optional/clamav/health.sh deleted file mode 100755 index c4c55044..00000000 --- a/optional/clamav/health.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -if [ "$(echo PING | nc localhost 3310)" = "PONG" ]; then - echo "ping successful" -else - echo "ping failed" - exit 1 -fi diff --git a/optional/clamav/start.py b/optional/clamav/start.py index 56e1bcfe..3d0c306d 100755 --- a/optional/clamav/start.py +++ b/optional/clamav/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import logging as log diff --git a/optional/fetchmail/Dockerfile b/optional/fetchmail/Dockerfile index 71075ced..12488213 100644 --- a/optional/fetchmail/Dockerfile +++ b/optional/fetchmail/Dockerfile @@ -1,23 +1,21 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION +# syntax=docker/dockerfile-upstream:1.4.3 -ENV TZ Etc/UTC +# fetchmail image +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip bash tzdata \ - && pip3 install --upgrade pip +RUN set -euxo pipefail \ + ; apk add --no-cache fetchmail openssl \ + ; mkdir -p /data -# Image specific layers under this line -RUN apk add --no-cache fetchmail ca-certificates openssl \ - && pip3 install requests +COPY fetchmail.py / -RUN mkdir -p /data +RUN echo $VERSION >/version -COPY fetchmail.py /fetchmail.py +HEALTHCHECK --start-period=350s CMD ["/bin/sh", "-c", "ps ax | grep [/]fetchmail.py"] + +VOLUME ["/var/lib/rspamd"] CMD ["/fetchmail.py"] -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 5459de59..32751ed7 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import time import os diff --git a/optional/fetchmail/requirements.txt b/optional/fetchmail/requirements.txt new file mode 100644 index 00000000..a8ed785e --- /dev/null +++ b/optional/fetchmail/requirements.txt @@ -0,0 +1 @@ +requests==2.26.0 diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index 30055a14..f9fd7598 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -1,27 +1,21 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION +# syntax=docker/dockerfile-upstream:1.4.3 -ENV TZ Etc/UTC +# webdav image +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip bash tzdata \ - && pip3 install --upgrade pip +RUN set -euxo pipefail \ + ; apk add --no-cache curl -# Image specific layers under this line -RUN apk add --no-cache curl \ - && pip3 install pytz radicale~=3.0 +COPY radicale.conf / - -COPY radicale.conf /radicale.conf +RUN echo $VERSION >/version EXPOSE 5232/tcp +HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 + VOLUME ["/data"] CMD radicale -S -C /radicale.conf - -HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 -RUN echo $VERSION >> /version diff --git a/optional/radicale/requirements.txt b/optional/radicale/requirements.txt new file mode 100644 index 00000000..fc61502c --- /dev/null +++ b/optional/radicale/requirements.txt @@ -0,0 +1,2 @@ +pytz==2021.3 +radicale~=3.0 diff --git a/optional/traefik-certdumper/Dockerfile b/optional/traefik-certdumper/Dockerfile index 829655f0..a94f32ba 100644 --- a/optional/traefik-certdumper/Dockerfile +++ b/optional/traefik-certdumper/Dockerfile @@ -1,16 +1,22 @@ +# syntax=docker/dockerfile-upstream:1.4.3 + +# cert dumper image FROM ldez/traefik-certs-dumper -ARG VERSION ENV TZ Etc/UTC +ENV LANG C.UTF-8 +ARG VERSION LABEL version=$VERSION -RUN apk --no-cache add inotify-tools util-linux bash tzdata +RUN set -euxo pipefail \ + ; apk add --no-cache bash inotify-tools tzdata util-linux COPY run.sh / +RUN echo $VERSION >/version + VOLUME ["/traefik"] VOLUME ["/output"] ENTRYPOINT ["/run.sh"] -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/unbound/Dockerfile b/optional/unbound/Dockerfile index 342ceebc..343326fe 100644 --- a/optional/unbound/Dockerfile +++ b/optional/unbound/Dockerfile @@ -1,33 +1,25 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION +# syntax=docker/dockerfile-upstream:1.4.3 -ENV TZ Etc/UTC +# resolver image +FROM base +ARG VERSION=local LABEL version=$VERSION -# python3 shared with most images -RUN apk add --no-cache \ - python3 py3-pip git bash py3-multidict tzdata \ - && pip3 install --upgrade pip +RUN set -euxo pipefail \ + ; apk add --no-cache bind-tools curl unbound \ + ; curl -so /etc/unbound/root.hints https://www.internic.net/domain/named.cache \ + ; chown root:unbound /etc/unbound \ + ; chmod 775 /etc/unbound \ + ; apk del --no-cache curl \ + ; /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key || true -# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, snappymail, roundcube -RUN pip3 install socrate==0.2.0 +COPY unbound.conf / +COPY start.py / -# Image specific layers under this line -RUN apk add --no-cache unbound curl bind-tools \ - && curl -o /etc/unbound/root.hints https://www.internic.net/domain/named.cache \ - && chown root:unbound /etc/unbound \ - && chmod 775 /etc/unbound \ - && apk del --no-cache curl \ - && /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key | true - -COPY start.py /start.py -COPY unbound.conf /unbound.conf +RUN echo $VERSION >/version EXPOSE 53/udp 53/tcp +HEALTHCHECK CMD dig @127.0.0.1 || exit 1 CMD /start.py - -HEALTHCHECK CMD dig @127.0.0.1 || exit 1 -RUN echo $VERSION >> /version \ No newline at end of file diff --git a/optional/unbound/start.py b/optional/unbound/start.py index 0e7d0fdc..f3a5bee7 100755 --- a/optional/unbound/start.py +++ b/optional/unbound/start.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import os import logging as log diff --git a/tests/build.hcl b/tests/build.hcl index c32da8d5..75e4f996 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -83,12 +83,16 @@ function "tag" { # ----------------------------------------------------------------------------------------- target "base" { inherits = ["defaults"] - context="core/base" + context = "core/base/" + contexts = { + core = "core/" + optional = "optional/" + } } target "assets" { inherits = ["defaults"] - context="core/admin/assets" + context = "core/admin/assets/" } # ----------------------------------------------------------------------------------------- @@ -96,7 +100,7 @@ target "assets" { # ----------------------------------------------------------------------------------------- target "docs" { inherits = ["defaults"] - context = "docs" + context = "docs/" tags = tag("docs") args = { version = "${MAILU_VERSION}" @@ -106,7 +110,7 @@ target "docs" { target "setup" { inherits = ["defaults"] - context="setup" + context = "setup/" tags = tag("setup") } @@ -115,8 +119,8 @@ target "setup" { # ----------------------------------------------------------------------------------------- target "none" { inherits = ["defaults"] - context="core/none" - contexts= { + context = "core/none/" + contexts = { base = "target:base" } tags = tag("none") @@ -124,8 +128,8 @@ target "none" { target "admin" { inherits = ["defaults"] - context="core/admin" - contexts= { + context = "core/admin/" + contexts = { base = "target:base" assets = "target:assets" } @@ -134,8 +138,8 @@ target "admin" { target "antispam" { inherits = ["defaults"] - context="core/rspamd" - contexts= { + context = "core/rspamd/" + contexts = { base = "target:base" } tags = tag("rspamd") @@ -143,8 +147,8 @@ target "antispam" { target "front" { inherits = ["defaults"] - context="core/nginx" - contexts= { + context = "core/nginx/" + contexts = { base = "target:base" } tags = tag("nginx") @@ -152,8 +156,8 @@ target "front" { target "imap" { inherits = ["defaults"] - context="core/dovecot" - contexts= { + context = "core/dovecot/" + contexts = { base = "target:base" } tags = tag("dovecot") @@ -161,8 +165,8 @@ target "imap" { target "smtp" { inherits = ["defaults"] - context="core/postfix" - contexts= { + context = "core/postfix/" + contexts = { base = "target:base" } tags = tag("postfix") @@ -173,13 +177,13 @@ target "smtp" { # ----------------------------------------------------------------------------------------- target "snappymail" { inherits = ["defaults"] - context="webmails/snappymail" + context = "webmails/snappymail/" tags = tag("snappymail") } target "roundcube" { inherits = ["defaults"] - context="webmails/roundcube" + context = "webmails/roundcube/" tags = tag("roundcube") } @@ -188,30 +192,42 @@ target "roundcube" { # ----------------------------------------------------------------------------------------- target "antivirus" { inherits = ["defaults"] - context="optional/clamav" + context = "optional/clamav/" + contexts = { + base = "target:base" + } tags = tag("clamav") } target "fetchmail" { inherits = ["defaults"] - context="optional/fetchmail" + context = "optional/fetchmail/" + contexts = { + base = "target:base" + } tags = tag("fetchmail") } target "resolver" { inherits = ["defaults"] - context="optional/unbound" + context = "optional/unbound/" + contexts = { + base = "target:base" + } tags = tag("unbound") } target "traefik-certdumper" { inherits = ["defaults"] - context="optional/traefik-certdumper" + context = "optional/traefik-certdumper/" tags = tag("traefik-certdumper") } target "webdav" { inherits = ["defaults"] - context="optional/radicale" + context = "optional/radicale/" + contexts = { + base = "target:base" + } tags = tag("radicale") } From 4c1071a4976182e98b275c156c8990843c57c353 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 14 Oct 2022 14:34:27 +0200 Subject: [PATCH 041/274] Move all requirements*.txt to base image --- core/admin/requirements-dev.txt | 28 ---------- core/base/Dockerfile | 32 ++++++----- core/base/requirements-build.txt | 3 ++ core/base/requirements-dev.txt | 53 +++++++++++++++++++ .../requirements-prod.txt} | 32 +++++++---- core/base/requirements.txt | 2 - core/dovecot/requirements.txt | 2 - core/nginx/requirements.txt | 2 - core/postfix/requirements.txt | 3 -- core/rspamd/requirements.txt | 1 - optional/fetchmail/requirements.txt | 1 - optional/radicale/requirements.txt | 2 - 12 files changed, 93 insertions(+), 68 deletions(-) delete mode 100644 core/admin/requirements-dev.txt create mode 100644 core/base/requirements-build.txt create mode 100644 core/base/requirements-dev.txt rename core/{admin/requirements.txt => base/requirements-prod.txt} (74%) delete mode 100644 core/base/requirements.txt delete mode 100644 core/dovecot/requirements.txt delete mode 100644 core/nginx/requirements.txt delete mode 100644 core/postfix/requirements.txt delete mode 100644 core/rspamd/requirements.txt delete mode 100644 optional/fetchmail/requirements.txt delete mode 100644 optional/radicale/requirements.txt diff --git a/core/admin/requirements-dev.txt b/core/admin/requirements-dev.txt deleted file mode 100644 index 5bf57874..00000000 --- a/core/admin/requirements-dev.txt +++ /dev/null @@ -1,28 +0,0 @@ -Flask -Flask-Login -Flask-SQLAlchemy -Flask-bootstrap -Flask-Babel -Flask-migrate -Flask-script -Flask-wtf -Flask-debugtoolbar -limits -redis -WTForms-Components -socrate -passlib -gunicorn -tabulate -PyYAML -PyOpenSSL -Pygments -dnspython -tenacity -mysql-connector-python -idna -srslib -marshmallow -flask-marshmallow -marshmallow-sqlalchemy -xmltodict diff --git a/core/base/Dockerfile b/core/base/Dockerfile index cd590596..52f0bd3a 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -4,11 +4,14 @@ ARG DISTRO=alpine:3.14.5 FROM $DISTRO as system -ENV TZ Etc/UTC -ENV LANG C.UTF-8 +ENV TZ=Etc/UTC LANG=C.UTF-8 + +ARG MAILU_UID=1000 +ARG MAILU_GID=1000 RUN set -euxo pipefail \ - ; adduser -s /bin/bash -Dh /app -k /var/empty -u 1000 -g mailu app \ + ; addgroup -Sg ${MAILU_GID} mailu \ + ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ ; apk add --no-cache bash ca-certificates python3 tzdata WORKDIR /app @@ -21,31 +24,26 @@ FROM system as build ENV VIRTUAL_ENV=/app/venv +COPY requirements-*.txt ./ + RUN set -euxo pipefail \ ; apk add --no-cache py3-pip \ ; python3 -m venv ${VIRTUAL_ENV} \ - ; venv/bin/pip install --no-cache-dir --upgrade --no-warn-script-location pip wheel + ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" COPY libs/ libs/ -COPY --from=core ./ core/ -COPY --from=optional ./ optional/ RUN set -euxo pipefail \ - ; grep -hEv '(podop|socrate)==' core/*/requirements.txt optional/*/requirements.txt \ - | sort -u >libs/requirements.txt \ -\ - ; venv/bin/pip install --no-cache-dir -r libs/requirements.txt \ + ; pip install --no-cache-dir -r requirements-prod.txt \ || ( \ apk add --no-cache --virtual .build-deps \ - build-base cargo gcc libffi-dev libressl-dev mariadb-connector-c-dev \ - musl-dev postgresql-dev python3-dev \ - ; venv/bin/pip install --no-cache-dir -r libs/requirements.txt \ - ; apk del .build-deps \ - ) \ -\ - ; venv/bin/pip freeze > venv/requirements.txt + build-base cargo gcc libffi-dev libressl-dev mariadb-connector-c-dev \ + musl-dev postgresql-dev python3-dev \ + ; pip install --no-cache-dir -r requirements-prod.txt \ + ; apk del .build-deps \ + ) # base mailu image diff --git a/core/base/requirements-build.txt b/core/base/requirements-build.txt new file mode 100644 index 00000000..2858969f --- /dev/null +++ b/core/base/requirements-build.txt @@ -0,0 +1,3 @@ +pip==22.2.2 +setuptools==65.4.1 +wheel==0.37.1 diff --git a/core/base/requirements-dev.txt b/core/base/requirements-dev.txt new file mode 100644 index 00000000..2fb4352f --- /dev/null +++ b/core/base/requirements-dev.txt @@ -0,0 +1,53 @@ +# core/base +libs/podop +libs/socrate + +# core/admin +alembic +Babel +click +dnspython +Flask +Flask-Babel +Flask-Bootstrap +Flask-DebugToolbar +Flask-Login +flask-marshmallow +Flask-Migrate +Flask-SQLAlchemy +Flask-WTF +gunicorn +idna +itsdangerous +limits +marshmallow +marshmallow-sqlalchemy +mysql-connector-python +passlib +psycopg2-binary +Pygments +pyOpenSSL +PyYAML +redis +SQLAlchemy +srslib +tabulate +tenacity +validators +Werkzeug +WTForms +WTForms-Components +xmltodict + +# core/nginx +watchdog + +# core/postfix +postfix-mta-sts-resolver + +# optional/fetchmail +requests + +# optional/radicale +radicale + diff --git a/core/admin/requirements.txt b/core/base/requirements-prod.txt similarity index 74% rename from core/admin/requirements.txt rename to core/base/requirements-prod.txt index 297c6902..fbbce5d5 100644 --- a/core/admin/requirements.txt +++ b/core/base/requirements-prod.txt @@ -1,19 +1,24 @@ +aiodns==3.0.0 +aiohttp==3.8.3 +aiosignal==1.2.0 alembic==1.7.4 appdirs==1.4.4 +async-timeout==4.0.2 +attrs==22.1.0 Babel==2.9.1 bcrypt==3.2.0 blinker==1.4 CacheControl==0.12.9 certifi==2021.10.8 -# cffi==1.15.0 +cffi==1.15.1 chardet==4.0.0 +charset-normalizer==2.0.12 click==8.0.3 colorama==0.4.4 contextlib2==21.6.0 cryptography==35.0.0 decorator==5.1.0 -# distlib==0.3.1 -# distro==1.5.0 +defusedxml==0.7.1 dnspython==2.1.0 dominate==2.6.0 email-validator==1.1.3 @@ -28,6 +33,7 @@ Flask-Migrate==3.1.0 Flask-Script==2.0.6 Flask-SQLAlchemy==2.5.1 Flask-WTF==0.15.1 +frozenlist==1.3.1 greenlet==1.1.2 gunicorn==20.1.0 html5lib==1.1 @@ -38,31 +44,34 @@ itsdangerous==2.0.1 Jinja2==3.0.2 limits==1.5.1 lockfile==0.12.2 -Mako==1.1.5 +Mako==1.2.3 MarkupSafe==2.0.1 marshmallow==3.14.0 marshmallow-sqlalchemy==0.26.1 msgpack==1.0.2 -# mysqlclient==2.0.3 +multidict==6.0.2 mysql-connector-python==8.0.25 ordered-set==4.0.2 -# packaging==20.9 passlib==1.7.4 -# pep517==0.10.0 +podop @ file:///app/libs/podop +postfix-mta-sts-resolver==1.0.1 progress==1.6 -#psycopg2==2.9.1 +protobuf==4.21.7 psycopg2-binary==2.9.3 +pycares==4.2.2 pycparser==2.20 Pygments==2.10.0 pyOpenSSL==21.0.0 pyparsing==3.0.4 +python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 +Radicale==3.1.8 redis==3.5.3 requests==2.26.0 retrying==1.3.3 -# six==1.15.0 -socrate==0.2.0 +six==1.16.0 +socrate @ file:///app/libs/socrate SQLAlchemy==1.4.26 srslib==0.1.4 tabulate==0.8.9 @@ -71,8 +80,11 @@ toml==0.10.2 urllib3==1.26.7 validators==0.18.2 visitor==0.1.3 +vobject==0.9.6.1 +watchdog==2.1.9 webencodings==0.5.1 Werkzeug==2.0.2 WTForms==2.3.3 WTForms-Components==0.10.5 xmltodict==0.12.0 +yarl==1.8.1 diff --git a/core/base/requirements.txt b/core/base/requirements.txt deleted file mode 100644 index 0e537e6a..00000000 --- a/core/base/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -libs/podop/ -libs/socrate/ diff --git a/core/dovecot/requirements.txt b/core/dovecot/requirements.txt deleted file mode 100644 index 16005f74..00000000 --- a/core/dovecot/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -podop==0.2.5 -socrate==0.2.0 diff --git a/core/nginx/requirements.txt b/core/nginx/requirements.txt deleted file mode 100644 index c96c3bb8..00000000 --- a/core/nginx/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -socrate==0.2.0 -watchdog==2.1.9 diff --git a/core/postfix/requirements.txt b/core/postfix/requirements.txt deleted file mode 100644 index 2d9b8135..00000000 --- a/core/postfix/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -podop==0.2.5 -postfix-mta-sts-resolver==1.1.4 -socrate==0.2.0 diff --git a/core/rspamd/requirements.txt b/core/rspamd/requirements.txt deleted file mode 100644 index be4b0107..00000000 --- a/core/rspamd/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -socrate==0.2.0 diff --git a/optional/fetchmail/requirements.txt b/optional/fetchmail/requirements.txt deleted file mode 100644 index a8ed785e..00000000 --- a/optional/fetchmail/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests==2.26.0 diff --git a/optional/radicale/requirements.txt b/optional/radicale/requirements.txt deleted file mode 100644 index fc61502c..00000000 --- a/optional/radicale/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pytz==2021.3 -radicale~=3.0 From 146921f6192a2950c62c98262ecb48097c974bae Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 14 Oct 2022 14:34:58 +0200 Subject: [PATCH 042/274] Move curl to base image --- core/admin/Dockerfile | 2 +- core/base/Dockerfile | 2 +- core/nginx/Dockerfile | 2 +- core/rspamd/Dockerfile | 2 +- optional/radicale/Dockerfile | 3 --- optional/unbound/Dockerfile | 3 +-- 6 files changed, 5 insertions(+), 9 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index ca98662d..02dcd968 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -7,7 +7,7 @@ ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache curl libressl mariadb-connector-c postgresql-libs + ; apk add --no-cache libressl mariadb-connector-c postgresql-libs COPY mailu/ ./mailu/ RUN set -euxo pipefail \ diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 52f0bd3a..ca87dd70 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -12,7 +12,7 @@ ARG MAILU_GID=1000 RUN set -euxo pipefail \ ; addgroup -Sg ${MAILU_GID} mailu \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ - ; apk add --no-cache bash ca-certificates python3 tzdata + ; apk add --no-cache bash ca-certificates curl python3 tzdata WORKDIR /app diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index cbb9cd7c..f271fc07 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -17,7 +17,7 @@ ARG VERSION LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache certbot curl nginx nginx-mod-mail openssl + ; apk add --no-cache certbot nginx nginx-mod-mail openssl COPY conf/ /conf/ COPY --from=static /static/ /static/ diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 2ccb5307..eca8e62b 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -7,7 +7,7 @@ ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache curl rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \ + ; apk add --no-cache rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \ ; mkdir /run/rspamd COPY conf/ /conf/ diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index f9fd7598..56606494 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -6,9 +6,6 @@ FROM base ARG VERSION=local LABEL version=$VERSION -RUN set -euxo pipefail \ - ; apk add --no-cache curl - COPY radicale.conf / RUN echo $VERSION >/version diff --git a/optional/unbound/Dockerfile b/optional/unbound/Dockerfile index 343326fe..831476ab 100644 --- a/optional/unbound/Dockerfile +++ b/optional/unbound/Dockerfile @@ -7,11 +7,10 @@ ARG VERSION=local LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache bind-tools curl unbound \ + ; apk add --no-cache bind-tools unbound \ ; curl -so /etc/unbound/root.hints https://www.internic.net/domain/named.cache \ ; chown root:unbound /etc/unbound \ ; chmod 775 /etc/unbound \ - ; apk del --no-cache curl \ ; /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key || true COPY unbound.conf / From 5c31120895dd6d23e52fc9e33034afee14ade74e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 14 Oct 2022 14:37:40 +0200 Subject: [PATCH 043/274] Remove obsolete contexts from base image --- tests/build.hcl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/build.hcl b/tests/build.hcl index 75e4f996..453e6d40 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -84,10 +84,6 @@ function "tag" { target "base" { inherits = ["defaults"] context = "core/base/" - contexts = { - core = "core/" - optional = "optional/" - } } target "assets" { From 7441a420c4eea54a336eae47ecc8b0ad17a95f4d Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 14 Oct 2022 16:17:46 +0200 Subject: [PATCH 044/274] Fix and speed-up arm build. Allow chosing of prod/dev env. --- core/base/Dockerfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index ca87dd70..d00ead10 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -22,28 +22,32 @@ CMD /bin/bash # build virtual env (intermediate) FROM system as build +ARG MAILU_ENV=prod + ENV VIRTUAL_ENV=/app/venv -COPY requirements-*.txt ./ +COPY requirements-build.txt ./ RUN set -euxo pipefail \ ; apk add --no-cache py3-pip \ ; python3 -m venv ${VIRTUAL_ENV} \ - ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt + ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt \ + ; apk del py3-pip ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" +COPY requirements-${MAILU_ENV}.txt ./ COPY libs/ libs/ RUN set -euxo pipefail \ - ; pip install --no-cache-dir -r requirements-prod.txt \ + ; pip install -r requirements-${MAILU_ENV}.txt \ || ( \ apk add --no-cache --virtual .build-deps \ - build-base cargo gcc libffi-dev libressl-dev mariadb-connector-c-dev \ - musl-dev postgresql-dev python3-dev \ - ; pip install --no-cache-dir -r requirements-prod.txt \ + build-base gcc libffi-dev python3-dev \ + ; pip install -r requirements-${MAILU_ENV}.txt \ ; apk del .build-deps \ - ) + ) \ + ; rm -rf /root/.cache /tmp/*.pem # base mailu image From d9bf6875e12255b7870448fd08d9012b41671752 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 17 Oct 2022 16:20:52 +0200 Subject: [PATCH 045/274] Optimize build order for better caching --- core/admin/Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 02dcd968..600c3e9f 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -9,16 +9,16 @@ LABEL version=$VERSION RUN set -euxo pipefail \ ; apk add --no-cache libressl mariadb-connector-c postgresql-libs -COPY mailu/ ./mailu/ -RUN set -euxo pipefail \ - ; venv/bin/pybabel compile -d mailu/translations - -COPY migrations/ ./migrations/ +COPY --from=assets /work/static/ ./mailu/static/ COPY audit.py / COPY start.py / -COPY --from=assets /work/static/ ./mailu/static/ +COPY migrations/ ./migrations/ + +COPY mailu/ ./mailu/ +RUN set -euxo pipefail \ + ; venv/bin/pybabel compile -d mailu/translations RUN echo $VERSION >/version @@ -27,5 +27,5 @@ HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index VOLUME ["/data","/dkim"] -ENV FLASK_APP mailu +ENV FLASK_APP=mailu CMD /start.py From 3e51d15b03f84ce5e74ea9f869a4c9649b8f9205 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 18 Oct 2022 15:58:53 +0200 Subject: [PATCH 046/274] Remove the strict anti-spoofing rule. --- core/admin/mailu/internal/views/postfix.py | 15 --------------- core/postfix/conf/main.cf | 1 - towncrier/newsfragments/2475.feature | 1 + 3 files changed, 1 insertion(+), 16 deletions(-) create mode 100644 towncrier/newsfragments/2475.feature diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index f8346bb1..8188270c 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -158,21 +158,6 @@ def postfix_sender_rate(sender): user = models.User.get(sender) or flask.abort(404) return flask.abort(404) if user.sender_limiter.hit() else flask.jsonify("450 4.2.1 You are sending too many emails too fast.") -@internal.route("/postfix/sender/access/") -def postfix_sender_access(sender): - """ Simply reject any sender that pretends to be from a local domain - """ - if '@' in sender: - if sender.startswith('<') and sender.endswith('>'): - sender = sender[1:-1] - try: - localpart, domain_name = models.Email.resolve_domain(sender) - if models.Domain.query.get(domain_name): - return flask.jsonify("REJECT") - except sqlalchemy.exc.StatementError: - pass - return flask.abort(404) - # idna encode domain part of each address in list of addresses def idna_encode(addresses): return [ diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index a892430c..f3b789f9 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -110,7 +110,6 @@ check_ratelimit = check_sasl_access ${podop}senderrate smtpd_client_restrictions = permit_mynetworks, - check_sender_access ${podop}senderaccess, reject_non_fqdn_sender, reject_unknown_sender_domain, reject_unknown_recipient_domain, diff --git a/towncrier/newsfragments/2475.feature b/towncrier/newsfragments/2475.feature new file mode 100644 index 00000000..e84bc68a --- /dev/null +++ b/towncrier/newsfragments/2475.feature @@ -0,0 +1 @@ +Remove the strict anti-spoofing rule. In 2022 we should have other controls (SPF/DKIM) for dealing with authorization and shouldn't assume that Mailu is the only MTA allowed to send emails on behalf of the domains it hosts. From 5ebcecf4dd6676637a5585d24d5a5853c7d23996 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 18 Oct 2022 16:05:57 +0200 Subject: [PATCH 047/274] Don't need that anymore either --- core/postfix/start.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/postfix/start.py b/core/postfix/start.py index 4faf2e2d..323ff1f6 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -27,7 +27,6 @@ def start_podop(): ("mailbox", "url", url + "mailbox/§"), ("recipientmap", "url", url + "recipient/map/§"), ("sendermap", "url", url + "sender/map/§"), - ("senderaccess", "url", url + "sender/access/§"), ("senderlogin", "url", url + "sender/login/§"), ("senderrate", "url", url + "sender/rate/§") ]) From 92cb8c146bc76d74e0f5254625df41b685f9c875 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 19 Oct 2022 11:02:22 +0000 Subject: [PATCH 048/274] Refine build_test_deploy.yml: Build base image before the other images. Change cache key to make it is re-used for all builds. This is not dangerous. The docker build process can determine itself whether a cache can be safely re-used or not. --- .github/workflows/build_test_deploy.yml | 161 +++++++++++++++++------- 1 file changed, 115 insertions(+), 46 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 2d721656..3172e411 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -89,6 +89,57 @@ jobs: run: | echo ${{ steps.targets.outputs.matrix }} +## This job buils the base image. The base image is used by all other images. + build-base-image: + name: Build base image + needs: + - targets + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v3 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/cache/base + key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + restore-keys: | + mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - uses: crazy-max/ghaction-github-runtime@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} + - name: Build all docker images + env: + DOCKER_ORG: ${{ env.DOCKER_ORG }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + uses: docker/bake-action@v2 + with: + files: ${{env.HCL_FILE}} + targets: base + load: false + push: false + set: | + *.cache-from=type=local,src=${{ runner.temp }}/cache/base + *.cache-to=type=local,dest=${{ runner.temp }}/cache/base,mode=max + *.platform=${{ inputs.architecture }} + # This job builds all the images. The build cache is stored in the github actions cache. # In further jobs, this cache is used to quickly rebuild the images. build: @@ -96,6 +147,7 @@ jobs: if: inputs.architecture == 'linux/amd64' needs: - targets + - build-base-image strategy: fail-fast: false matrix: @@ -113,11 +165,18 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/cache/base + key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + restore-keys: | + mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/${{ matrix.target }} - key: ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }}-${{ github.run_id }} + key: ${{ matrix.target }}-${{ inputs.architecture }}-${{ github.run_id }} restore-keys: | ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }} - name: Set up QEMU @@ -143,6 +202,7 @@ jobs: push: false set: | *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }} + *.cache-from=type=local,src=${{ runner.temp }}/cache/base *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max *.platform=${{ inputs.architecture }} @@ -153,6 +213,7 @@ jobs: if: inputs.architecture != 'linux/amd64' needs: - targets + - build-base-image strategy: fail-fast: false matrix: @@ -170,13 +231,20 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/cache/base + key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + restore-keys: | + mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/${{ matrix.target }} - key: ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }}-${{ github.run_id }} + key: ${{ matrix.target }}-${{ inputs.architecture }}-${{ github.run_id }} restore-keys: | - ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }} + ${{ matrix.target }}-${{ inputs.architecture }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -200,6 +268,7 @@ jobs: push: false set: | *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }} + *.cache-from=type=local,src=${{ runner.temp }}/cache/base *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max *.platform=${{ inputs.architecture }} @@ -238,72 +307,72 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/docs - key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }} + key: docs-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image setup uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/setup - key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }} + key: setup-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image admin uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/admin - key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }} + key: admin-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antispam uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antispam - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }} + key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image front uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/front - key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }} + key: front-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image imap uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/imap - key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }} + key: imap-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image smtp uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/smtp - key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }} + key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image snappymail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/snappymail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }} + key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image roundcube uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/roundcube - key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }} + key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antivirus uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antivirus - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }} + key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image fetchmail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/fetchmail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }} + key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image resolver uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/resolver - key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }} + key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image traefik-certdumper uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/traefik-certdumper - key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }} + key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image webdav uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/webdav - key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }} + key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -370,76 +439,76 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs + - name: Configure /cache for image docs uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/docs - key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }} + key: docs-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image setup uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/setup - key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }} + key: setup-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image admin uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/admin - key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }} + key: admin-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antispam uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antispam - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }} + key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image front uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/front - key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }} + key: front-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image imap uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/imap - key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }} + key: imap-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image smtp uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/smtp - key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }} + key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image snappymail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/snappymail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }} + key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image roundcube uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/roundcube - key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }} + key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antivirus uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antivirus - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }} + key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image fetchmail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/fetchmail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }} + key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image resolver uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/resolver - key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }} + key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image traefik-certdumper uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/traefik-certdumper - key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }} + key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image webdav uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/webdav - key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }} + key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -496,72 +565,72 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/docs - key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }} + key: docs-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image setup uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/setup - key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }} + key: setup-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image admin uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/admin - key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }} + key: admin-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antispam uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antispam - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }} + key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image front uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/front - key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }} + key: front-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image imap uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/imap - key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }} + key: imap-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image smtp uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/smtp - key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }} + key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image snappymail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/snappymail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }} + key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image roundcube uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/roundcube - key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }} + key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image antivirus uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/antivirus - key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }} + key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image fetchmail uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/fetchmail - key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }} + key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image resolver uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/resolver - key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }} + key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image traefik-certdumper uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/traefik-certdumper - key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }} + key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - name: Configure /cache for image webdav uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/webdav - key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }} + key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 From f9ba0e688fa3089b98db4dc8944cad9ea11c605e Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 19 Oct 2022 11:25:42 +0000 Subject: [PATCH 049/274] Removed syntax error --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 3172e411..8aee5c5f 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -439,7 +439,7 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs + - name: Configure /cache for image docs uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/docs From 451738e32b026b7b3e93a3c732e99427ef61add0 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 19 Oct 2022 11:35:57 +0000 Subject: [PATCH 050/274] We want the function result. Not the function statement. --- .github/workflows/build_test_deploy.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 8aee5c5f..7b7c357a 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -111,9 +111,9 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} restore-keys: | - mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') + mailu-base-${{ inputs.architecture }}-${{hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -169,9 +169,9 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} restore-keys: | - mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') + mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder uses: actions/cache@v3 with: @@ -235,9 +235,9 @@ jobs: uses: actions/cache@v3 with: path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt')-${{ github.run_id }} + key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} restore-keys: | - mailu-base-${{ inputs.architecture }}-hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') + mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder uses: actions/cache@v3 with: From 8775a2bf04cc7e730de02dc36a18e78b9d71cfbe Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Oct 2022 15:28:20 +0200 Subject: [PATCH 051/274] untested code that may just work --- core/admin/mailu/internal/views/rspamd.py | 5 +++++ core/rspamd/conf/whitelist.conf | 8 ++++++++ 2 files changed, 13 insertions(+) create mode 100644 core/rspamd/conf/whitelist.conf diff --git a/core/admin/mailu/internal/views/rspamd.py b/core/admin/mailu/internal/views/rspamd.py index 458dbb81..3d3f8719 100644 --- a/core/admin/mailu/internal/views/rspamd.py +++ b/core/admin/mailu/internal/views/rspamd.py @@ -25,3 +25,8 @@ def rspamd_dkim_key(domain_name): } ) return flask.jsonify({'data': {'selectors': selectors}}) + +@internal.route("/rspamd/local_domains", methods=['GET']) +def rspamd_local_domains(): + domains = set(models.Domain.query.all() + models.Alternative.query.all()) + return '\n'.join(domains) diff --git a/core/rspamd/conf/whitelist.conf b/core/rspamd/conf/whitelist.conf new file mode 100644 index 00000000..fcde167a --- /dev/null +++ b/core/rspamd/conf/whitelist.conf @@ -0,0 +1,8 @@ +rules { + BLACKLIST_ANTISPOOF = { + valid_dmarc = true; + blacklist = true; + domains = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; + score = 15.0; + } +} From cc2c308d1d2fb1674627e6a035ee483166d4a122 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Oct 2022 15:33:37 +0200 Subject: [PATCH 052/274] update the towncrier entry --- towncrier/newsfragments/2475.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/towncrier/newsfragments/2475.feature b/towncrier/newsfragments/2475.feature index e84bc68a..d5340380 100644 --- a/towncrier/newsfragments/2475.feature +++ b/towncrier/newsfragments/2475.feature @@ -1 +1 @@ -Remove the strict anti-spoofing rule. In 2022 we should have other controls (SPF/DKIM) for dealing with authorization and shouldn't assume that Mailu is the only MTA allowed to send emails on behalf of the domains it hosts. +Upgrade the anti-spoofing rule. We shouldn't assume that Mailu is the only MTA allowed to send emails on behalf of the domains it hosts... but we should also ensure that both the envelope from and header from are checked. From 0204c9e59d5202090d44d6747b635201f399c99e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Oct 2022 16:08:29 +0200 Subject: [PATCH 053/274] doh --- core/admin/mailu/internal/views/rspamd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/rspamd.py b/core/admin/mailu/internal/views/rspamd.py index 3d3f8719..e2e453ad 100644 --- a/core/admin/mailu/internal/views/rspamd.py +++ b/core/admin/mailu/internal/views/rspamd.py @@ -29,4 +29,4 @@ def rspamd_dkim_key(domain_name): @internal.route("/rspamd/local_domains", methods=['GET']) def rspamd_local_domains(): domains = set(models.Domain.query.all() + models.Alternative.query.all()) - return '\n'.join(domains) + return '\n'.join(map(str,domains)) From 6ea2d84a3cd0bd5b460fa3ea2b9f9939800adadb Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 19 Oct 2022 14:55:22 +0000 Subject: [PATCH 054/274] Remove outdated wrong documentation --- docs/contributors/environment.rst | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index 47f9f1af..1b53afe6 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -312,35 +312,6 @@ The following must be done on every PR or after every new commit to an existing If git opens a editor for a commit message just save and exit as-is. If you have a merge conflict, see above and do the complete procedure from ``git fetch`` onward again. -Web administration ------------------- - -The administration Web interface requires a proper dev environment that can easily be setup using -``virtualenv`` (make sure you are using Python 3) : - -.. code-block:: bash - - cd core/admin - virtualenv . - source bin/activate - pip install -r requirements.txt - -You can then export the path to the development database (use four slashes for absolute path): - -.. code-block:: bash - - export SQLALCHEMY_DATABASE_URI=sqlite:///path/to/dev.db - -And finally run the server with debug enabled: - -.. code-block:: bash - - python run.py - -Any change to the files will automatically restart the Web server and reload the files. - -When using the development environment, a debugging toolbar is displayed on the right side -of the screen, that you can open to access query details, internal variables, etc. Documentation ------------- From f7b3aad8316abe8fb5ac987c09656d888c847558 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Oct 2022 17:53:32 +0200 Subject: [PATCH 055/274] Ensure we REJECT when we don't have a DMARC policy This restores the old behaviour --- core/rspamd/conf/force_actions.conf | 7 +++++++ core/rspamd/conf/multimap.conf | 5 +++++ 2 files changed, 12 insertions(+) create mode 100644 core/rspamd/conf/force_actions.conf create mode 100644 core/rspamd/conf/multimap.conf diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf new file mode 100644 index 00000000..64c992c4 --- /dev/null +++ b/core/rspamd/conf/force_actions.conf @@ -0,0 +1,7 @@ +rules { + WHITELIST_EXCEPTION { + action = "reject"; + expression = "(AUTH_NA|BLACKLIST_ANTISPOOF) & IS_LOCAL_DOMAIN"; + message = "Rejected (anti-spoofing)"; + } +} diff --git a/core/rspamd/conf/multimap.conf b/core/rspamd/conf/multimap.conf new file mode 100644 index 00000000..dd2ffa61 --- /dev/null +++ b/core/rspamd/conf/multimap.conf @@ -0,0 +1,5 @@ +IS_LOCAL_DOMAIN { + type = "from"; + filter = "email:domain"; + map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; +} From be4dd6d84a747f6b80acb83cf89c8db1426ceea7 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 19 Oct 2022 18:22:33 +0200 Subject: [PATCH 056/274] Spell it out --- core/rspamd/conf/force_actions.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf index 64c992c4..20bf9245 100644 --- a/core/rspamd/conf/force_actions.conf +++ b/core/rspamd/conf/force_actions.conf @@ -1,7 +1,7 @@ rules { - WHITELIST_EXCEPTION { + ANTISPOOF { action = "reject"; - expression = "(AUTH_NA|BLACKLIST_ANTISPOOF) & IS_LOCAL_DOMAIN"; + expression = "((R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA)|BLACKLIST_ANTISPOOF) & IS_LOCAL_DOMAIN"; message = "Rejected (anti-spoofing)"; } } From 23d06a5761096d6239db47bea312e9dbd518ddff Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Wed, 19 Oct 2022 19:36:13 +0200 Subject: [PATCH 057/274] Fix a bunch of typos --- CONTRIBUTING.md | 2 +- ISSUE_TEMPLATE.md | 2 +- core/admin/mailu/internal/nginx.py | 2 +- core/admin/mailu/schemas.py | 2 +- .../mailu/translations/ca/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/da/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/de/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/en/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/es/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/eu/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/fr/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/he/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/hu/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/is/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/it/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/ja/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/nb_NO/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/nl/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/pl/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/pt/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/ru/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/sv/LC_MESSAGES/messages.po | 2 +- .../mailu/translations/zh/LC_MESSAGES/messages.po | 2 +- core/admin/mailu/ui/templates/relay/edit.html | 2 +- core/admin/mailu/utils.py | 2 +- core/admin/messages.pot | 2 +- core/admin/start.py | 4 ++-- core/dovecot/conf/dovecot.conf | 2 +- core/nginx/conf/nginx.conf | 2 +- docs/antispam.rst | 2 +- docs/arm_images.rst | 2 +- docs/compose/.env | 2 +- docs/compose/requirements.rst | 2 +- docs/compose/setup.rst | 2 +- docs/configuration.rst | 6 +++--- docs/contributors/guidelines.rst | 6 +++--- docs/contributors/workflow.rst | 2 +- docs/dns.rst | 4 ++-- docs/faq.rst | 12 ++++++------ docs/releases.rst | 12 ++++++------ docs/swarm/1.5/README.md | 6 +++--- docs/swarm/master/README.md | 6 +++--- docs/swarm/master/README_nfs_example.md | 6 +++--- docs/webadministration.rst | 2 +- setup/README.md | 6 +++--- setup/templates/setup.html | 2 +- setup/templates/steps/compose/03_expose.html | 4 ++-- tests/compose/core/00_create_users.sh | 2 +- webmails/roundcube/config.inc.php | 2 +- webmails/roundcube/start.py | 2 +- 50 files changed, 75 insertions(+), 75 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 094506b6..fca178b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,4 +4,4 @@ This project is open source, and your contributions are all welcome. There are m 2. contribute code and/or configuration to the repository (see [the development guidelines](https://mailu.io/master/contributors/workflow.html) for details); 3. contribute localization to your native language (see [the localization docs](https://mailu.io/master/contributors/localization.html) for details); -Either way, keep in mind that the code you write or the translation you produce muts be licensed under the same conditions as the project itself. Additionally, all contributors are considered equal co-authors of the project. +Either way, keep in mind that the code you write or the translation you produce must be licensed under the same conditions as the project itself. Additionally, all contributors are considered equal co-authors of the project. diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 10b9134f..ce972925 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -6,7 +6,7 @@ To be able to help you best, we need some more information. ## Before you open your issue - [ ] Check if no issue or pull-request for this already exists. - [ ] Check [documentation](https://mailu.io/master/) and [FAQ](https://mailu.io/master/faq.html). (Tip, use the search function on the documentation page) -- [ ] You understand `Mailu` is made by volunteers in their **free time** — be conscise, civil and accept that delays can occur. +- [ ] You understand `Mailu` is made by volunteers in their **free time** — be concise, civil and accept that delays can occur. - [ ] The title of the issue should be short and simple. It should contain specific terms related to the actual issue. Be specific while writing the title. ## Environment & Versions diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 2ee6d9b3..870cc76d 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -142,7 +142,7 @@ def get_server(protocol, authenticated=False): else: hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25) try: - # test if hostname is already resolved to an ip adddress + # test if hostname is already resolved to an ip address ipaddress.ip_address(hostname) except: # hostname is not an ip address - so we need to resolve it diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 00cbf464..ca3530fa 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -968,7 +968,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): ) from exc # sort list of new values data[key] = sorted(new_value) - # log backref modification not catched by modify hook + # log backref modification not caught by modify hook if isinstance(self.fields[key], RelatedList): if callback := self.context.get('callback'): before = {str(v) for v in getattr(instance, key)} diff --git a/core/admin/mailu/translations/ca/LC_MESSAGES/messages.po b/core/admin/mailu/translations/ca/LC_MESSAGES/messages.po index c194ed16..52a47a82 100644 --- a/core/admin/mailu/translations/ca/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/ca/LC_MESSAGES/messages.po @@ -660,7 +660,7 @@ msgid "New relay domain" msgstr "Nou domini llegat (relayed)" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Editeu domini llegat (relayed)" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/da/LC_MESSAGES/messages.po b/core/admin/mailu/translations/da/LC_MESSAGES/messages.po index 23c5030d..b386d712 100644 --- a/core/admin/mailu/translations/da/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/da/LC_MESSAGES/messages.po @@ -649,7 +649,7 @@ msgid "New relay domain" msgstr "" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/de/LC_MESSAGES/messages.po b/core/admin/mailu/translations/de/LC_MESSAGES/messages.po index b6d3caf7..afd9abda 100644 --- a/core/admin/mailu/translations/de/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/de/LC_MESSAGES/messages.po @@ -655,7 +655,7 @@ msgid "New relay domain" msgstr "Neue Relay-Domain" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Relay-Domain bearbeiten" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/en/LC_MESSAGES/messages.po b/core/admin/mailu/translations/en/LC_MESSAGES/messages.po index 47253a04..4ec33e05 100644 --- a/core/admin/mailu/translations/en/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/en/LC_MESSAGES/messages.po @@ -649,7 +649,7 @@ msgid "New relay domain" msgstr "New relay domain" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/es/LC_MESSAGES/messages.po b/core/admin/mailu/translations/es/LC_MESSAGES/messages.po index 3f2a9439..c385905e 100644 --- a/core/admin/mailu/translations/es/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/es/LC_MESSAGES/messages.po @@ -651,7 +651,7 @@ msgid "New relay domain" msgstr "Nuevo dominio externo (relay)" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Editar dominio externo (relay)" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/eu/LC_MESSAGES/messages.po b/core/admin/mailu/translations/eu/LC_MESSAGES/messages.po index f7c2faa1..d47229aa 100644 --- a/core/admin/mailu/translations/eu/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/eu/LC_MESSAGES/messages.po @@ -649,7 +649,7 @@ msgid "New relay domain" msgstr "Igorritako domeinu berria" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Editatu igorritako domeinua" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/fr/LC_MESSAGES/messages.po b/core/admin/mailu/translations/fr/LC_MESSAGES/messages.po index 56903d90..def764ac 100644 --- a/core/admin/mailu/translations/fr/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/fr/LC_MESSAGES/messages.po @@ -653,7 +653,7 @@ msgid "New relay domain" msgstr "Nouveau domaine relayé" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Modifier le domaine relayé" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/he/LC_MESSAGES/messages.po b/core/admin/mailu/translations/he/LC_MESSAGES/messages.po index 113872c7..106b981d 100644 --- a/core/admin/mailu/translations/he/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/he/LC_MESSAGES/messages.po @@ -658,7 +658,7 @@ msgid "New relay domain" msgstr "שם תחום מועבר" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "עריכת שמות תחום מועברים" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/hu/LC_MESSAGES/messages.po b/core/admin/mailu/translations/hu/LC_MESSAGES/messages.po index 676c975a..d5d1f496 100644 --- a/core/admin/mailu/translations/hu/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/hu/LC_MESSAGES/messages.po @@ -652,7 +652,7 @@ msgid "New relay domain" msgstr "Új továbbító tartomány" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Továbbított tartomány szerkesztése" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/is/LC_MESSAGES/messages.po b/core/admin/mailu/translations/is/LC_MESSAGES/messages.po index d48a5bab..81cf2039 100644 --- a/core/admin/mailu/translations/is/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/is/LC_MESSAGES/messages.po @@ -648,7 +648,7 @@ msgid "New relay domain" msgstr "" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/it/LC_MESSAGES/messages.po b/core/admin/mailu/translations/it/LC_MESSAGES/messages.po index 240e80cd..50630dc6 100644 --- a/core/admin/mailu/translations/it/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/it/LC_MESSAGES/messages.po @@ -655,7 +655,7 @@ msgid "New relay domain" msgstr "Nuovo dominio affidato" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Editare dominio affidato" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/ja/LC_MESSAGES/messages.po b/core/admin/mailu/translations/ja/LC_MESSAGES/messages.po index d6bad326..2bedfa54 100644 --- a/core/admin/mailu/translations/ja/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/ja/LC_MESSAGES/messages.po @@ -649,7 +649,7 @@ msgid "New relay domain" msgstr "リレードメイン名を追加" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "リレードメイン名を編集" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/nb_NO/LC_MESSAGES/messages.po b/core/admin/mailu/translations/nb_NO/LC_MESSAGES/messages.po index ab1034bb..07dfeea7 100644 --- a/core/admin/mailu/translations/nb_NO/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/nb_NO/LC_MESSAGES/messages.po @@ -657,7 +657,7 @@ msgid "New relay domain" msgstr "Nytt videresendt domene" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Endre videresendt domene" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/nl/LC_MESSAGES/messages.po b/core/admin/mailu/translations/nl/LC_MESSAGES/messages.po index cc43c02f..18728723 100644 --- a/core/admin/mailu/translations/nl/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/nl/LC_MESSAGES/messages.po @@ -650,7 +650,7 @@ msgid "New relay domain" msgstr "Nieuw relay domein" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Bewerk relay domein" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/pl/LC_MESSAGES/messages.po b/core/admin/mailu/translations/pl/LC_MESSAGES/messages.po index 2d423020..1312cbc6 100644 --- a/core/admin/mailu/translations/pl/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/pl/LC_MESSAGES/messages.po @@ -659,7 +659,7 @@ msgstr "Nowa domena do przekierowania" #: mailu/ui/templates/relay/edit.html:4 #, fuzzy -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Edycja domeny" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/pt/LC_MESSAGES/messages.po b/core/admin/mailu/translations/pt/LC_MESSAGES/messages.po index 1927fa05..e210e2d4 100644 --- a/core/admin/mailu/translations/pt/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/pt/LC_MESSAGES/messages.po @@ -650,7 +650,7 @@ msgid "New relay domain" msgstr "Novo domínio para encaminhamento" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Editar domínio de encaminhamento" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/ru/LC_MESSAGES/messages.po b/core/admin/mailu/translations/ru/LC_MESSAGES/messages.po index eb70db2f..1133763c 100644 --- a/core/admin/mailu/translations/ru/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/ru/LC_MESSAGES/messages.po @@ -656,7 +656,7 @@ msgid "New relay domain" msgstr "Новый релейный домен" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Изменить релейный домен" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/sv/LC_MESSAGES/messages.po b/core/admin/mailu/translations/sv/LC_MESSAGES/messages.po index 5b1bcce8..93eaa139 100644 --- a/core/admin/mailu/translations/sv/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/sv/LC_MESSAGES/messages.po @@ -645,7 +645,7 @@ msgid "New relay domain" msgstr "Ny relä-domän" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "Redigera reläade domäner" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/translations/zh/LC_MESSAGES/messages.po b/core/admin/mailu/translations/zh/LC_MESSAGES/messages.po index 9544091b..80f3435d 100644 --- a/core/admin/mailu/translations/zh/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/zh/LC_MESSAGES/messages.po @@ -646,7 +646,7 @@ msgid "New relay domain" msgstr "新的中继域" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "编辑中继域" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/mailu/ui/templates/relay/edit.html b/core/admin/mailu/ui/templates/relay/edit.html index df2418a4..18b24519 100644 --- a/core/admin/mailu/ui/templates/relay/edit.html +++ b/core/admin/mailu/ui/templates/relay/edit.html @@ -1,7 +1,7 @@ {%- extends "form.html" %} {%- block title %} -{% trans %}Edit relayd domain{% endtrans %} +{% trans %}Edit relayed domain{% endtrans %} {%- endblock %} {%- block subtitle %} diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 4469ae8a..7e208702 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -472,7 +472,7 @@ class MailuSessionExtension: redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL']) ) - # clean expired sessions oonce on first use in case lifetime was changed + # clean expired sessions once on first use in case lifetime was changed def cleaner(): with cleaned.get_lock(): if not cleaned.value: diff --git a/core/admin/messages.pot b/core/admin/messages.pot index 0eec2f6a..9cdd91df 100644 --- a/core/admin/messages.pot +++ b/core/admin/messages.pot @@ -647,7 +647,7 @@ msgid "New relay domain" msgstr "" #: mailu/ui/templates/relay/edit.html:4 -msgid "Edit relayd domain" +msgid "Edit relayed domain" msgstr "" #: mailu/ui/templates/relay/list.html:4 diff --git a/core/admin/start.py b/core/admin/start.py index 8bb1cef1..f5378d35 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -15,7 +15,7 @@ password = os.environ.get("INITIAL_ADMIN_PW") if account is not None and domain is not None and password is not None: mode = os.environ.get("INITIAL_ADMIN_MODE", default="ifmissing") - log.info("Creating initial admin accout %s@%s with mode %s",account,domain,mode) + log.info("Creating initial admin account %s@%s with mode %s", account, domain, mode) os.system("flask mailu admin %s %s '%s' --mode %s" % (account, domain, password, mode)) def test_DNS(): @@ -37,7 +37,7 @@ def test_DNS(): try: result = resolver.resolve('example.org', dns.rdatatype.A, dns.rdataclass.IN, lifetime=10) except Exception as e: - log.critical("Your DNS resolver at %s is not working (%s). Please see https://mailu.io/master/faq.html#the-admin-container-won-t-start-and-its-log-says-critical-your-dns-resolver-isn-t-doing-dnssec-validation", ns, e); + log.critical("Your DNS resolver at %s is not working (%s). Please see https://mailu.io/master/faq.html#the-admin-container-won-t-start-and-its-log-says-critical-your-dns-resolver-isn-t-doing-dnssec-validation", ns, e) else: if result.response.flags & dns.flags.AD: break diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index ed2c4cdf..7a987582 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -151,7 +151,7 @@ plugin { # Send vacation replies even for aliases # See the Pigeonhole documentation about warnings: http://wiki2.dovecot.org/Pigeonhole/Sieve/Extensions/Vacation - # It appears that our implemntation of mail delivery meets criteria of section 4.5 + # It appears that our implementation of mail delivery meets criteria of section 4.5 # from RFC 5230 and that disabling the recipient checks is not an issue here. sieve_vacation_dont_check_recipient = yes diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 8fcaef26..7c831f1d 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -287,7 +287,7 @@ mail { ssl_session_cache shared:SSLMAIL:50m; {% endif %} - # Advertise real capabilites of backends (postfix/dovecot) + # Advertise real capabilities of backends (postfix/dovecot) smtp_capabilities PIPELINING SIZE {{ MESSAGE_SIZE_LIMIT }} ETRN ENHANCEDSTATUSCODES 8BITMIME DSN; pop3_capabilities TOP UIDL RESP-CODES PIPELINING AUTH-RESP-CODE USER; imap_capabilities IMAP4 IMAP4rev1 UIDPLUS SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+; diff --git a/docs/antispam.rst b/docs/antispam.rst index 7cadde1d..c7aab042 100644 --- a/docs/antispam.rst +++ b/docs/antispam.rst @@ -8,7 +8,7 @@ How does spam filtering work in Mailu? Mailu uses Rspamd for spam filtering. Rspamd is a Fast, free and open-source spam filtering system. -Rspamd rejects non-compliant email messages and email messages that contain virusses. In Mailu Rspamd uses a scoring scale from 0 to 15. The following values are the default values, and can be changed inside the Rspamd WebUI under the tab configuration: +Rspamd rejects non-compliant email messages and email messages that contain viruses. In Mailu Rspamd uses a scoring scale from 0 to 15. The following values are the default values, and can be changed inside the Rspamd WebUI under the tab configuration: * Email messages with a score of 15 or higher will be rejected. diff --git a/docs/arm_images.rst b/docs/arm_images.rst index 2a1a1574..72cbdb56 100644 --- a/docs/arm_images.rst +++ b/docs/arm_images.rst @@ -31,7 +31,7 @@ all images from the mailu docker repository. Build manually -------------- -It is possible to build the images manually. There are two possiblities for this. +It is possible to build the images manually. There are two possibilities for this. Github actions `````````````` diff --git a/docs/compose/.env b/docs/compose/.env index fa952270..265b4bad 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -153,7 +153,7 @@ REAL_IP_HEADER= # IPs for nginx set_real_ip_from (CIDR list separated by commas) REAL_IP_FROM= -# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no) +# choose whether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no) REJECT_UNLISTED_RECIPIENT= # Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET) diff --git a/docs/compose/requirements.rst b/docs/compose/requirements.rst index 84666f21..a9ec6eab 100644 --- a/docs/compose/requirements.rst +++ b/docs/compose/requirements.rst @@ -92,7 +92,7 @@ about setting up a proper Docker install. Default configuration should be suited for Mailu. Additionally, you must install ``docker-compose`` by following the instructions -from the `Docker website`_ if you plan on using the CompComposesoe flavor. Compose is a +from the `Docker website`_ if you plan on using the Compose flavor. Compose is a management tool for Docker, especially suited for multiple containers systems like Mailu. diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst index 57be78c9..57c9b761 100644 --- a/docs/compose/setup.rst +++ b/docs/compose/setup.rst @@ -41,7 +41,7 @@ values: .. note:: When using *Letsencrypt!* you have to make sure that the DNS ``A`` and ``AAAA`` records for the - all hostnames mentioned in the ``HOSTNAMES`` variable match with the ip adresses of you server. + all hostnames mentioned in the ``HOSTNAMES`` variable match with the ip addresses of you server. Or else certificate generation will fail! See also: :ref:`dns_setup`. Bind address diff --git a/docs/configuration.rst b/docs/configuration.rst index 80f97d3a..e1eca2e5 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -90,10 +90,10 @@ by setting ``OUTBOUND_TLS_LEVEL`` to ``encrypt`` or ``secure``. This setting is highly recommended if you are using a relayhost that supports TLS but discouraged otherwise. ``DEFER_ON_TLS_ERROR`` (default: True) controls whether incomplete policies (DANE without DNSSEC or "testing" MTA-STS policies) will be taken into -account and whether emails will be defered if the additional checks enforced by +account and whether emails will be deferred if the additional checks enforced by those policies fail. -Similarily by default nginx uses "opportunistic TLS" for inbound mail. This can be changed +Similarly by default nginx uses "opportunistic TLS" for inbound mail. This can be changed by setting ``INBOUND_TLS_ENFORCE`` to ``True``. Please note that this is forbidden for internet facing hosts according to e.g. `RFC 3207`_ , because this prevents MTAs without STARTTLS support or e.g. mismatching TLS versions to deliver emails to Mailu. @@ -146,7 +146,7 @@ in the admin interface, while ``SITENAME`` is a customization option for every Web interface. - ``LOGO_BACKGROUND`` sets a custom background colour for the brand logo - in the topleft of the main admin interface. + in the top left of the main admin interface. For a list of colour codes refer to this page of `w3schools`_. - ``LOGO_URL`` sets a URL for a custom logo. This logo replaces the Mailu diff --git a/docs/contributors/guidelines.rst b/docs/contributors/guidelines.rst index 31e29e54..d55f9adf 100644 --- a/docs/contributors/guidelines.rst +++ b/docs/contributors/guidelines.rst @@ -25,7 +25,7 @@ What features should be included Mailu is a mail server. That said, mail works in an ecosystem and we should set clear boundaries on what Mailu is and what it is not. Beyond mail, we include -features that mail is dependent on in some way, not the other way aroud. +features that mail is dependent on in some way, not the other way around. For instance, we include contact management and a dav service for synchronizing contacts, because sending mail is made easier with manageable contacts. We do @@ -97,7 +97,7 @@ lifecycle management. Anything that is not static, i.e. able to change at runtime, either due to configuration in the admin UI or user behavior, should take advantage of the -admin API. The `podop` package binds mail specific sofware (Postfix and Dovecot +admin API. The `podop` package binds mail specific software (Postfix and Dovecot at the moment) to the admin API, other containers should use specific API calls. What traffic should go through the nginx container @@ -155,7 +155,7 @@ Should we use default configuration Some tools ship with default configuration, that handles the standard behavior. Using this configuration is prone to later changes and unexpected side effects. We should always provide all required configuration, including the base files, -and not rely on default configuration files froms the distribution. +and not rely on default configuration files from the distribution. For that reason, in case the tool looks for specific files and include them automatically, we should overwrite them or delete them. diff --git a/docs/contributors/workflow.rst b/docs/contributors/workflow.rst index 00a447aa..957d4a9b 100644 --- a/docs/contributors/workflow.rst +++ b/docs/contributors/workflow.rst @@ -75,7 +75,7 @@ Supported file extensions are: Forked projects --------------- -If you find yourself forking the project for a specific independant purpose +If you find yourself forking the project for a specific independent purpose (commercial use, different philosophy or incompatible point of view), we would be glad if you let us know so that we can list interesting known forks and their specific features (and backport some of them if your implementation diff --git a/docs/dns.rst b/docs/dns.rst index 82d21ee9..907aa5ea 100644 --- a/docs/dns.rst +++ b/docs/dns.rst @@ -52,7 +52,7 @@ Note that both point to the same mail server hostname, which is unique to your s Reverse DNS entries ------------------- -For a mail system, it's higly recommended to set up reverse DNS as well. That means, if your hostname +For a mail system, it's highly recommended to set up reverse DNS as well. That means, if your hostname ``mail.mydomain.com`` resolves to ``a.b.c.d``, the IP ``a.b.c.d`` should also resolve back to the same hostname. You can verify this with @@ -71,7 +71,7 @@ DKIM/SPF & DMARC Entries Finally, you'll need to visit the admin dashboard (or use the cli) to regenerate your DMARC, SPF, and DKIM records. -Once the DNS changes to your host have propogated (and if SSL / domain rules were setup correctly), visit your admin +Once the DNS changes to your host have propagated (and if SSL / domain rules were setup correctly), visit your admin dashboard at https://example.com/admin/domain/details/example.com. Click on `regenerate keys` and add the required records to your DNS provider. If you've enabled DKIM/SPF / DMARC and haven't added these entries, your mail might not get delivered. diff --git a/docs/faq.rst b/docs/faq.rst index 7b888867..7782e93c 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -71,11 +71,11 @@ We thank you for your understanding and patience. I would like to donate (for a feature) `````````````````````````````````````` -We maintain a `Communtity Bridge`_ project through which you can donate. +We maintain a `Community Bridge`_ project through which you can donate. This budget will be used to pay for development of features, mentorship and hopefully future events. Contributing companies or individuals can be paid from this budget to support their development efforts. -We are also looking into GitHub's integrated sponorship program for individual contributors. +We are also looking into GitHub's integrated sponsorship program for individual contributors. Once those become available, we will add them to the project. Please click the |sponsor| button on top of our GitHub Page for current possibilities. @@ -92,7 +92,7 @@ Please click the |sponsor| button on top of our GitHub Page for current possibil .. _`Enhancement issues`: https://github.com/Mailu/Mailu/issues?q=is%3Aissue+is%3Aopen+label%3Atype%2Fenhancement .. _`Feature request issues`: https://github.com/Mailu/Mailu/issues?q=is%3Aopen+is%3Aissue+label%3Atype%2Ffeature .. _`GitHub`: https://github.com/Mailu/Mailu -.. _`Communtity Bridge`: https://funding.communitybridge.org/projects/mailu +.. _`Community Bridge`: https://funding.communitybridge.org/projects/mailu Deployment related ------------------ @@ -216,7 +216,7 @@ Therefore it is advised to create backups on a regular base! A backup MX can be configured as **failover**. For this you need a separate server running Mailu. On that server, your domains will need to be setup as "Relayed domains", pointing -to you mainr server. MX records for the mail domains with a higher priority number will have +to you main server. MX records for the mail domains with a higher priority number will have to point to this server. Please be aware that a backup MX can act as a `spam magnet`_ (archive.org). For **service** HA, please see: `How does Mailu scale up?`_ @@ -296,7 +296,7 @@ I want to integrate Nextcloud 15 (and newer) with Mailu If a domain name (e.g. example.com) is specified, then this makes sure that only users from this domain will be allowed to login. -After successfull login the domain part will be stripped and the rest used as username in Nextcloud. e.g. 'username@example.com' will be 'username' in Nextcloud. Disable this behaviour by changing true (the fifth parameter) to false. +After successfull login the domain part will be stripped and the rest used as username in Nextcloud. e.g. 'username@example.com' will be 'username' in Nextcloud. Disable this behaviour by changing true (the fifth parameter) to false. *Issue reference:* `575`_. @@ -798,7 +798,7 @@ Very often, SPF failure is related to Mailu sending emails with a different IP a This is mostly due to using a separate IP address for Mailu and still having masquerading NAT setup for Docker, which results in a different outbound IP address. You can simply check the email headers on the receiving side to confirm this. -If you wish to explicitely NAT Mailu outbound traffic, it is usually easy to source-NAT outgoing SMTP traffic using iptables : +If you wish to explicitly NAT Mailu outbound traffic, it is usually easy to source-NAT outgoing SMTP traffic using iptables : ``` iptables -t nat -A POSTROUTING -o eth0 -p tcp --dport 25 -j SNAT --to diff --git a/docs/releases.rst b/docs/releases.rst index 8f4c2bf0..51d904ed 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -99,13 +99,13 @@ A short summary of the new features: - Domain details page is enhanced with DNS client auto-configuration (RFC6186) entries. - Centralize the authentication of webmails behind the admin interface. - - The new single sign on page opens up the possiblity to introduce 2 factor authentication in the future. + - The new single sign on page opens up the possibility to introduce 2 factor authentication in the future. - Add sending quotas per user (configured in mailu.env). This determines how many emails each user can send every day. - Allow specific users to send emails from any address using the WILDCARD_SENDERS setting (mailu.env.). - Use semantic versioning for building releases. -- Internal improviments to improve performance of authentication requests. -- Introduded a language selector for the Admin interface. +- Internal improvements to improve performance of authentication requests. +- Introduced a language selector for the Admin interface. - Add cli commands config-import and config-export for importing/exporting Mailu config via YAML. - Enable support of all hash types passlib supports. - Switch to bcrypt_sha256 (stronger hashing of passwords in Mailu database)/ @@ -274,7 +274,7 @@ Other changes include software updates with some new features in Rainloop Back-end ```````` -One of the big tasks was upgradig to latest Alpine (3.10), which is now finished. +One of the big tasks was upgrading to latest Alpine (3.10), which is now finished. Also, a lot was improved about the environment variables meant to provide specific hosts in custom setups. @@ -434,7 +434,7 @@ and bug fixes. **If you are upgrading**, please go through the setup guide and download the latest ``docker-compose.yml`` and ``.env``, then update them with your -sepcific settings, because more than 50% of these templates was rewritten. +specific settings, because more than 50% of these templates was rewritten. You should then be able to pull and start your new e-mail stack with no issue, simply remove orphaned container, since some were renamed and others were removed (e.g. rmilter): @@ -522,7 +522,7 @@ When creating an alias, one may now enable the "SQL LIKE" syntax then use standard SQL wildcards ``%`` and ``_`` to specify matches for a given alias. For instance : -- ``%@domain.tld`` will match any uncatched email sent to that domain (catch-all) +- ``%@domain.tld`` will match any uncaught email sent to that domain (catch-all) - ``support-%@domain.tld`` will match any email sent to an address starting with ``support-`` - ``_@domain.tld`` will match any email sent to a one-character address diff --git a/docs/swarm/1.5/README.md b/docs/swarm/1.5/README.md index 6b56e642..5af4c9dd 100644 --- a/docs/swarm/1.5/README.md +++ b/docs/swarm/1.5/README.md @@ -1,6 +1,6 @@ # Install Mailu on a docker swarm -## Prequisites +## Prerequisites ### Swarm @@ -28,7 +28,7 @@ mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active ``` ### Volume definition -For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. +For data persistence (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. Hereafter we will use a NFS share: ```bash core@coreos-01 ~ $ showmount -e 192.168.0.30 @@ -358,7 +358,7 @@ tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 ``` ### Remove the stack -Run the follwoing command: +Run the following command: ```bash core@coreos-01 ~ $ docker stack rm mailu ``` diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index e82f7406..56cec36d 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -38,7 +38,7 @@ Although the above situation is less-likely to occur on a stable (local) network -- @muhlemmer, 17th of January 2019. -## Prequisites +## Prerequisites ### Swarm @@ -66,7 +66,7 @@ mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active ``` ### Volume definition -For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. +For data persistence (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT" (GlusterFS and nfs shares have been successfully used). @@ -290,7 +290,7 @@ core@coreos-01 ~ $ docker service logs -f mailu_fetchmail ``` ## Remove the stack -Run the follwoing command: +Run the following command: ```bash core@coreos-01 ~ $ docker stack rm mailu ``` diff --git a/docs/swarm/master/README_nfs_example.md b/docs/swarm/master/README_nfs_example.md index ac52b84e..2c4b8145 100644 --- a/docs/swarm/master/README_nfs_example.md +++ b/docs/swarm/master/README_nfs_example.md @@ -28,7 +28,7 @@ mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active ``` ### Volume definition -For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. +For data persistence (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. Hereafter we will use a NFS share: ```bash core@coreos-01 ~ $ showmount -e 192.168.0.30 @@ -58,7 +58,7 @@ core@coreos-01 ~ $ sudo umount /mnt/local/ ## Networking mode On this example, we are using: -- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. +- the mesh routing mode (default mode). With this mode, each service is given a virtual IP address and docker manages the routing between this virtual IP and the container(s) providing this service. - the default ingress mode. ### Allow authentification with the mesh routing @@ -351,7 +351,7 @@ tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:master coreos-01 ``` ## Remove the stack -Run the follwoing command: +Run the following command: ```bash core@coreos-01 ~ $ docker stack rm mailu ``` diff --git a/docs/webadministration.rst b/docs/webadministration.rst index 6e7525dc..e17d12f0 100644 --- a/docs/webadministration.rst +++ b/docs/webadministration.rst @@ -327,7 +327,7 @@ This page also shows an overview of the following settings of an user: * Sending Quota. The sending quota is the limit of messages a single user can send per day. -* Comment. A desription for the user. +* Comment. A description for the user. * Created. Date when the user was created. diff --git a/setup/README.md b/setup/README.md index 500968bc..7c3e3ab4 100644 --- a/setup/README.md +++ b/setup/README.md @@ -2,8 +2,8 @@ (Everything will go under setup/ directory - using Kubernetes flavor as example) Until this point, the app is working as it follows: -- when accesing the setup page it will display the flavors selection step (`templates/steps/flavor.html`) -- after you choose your desired flavor it will iterare over the files in the flavor directory and building the page +- when accessing the setup page it will display the flavors selection step (`templates/steps/flavor.html`) +- after you choose your desired flavor it will iterate over the files in the flavor directory and building the page (`templates/steps/config.html is general for all flavors`) - when you complete all required fields and press "Setup Mailu" button it will redirect you to the setup page (`flavors/choosen-flavor/setup.html`) @@ -29,7 +29,7 @@ In the example below the string entered in the input field is stored in the vari ``` -In order to use the variable furter you use it like `{{ var_test }}` +In order to use the variable further you use it like `{{ var_test }}` In the setup page (`flavors/kubernetes/setup.html`) you can add steps by importing macros diff --git a/setup/templates/setup.html b/setup/templates/setup.html index fa1be8aa..b4bca029 100644 --- a/setup/templates/setup.html +++ b/setup/templates/setup.html @@ -10,7 +10,7 @@ file before running anything based on it.

If you encounter issues while setting Mailu up, please review the documentation first, then check if an issue is open for that specific - problem. If not, you may either use Github to open an issue and detail what + problem. If not, you may either use GitHub to open an issue and detail what your problem or bug looks like, or join us on Matrix and discuss it with contributors.

{% endcall %} diff --git a/setup/templates/steps/compose/03_expose.html b/setup/templates/steps/compose/03_expose.html index 6d2150b6..201ac3eb 100644 --- a/setup/templates/steps/compose/03_expose.html +++ b/setup/templates/steps/compose/03_expose.html @@ -5,8 +5,8 @@ you expose it to the world.

Among Mailu services, the front server is the one accepting connections, be it directly from the outside world, through a reverse proxy or in any -complex configuration that you might want to setup. It needs to listen on some -IP addresses in order to expose its public services. You must at least setup +complex configuration that you might want to set up. It needs to listen on some +IP addresses in order to expose its public services. You must at least set up an IPv4 or an IPv6 address if you wish to access Mailu.

Warning You must use specific addresses, please diff --git a/tests/compose/core/00_create_users.sh b/tests/compose/core/00_create_users.sh index f5108302..2e36189f 100755 --- a/tests/compose/core/00_create_users.sh +++ b/tests/compose/core/00_create_users.sh @@ -8,4 +8,4 @@ docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mail docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu admin admin mailu.io 'password' --mode=update || exit 1 docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user user mailu.io 'password' || exit 1 docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu user 'user/with/slash' mailu.io 'password' || exit 1 -echo "User testing succesfull!" +echo "User testing successful!" diff --git a/webmails/roundcube/config.inc.php b/webmails/roundcube/config.inc.php index a7b742f5..e37666bc 100644 --- a/webmails/roundcube/config.inc.php +++ b/webmails/roundcube/config.inc.php @@ -29,7 +29,7 @@ $config['managesieve_usetls'] = false; // We access the IMAP and SMTP servers locally with internal names, SSL // will obviously fail but this sounds better than allowing insecure login -// from the outter world +// from the outer world $ssl_no_check = array( 'ssl' => array( 'verify_peer' => false, diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 1ee1ed50..372507ee 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -83,7 +83,7 @@ try: except subprocess.CalledProcessError as exc: err = exc.stdout.decode() if "already exists" in err: - print("Already initialzed") + print("Already initialized") else: print(err) exit(3) From af87456faf65ffbe133e451fa7f002983d7e282a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 20 Oct 2022 10:24:10 +0200 Subject: [PATCH 058/274] this works for me --- core/rspamd/conf/force_actions.conf | 16 +++++++++++++--- core/rspamd/conf/multimap.conf | 12 +++++++++--- core/rspamd/conf/whitelist.conf | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf index 20bf9245..0d1d1a2b 100644 --- a/core/rspamd/conf/force_actions.conf +++ b/core/rspamd/conf/force_actions.conf @@ -1,7 +1,17 @@ rules { - ANTISPOOF { + ANTISPOOF_NOAUTH { action = "reject"; - expression = "((R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA)|BLACKLIST_ANTISPOOF) & IS_LOCAL_DOMAIN"; - message = "Rejected (anti-spoofing)"; + expression = "(IS_LOCAL_DOMAIN_E & MISSING_FROM) | (IS_LOCAL_DOMAIN_H & (R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA))"; + message = "Rejected (anti-spoofing noauth)"; + } + ANTISPOOF_DMARC_ENFORCE_LOCAL { + action = "reject"; + expression = "((IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE)"; + message = "Rejected (anti-spoofing DMARC-enforce for local domains)"; + } + ANTISPOOF_AUTH_FAILED { + action = "reject"; + expression = "BLACKLIST_ANTISPOOF"; + message = "Rejected (anti-spoofing auth-failed)"; } } diff --git a/core/rspamd/conf/multimap.conf b/core/rspamd/conf/multimap.conf index dd2ffa61..dd25c08e 100644 --- a/core/rspamd/conf/multimap.conf +++ b/core/rspamd/conf/multimap.conf @@ -1,5 +1,11 @@ -IS_LOCAL_DOMAIN { - type = "from"; - filter = "email:domain"; +IS_LOCAL_DOMAIN_H { + type = "selector" + selector = "from('mime'):domain"; + map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; +} + +IS_LOCAL_DOMAIN_E { + type = "selector" + selector = "from('smtp'):domain"; map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; } diff --git a/core/rspamd/conf/whitelist.conf b/core/rspamd/conf/whitelist.conf index fcde167a..208ca203 100644 --- a/core/rspamd/conf/whitelist.conf +++ b/core/rspamd/conf/whitelist.conf @@ -3,6 +3,6 @@ rules { valid_dmarc = true; blacklist = true; domains = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; - score = 15.0; + score = 99.0; } } From 8da6117bb92aac66f0ab422ead887e9314d079c4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 20 Oct 2022 10:35:43 +0200 Subject: [PATCH 059/274] clarify --- core/rspamd/conf/force_actions.conf | 6 +++--- core/rspamd/conf/whitelist.conf | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf index 0d1d1a2b..54e893a9 100644 --- a/core/rspamd/conf/force_actions.conf +++ b/core/rspamd/conf/force_actions.conf @@ -2,16 +2,16 @@ rules { ANTISPOOF_NOAUTH { action = "reject"; expression = "(IS_LOCAL_DOMAIN_E & MISSING_FROM) | (IS_LOCAL_DOMAIN_H & (R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA))"; - message = "Rejected (anti-spoofing noauth)"; + message = "Rejected (anti-spoofing: noauth). Please setup DMARC with DKIM or SPF if you want to send emails from your domain from other servers."; } ANTISPOOF_DMARC_ENFORCE_LOCAL { action = "reject"; expression = "((IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE)"; - message = "Rejected (anti-spoofing DMARC-enforce for local domains)"; + message = "Rejected (anti-spoofing: DMARC is enforced for local domains)"; } ANTISPOOF_AUTH_FAILED { action = "reject"; expression = "BLACKLIST_ANTISPOOF"; - message = "Rejected (anti-spoofing auth-failed)"; + message = "Rejected (anti-spoofing: auth-failed)"; } } diff --git a/core/rspamd/conf/whitelist.conf b/core/rspamd/conf/whitelist.conf index 208ca203..56f8a83d 100644 --- a/core/rspamd/conf/whitelist.conf +++ b/core/rspamd/conf/whitelist.conf @@ -3,6 +3,6 @@ rules { valid_dmarc = true; blacklist = true; domains = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; - score = 99.0; + score = 0.0; } } From 8929f54de5ff960c316d2c76292cb32e2c716b5c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 20 Oct 2022 11:32:58 +0200 Subject: [PATCH 060/274] clarify Also cover the case where the DKIM sig is for another domain and there is no explicit DMARC policy --- core/rspamd/conf/force_actions.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf index 54e893a9..5c4e8eea 100644 --- a/core/rspamd/conf/force_actions.conf +++ b/core/rspamd/conf/force_actions.conf @@ -6,8 +6,8 @@ rules { } ANTISPOOF_DMARC_ENFORCE_LOCAL { action = "reject"; - expression = "((IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE)"; - message = "Rejected (anti-spoofing: DMARC is enforced for local domains)"; + expression = "((IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)"; + message = "Rejected (anti-spoofing: DMARC compliance is enforced for local domains, regardless of the policy setting)"; } ANTISPOOF_AUTH_FAILED { action = "reject"; From 8a60b658b46dd8b8f9a99581849d8fa04dc76fdb Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Thu, 20 Oct 2022 13:41:35 +0200 Subject: [PATCH 061/274] Implement FETCHMAIL_ENABLED --- core/admin/mailu/configuration.py | 5 +++-- core/admin/mailu/ui/templates/sidebar.html | 2 ++ core/admin/mailu/ui/views/fetches.py | 9 +++++++++ docs/configuration.rst | 3 +++ setup/flavors/compose/mailu.env | 3 +++ towncrier/newsfragments/2127.feature | 1 + 6 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/2127.feature diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 2f137c0c..e7de2a40 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -16,13 +16,14 @@ DEFAULT_CONFIG = { 'DOMAIN_REGISTRATION': False, 'TEMPLATES_AUTO_RELOAD': True, 'MEMORY_SESSIONS': False, + 'FETCHMAIL_ENABLED': False, # Database settings 'DB_FLAVOR': None, 'DB_USER': 'mailu', 'DB_PW': None, 'DB_HOST': 'database', 'DB_NAME': 'mailu', - 'SQLITE_DATABASE_FILE':'data/main.db', + 'SQLITE_DATABASE_FILE': 'data/main.db', 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', 'SQLALCHEMY_TRACK_MODIFICATIONS': False, # Statistics management @@ -59,7 +60,7 @@ DEFAULT_CONFIG = { # Web settings 'SITENAME': 'Mailu', 'WEBSITE': 'https://mailu.io', - 'ADMIN' : 'none', + 'ADMIN': 'none', 'WEB_ADMIN': '/admin', 'WEB_WEBMAIL': '/webmail', 'WEBMAIL': 'none', diff --git a/core/admin/mailu/ui/templates/sidebar.html b/core/admin/mailu/ui/templates/sidebar.html index a57a2abe..54448c8b 100644 --- a/core/admin/mailu/ui/templates/sidebar.html +++ b/core/admin/mailu/ui/templates/sidebar.html @@ -31,12 +31,14 @@

{% trans %}Auto-reply{% endtrans %}

+ {%- if config["FETCHMAIL_ENABLED"] %}

  • {% trans %}Fetched accounts{% endtrans %}
    • + {%- endif %}
  • diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index ec208af1..ca837a8e 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -1,5 +1,6 @@ from mailu import models from mailu.ui import ui, forms, access +from flask import current_app as app import flask import flask_login @@ -10,6 +11,8 @@ import wtforms @ui.route('/fetch/list/', methods=['GET']) @access.owner(models.User, 'user_email') def fetch_list(user_email): + if not app.config['FETCHMAIL_ENABLED']: + flask.abort(404) user_email = user_email or flask_login.current_user.email user = models.User.query.get(user_email) or flask.abort(404) return flask.render_template('fetch/list.html', user=user) @@ -19,6 +22,8 @@ def fetch_list(user_email): @ui.route('/fetch/create/', methods=['GET', 'POST']) @access.owner(models.User, 'user_email') def fetch_create(user_email): + if not app.config['FETCHMAIL_ENABLED']: + flask.abort(404) user_email = user_email or flask_login.current_user.email user = models.User.query.get(user_email) or flask.abort(404) form = forms.FetchForm() @@ -37,6 +42,8 @@ def fetch_create(user_email): @ui.route('/fetch/edit/', methods=['GET', 'POST']) @access.owner(models.Fetch, 'fetch_id') def fetch_edit(fetch_id): + if not app.config['FETCHMAIL_ENABLED']: + flask.abort(404) fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) form = forms.FetchForm(obj=fetch) if form.validate_on_submit(): @@ -55,6 +62,8 @@ def fetch_edit(fetch_id): @access.confirmation_required("delete a fetched account") @access.owner(models.Fetch, 'fetch_id') def fetch_delete(fetch_id): + if not app.config['FETCHMAIL_ENABLED']: + flask.abort(404) fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) user = fetch.user models.db.session.delete(fetch) diff --git a/docs/configuration.rst b/docs/configuration.rst index 80f97d3a..58965dff 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -102,6 +102,9 @@ support or e.g. mismatching TLS versions to deliver emails to Mailu. .. _fetchmail: +When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is shown in the admin interface. +The container itself still needs to be deployed manually. As fetchmail is optional, ``FETCHMAIL_ENABLED`` defaults to ``False``. + The ``FETCHMAIL_DELAY`` is a delay (in seconds) for the fetchmail service to go and fetch new email if available. Do not use too short delays if you do not want to be blacklisted by external services, but not too long delays if you diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 2963c882..4e13a8c9 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -79,6 +79,9 @@ RELAYNETS= # Will relay all outgoing mails if configured RELAYHOST={{ relayhost }} +# Show fetchmail functionality in admin interface +FETCHMAIL_ENABLED={{ fetchmail_enabled or 'False' }} + # Fetchmail delay FETCHMAIL_DELAY={{ fetchmail_delay or '600' }} diff --git a/towncrier/newsfragments/2127.feature b/towncrier/newsfragments/2127.feature new file mode 100644 index 00000000..dd4951dd --- /dev/null +++ b/towncrier/newsfragments/2127.feature @@ -0,0 +1 @@ +Add FETCHMAIL_ENABLED to toggle the fetchmail functionality in the admin interface \ No newline at end of file From c4fcaed7d42c68f8ff13ad37a96d9434cd8bf500 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 20 Oct 2022 16:01:18 +0200 Subject: [PATCH 062/274] doh --- core/rspamd/conf/force_actions.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf index 5c4e8eea..c0ffe728 100644 --- a/core/rspamd/conf/force_actions.conf +++ b/core/rspamd/conf/force_actions.conf @@ -6,7 +6,7 @@ rules { } ANTISPOOF_DMARC_ENFORCE_LOCAL { action = "reject"; - expression = "((IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)"; + expression = "(IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)"; message = "Rejected (anti-spoofing: DMARC compliance is enforced for local domains, regardless of the policy setting)"; } ANTISPOOF_AUTH_FAILED { From 56617bbe128f6115863ae5489b1bceb4a05ab8c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Zupan?= Date: Fri, 21 Oct 2022 16:42:33 -0700 Subject: [PATCH 063/274] Quote SMTP SIZE to avoid splitting keyword and parameter in EHLO response --- core/nginx/conf/nginx.conf | 2 +- towncrier/newsfragments/2485.bugfix | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 towncrier/newsfragments/2485.bugfix diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 7c831f1d..f9278f38 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -288,7 +288,7 @@ mail { {% endif %} # Advertise real capabilities of backends (postfix/dovecot) - smtp_capabilities PIPELINING SIZE {{ MESSAGE_SIZE_LIMIT }} ETRN ENHANCEDSTATUSCODES 8BITMIME DSN; + smtp_capabilities PIPELINING "SIZE {{ MESSAGE_SIZE_LIMIT }}" ETRN ENHANCEDSTATUSCODES 8BITMIME DSN; pop3_capabilities TOP UIDL RESP-CODES PIPELINING AUTH-RESP-CODE USER; imap_capabilities IMAP4 IMAP4rev1 UIDPLUS SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+; diff --git a/towncrier/newsfragments/2485.bugfix b/towncrier/newsfragments/2485.bugfix new file mode 100644 index 00000000..2e12fe89 --- /dev/null +++ b/towncrier/newsfragments/2485.bugfix @@ -0,0 +1 @@ +Quote SMTP SIZE to avoid splitting keyword and parameter in EHLO response From bd1b73032c8e439c7d1be640505853170f311951 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Oct 2022 09:48:51 +0200 Subject: [PATCH 064/274] Poke a hole for mailing lists --- core/rspamd/conf/force_actions.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf index c0ffe728..a5c342b8 100644 --- a/core/rspamd/conf/force_actions.conf +++ b/core/rspamd/conf/force_actions.conf @@ -1,17 +1,17 @@ rules { ANTISPOOF_NOAUTH { action = "reject"; - expression = "(IS_LOCAL_DOMAIN_E & MISSING_FROM) | (IS_LOCAL_DOMAIN_H & (R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA))"; + expression = "-MAILLIST & ((IS_LOCAL_DOMAIN_E & MISSING_FROM) | (IS_LOCAL_DOMAIN_H & (R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA)))"; message = "Rejected (anti-spoofing: noauth). Please setup DMARC with DKIM or SPF if you want to send emails from your domain from other servers."; } ANTISPOOF_DMARC_ENFORCE_LOCAL { action = "reject"; - expression = "(IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)"; + expression = "-MAILLIST & (IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)"; message = "Rejected (anti-spoofing: DMARC compliance is enforced for local domains, regardless of the policy setting)"; } ANTISPOOF_AUTH_FAILED { action = "reject"; - expression = "BLACKLIST_ANTISPOOF"; + expression = "-MAILLIST & BLACKLIST_ANTISPOOF"; message = "Rejected (anti-spoofing: auth-failed)"; } } From 07bf8ce6df23d588ecb5312caf3defba0f7af1ae Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Oct 2022 09:49:58 +0200 Subject: [PATCH 065/274] Add anti-spoofing to the feature list --- README.md | 2 +- docs/index.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0d8402f5..0fd737b6 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Main features include: - **User features**, aliases, auto-reply, auto-forward, fetched accounts - **Admin features**, global admins, announcements, per-domain delegation, quotas - **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner -- **Antispam**, auto-learn, greylisting, DMARC and SPF +- **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing - **Freedom**, all FOSS components, no tracker included ![Domains](docs/assets/screenshots/domains.png) diff --git a/docs/index.rst b/docs/index.rst index 4e4bb416..5c004dc1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,8 +28,8 @@ Main features include: - **Web access**, multiple Webmails and administration interface - **User features**, aliases, auto-reply, auto-forward, fetched accounts - **Admin features**, global admins, announcements, per-domain delegation, quotas -- **Security**, enforced TLS, Letsencrypt!, outgoing DKIM, anti-virus scanner -- **Antispam**, auto-learn, greylisting, DMARC and SPF +- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner +- **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing - **Freedom**, all FOSS components, no tracker included .. image:: assets/screenshots/create.png From 84a722eabcad706616f1de893a587ee032d86281 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 24 Oct 2022 19:31:53 +0200 Subject: [PATCH 066/274] Optimize the query --- core/admin/mailu/internal/views/rspamd.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/admin/mailu/internal/views/rspamd.py b/core/admin/mailu/internal/views/rspamd.py index e2e453ad..b6ead86b 100644 --- a/core/admin/mailu/internal/views/rspamd.py +++ b/core/admin/mailu/internal/views/rspamd.py @@ -28,5 +28,4 @@ def rspamd_dkim_key(domain_name): @internal.route("/rspamd/local_domains", methods=['GET']) def rspamd_local_domains(): - domains = set(models.Domain.query.all() + models.Alternative.query.all()) - return '\n'.join(map(str,domains)) + return '\n'.join(domain[0] for domain in models.Domain.query.with_entities(models.Domain.name).all() + models.Alternative.query.with_entities(models.Alternative.name).all()) From 95a3a3d3423163267d2d1207b6e87a8b2ce7da27 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 25 Oct 2022 12:05:25 +0200 Subject: [PATCH 067/274] doh --- core/rspamd/conf/force_actions.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf index a5c342b8..9f803405 100644 --- a/core/rspamd/conf/force_actions.conf +++ b/core/rspamd/conf/force_actions.conf @@ -1,17 +1,17 @@ rules { ANTISPOOF_NOAUTH { action = "reject"; - expression = "-MAILLIST & ((IS_LOCAL_DOMAIN_E & MISSING_FROM) | (IS_LOCAL_DOMAIN_H & (R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA)))"; + expression = "!MAILLIST & ((IS_LOCAL_DOMAIN_E & MISSING_FROM) | (IS_LOCAL_DOMAIN_H & (R_DKIM_NA & R_SPF_NA & DMARC_NA & ARC_NA)))"; message = "Rejected (anti-spoofing: noauth). Please setup DMARC with DKIM or SPF if you want to send emails from your domain from other servers."; } ANTISPOOF_DMARC_ENFORCE_LOCAL { action = "reject"; - expression = "-MAILLIST & (IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)"; + expression = "!MAILLIST & (IS_LOCAL_DOMAIN_H | IS_LOCAL_DOMAIN_E) & (DMARC_POLICY_SOFTFAIL | DMARC_POLICY_REJECT | DMARC_POLICY_QUARANTINE | DMARC_NA)"; message = "Rejected (anti-spoofing: DMARC compliance is enforced for local domains, regardless of the policy setting)"; } ANTISPOOF_AUTH_FAILED { action = "reject"; - expression = "-MAILLIST & BLACKLIST_ANTISPOOF"; + expression = "!MAILLIST & BLACKLIST_ANTISPOOF"; message = "Rejected (anti-spoofing: auth-failed)"; } } From 0f17299b4e6c6deba13a7755fb547409aa235bd2 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 25 Oct 2022 14:43:47 +0200 Subject: [PATCH 068/274] Admin may not have started up when this loads --- core/rspamd/conf/multimap.conf | 8 ++++++-- core/rspamd/conf/whitelist.conf | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/rspamd/conf/multimap.conf b/core/rspamd/conf/multimap.conf index dd25c08e..7592fee0 100644 --- a/core/rspamd/conf/multimap.conf +++ b/core/rspamd/conf/multimap.conf @@ -1,11 +1,15 @@ IS_LOCAL_DOMAIN_H { type = "selector" selector = "from('mime'):domain"; - map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; + map = [ +"http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains", +"fallback+file:///dev/null"]; } IS_LOCAL_DOMAIN_E { type = "selector" selector = "from('smtp'):domain"; - map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; + map = [ +"http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains", +"fallback+file:///dev/null"]; } diff --git a/core/rspamd/conf/whitelist.conf b/core/rspamd/conf/whitelist.conf index 56f8a83d..01efd80d 100644 --- a/core/rspamd/conf/whitelist.conf +++ b/core/rspamd/conf/whitelist.conf @@ -2,7 +2,7 @@ rules { BLACKLIST_ANTISPOOF = { valid_dmarc = true; blacklist = true; - domains = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; + domains = ["http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains", "fallback+file:///dev/null"]; score = 0.0; } } From d8cf0c3848475221288f46a8264e9252c93e0156 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 27 Oct 2022 10:21:19 +0200 Subject: [PATCH 069/274] Revert "Admin may not have started up when this loads" This reverts commit 0f17299b4e6c6deba13a7755fb547409aa235bd2. --- core/rspamd/conf/multimap.conf | 8 ++------ core/rspamd/conf/whitelist.conf | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core/rspamd/conf/multimap.conf b/core/rspamd/conf/multimap.conf index 7592fee0..dd25c08e 100644 --- a/core/rspamd/conf/multimap.conf +++ b/core/rspamd/conf/multimap.conf @@ -1,15 +1,11 @@ IS_LOCAL_DOMAIN_H { type = "selector" selector = "from('mime'):domain"; - map = [ -"http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains", -"fallback+file:///dev/null"]; + map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; } IS_LOCAL_DOMAIN_E { type = "selector" selector = "from('smtp'):domain"; - map = [ -"http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains", -"fallback+file:///dev/null"]; + map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; } diff --git a/core/rspamd/conf/whitelist.conf b/core/rspamd/conf/whitelist.conf index 01efd80d..56f8a83d 100644 --- a/core/rspamd/conf/whitelist.conf +++ b/core/rspamd/conf/whitelist.conf @@ -2,7 +2,7 @@ rules { BLACKLIST_ANTISPOOF = { valid_dmarc = true; blacklist = true; - domains = ["http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains", "fallback+file:///dev/null"]; + domains = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; score = 0.0; } } From 89f7d983b41a41f17dcd4d1f8a2adf5696682ed2 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 27 Oct 2022 10:51:45 +0200 Subject: [PATCH 070/274] Don't start rspamd until admin is up and working --- core/rspamd/start.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index fcb33a97..b03a602f 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -3,7 +3,9 @@ import os import glob import logging as log +import requests import sys +import time from socrate import system, conf log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) @@ -19,5 +21,16 @@ if os.environ.get("ANTIVIRUS") == 'clamav': for rspamd_file in glob.glob("/conf/*"): conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file))) +# Admin may not be up just yet +healthcheck = f'http://{os.environ["ADMIN_ADDRESS"]}/internal/rspamd/local_domains' +while True: + time.sleep(1) + try: + if requests.get(healthcheck,timeout=2).ok: + break + except: + pass + print("Admin is not up just yet, retrying in 1 second") + # Run rspamd os.execv("/usr/sbin/rspamd", ["rspamd", "-i", "-f"]) From 005a8fa1fc85ab8e3f68dd46aaebb15116cbb6c5 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Oct 2022 17:18:23 +0200 Subject: [PATCH 071/274] Improve ISSUE_TEMPLATE.md --- ISSUE_TEMPLATE.md | 61 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index ce972925..7c05deae 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,44 +1,71 @@ + + +## Environment & Version -## Environment & Versions ### Environment - - [ ] docker-compose - - [ ] kubernetes - - [ ] docker swarm -### Versions +- [ ] docker-compose +- [ ] kubernetes +- [ ] docker swarm + +### Version + +- Version: `master` + + ## Description + ## Replication Steps + + +## Observed behaviour + ## Expected behaviour -Explain what results you expected - be as specific as possible. Just saying "it doesn’t work as expected" is not useful. It's also helpful to describe what you actually experienced. + ## Logs -Often it is very useful to include log fragments of the involved component. You can get the logs via `docker logs --tail 1000`. For example for the admin container: -`docker logs mailu_admin_1 --tail 1000` + From 6b785abb01913092bc134898982a2700b9657e72 Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Fri, 28 Oct 2022 10:05:47 +0200 Subject: [PATCH 072/274] Rename flag_updated_at_as_modified to dont_change_updated_at --- core/admin/mailu/internal/views/dovecot.py | 2 +- core/admin/mailu/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index e949e2eb..0d56950b 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -33,7 +33,7 @@ def dovecot_quota(ns, user_email): user = models.User.query.get(user_email) or flask.abort(404) if ns == "storage": user.quota_bytes_used = flask.request.get_json() - user.flag_updated_at_as_modified() + user.dont_change_updated_at() models.db.session.commit() return flask.jsonify(None) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index edcdcf27..0071bc77 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -155,7 +155,7 @@ class Base(db.Model): self.__hashed = id(self) if primary is None else hash(primary) return self.__hashed - def flag_updated_at_as_modified(self): + def dont_change_updated_at(self): """ Mark updated_at as modified, but keep the old date when updating the model""" flag_modified(self, 'updated_at') From 6363acf30a063551976a1d6b3372d0f39cf208ee Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Fri, 28 Oct 2022 10:19:30 +0200 Subject: [PATCH 073/274] Add dont_change_updated_at to fetch_done --- core/admin/mailu/internal/views/fetch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/mailu/internal/views/fetch.py b/core/admin/mailu/internal/views/fetch.py index ccd3a159..1945b9c7 100644 --- a/core/admin/mailu/internal/views/fetch.py +++ b/core/admin/mailu/internal/views/fetch.py @@ -27,6 +27,7 @@ def fetch_done(fetch_id): fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) fetch.last_check = datetime.datetime.now() fetch.error_message = str(flask.request.get_json()) + fetch.dont_change_updated_at() models.db.session.add(fetch) models.db.session.commit() return "" From 4be0cbf4da68f56759bf6a458d1889df57650047 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 28 Oct 2022 11:52:49 +0000 Subject: [PATCH 074/274] Switch workflow to ghcr.io - Build images & build cache are pushed to ghcr.io. - Tests will make use of the images pushed to ghcr.io. - Deploy step only copies images from ghcr.io to docker.io. - Resolves strange build errors tied to buildx+intermediate builds - Results in quicker build times. --- .github/workflows/arm.yml | 4 +- .github/workflows/build_test_deploy.yml | 436 +++++------------------- .github/workflows/x64.yml | 4 +- 3 files changed, 90 insertions(+), 354 deletions(-) diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 22d58cd5..7125737c 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -61,7 +61,7 @@ jobs: echo "RELEASE=true" >> $GITHUB_ENV echo "DEPLOY=true" >> $GITHUB_ENV echo "RELEASE=true" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging for master + - name: Derive PINNED_MAILU_VERSION for master if: env.BRANCH == 'master' shell: bash env: @@ -98,4 +98,4 @@ jobs: #else # pinned_version=$root_version.$(expr $patch_version + 1) #fi -#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV \ No newline at end of file +#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 7b7c357a..e8bb1a0a 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -58,15 +58,15 @@ on: required: true type: string deploy: - description: Deploy to docker hub. Happens for all branches but staging + description: Deploy to docker hub. Happens for all branches but staging. Use string true or false. default: true required: false - type: boolean + type: string release: - description: 'Tag and create the github release. Only happens for branch x.y (release branch)' + description: Tag and create the github release. Use string true or false. default: false required: false - type: boolean + type: string env: HCL_FILE: ./tests/build.hcl @@ -84,7 +84,7 @@ jobs: - name: Create matrix id: targets run: | - echo ::set-output name=matrix::$(docker buildx bake -f ${{env.HCL_FILE}} --print | jq -cr '.group.default.targets') + echo matrix=$(docker buildx bake -f ${{env.HCL_FILE}} --print | jq -cr '.group.default.targets') >> $GITHUB_OUTPUT - name: Show matrix run: | echo ${{ steps.targets.outputs.matrix }} @@ -107,26 +107,25 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} - restore-keys: | - mailu-base-${{ inputs.architecture }}-${{hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 + with: + string: ${{ github.repository_owner }} - name: Build all docker images env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} uses: docker/bake-action@v2 @@ -136,8 +135,8 @@ jobs: load: false push: false set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/base - *.cache-to=type=local,dest=${{ runner.temp }}/cache/base,mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }},mode=max *.platform=${{ inputs.architecture }} # This job builds all the images. The build cache is stored in the github actions cache. @@ -165,33 +164,25 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} - restore-keys: | - mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/${{ matrix.target }} - key: ${{ matrix.target }}-${{ inputs.architecture }}-${{ github.run_id }} - restore-keys: | - ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 + with: + string: ${{ github.repository_owner }} - name: Build all docker images env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} uses: docker/bake-action@v2 @@ -199,11 +190,11 @@ jobs: files: ${{env.HCL_FILE}} targets: ${{ matrix.target }} load: false - push: false + push: true set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }} - *.cache-from=type=local,src=${{ runner.temp }}/cache/base - *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} *.platform=${{ inputs.architecture }} # This job builds all the images. The build cache is stored in the github actions cache. @@ -231,33 +222,25 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure actions/cache@v3 action for storing build cache of base image in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/base - key: mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-${{ github.run_id }} - restore-keys: | - mailu-base-${{ inputs.architecture }}-${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }} - - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/${{ matrix.target }} - key: ${{ matrix.target }}-${{ inputs.architecture }}-${{ github.run_id }} - restore-keys: | - ${{ matrix.target }}-${{ inputs.architecture }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 + with: + string: ${{ github.repository_owner }} - name: Build all docker images env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} uses: docker/bake-action@v2 @@ -267,9 +250,10 @@ jobs: load: false push: false set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }} - *.cache-from=type=local,src=${{ runner.temp }}/cache/base - *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max *.platform=${{ inputs.architecture }} # This job runs all the tests. @@ -303,112 +287,22 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/docs - key: docs-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image setup - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/setup - key: setup-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image admin - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/admin - key: admin-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antispam - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antispam - key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image front - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/front - key: front-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image imap - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/imap - key: imap-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image smtp - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/smtp - key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image snappymail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/snappymail - key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image roundcube - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/roundcube - key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antivirus - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antivirus - key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image fetchmail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/fetchmail - key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image resolver - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/resolver - key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image traefik-certdumper - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/traefik-certdumper - key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image webdav - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/webdav - key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Login to Docker Hub + - name: Login to GitHub Container Registry uses: docker/login-action@v2 with: - username: ${{ secrets.Docker_Login }} - password: ${{ secrets.Docker_Password }} - - name: Build docker images for testing from cache - env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 with: - files: ${{env.HCL_FILE}} - load: true - push: false - set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/docs - *.cache-from=type=local,src=${{ runner.temp }}/cache/setup - *.cache-from=type=local,src=${{ runner.temp }}/cache/admin - *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam - *.cache-from=type=local,src=${{ runner.temp }}/cache/front - *.cache-from=type=local,src=${{ runner.temp }}/cache/imap - *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp - *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail - *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube - *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus - *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail - *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver - *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper - *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav - *.platform=${{ inputs.architecture }} + string: ${{ github.repository_owner }} - name: Install python packages run: python3 -m pip install -r tests/requirements.txt - name: Copy all certs @@ -416,12 +310,10 @@ jobs: - name: Test ${{ matrix.target }} run: python tests/compose/test.py ${{ matrix.target }} ${{ matrix.time }} env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} MAILU_VERSION: ${{ env.MAILU_VERSION }} PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} -# This job deploys the docker images to the docker repository. The build.hcl file contains logic that determines what tags are pushed. -# E.g. for master only the :master and :latest tags are pushed. deploy: name: Deploy images # Deploying is not required for staging @@ -430,6 +322,10 @@ jobs: needs: - build - tests + strategy: + fail-fast: false + matrix: + target: ["setup", "docs", "fetchmail", "roundcube", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx", "snappymail"] steps: - uses: actions/checkout@v3 - name: Retrieve global variables @@ -439,76 +335,6 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/docs - key: docs-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image setup - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/setup - key: setup-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image admin - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/admin - key: admin-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antispam - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antispam - key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image front - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/front - key: front-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image imap - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/imap - key: imap-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image smtp - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/smtp - key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image snappymail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/snappymail - key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image roundcube - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/roundcube - key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antivirus - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antivirus - key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image fetchmail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/fetchmail - key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image resolver - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/resolver - key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image traefik-certdumper - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/traefik-certdumper - key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image webdav - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/webdav - key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -519,31 +345,19 @@ jobs: with: username: ${{ secrets.Docker_Login }} password: ${{ secrets.Docker_Password }} - - name: Deploy images to docker hub. Build.hcl contains the logic for the tags that are pushed. - env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 with: - files: ${{env.HCL_FILE}} - push: true - set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/docs - *.cache-from=type=local,src=${{ runner.temp }}/cache/setup - *.cache-from=type=local,src=${{ runner.temp }}/cache/admin - *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam - *.cache-from=type=local,src=${{ runner.temp }}/cache/front - *.cache-from=type=local,src=${{ runner.temp }}/cache/imap - *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp - *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail - *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube - *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus - *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail - *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver - *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper - *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav - *.platform=${{ inputs.architecture }} + string: ${{ github.repository_owner }} + - name: Push image to Docker + shell: bash + run: | + if [ '${{ env.MAILU_VERSION }}' == 'master' ]; then pinned_mailu_version='master'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi; + docker buildx imagetools create \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} deploy-arm: name: Deploy images for arm @@ -552,6 +366,10 @@ jobs: runs-on: self-hosted needs: - build-arm + strategy: + fail-fast: false + matrix: + target: ["setup", "docs", "fetchmail", "roundcube", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx", "snappymail"] steps: - uses: actions/checkout@v3 - name: Retrieve global variables @@ -561,76 +379,6 @@ jobs: echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV - - name: Configure /cache for image docs - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/docs - key: docs-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image setup - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/setup - key: setup-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image admin - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/admin - key: admin-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antispam - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antispam - key: antispam-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image front - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/front - key: front-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image imap - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/imap - key: imap-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image smtp - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/smtp - key: smtp-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image snappymail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/snappymail - key: snappymail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image roundcube - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/roundcube - key: roundcube-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image antivirus - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/antivirus - key: antivirus-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image fetchmail - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/fetchmail - key: fetchmail-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image resolver - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/resolver - key: resolver-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image traefik-certdumper - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/traefik-certdumper - key: traefik-certdumper-${{ inputs.architecture }}-${{ github.run_id }} - - name: Configure /cache for image webdav - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/cache/webdav - key: webdav-${{ inputs.architecture }}-${{ github.run_id }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - uses: crazy-max/ghaction-github-runtime@v2 @@ -641,31 +389,19 @@ jobs: with: username: ${{ secrets.Docker_Login }} password: ${{ secrets.Docker_Password }} - - name: Deploy images to docker hub. Build.hcl contains the logic for the tags that are pushed. - env: - DOCKER_ORG: ${{ env.DOCKER_ORG }} - MAILU_VERSION: ${{ env.MAILU_VERSION }} - PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} - uses: docker/bake-action@v2 + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 with: - files: ${{env.HCL_FILE}} - push: true - set: | - *.cache-from=type=local,src=${{ runner.temp }}/cache/docs - *.cache-from=type=local,src=${{ runner.temp }}/cache/setup - *.cache-from=type=local,src=${{ runner.temp }}/cache/admin - *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam - *.cache-from=type=local,src=${{ runner.temp }}/cache/front - *.cache-from=type=local,src=${{ runner.temp }}/cache/imap - *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp - *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail - *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube - *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus - *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail - *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver - *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper - *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav - *.platform=${{ inputs.architecture }} + string: ${{ github.repository_owner }} + - name: Push image to Docker + shell: bash + run: | + if [ '${{ env.MAILU_VERSION }}' == 'master-arm' ]; then pinned_mailu_version='master-arm'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi; + docker buildx imagetools create \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} #This job creates a tagged release. A tag is created for the pinned version x.y.z. The GH release refers to this tag. tag-release: diff --git a/.github/workflows/x64.yml b/.github/workflows/x64.yml index e1691756..97b6beac 100644 --- a/.github/workflows/x64.yml +++ b/.github/workflows/x64.yml @@ -82,7 +82,7 @@ jobs: echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV echo "DEPLOY=false" >> $GITHUB_ENV echo "RELEASE=false" >> $GITHUB_ENV - - name: Derive PINNED_MAILU_VERSION for staging for master + - name: Derive PINNED_MAILU_VERSION for master if: env.BRANCH == 'master' shell: bash env: @@ -131,4 +131,4 @@ jobs: #else # pinned_version=$root_version.$(expr $patch_version + 1) #fi -#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV \ No newline at end of file +#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV From 024b0573b385f65c7751ed7109cc7d25e6abf878 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 28 Oct 2022 15:47:48 +0200 Subject: [PATCH 075/274] Update build reqs and fix armv7 build --- core/base/Dockerfile | 13 ++++++------- core/base/requirements-build.txt | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index d00ead10..2cade640 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -32,7 +32,7 @@ RUN set -euxo pipefail \ ; apk add --no-cache py3-pip \ ; python3 -m venv ${VIRTUAL_ENV} \ ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt \ - ; apk del py3-pip + ; apk del -r py3-pip ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" @@ -40,13 +40,12 @@ COPY requirements-${MAILU_ENV}.txt ./ COPY libs/ libs/ RUN set -euxo pipefail \ + ; machine="$(uname -m)"; deps="" \ + ; [[ "${machine}" == arm* || "${machine}" == aarch64 ]] && deps="${deps} build-base gcc libffi-dev python3-dev" \ + ; [[ "${machine}" == armv7* ]] && deps="${deps} cargo libressl-dev mariadb-connector-c-dev postgresql-dev" \ + ; [[ "${deps}" ]] && apk add --virtual .build-deps ${deps} \ ; pip install -r requirements-${MAILU_ENV}.txt \ - || ( \ - apk add --no-cache --virtual .build-deps \ - build-base gcc libffi-dev python3-dev \ - ; pip install -r requirements-${MAILU_ENV}.txt \ - ; apk del .build-deps \ - ) \ + ; apk -e info -q .build-deps && apk del -r .build-deps \ ; rm -rf /root/.cache /tmp/*.pem diff --git a/core/base/requirements-build.txt b/core/base/requirements-build.txt index 2858969f..406eb725 100644 --- a/core/base/requirements-build.txt +++ b/core/base/requirements-build.txt @@ -1,3 +1,3 @@ -pip==22.2.2 -setuptools==65.4.1 +pip==22.3 +setuptools==65.5.0 wheel==0.37.1 From ec4224123baaabcaac12eae8f65a3b268165bead Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 28 Oct 2022 18:28:31 +0200 Subject: [PATCH 076/274] Use the logger --- core/rspamd/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index b03a602f..031aaf88 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -30,7 +30,7 @@ while True: break except: pass - print("Admin is not up just yet, retrying in 1 second") + log.warning("Admin is not up just yet, retrying in 1 second") # Run rspamd os.execv("/usr/sbin/rspamd", ["rspamd", "-i", "-f"]) From 06b784da5705b0bf7a471da6d73c46ef27ac1f46 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 29 Oct 2022 08:57:48 +0000 Subject: [PATCH 077/274] Shorten default function by using lambda --- core/admin/mailu/models.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 1e703e1a..2d276a8a 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -509,12 +509,7 @@ class User(Base, Email): displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean, nullable=False, default=True) spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True) - def default_spam_threshold(cls): - if app: - return app.config['DEFAULT_SPAM_THRESHOLD'] - else: - return 80 - spam_threshold = db.Column(db.Integer, nullable=False, default=default_spam_threshold) + spam_threshold = db.Column(db.Integer, nullable=False, default=lambda:int(app.config.get("DEFAULT_SPAM_THRESHOLD", 80))) # Flask-login attributes is_authenticated = True From 89424485619585cc7f27a37e6efcd83e3eb2ab4e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 29 Oct 2022 11:40:14 +0200 Subject: [PATCH 078/274] Upgrade to alpine 3.16.2 This may fix the build issues on arm --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 2cade640..721b5db1 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile-upstream:1.4.3 # base system image (intermediate) -ARG DISTRO=alpine:3.14.5 +ARG DISTRO=alpine:3.16.2 FROM $DISTRO as system ENV TZ=Etc/UTC LANG=C.UTF-8 From a63bad6bf2c7d070bc3f7507f44633cc8c67c2cb Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 29 Oct 2022 12:30:38 +0200 Subject: [PATCH 079/274] towncrier --- towncrier/newsfragments/2497.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/2497.bugfix diff --git a/towncrier/newsfragments/2497.bugfix b/towncrier/newsfragments/2497.bugfix new file mode 100644 index 00000000..67752060 --- /dev/null +++ b/towncrier/newsfragments/2497.bugfix @@ -0,0 +1 @@ +Upgrade to alpine 3.16.2 From 2e467092a2a61e585e890815fa16374c16fee765 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 29 Oct 2022 15:53:55 +0200 Subject: [PATCH 080/274] The newer dovecot sends data podop should ignore --- core/base/libs/podop/podop/dovecot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 68bfa172..b1319f51 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -75,10 +75,10 @@ class DictProtocol(asyncio.Protocol): logging.debug("Client {}.{} type {}, user {}, dict {}".format( self.major, self.minor, self.value_type, self.user, dict_name)) - async def process_lookup(self, key): + async def process_lookup(self, key, user=None): """ Process a dict lookup message """ - logging.debug("Looking up {}".format(key)) + logging.debug("Looking up {} for {}".format(key, user)) # Priv and shared keys are handled slighlty differently key_type, key = key.decode("utf8").split("/", 1) try: @@ -95,7 +95,7 @@ class DictProtocol(asyncio.Protocol): except KeyError: return self.reply(b"N") - def process_begin(self, transaction_id): + def process_begin(self, transaction_id, user=None): """ Process a dict begin message """ self.transactions[transaction_id] = {} From 076d67b51331efbac9cbb7685f34e6dbacfbc30c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 29 Oct 2022 17:02:52 +0200 Subject: [PATCH 081/274] follow the protocol --- core/base/libs/podop/podop/dovecot.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index b1319f51..c90e188d 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -33,6 +33,8 @@ class DictProtocol(asyncio.Protocol): self.dict = None # Dictionary of active transaction lists per transaction id self.transactions = {} + # Dictionary of user per transaction id + self.transactions_user = {} super(DictProtocol, self).__init__() def connection_made(self, transport): @@ -83,7 +85,7 @@ class DictProtocol(asyncio.Protocol): key_type, key = key.decode("utf8").split("/", 1) try: result = await self.dict.get( - key, ns=(self.user if key_type == "priv" else None) + key, ns=((user if user else self.user) if key_type == "priv" else None) ) if type(result) is str: response = result.encode("utf8") @@ -99,6 +101,7 @@ class DictProtocol(asyncio.Protocol): """ Process a dict begin message """ self.transactions[transaction_id] = {} + self.transactions_user[transaction_id] = user if user else self.user def process_set(self, transaction_id, key, value): """ Process a dict set message @@ -116,10 +119,11 @@ class DictProtocol(asyncio.Protocol): key_type, key = key.decode("utf8").split("/", 1) result = await self.dict.set( key, json.loads(value), - ns=(self.user if key_type == "priv" else None) + ns=(self.transactions_user[transaction_id] if key_type == "priv" else None) ) # Remove stored transaction del self.transactions[transaction_id] + del self.transactions_user[transaction_id] return self.reply(b"O", transaction_id) def reply(self, command, *args): From 340e35909627577d10ca304ebd3362116d3aefdf Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 29 Oct 2022 17:13:58 +0200 Subject: [PATCH 082/274] doh --- core/base/libs/podop/podop/dovecot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index c90e188d..40b48eef 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -85,7 +85,7 @@ class DictProtocol(asyncio.Protocol): key_type, key = key.decode("utf8").split("/", 1) try: result = await self.dict.get( - key, ns=((user if user else self.user) if key_type == "priv" else None) + key, ns=((user.decode("utf8") if user else self.user) if key_type == "priv" else None) ) if type(result) is str: response = result.encode("utf8") @@ -101,7 +101,7 @@ class DictProtocol(asyncio.Protocol): """ Process a dict begin message """ self.transactions[transaction_id] = {} - self.transactions_user[transaction_id] = user if user else self.user + self.transactions_user[transaction_id] = user.decode("utf8") if user else self.user def process_set(self, transaction_id, key, value): """ Process a dict set message From cf34be967cf45b06d1d98387ce3c80547671c5fb Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 20:15:10 +0100 Subject: [PATCH 083/274] Implement ITERATE --- core/admin/mailu/internal/views/dovecot.py | 5 ++++ core/base/libs/podop/podop/dovecot.py | 33 +++++++++++++++++++--- core/dovecot/conf/auth.conf | 1 + towncrier/newsfragments/2498.feature | 1 + 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 towncrier/newsfragments/2498.feature diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 0d56950b..10f3d5fb 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -19,6 +19,11 @@ def dovecot_passdb_dict(user_email): "allow_nets": ",".join(allow_nets) }) +@internal.route("/dovecot/userdb/") +def dovecot_userdb_dict_list(): + return flask.jsonify([ + user[0] for user in models.User.query.filter(models.User.enabled.is_(True)).with_entities(models.User.email).all() + ]) @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 40b48eef..18956dd4 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -77,7 +77,7 @@ class DictProtocol(asyncio.Protocol): logging.debug("Client {}.{} type {}, user {}, dict {}".format( self.major, self.minor, self.value_type, self.user, dict_name)) - async def process_lookup(self, key, user=None): + async def process_lookup(self, key, user=None, is_iter=False): """ Process a dict lookup message """ logging.debug("Looking up {} for {}".format(key, user)) @@ -93,10 +93,33 @@ class DictProtocol(asyncio.Protocol): response = result else: response = json.dumps(result).encode("ascii") - return self.reply(b"O", response) + logging.debug("Replying {}".format(key)) + return self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response) except KeyError: return self.reply(b"N") + async def process_iterate(self, flags, max_rows, path, user=None): + """ Process an iterate command + """ + logging.debug("Iterate flags {} max_rows {} on {} for {}".format(flags, max_rows, path, user)) + # Priv and shared keys are handled slighlty differently + key_type, key = path.decode("utf8").split("/", 1) + max_rows = int(max_rows.decode("utf-8")) + flags = int(flags.decode("utf-8")) + if flags != 0: # not implemented + return self.reply(b"F") + try: + result = await self.dict.iter(key) + logging.debug("Found {} entries: {}".format(len(result), result)) + returned_results = 0 + for k in result: + if max_rows == 0 or returned_results < max_rows: + await self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True) + returned_results += 1 + return self.reply(b"\n") # ITER_FINISHED + except KeyError: + return self.reply(b"F") + def process_begin(self, transaction_id, user=None): """ Process a dict begin message """ @@ -126,11 +149,12 @@ class DictProtocol(asyncio.Protocol): del self.transactions_user[transaction_id] return self.reply(b"O", transaction_id) - def reply(self, command, *args): + def reply(self, command, *args, end=True): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) - self.transport.write(b"\n") + if end: + self.transport.write(b"\n") @classmethod def factory(cls, table_map): @@ -141,6 +165,7 @@ class DictProtocol(asyncio.Protocol): COMMANDS = { ord("H"): process_hello, ord("L"): process_lookup, + ord("I"): process_iterate, ord("B"): process_begin, ord("C"): process_commit, ord("S"): process_set diff --git a/core/dovecot/conf/auth.conf b/core/dovecot/conf/auth.conf index 44a874ba..1f122416 100644 --- a/core/dovecot/conf/auth.conf +++ b/core/dovecot/conf/auth.conf @@ -1,5 +1,6 @@ uri = proxy:/tmp/podop.socket:auth iterate_disable = yes +iterate_prefix = 'userdb/' default_pass_scheme = plain password_key = passdb/%u user_key = userdb/%u diff --git a/towncrier/newsfragments/2498.feature b/towncrier/newsfragments/2498.feature new file mode 100644 index 00000000..961b6a84 --- /dev/null +++ b/towncrier/newsfragments/2498.feature @@ -0,0 +1 @@ +Implement the required glue to make "doveadm -A" work From 5ec4277e1ed838c6645efae7335457a6a0aa464f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:11:45 +0100 Subject: [PATCH 084/274] Make it async. I'm not sure it's a good idea --- core/base/libs/podop/podop/dovecot.py | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 18956dd4..6dc25af5 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -40,6 +40,7 @@ class DictProtocol(asyncio.Protocol): def connection_made(self, transport): logging.info('Connect {}'.format(transport.get_extra_info('peername'))) self.transport = transport + self.transport_lock = asyncio.Lock() def data_received(self, data): logging.debug("Received {}".format(data)) @@ -94,9 +95,9 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response) + return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response)) except KeyError: - return self.reply(b"N") + return await self.reply(b"N") async def process_iterate(self, flags, max_rows, path, user=None): """ Process an iterate command @@ -107,18 +108,24 @@ class DictProtocol(asyncio.Protocol): max_rows = int(max_rows.decode("utf-8")) flags = int(flags.decode("utf-8")) if flags != 0: # not implemented - return self.reply(b"F") + return await self.reply(b"F") + rows = [] try: result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) returned_results = 0 for k in result: if max_rows == 0 or returned_results < max_rows: - await self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True) + rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) returned_results += 1 - return self.reply(b"\n") # ITER_FINISHED + await asyncio.gather(*rows) + return await self.reply(b"\n") # ITER_FINISHED except KeyError: - return self.reply(b"F") + return await self.reply(b"F") + except Exception as e: + logging.error(f"Got {e}, cancelling remaining tasks") + for task in rows: + task.cancel() def process_begin(self, transaction_id, user=None): """ Process a dict begin message @@ -147,14 +154,15 @@ class DictProtocol(asyncio.Protocol): # Remove stored transaction del self.transactions[transaction_id] del self.transactions_user[transaction_id] - return self.reply(b"O", transaction_id) + return await self.reply(b"O", transaction_id) - def reply(self, command, *args, end=True): + async def reply(self, command, *args, end=True): logging.debug("Replying {} with {}".format(command, args)) - self.transport.write(command) - self.transport.write(b"\t".join(map(tabescape, args))) - if end: - self.transport.write(b"\n") + async with self.transport_lock: + self.transport.write(command) + self.transport.write(b"\t".join(map(tabescape, args))) + if end: + self.transport.write(b"\n") @classmethod def factory(cls, table_map): From 1ae4c37cb9351828b2d1687f5c59c9407c11fb1f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:25:34 +0100 Subject: [PATCH 085/274] Don't do fancy, just re-raise it --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 6dc25af5..fcf01fab 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -123,9 +123,9 @@ class DictProtocol(asyncio.Protocol): except KeyError: return await self.reply(b"F") except Exception as e: - logging.error(f"Got {e}, cancelling remaining tasks") for task in rows: task.cancel() + raise e def process_begin(self, transaction_id, user=None): """ Process a dict begin message From e10527a4bf6afde33154c4486324a640eda7f4de Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:33:10 +0100 Subject: [PATCH 086/274] This is not used anymore --- core/base/libs/podop/podop/dovecot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index fcf01fab..94385a75 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -95,7 +95,7 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response)) + return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") @@ -156,7 +156,7 @@ class DictProtocol(asyncio.Protocol): del self.transactions_user[transaction_id] return await self.reply(b"O", transaction_id) - async def reply(self, command, *args, end=True): + async def reply(self, command, *args): logging.debug("Replying {} with {}".format(command, args)) async with self.transport_lock: self.transport.write(command) From 1ce889b91b83f444bbdcf755afd6a17b62295218 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:43:34 +0100 Subject: [PATCH 087/274] Do it the pythonic way --- core/base/libs/podop/podop/dovecot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 94385a75..8c074186 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -82,6 +82,7 @@ class DictProtocol(asyncio.Protocol): """ Process a dict lookup message """ logging.debug("Looking up {} for {}".format(key, user)) + orig_key = key # Priv and shared keys are handled slighlty differently key_type, key = key.decode("utf8").split("/", 1) try: @@ -95,7 +96,7 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response) if is_iter else self.reply(b"O", response)) + return await (self.reply(b"O", orig_key, response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") @@ -113,11 +114,10 @@ class DictProtocol(asyncio.Protocol): try: result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) - returned_results = 0 - for k in result: - if max_rows == 0 or returned_results < max_rows: - rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) - returned_results += 1 + for i,k in enumerate(result): + if max_rows > 0 and max_rows >= i: + break + rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) return await self.reply(b"\n") # ITER_FINISHED except KeyError: From 2a417dbfc23e5c00d193a889c35fef8bd7a80d70 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:51:29 +0100 Subject: [PATCH 088/274] doesn't belong here --- core/base/libs/podop/podop/dovecot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 8c074186..7b27fa86 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -95,7 +95,6 @@ class DictProtocol(asyncio.Protocol): response = result else: response = json.dumps(result).encode("ascii") - logging.debug("Replying {}".format(key)) return await (self.reply(b"O", orig_key, response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") From cdc9b63a46561beabceed0bbb40a21ccc8b508dc Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:54:03 +0100 Subject: [PATCH 089/274] Guard the message logging too --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 7b27fa86..8fc06d6d 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -156,8 +156,8 @@ class DictProtocol(asyncio.Protocol): return await self.reply(b"O", transaction_id) async def reply(self, command, *args): - logging.debug("Replying {} with {}".format(command, args)) async with self.transport_lock: + logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) if end: From 96d928963085aae311a20c84ac74578879b3632d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 22:12:15 +0100 Subject: [PATCH 090/274] No need to send an extra \n --- core/base/libs/podop/podop/dovecot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 8fc06d6d..1206b5ef 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -118,7 +118,9 @@ class DictProtocol(asyncio.Protocol): break rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) - return await self.reply(b"\n") # ITER_FINISHED + async with self.transport_lock: + self.transport.write(b"\n") # ITER_FINISHED + return except KeyError: return await self.reply(b"F") except Exception as e: @@ -160,8 +162,7 @@ class DictProtocol(asyncio.Protocol): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) - if end: - self.transport.write(b"\n") + self.transport.write(b"\n") @classmethod def factory(cls, table_map): From c1f571a4c342bc45714123a08a479feeac72655a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 08:48:55 +0100 Subject: [PATCH 091/274] Speed things up. If we want to go further than this we should change podop's incr(), pass the flags and make admin process the results. --- core/admin/mailu/internal/views/dovecot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 10f3d5fb..783a14f4 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,9 +27,9 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - user = models.User.query.get(user_email) or flask.abort(404) + quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes={}".format(user.quota_bytes) + "quota_rule": "*:bytes="+quota[0]) }) From 6def1b555b086bde18d3aa6604fe8c8e20ffb008 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 10:06:55 +0100 Subject: [PATCH 092/274] doh --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 1206b5ef..0afa3dd4 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -114,7 +114,7 @@ class DictProtocol(asyncio.Protocol): result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) for i,k in enumerate(result): - if max_rows > 0 and max_rows >= i: + if max_rows > 0 and i >= max_rows: break rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) From afbaabd8cdd82f041a4a6bc77e397dfe684c0128 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 19:41:40 +0100 Subject: [PATCH 093/274] v1 --- core/admin/assets/content/assets/app.js | 57 +++++++++++++++++++++++++ core/admin/mailu/sso/forms.py | 1 + core/admin/mailu/sso/views/base.py | 5 ++- core/admin/mailu/ui/forms.py | 4 ++ core/admin/mailu/ui/views/domains.py | 4 ++ core/admin/mailu/ui/views/users.py | 18 ++++++++ 6 files changed, 88 insertions(+), 1 deletion(-) diff --git a/core/admin/assets/content/assets/app.js b/core/admin/assets/content/assets/app.js index 03ea6215..b0d48470 100644 --- a/core/admin/assets/content/assets/app.js +++ b/core/admin/assets/content/assets/app.js @@ -3,6 +3,56 @@ require('./app.css'); import logo from './mailu.png'; import modules from "./*.json"; +// Inspired from https://github.com/mehdibo/hibp-js/blob/master/hibp.js +function sha1(string){ + var buffer = new TextEncoder("utf-8").encode(string); + return crypto.subtle.digest("SHA-1", buffer).then(function (buffer) { + // Get the hex code + var hexCodes = []; + var view = new DataView(buffer); + for (var i = 0; i < view.byteLength; i += 4) { + // Using getUint32 reduces the number of iterations needed (we process 4 bytes each time) + var value = view.getUint32(i) + // toString(16) will give the hex representation of the number without padding + var stringValue = value.toString(16) + // We use concatenation and slice for padding + var padding = '00000000' + var paddedValue = (padding + stringValue).slice(-padding.length) + hexCodes.push(paddedValue); + } + // Join all the hex strings into one + return hexCodes.join(""); + }); +} + +function hibpCheck(pwd){ + // We hash the pwd first + sha1(pwd).then(function(hash){ + // We send the first 5 chars of the hash to hibp's API + const req = new XMLHttpRequest(); + req.addEventListener("load", function(){ + // When we get back a response from the server + // We create an array of lines and loop through them + const resp = this.responseText.split('\n'); + const hashSub = hash.slice(5).toUpperCase(); + for(index in resp){ + // Check if the line matches the rest of the hash + if(resp[index].substring(0, 35) == hashSub){ + var val = resp[index].split(":")[1] + if (val > 0) { + $("#pwned").value = val; + } + return; // If found no need to continue the loop + } + } + $("#pwned").value = 0; + }); + req.open('GET', 'https://api.pwnedpasswords.com/range/'+hash.substr(0, 5)); + req.setRequestHeader('Add-Padding', 'true'); + req.send(); + }); +} + // TODO: conditionally (or lazy) load select2 and dataTable $('document').ready(function() { @@ -75,5 +125,12 @@ $('document').ready(function() { $('form :input').prop('disabled', true); } + if (window.isSecureContext) { + $("#HIBPpw").change(function(){ + hibpCheck($("#HIBPpw").value); + return true; + }) + } + }); diff --git a/core/admin/mailu/sso/forms.py b/core/admin/mailu/sso/forms.py index 5cf38dbe..530fc1c2 100644 --- a/core/admin/mailu/sso/forms.py +++ b/core/admin/mailu/sso/forms.py @@ -7,5 +7,6 @@ class LoginForm(flask_wtf.FlaskForm): csrf = False email = fields.StringField(_('E-mail'), [validators.Email(), validators.DataRequired()]) pw = fields.PasswordField(_('Password'), [validators.DataRequired()]) + pwned = fields.HiddenField(label='',default=-1) submitWebmail = fields.SubmitField(_('Sign in')) submitAdmin = fields.SubmitField(_('Sign in')) diff --git a/core/admin/mailu/sso/views/base.py b/core/admin/mailu/sso/views/base.py index 00b6912d..34ba98b5 100644 --- a/core/admin/mailu/sso/views/base.py +++ b/core/admin/mailu/sso/views/base.py @@ -40,7 +40,10 @@ def login(): flask_login.login_user(user) response = flask.redirect(destination) response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login'), secure=app.config['SESSION_COOKIE_SECURE'], httponly=True) - flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip}.') + flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip} pwned={form.pwned.data}.') + breaches = int(form.pwned.data) + if breaches > 0: + flask.flash(f"Your password appears in {breaches} data breaches! Please change it.", "error") return response else: utils.limiter.rate_limit_user(username, client_ip, device_cookie, device_cookie_username) if models.User.get(username) else utils.limiter.rate_limit_ip(client_ip) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index 683db7ae..67e4b674 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -59,6 +59,7 @@ class DomainSignupForm(flask_wtf.FlaskForm): localpart = fields.StringField(_('Initial admin'), [validators.DataRequired()]) pw = fields.PasswordField(_('Admin password'), [validators.DataRequired()]) pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')]) + pwned = fields.HiddenField(label='',default=-1) captcha = flask_wtf.RecaptchaField() submit = fields.SubmitField(_('Create')) @@ -79,6 +80,7 @@ class UserForm(flask_wtf.FlaskForm): localpart = fields.StringField(_('E-mail'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)]) pw = fields.PasswordField(_('Password')) pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')]) + pwned = fields.HiddenField(label='',default=-1) quota_bytes = fields_.IntegerSliderField(_('Quota'), default=10**9) enable_imap = fields.BooleanField(_('Allow IMAP access'), default=True) enable_pop = fields.BooleanField(_('Allow POP3 access'), default=True) @@ -92,6 +94,7 @@ class UserSignupForm(flask_wtf.FlaskForm): localpart = fields.StringField(_('Email address'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)]) pw = fields.PasswordField(_('Password'), [validators.DataRequired()]) pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')]) + pwned = fields.HiddenField(label='',default=-1) submit = fields.SubmitField(_('Sign up')) class UserSignupFormCaptcha(UserSignupForm): @@ -111,6 +114,7 @@ class UserSettingsForm(flask_wtf.FlaskForm): class UserPasswordForm(flask_wtf.FlaskForm): pw = fields.PasswordField(_('Password'), [validators.DataRequired()]) pw2 = fields.PasswordField(_('Password check'), [validators.DataRequired()]) + pwned = fields.HiddenField(label='',default=-1) submit = fields.SubmitField(_('Update password')) diff --git a/core/admin/mailu/ui/views/domains.py b/core/admin/mailu/ui/views/domains.py index 4010e2ae..4b237ca0 100644 --- a/core/admin/mailu/ui/views/domains.py +++ b/core/admin/mailu/ui/views/domains.py @@ -93,6 +93,10 @@ def domain_signup(domain_name=None): del form.pw del form.pw2 if form.validate_on_submit(): + breaches = int(form.pwned.data) + if breaches > 0: + flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + return flask.render_template('domain/signup.html', form=form) conflicting_domain = models.Domain.query.get(form.name.data) conflicting_alternative = models.Alternative.query.get(form.name.data) conflicting_relay = models.Relay.query.get(form.name.data) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index c4b26036..2bd664b5 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -28,6 +28,11 @@ def user_create(domain_name): form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=domain.max_quota_bytes)] if form.validate_on_submit(): + breaches = int(form.pwned.data) + if breaches > 0: + flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + return flask.render_template('user/create.html', + domain=domain, form=form) if domain.has_email(form.localpart.data): flask.flash('Email is already used', 'error') else: @@ -60,6 +65,11 @@ def user_edit(user_email): form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=max_quota_bytes)] if form.validate_on_submit(): + breaches = int(form.pwned.data) + if breaches > 0: + flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + return flask.render_template('user/edit.html', form=form, user=user, + domain=user.domain, max_quota_bytes=max_quota_bytes) form.populate_obj(user) if form.pw.data: user.set_password(form.pw.data) @@ -119,6 +129,10 @@ def user_password(user_email): if form.pw.data != form.pw2.data: flask.flash('Passwords do not match', 'error') else: + breaches = int(form.pwned.data) + if breaches > 0: + flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + return flask.render_template('user/password.html', form=form, user=user) flask.session.regenerate() user.set_password(form.pw.data) models.db.session.commit() @@ -170,6 +184,10 @@ def user_signup(domain_name=None): if domain.has_email(form.localpart.data) or models.Alias.resolve(form.localpart.data, domain_name): flask.flash('Email is already used', 'error') else: + breaches = int(form.pwned.data) + if breaches > 0: + flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + return flask.render_template('user/signup.html', domain=domain, form=form) flask.session.regenerate() user = models.User(domain=domain) form.populate_obj(user) From 9cb8df57c683c7fb01e2adda8a23028559944e00 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 19:48:13 +0100 Subject: [PATCH 094/274] enforce at least 8 chars --- core/admin/mailu/ui/views/domains.py | 3 +++ core/admin/mailu/ui/views/users.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/core/admin/mailu/ui/views/domains.py b/core/admin/mailu/ui/views/domains.py index 4b237ca0..b39657f1 100644 --- a/core/admin/mailu/ui/views/domains.py +++ b/core/admin/mailu/ui/views/domains.py @@ -93,6 +93,9 @@ def domain_signup(domain_name=None): del form.pw del form.pw2 if form.validate_on_submit(): + if not flask_login.current_user.is_authenticated and len(form.pw.data) < 8: + flask.flash("This password is too short.", "error") + return flask.render_template('domain/signup.html', form=form) breaches = int(form.pwned.data) if breaches > 0: flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index 2bd664b5..3fe96109 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -28,6 +28,10 @@ def user_create(domain_name): form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=domain.max_quota_bytes)] if form.validate_on_submit(): + if len(form.pw.data) < 8: + flask.flash("This password is too short.", "error") + return flask.render_template('user/create.html', + domain=domain, form=form) breaches = int(form.pwned.data) if breaches > 0: flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") @@ -65,6 +69,10 @@ def user_edit(user_email): form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=max_quota_bytes)] if form.validate_on_submit(): + if len(form.pw.data) < 8: + flask.flash("This password is too short.", "error") + return flask.render_template('user/edit.html', form=form, user=user, + domain=user.domain, max_quota_bytes=max_quota_bytes) breaches = int(form.pwned.data) if breaches > 0: flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") @@ -129,6 +137,9 @@ def user_password(user_email): if form.pw.data != form.pw2.data: flask.flash('Passwords do not match', 'error') else: + if len(form.pw.data) < 8: + flask.flash("This password is too short.", "error") + return flask.render_template('user/password.html', form=form, user=user) breaches = int(form.pwned.data) if breaches > 0: flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") @@ -184,6 +195,9 @@ def user_signup(domain_name=None): if domain.has_email(form.localpart.data) or models.Alias.resolve(form.localpart.data, domain_name): flask.flash('Email is already used', 'error') else: + if len(form.pw.data) < 8: + flask.flash("This password is too short.", "error") + return flask.render_template('user/signup.html', domain=domain, form=form) breaches = int(form.pwned.data) if breaches > 0: flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") From 5d314c49aea869bd54324b464d6162e2478d7ade Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 19:50:08 +0100 Subject: [PATCH 095/274] towncrier --- towncrier/newsfragments/2500.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/2500.feature diff --git a/towncrier/newsfragments/2500.feature b/towncrier/newsfragments/2500.feature new file mode 100644 index 00000000..3c37934e --- /dev/null +++ b/towncrier/newsfragments/2500.feature @@ -0,0 +1 @@ +Implement a minimum length for passwords of 8 characters. Check passwords upon login against HaveIBeenPwned and warn users if their passwords are compromised. From bba98b320e0d10ac8cc8333d991d215a0c6b5eb1 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 31 Oct 2022 23:40:51 +0100 Subject: [PATCH 096/274] Fix armv7 build by manually downloading crates.io index --- core/base/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 721b5db1..6f6cb7a2 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -32,7 +32,8 @@ RUN set -euxo pipefail \ ; apk add --no-cache py3-pip \ ; python3 -m venv ${VIRTUAL_ENV} \ ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt \ - ; apk del -r py3-pip + ; apk del -r py3-pip \ + ; rm -f /tmp/*.pem ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" @@ -42,11 +43,12 @@ COPY libs/ libs/ RUN set -euxo pipefail \ ; machine="$(uname -m)"; deps="" \ ; [[ "${machine}" == arm* || "${machine}" == aarch64 ]] && deps="${deps} build-base gcc libffi-dev python3-dev" \ - ; [[ "${machine}" == armv7* ]] && deps="${deps} cargo libressl-dev mariadb-connector-c-dev postgresql-dev" \ + ; [[ "${machine}" == armv7* ]] && deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ ; [[ "${deps}" ]] && apk add --virtual .build-deps ${deps} \ + ; [[ "${machine}" == armv7* ]] && mkdir -p /root/.cargo/registry/index && git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ ; pip install -r requirements-${MAILU_ENV}.txt \ ; apk -e info -q .build-deps && apk del -r .build-deps \ - ; rm -rf /root/.cache /tmp/*.pem + ; rm -rf /root/.cargo /root/.cache /tmp/*.pem # base mailu image From 91f86a4c2ae64ea05e890862ad49f5d6e8094c89 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 31 Oct 2022 23:57:51 +0100 Subject: [PATCH 097/274] Resolve using socrate function --- core/admin/mailu/internal/nginx.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 870cc76d..43e4dd6a 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -1,12 +1,11 @@ from mailu import models, utils from flask import current_app as app +from socrate import system import re import urllib import ipaddress -import socket import sqlalchemy.exc -import tenacity SUPPORTED_AUTH_METHODS = ["none", "plain"] @@ -146,13 +145,5 @@ def get_server(protocol, authenticated=False): ipaddress.ip_address(hostname) except: # hostname is not an ip address - so we need to resolve it - hostname = resolve_hostname(hostname) + hostname = system.resolve_hostname(hostname) return hostname, port - -@tenacity.retry(stop=tenacity.stop_after_attempt(100), - wait=tenacity.wait_random(min=2, max=5)) -def resolve_hostname(hostname): - """ This function uses system DNS to resolve a hostname. - It is capable of retrying in case the host is not immediately available - """ - return socket.gethostbyname(hostname) From e915e444e95be5b702e345e7942cea4f1533405b Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 09:38:14 +0000 Subject: [PATCH 098/274] Remove superfluous cache export entry for arm --- .github/workflows/build_test_deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index e8bb1a0a..487470b1 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -253,7 +253,6 @@ jobs: *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm - *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max *.platform=${{ inputs.architecture }} # This job runs all the tests. From a2d43be6def679d5f9afbb90217bfd8248e201e4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 1 Nov 2022 11:02:21 +0100 Subject: [PATCH 099/274] Fix building wheels when deps need to compile --- core/base/Dockerfile | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 6f6cb7a2..a661877b 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -22,7 +22,7 @@ CMD /bin/bash # build virtual env (intermediate) FROM system as build -ARG MAILU_ENV=prod +ARG MAILU_DEPS=prod ENV VIRTUAL_ENV=/app/venv @@ -37,18 +37,25 @@ RUN set -euxo pipefail \ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" -COPY requirements-${MAILU_ENV}.txt ./ +COPY requirements-${MAILU_DEPS}.txt ./ COPY libs/ libs/ RUN set -euxo pipefail \ - ; machine="$(uname -m)"; deps="" \ - ; [[ "${machine}" == arm* || "${machine}" == aarch64 ]] && deps="${deps} build-base gcc libffi-dev python3-dev" \ - ; [[ "${machine}" == armv7* ]] && deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ - ; [[ "${deps}" ]] && apk add --virtual .build-deps ${deps} \ - ; [[ "${machine}" == armv7* ]] && mkdir -p /root/.cargo/registry/index && git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ - ; pip install -r requirements-${MAILU_ENV}.txt \ - ; apk -e info -q .build-deps && apk del -r .build-deps \ - ; rm -rf /root/.cargo /root/.cache /tmp/*.pem + ; pip install -r requirements-${MAILU_DEPS}.txt || \ + { \ + machine="$(uname -m)" \ + ; deps="build-base gcc libffi-dev python3-dev" \ + ; [[ "${machine}" == armv7 ]] && \ + deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ + ; apk add --virtual .build-deps ${deps} \ + ; [[ "${machine}" == armv7 ]] && \ + mkdir -p /root/.cargo/registry/index && \ + git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ + ; pip install -r requirements-${MAILU_DEPS}.txt \ + ; apk del -r .build-deps \ + ; rm -rf /root/.cargo /tmp/*.pem \ + ; } \ + ; rm -rf /root/.cache # base mailu image From c7cba1b07553570122cc835d39994393a3b4c344 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 10:49:44 +0000 Subject: [PATCH 100/274] Finishing touches for fixing arm builds - Use self-hosted runners for arm base image - Use seperate docker image cache for arm build - Remove unneeded needs items. --- .github/workflows/build_test_deploy.yml | 64 +++++++++++++++++++++---- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 487470b1..c01f67fa 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -89,9 +89,10 @@ jobs: run: | echo ${{ steps.targets.outputs.matrix }} -## This job buils the base image. The base image is used by all other images. - build-base-image: +## This job builds the base image. The base image is used by all other images. + build-base-image-x64: name: Build base image + if: inputs.architecture == 'linux/amd64' needs: - targets runs-on: ubuntu-latest @@ -139,14 +140,64 @@ jobs: *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }},mode=max *.platform=${{ inputs.architecture }} +## This job builds the base image. The base image is used by all other images. + build-base-image-arm: + name: Build base image + if: inputs.architecture != 'linux/amd64' + needs: + - targets + runs-on: self-hosted + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v3 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - uses: crazy-max/ghaction-github-runtime@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 + with: + string: ${{ github.repository_owner }} + - name: Build all docker images + env: + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + uses: docker/bake-action@v2 + with: + files: ${{env.HCL_FILE}} + targets: base + load: false + push: false + set: | + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max + *.platform=${{ inputs.architecture }} + # This job builds all the images. The build cache is stored in the github actions cache. # In further jobs, this cache is used to quickly rebuild the images. build: name: Build images for linux/amd64 if: inputs.architecture == 'linux/amd64' needs: - - targets - - build-base-image + - build-base-image-x64 strategy: fail-fast: false matrix: @@ -203,8 +254,7 @@ jobs: name: Build images for ARM64 & ARM/V7 if: inputs.architecture != 'linux/amd64' needs: - - targets - - build-base-image + - build-base-image-arm strategy: fail-fast: false matrix: @@ -264,7 +314,6 @@ jobs: contents: read packages: read needs: - - targets - build strategy: fail-fast: false @@ -319,7 +368,6 @@ jobs: if: inputs.deploy == 'true' runs-on: ubuntu-latest needs: - - build - tests strategy: fail-fast: false From 6549dbf24788e8ec949eac5bb22b57c8424ce6f4 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 10:56:33 +0000 Subject: [PATCH 101/274] Sigh. needs.* context is only available if you include it in needs: --- .github/workflows/build_test_deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index c01f67fa..f3977e1f 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -197,6 +197,7 @@ jobs: name: Build images for linux/amd64 if: inputs.architecture == 'linux/amd64' needs: + - targets - build-base-image-x64 strategy: fail-fast: false @@ -254,6 +255,7 @@ jobs: name: Build images for ARM64 & ARM/V7 if: inputs.architecture != 'linux/amd64' needs: + - targets - build-base-image-arm strategy: fail-fast: false From 5137b235e9df6d5a1a44b4d1a04f701e209b2cd8 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 1 Nov 2022 13:47:21 +0100 Subject: [PATCH 102/274] whitelist what we know works If other people use other arch and want their builds to go faster we can whitelist them too after they have confirmed it works. --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index a661877b..29223182 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -45,7 +45,7 @@ RUN set -euxo pipefail \ { \ machine="$(uname -m)" \ ; deps="build-base gcc libffi-dev python3-dev" \ - ; [[ "${machine}" == armv7 ]] && \ + ; [[ "${machine}" != x86_64 ]] && \ deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ ; apk add --virtual .build-deps ${deps} \ ; [[ "${machine}" == armv7 ]] && \ From ff9f152a529e47be7bd723dada8937f1ec1957e5 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 1 Nov 2022 14:11:59 +0100 Subject: [PATCH 103/274] This may be helpful too --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 29223182..d5be6a90 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -48,7 +48,7 @@ RUN set -euxo pipefail \ ; [[ "${machine}" != x86_64 ]] && \ deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ ; apk add --virtual .build-deps ${deps} \ - ; [[ "${machine}" == armv7 ]] && \ + ; [[ "${machine}" == armv7* ]] && \ mkdir -p /root/.cargo/registry/index && \ git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ ; pip install -r requirements-${MAILU_DEPS}.txt \ From b3151e99047fd3b529117833eb4450ca81abb0e2 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 14:36:17 +0000 Subject: [PATCH 104/274] Actually push the build arm images to ghcr.io --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index f3977e1f..2f710f4e 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -300,7 +300,7 @@ jobs: files: ${{env.HCL_FILE}} targets: ${{ matrix.target }} load: false - push: false + push: true set: | *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max From db7ce8c83e083bd0a086dbc9855f344eae656306 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 15:18:03 +0000 Subject: [PATCH 105/274] Login docker.io to prevent rate limiting for pulling images --- .github/workflows/build_test_deploy.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 2f710f4e..93e1214d 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -91,7 +91,7 @@ jobs: ## This job builds the base image. The base image is used by all other images. build-base-image-x64: - name: Build base image + name: Build base image x64 if: inputs.architecture == 'linux/amd64' needs: - targets @@ -119,6 +119,11 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v2 @@ -142,7 +147,7 @@ jobs: ## This job builds the base image. The base image is used by all other images. build-base-image-arm: - name: Build base image + name: Build base image arm if: inputs.architecture != 'linux/amd64' needs: - targets @@ -170,6 +175,11 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v2 @@ -227,6 +237,11 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v2 @@ -285,6 +300,11 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v2 From d920b3d0370e9a54a24a02244c8de46e05bb38c3 Mon Sep 17 00:00:00 2001 From: wkr Date: Wed, 2 Nov 2022 17:48:22 +0100 Subject: [PATCH 106/274] fix(auto-reply): include start and end dates in the auto-reply period; issue #2512 --- core/admin/mailu/models.py | 4 ++-- towncrier/newsfragments/2512.bugfix | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/2512.bugfix diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index f30ef387..48ce8b33 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -546,8 +546,8 @@ class User(Base, Email): now = date.today() return ( self.reply_enabled and - self.reply_startdate < now and - self.reply_enddate > now + self.reply_startdate <= now and + self.reply_enddate >= now ) @property diff --git a/towncrier/newsfragments/2512.bugfix b/towncrier/newsfragments/2512.bugfix new file mode 100644 index 00000000..b1b6aa99 --- /dev/null +++ b/towncrier/newsfragments/2512.bugfix @@ -0,0 +1 @@ +Fix: include start and end dates in the auto-reply period \ No newline at end of file From 66250e396c4d81bb3f1ffe9121bf292d1cb6ba63 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 3 Nov 2022 16:19:44 +0100 Subject: [PATCH 107/274] refactor --- core/admin/mailu/ui/views/domains.py | 10 +++----- core/admin/mailu/ui/views/users.py | 36 +++++++--------------------- core/admin/mailu/utils.py | 11 +++++++++ 3 files changed, 23 insertions(+), 34 deletions(-) diff --git a/core/admin/mailu/ui/views/domains.py b/core/admin/mailu/ui/views/domains.py index b39657f1..4cdd830a 100644 --- a/core/admin/mailu/ui/views/domains.py +++ b/core/admin/mailu/ui/views/domains.py @@ -1,4 +1,4 @@ -from mailu import models +from mailu import models, utils from mailu.ui import ui, forms, access from flask import current_app as app @@ -93,12 +93,8 @@ def domain_signup(domain_name=None): del form.pw del form.pw2 if form.validate_on_submit(): - if not flask_login.current_user.is_authenticated and len(form.pw.data) < 8: - flask.flash("This password is too short.", "error") - return flask.render_template('domain/signup.html', form=form) - breaches = int(form.pwned.data) - if breaches > 0: - flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + if msg := utils.isBadOrPwned(form): + flask.flash(msg, "error") return flask.render_template('domain/signup.html', form=form) conflicting_domain = models.Domain.query.get(form.name.data) conflicting_alternative = models.Alternative.query.get(form.name.data) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index 3fe96109..85a5c2db 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -1,4 +1,4 @@ -from mailu import models +from mailu import models, utils from mailu.ui import ui, access, forms from flask import current_app as app @@ -28,13 +28,8 @@ def user_create(domain_name): form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=domain.max_quota_bytes)] if form.validate_on_submit(): - if len(form.pw.data) < 8: - flask.flash("This password is too short.", "error") - return flask.render_template('user/create.html', - domain=domain, form=form) - breaches = int(form.pwned.data) - if breaches > 0: - flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + if msg := utils.isBadOrPwned(form): + flask.flash(msg, "error") return flask.render_template('user/create.html', domain=domain, form=form) if domain.has_email(form.localpart.data): @@ -69,13 +64,8 @@ def user_edit(user_email): form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=max_quota_bytes)] if form.validate_on_submit(): - if len(form.pw.data) < 8: - flask.flash("This password is too short.", "error") - return flask.render_template('user/edit.html', form=form, user=user, - domain=user.domain, max_quota_bytes=max_quota_bytes) - breaches = int(form.pwned.data) - if breaches > 0: - flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + if msg := utils.isBadOrPwned(form): + flask.flash(msg, "error") return flask.render_template('user/edit.html', form=form, user=user, domain=user.domain, max_quota_bytes=max_quota_bytes) form.populate_obj(user) @@ -137,12 +127,8 @@ def user_password(user_email): if form.pw.data != form.pw2.data: flask.flash('Passwords do not match', 'error') else: - if len(form.pw.data) < 8: - flask.flash("This password is too short.", "error") - return flask.render_template('user/password.html', form=form, user=user) - breaches = int(form.pwned.data) - if breaches > 0: - flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + if msg := utils.isBadOrPwned(form): + flask.flash(msg, "error") return flask.render_template('user/password.html', form=form, user=user) flask.session.regenerate() user.set_password(form.pw.data) @@ -195,12 +181,8 @@ def user_signup(domain_name=None): if domain.has_email(form.localpart.data) or models.Alias.resolve(form.localpart.data, domain_name): flask.flash('Email is already used', 'error') else: - if len(form.pw.data) < 8: - flask.flash("This password is too short.", "error") - return flask.render_template('user/signup.html', domain=domain, form=form) - breaches = int(form.pwned.data) - if breaches > 0: - flask.flash(f"This password appears in {breaches} data breaches! Please change it.", "error") + if msg := utils.isBadOrPwned(form): + flask.flash(msg, "error") return flask.render_template('user/signup.html', domain=domain, form=form) flask.session.regenerate() user = models.User(domain=domain) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 7e208702..4c55e27d 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -507,3 +507,14 @@ def gen_temp_token(email, session): app.config['PERMANENT_SESSION_LIFETIME'], ) return token + +def isBadOrPwned(form): + try: + if len(form.pw.data) < 8: + return "This password is too short." + breaches = int(form.pwned.data) + except ValueError: + breaches = -1 + if breaches > 0: + return f"This password appears in {breaches} data breaches! It is not unique; please change it." + return None From 24b2c7c04aee19c8e06fb454aa2507aad2ac2619 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 3 Nov 2022 16:25:10 +0100 Subject: [PATCH 108/274] doh --- core/admin/assets/content/assets/app.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/assets/content/assets/app.js b/core/admin/assets/content/assets/app.js index b0d48470..96d6deb7 100644 --- a/core/admin/assets/content/assets/app.js +++ b/core/admin/assets/content/assets/app.js @@ -38,7 +38,7 @@ function hibpCheck(pwd){ for(index in resp){ // Check if the line matches the rest of the hash if(resp[index].substring(0, 35) == hashSub){ - var val = resp[index].split(":")[1] + const val = resp[index].split(":")[1] if (val > 0) { $("#pwned").value = val; } @@ -126,8 +126,8 @@ $('document').ready(function() { } if (window.isSecureContext) { - $("#HIBPpw").change(function(){ - hibpCheck($("#HIBPpw").value); + $("#pw").change(function(){ + hibpCheck($("#pw").value); return true; }) } From 6b7026ef696c9560006c2775862b1ab7949cf522 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 3 Nov 2022 16:28:07 +0100 Subject: [PATCH 109/274] Here too --- core/admin/mailu/sso/views/base.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/sso/views/base.py b/core/admin/mailu/sso/views/base.py index 34ba98b5..6fa9403f 100644 --- a/core/admin/mailu/sso/views/base.py +++ b/core/admin/mailu/sso/views/base.py @@ -41,9 +41,8 @@ def login(): response = flask.redirect(destination) response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login'), secure=app.config['SESSION_COOKIE_SECURE'], httponly=True) flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip} pwned={form.pwned.data}.') - breaches = int(form.pwned.data) - if breaches > 0: - flask.flash(f"Your password appears in {breaches} data breaches! Please change it.", "error") + if msg := utils.isBadOrPwned(form): + flask.flash(msg, "error") return response else: utils.limiter.rate_limit_user(username, client_ip, device_cookie, device_cookie_username) if models.User.get(username) else utils.limiter.rate_limit_ip(client_ip) From dec5309ef97ce0bc02dfb9b70f796c8a22223f84 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 16:39:29 +0100 Subject: [PATCH 110/274] Fix typo --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 783a14f4..7aeeeb6b 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -29,7 +29,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes="+quota[0]) + "quota_rule": "*:bytes="+quota[0] }) From 001acd60ac9e2aed1b9b36ba05b374e0130c43a2 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 3 Nov 2022 16:44:18 +0100 Subject: [PATCH 111/274] doh2 --- core/admin/mailu/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 4c55e27d..f160fe3f 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -509,12 +509,12 @@ def gen_temp_token(email, session): return token def isBadOrPwned(form): - try: - if len(form.pw.data) < 8: - return "This password is too short." - breaches = int(form.pwned.data) - except ValueError: - breaches = -1 - if breaches > 0: - return f"This password appears in {breaches} data breaches! It is not unique; please change it." + try: + if len(form.pw.data) < 8: + return "This password is too short." + breaches = int(form.pwned.data) + except ValueError: + breaches = -1 + if breaches > 0: + return f"This password appears in {breaches} data breaches! It is not unique; please change it." return None From bec0b1c3b221fc4771583f41ce6a7fc06350a511 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:26:27 +0100 Subject: [PATCH 112/274] Fix variable name --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 7aeeeb6b..2cb2d526 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,7 +27,7 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ "quota_rule": "*:bytes="+quota[0] }) From 595b32cf97445e60f9b6fbad3528908b1c478b48 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:37:21 +0100 Subject: [PATCH 113/274] Fix quota return value --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 2cb2d526..ab33b220 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -29,7 +29,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes="+quota[0] + "quota_rule": f"*:bytes={quota[0]}" }) From 46773f639b153d07d03f014483ff60cc3847cef8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:45:21 +0100 Subject: [PATCH 114/274] Return 404 is user-id cannot be parsed --- core/admin/mailu/internal/views/dovecot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index ab33b220..3df3839e 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,7 +27,10 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + try: + quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + except ValueError: + flask.abort(404) return flask.jsonify({ "quota_rule": f"*:bytes={quota[0]}" }) From c57706ad279cc32da3e7f2140d6f3cc67d84cb3a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:50:39 +0100 Subject: [PATCH 115/274] Duh --- core/admin/mailu/internal/views/dovecot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 3df3839e..07fce5b2 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -5,6 +5,7 @@ from flask import current_app as app import flask import socket import os +import sqlalchemy.exc @internal.route("/dovecot/passdb/") def dovecot_passdb_dict(user_email): @@ -29,7 +30,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): try: quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) - except ValueError: + except sqlalchemy.exc.StatementError as exc: flask.abort(404) return flask.jsonify({ "quota_rule": f"*:bytes={quota[0]}" From 14f802fb4a314c63db19ba9d4f2eca8a24ae13d5 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 3 Nov 2022 18:38:55 +0100 Subject: [PATCH 116/274] untested but that should work --- core/admin/assets/content/assets/app.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/admin/assets/content/assets/app.js b/core/admin/assets/content/assets/app.js index 96d6deb7..81f1802a 100644 --- a/core/admin/assets/content/assets/app.js +++ b/core/admin/assets/content/assets/app.js @@ -129,7 +129,13 @@ $('document').ready(function() { $("#pw").change(function(){ hibpCheck($("#pw").value); return true; - }) + }); + $("#pw").closest("form").submit(function(event){ + if($("#pwned").value > -1) {return;}; + event.preventDefault(); + hibpCheck($("#pw").value) + event.trigger(); + }); } }); From 54e9858633bfdc210ef68131daf5c2379b35facf Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 3 Nov 2022 18:42:19 +0100 Subject: [PATCH 117/274] this --- core/admin/assets/content/assets/app.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/assets/content/assets/app.js b/core/admin/assets/content/assets/app.js index 81f1802a..7c13ff24 100644 --- a/core/admin/assets/content/assets/app.js +++ b/core/admin/assets/content/assets/app.js @@ -133,7 +133,7 @@ $('document').ready(function() { $("#pw").closest("form").submit(function(event){ if($("#pwned").value > -1) {return;}; event.preventDefault(); - hibpCheck($("#pw").value) + hibpCheck($("#pw").value); event.trigger(); }); } From 3e9def6cd9530843e325ac84b130b27c6732911e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 4 Nov 2022 10:46:45 +0100 Subject: [PATCH 118/274] Use the new notation: arm64/v8 instead of arm64 --- .github/workflows/arm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 7125737c..1240c630 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -76,7 +76,7 @@ jobs: - derive-variables uses: ./.github/workflows/build_test_deploy.yml with: - architecture: 'linux/arm64,linux/arm/v7' + architecture: 'linux/arm64/v8,linux/arm/v7' mailu_version: ${{needs.derive-variables.outputs.MAILU_VERSION}}-arm pinned_mailu_version: ${{needs.derive-variables.outputs.PINNED_MAILU_VERSION}}-arm docker_org: ${{needs.derive-variables.outputs.DOCKER_ORG}} From 83fdc07a6fc2c8b3fad75619c4e8ce988dff8621 Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Fri, 4 Nov 2022 11:59:26 +0100 Subject: [PATCH 119/274] Default FETCHMAIL_ENABLED to True --- core/admin/mailu/configuration.py | 2 +- docs/configuration.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index e7de2a40..060edc8e 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -16,7 +16,7 @@ DEFAULT_CONFIG = { 'DOMAIN_REGISTRATION': False, 'TEMPLATES_AUTO_RELOAD': True, 'MEMORY_SESSIONS': False, - 'FETCHMAIL_ENABLED': False, + 'FETCHMAIL_ENABLED': True, # Database settings 'DB_FLAVOR': None, 'DB_USER': 'mailu', diff --git a/docs/configuration.rst b/docs/configuration.rst index 58965dff..6bf0845b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -102,8 +102,8 @@ support or e.g. mismatching TLS versions to deliver emails to Mailu. .. _fetchmail: -When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is shown in the admin interface. -The container itself still needs to be deployed manually. As fetchmail is optional, ``FETCHMAIL_ENABLED`` defaults to ``False``. +When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is enabled in the admin interface. +The container itself still needs to be deployed manually. ``FETCHMAIL_ENABLED`` defaults to ``True``. The ``FETCHMAIL_DELAY`` is a delay (in seconds) for the fetchmail service to go and fetch new email if available. Do not use too short delays if you do not From 27a5f9db6525ce4780bd6e2cf8e758ceb2f9ae38 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 13:35:13 +0100 Subject: [PATCH 120/274] Reformatting --- core/admin/mailu/sso/forms.py | 2 +- core/admin/mailu/ui/forms.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/sso/forms.py b/core/admin/mailu/sso/forms.py index 530fc1c2..ca124c02 100644 --- a/core/admin/mailu/sso/forms.py +++ b/core/admin/mailu/sso/forms.py @@ -7,6 +7,6 @@ class LoginForm(flask_wtf.FlaskForm): csrf = False email = fields.StringField(_('E-mail'), [validators.Email(), validators.DataRequired()]) pw = fields.PasswordField(_('Password'), [validators.DataRequired()]) - pwned = fields.HiddenField(label='',default=-1) + pwned = fields.HiddenField(label='', default=-1) submitWebmail = fields.SubmitField(_('Sign in')) submitAdmin = fields.SubmitField(_('Sign in')) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index 67e4b674..59097c71 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -59,7 +59,7 @@ class DomainSignupForm(flask_wtf.FlaskForm): localpart = fields.StringField(_('Initial admin'), [validators.DataRequired()]) pw = fields.PasswordField(_('Admin password'), [validators.DataRequired()]) pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')]) - pwned = fields.HiddenField(label='',default=-1) + pwned = fields.HiddenField(label='', default=-1) captcha = flask_wtf.RecaptchaField() submit = fields.SubmitField(_('Create')) @@ -80,7 +80,7 @@ class UserForm(flask_wtf.FlaskForm): localpart = fields.StringField(_('E-mail'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)]) pw = fields.PasswordField(_('Password')) pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')]) - pwned = fields.HiddenField(label='',default=-1) + pwned = fields.HiddenField(label='', default=-1) quota_bytes = fields_.IntegerSliderField(_('Quota'), default=10**9) enable_imap = fields.BooleanField(_('Allow IMAP access'), default=True) enable_pop = fields.BooleanField(_('Allow POP3 access'), default=True) @@ -94,7 +94,7 @@ class UserSignupForm(flask_wtf.FlaskForm): localpart = fields.StringField(_('Email address'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)]) pw = fields.PasswordField(_('Password'), [validators.DataRequired()]) pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')]) - pwned = fields.HiddenField(label='',default=-1) + pwned = fields.HiddenField(label='', default=-1) submit = fields.SubmitField(_('Sign up')) class UserSignupFormCaptcha(UserSignupForm): @@ -114,7 +114,7 @@ class UserSettingsForm(flask_wtf.FlaskForm): class UserPasswordForm(flask_wtf.FlaskForm): pw = fields.PasswordField(_('Password'), [validators.DataRequired()]) pw2 = fields.PasswordField(_('Password check'), [validators.DataRequired()]) - pwned = fields.HiddenField(label='',default=-1) + pwned = fields.HiddenField(label='', default=-1) submit = fields.SubmitField(_('Update password')) From 311f41c331e12668e57fed59ca283e15d363c9fb Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 13:35:38 +0100 Subject: [PATCH 121/274] Add missing hidden fields --- core/admin/mailu/sso/templates/form_sso.html | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/mailu/sso/templates/form_sso.html b/core/admin/mailu/sso/templates/form_sso.html index d2451597..d713251e 100644 --- a/core/admin/mailu/sso/templates/form_sso.html +++ b/core/admin/mailu/sso/templates/form_sso.html @@ -3,6 +3,7 @@ {%- block content %} {%- call macros.card() %}
    + {{ form.hidden_tag() }} {{ macros.form_field(form.email) }} {{ macros.form_field(form.pw) }} {{ macros.form_fields(fields, label=False, class="btn btn-default") }} From b2e47642f7b547e76a9bc8363a63027b5ba0ce26 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 4 Nov 2022 13:49:05 +0100 Subject: [PATCH 122/274] Tag the images with latest tag as well. --- .github/workflows/build_test_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 93e1214d..3e883432 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -470,6 +470,7 @@ jobs: docker buildx imagetools create \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \ ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} #This job creates a tagged release. A tag is created for the pinned version x.y.z. The GH release refers to this tag. From 2a3266b6b8061e5a1a070078319798a11e3e1fb3 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 4 Nov 2022 14:13:06 +0100 Subject: [PATCH 123/274] Forgot to update both deploy jobs --- .github/workflows/build_test_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 3e883432..d1395bec 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -426,6 +426,7 @@ jobs: docker buildx imagetools create \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \ ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} deploy-arm: From ea636a183546074c19c714d3c905b3810eb27b07 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 15:13:56 +0100 Subject: [PATCH 124/274] Fix hibp test --- core/admin/assets/content/assets/app.js | 43 +++++++++++++------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/core/admin/assets/content/assets/app.js b/core/admin/assets/content/assets/app.js index 7c13ff24..cac55971 100644 --- a/core/admin/assets/content/assets/app.js +++ b/core/admin/assets/content/assets/app.js @@ -4,7 +4,7 @@ import logo from './mailu.png'; import modules from "./*.json"; // Inspired from https://github.com/mehdibo/hibp-js/blob/master/hibp.js -function sha1(string){ +function sha1(string) { var buffer = new TextEncoder("utf-8").encode(string); return crypto.subtle.digest("SHA-1", buffer).then(function (buffer) { // Get the hex code @@ -12,12 +12,12 @@ function sha1(string){ var view = new DataView(buffer); for (var i = 0; i < view.byteLength; i += 4) { // Using getUint32 reduces the number of iterations needed (we process 4 bytes each time) - var value = view.getUint32(i) + var value = view.getUint32(i); // toString(16) will give the hex representation of the number without padding - var stringValue = value.toString(16) + var stringValue = value.toString(16); // We use concatenation and slice for padding - var padding = '00000000' - var paddedValue = (padding + stringValue).slice(-padding.length) + var padding = '00000000'; + var paddedValue = (padding + stringValue).slice(-padding.length); hexCodes.push(paddedValue); } // Join all the hex strings into one @@ -25,30 +25,30 @@ function sha1(string){ }); } -function hibpCheck(pwd){ +function hibpCheck(pwd) { // We hash the pwd first sha1(pwd).then(function(hash){ // We send the first 5 chars of the hash to hibp's API const req = new XMLHttpRequest(); + req.open('GET', 'https://api.pwnedpasswords.com/range/'+hash.substr(0, 5)); + req.setRequestHeader('Add-Padding', 'true'); req.addEventListener("load", function(){ // When we get back a response from the server // We create an array of lines and loop through them - const resp = this.responseText.split('\n'); + const lines = this.responseText.split("\n"); const hashSub = hash.slice(5).toUpperCase(); - for(index in resp){ + for (var i in lines){ // Check if the line matches the rest of the hash - if(resp[index].substring(0, 35) == hashSub){ - const val = resp[index].split(":")[1] + if (lines[i].substring(0, 35) == hashSub){ + const val = parseInt(lines[i].trimEnd("\r").split(":")[1]); if (val > 0) { - $("#pwned").value = val; - } + $("#pwned").val(val); + } return; // If found no need to continue the loop } } - $("#pwned").value = 0; + $("#pwned").val(0); }); - req.open('GET', 'https://api.pwnedpasswords.com/range/'+hash.substr(0, 5)); - req.setRequestHeader('Add-Padding', 'true'); req.send(); }); } @@ -126,15 +126,16 @@ $('document').ready(function() { } if (window.isSecureContext) { - $("#pw").change(function(){ - hibpCheck($("#pw").value); + $("#pw").on("change paste", function(){ + hibpCheck($(this).val()); return true; }); $("#pw").closest("form").submit(function(event){ - if($("#pwned").value > -1) {return;}; - event.preventDefault(); - hibpCheck($("#pw").value); - event.trigger(); + if (parseInt($("#pwned").val()) < 0) { + event.preventDefault(); + hibpCheck($("#pw").val()); + event.trigger(); + } }); } From 8732b70b309de278478b0c9a7393d3b12f4a3c78 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 15:15:19 +0100 Subject: [PATCH 125/274] Add shell script to run admin dev environment --- core/admin/.gitignore | 1 + core/admin/mailu/__init__.py | 6 ++- core/admin/mailu/configuration.py | 2 + core/admin/run_dev.sh | 83 +++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100755 core/admin/run_dev.sh diff --git a/core/admin/.gitignore b/core/admin/.gitignore index 5bb3bd8e..10ac2b76 100644 --- a/core/admin/.gitignore +++ b/core/admin/.gitignore @@ -2,3 +2,4 @@ lib64 .vscode tags +dev diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index fe1f376c..5bb40447 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -44,8 +44,10 @@ def create_app_from_config(config): # Initialize debugging tools if app.config.get("DEBUG"): debug.toolbar.init_app(app) - # TODO: add a specific configuration variable for profiling - # debug.profiler.init_app(app) + if app.config.get("DEBUG_PROFILER"): + debug.profiler.init_app(app) + if assets := app.config.get('DEBUG_ASSETS'): + app.static_folder = assets # Inject the default variables in the Jinja parser # TODO: move this to blueprints when needed diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 081c4ee0..a4c8c537 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -13,6 +13,8 @@ DEFAULT_CONFIG = { 'RATELIMIT_STORAGE_URL': '', 'QUOTA_STORAGE_URL': '', 'DEBUG': False, + 'DEBUG_PROFILER': False, + 'DEBUG_ASSETS': '', 'DOMAIN_REGISTRATION': False, 'TEMPLATES_AUTO_RELOAD': True, 'MEMORY_SESSIONS': False, diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh new file mode 100755 index 00000000..74bce531 --- /dev/null +++ b/core/admin/run_dev.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +### CONFIG + +DEV_PORT="${DEV_PORT:-8080}" +DEV_NAME="${DEV_NAME:-mailu-dev}" +DEV_PROFILE="${DEV_PROFILE:-false}" + + +### MAIN + +here="$(realpath "$(pwd)/${0%/*}")" +cd "${here}" + +docker="$(command -v podman || command -v docker || echo echo docker)" + +[[ -d dev ]] && rm -rf dev +mkdir -p dev/data || exit 1 + +# base +cp ../base/requirements-* dev/ +cp -r ../base/libs dev/ +sed -E '/^#/d;s:^FROM system$:FROM system AS base:' ../base/Dockerfile > dev/Dockerfile + +# assets +cp -r assets/content dev/ +sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' assets/Dockerfile >> dev/Dockerfile + +cat >> dev/Dockerfile <> dev/Dockerfile + +cat >> dev/Dockerfile < Date: Fri, 4 Nov 2022 18:29:45 +0100 Subject: [PATCH 126/274] Improve dev runner --- core/admin/run_dev.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 74bce531..fc553cc4 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -11,11 +11,16 @@ DEV_PROFILE="${DEV_PROFILE:-false}" ### MAIN +docker="$(command -v podman || command -v docker || echo false)" +[[ "${docker}" == "false" ]] && { + echo "Sorry, you'll need podman or docker to run this." + exit 1 +} + here="$(realpath "$(pwd)/${0%/*}")" cd "${here}" -docker="$(command -v podman || command -v docker || echo echo docker)" - +# TODO: use /tmp/... folder [[ -d dev ]] && rm -rf dev mkdir -p dev/data || exit 1 @@ -36,6 +41,12 @@ EOF # admin sed -E '/^#/d;/^(COPY|EXPOSE|HEALTHCHECK|VOLUME|CMD) /d; s:^(.* )[^ ]*pybabel[^\\]*(.*):\1true \2:' Dockerfile >> dev/Dockerfile +DEV_URI="http://" +[[ "${DEV_PORT}" == *:* ]] || DEV_URI="${DEV_URI}localhost:" +DEV_URI="${DEV_URI}${DEV_PORT}/admin/ui/" + +MSG="\\n======================================================================\\nUI is found here: ${DEV_URI}\\nLog in with: admin@example.com and password admin if this is a new DB.\\n======================================================================\\n" + cat >> dev/Dockerfile </dev/null && flask mailu admin admin example.com admin --mode ifmissing >/dev/null && echo -e '${MSG}' 1>&2 && flask run --host=0.0.0.0 --port=8080"] EOF # TODO: re-compile assets on change? # TODO: re-run babel on change? # build -${docker} build --tag "${DEV_NAME}:latest" dev/ +chmod -R u+rwX,go+rX dev/ +"${docker}" build --tag "${DEV_NAME}:latest" dev/ # run args=( --rm -it --name "${DEV_NAME}" --publish "${DEV_PORT}:8080" --volume "${here}/dev/data/:/data/" ) @@ -78,6 +91,6 @@ for file in "${here}"/assets/content/assets/*; do args+=( --volume "${file}:/app/static/${file/*\//}" ) done -${docker} run "${args[@]}" "${DEV_NAME}" +"${docker}" run "${args[@]}" "${DEV_NAME}" # TODO: remove dev folder? From 1d90dc3ea382ec9f45f7d06a38ef905141f53ba4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 18:54:59 +0100 Subject: [PATCH 127/274] Allow running without redis --- core/admin/mailu/configuration.py | 5 +++-- core/admin/run_dev.sh | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index a4c8c537..e18eb0b2 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -151,8 +151,9 @@ class ConfigManager: template = self.DB_TEMPLATES[self.config['DB_FLAVOR']] self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config) - self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' - self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' +# remove? self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' + if not self.config.get('RATELIMIT_STORAGE_URL'): + self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3' self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index fc553cc4..62919f6f 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -56,6 +56,7 @@ RUN set -euxo pipefail \ ENV FLASK_ENV=development ENV MEMORY_SESSIONS=true +ENV RATELIMIT_STORAGE_URL="memory://" ENV SESSION_COOKIE_SECURE=false ENV DEBUG=true From bbeb211d7262ea52f4501facb4c7b9d7fa0e1970 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 21:41:31 +0100 Subject: [PATCH 128/274] Listen to localhost by default --- core/admin/run_dev.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 62919f6f..03003348 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -4,10 +4,10 @@ set -euxo pipefail ### CONFIG -DEV_PORT="${DEV_PORT:-8080}" DEV_NAME="${DEV_NAME:-mailu-dev}" -DEV_PROFILE="${DEV_PROFILE:-false}" - +DEV_PROFILER="${DEV_PROFILER:-false}" +DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}" +[[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}" ### MAIN @@ -41,11 +41,7 @@ EOF # admin sed -E '/^#/d;/^(COPY|EXPOSE|HEALTHCHECK|VOLUME|CMD) /d; s:^(.* )[^ ]*pybabel[^\\]*(.*):\1true \2:' Dockerfile >> dev/Dockerfile -DEV_URI="http://" -[[ "${DEV_PORT}" == *:* ]] || DEV_URI="${DEV_URI}localhost:" -DEV_URI="${DEV_URI}${DEV_PORT}/admin/ui/" - -MSG="\\n======================================================================\\nUI is found here: ${DEV_URI}\\nLog in with: admin@example.com and password admin if this is a new DB.\\n======================================================================\\n" +MSG="\\n======================================================================\\nUI can be found here: http://${DEV_LISTEN}/sso/login\\nLog in with: admin@example.com and password admin if this is a new DB.\\n======================================================================\\n" cat >> dev/Dockerfile < Date: Fri, 4 Nov 2022 22:20:08 +0100 Subject: [PATCH 129/274] Remove unused QUOTA_STORAGE_URL --- core/admin/mailu/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index e18eb0b2..b941e95c 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -11,7 +11,6 @@ DEFAULT_CONFIG = { 'BABEL_DEFAULT_TIMEZONE': 'UTC', 'BOOTSTRAP_SERVE_LOCAL': True, 'RATELIMIT_STORAGE_URL': '', - 'QUOTA_STORAGE_URL': '', 'DEBUG': False, 'DEBUG_PROFILER': False, 'DEBUG_ASSETS': '', @@ -151,9 +150,9 @@ class ConfigManager: template = self.DB_TEMPLATES[self.config['DB_FLAVOR']] self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config) -# remove? self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' if not self.config.get('RATELIMIT_STORAGE_URL'): self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' + self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3' self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True @@ -162,9 +161,9 @@ class ConfigManager: self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME']) self.config['AUTH_RATELIMIT_IP_V4_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V4_MASK']) self.config['AUTH_RATELIMIT_IP_V6_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V6_MASK']) - hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')] self.config['AUTH_RATELIMIT_EXEMPTION'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['AUTH_RATELIMIT_EXEMPTION'].split(',')) if cidr) self.config['MESSAGE_RATELIMIT_EXEMPTION'] = set([s for s in self.config['MESSAGE_RATELIMIT_EXEMPTION'].lower().replace(' ', '').split(',') if s]) + hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')] self.config['HOSTNAMES'] = ','.join(hostnames) self.config['HOSTNAME'] = hostnames[0] self.config['DEFAULT_SPAM_THRESHOLD'] = int(self.config['DEFAULT_SPAM_THRESHOLD']) From 71263f1a8cd687cfad4b29f75f9f5d5aa03b62db Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 23:21:11 +0100 Subject: [PATCH 130/274] Add more env variables and restyle code --- core/admin/run_dev.sh | 154 +++++++++++++++++++++++++++--------------- 1 file changed, 100 insertions(+), 54 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 03003348..a2e32912 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -1,93 +1,139 @@ #!/usr/bin/env bash -set -euxo pipefail +set -euo pipefail ### CONFIG DEV_NAME="${DEV_NAME:-mailu-dev}" +DEV_DB="${DEV_DB:-}" DEV_PROFILER="${DEV_PROFILER:-false}" DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}" [[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}" +DEV_ADMIN="${DEV_ADMIN:-admin@example.com}" +DEV_PASSWORD="${DEV_PASSWORD:-admin}" ### MAIN -docker="$(command -v podman || command -v docker || echo false)" -[[ "${docker}" == "false" ]] && { - echo "Sorry, you'll need podman or docker to run this." - exit 1 +[[ -n "${DEV_DB}" ]] && { + [[ -f "${DEV_DB}" ]] || { + echo "Sorry, can't find DEV_DB: '${DEV_DB}'" + exit 1 + } + DEV_DB="$(realpath "${DEV_DB}")" } -here="$(realpath "$(pwd)/${0%/*}")" -cd "${here}" +docker="$(command -v podman || command -v docker || echo false)" +[[ "${docker}" == "false" ]] && { + echo "Sorry, you'll need podman or docker to run this." + exit 1 +} -# TODO: use /tmp/... folder -[[ -d dev ]] && rm -rf dev -mkdir -p dev/data || exit 1 +tmp="$(mktemp -d)" +[[ -n "${tmp}" && -d "${tmp}" ]] || { + echo "Sorry, can't create temporary folder." + exit 1 +} +trap "rm -rf '${tmp}'" INT TERM EXIT + +admin="$(realpath "$(pwd)/${0%/*}")" +base="${admin}/../base" +assets="${admin}/assets" + +cd "${tmp}" # base -cp ../base/requirements-* dev/ -cp -r ../base/libs dev/ -sed -E '/^#/d;s:^FROM system$:FROM system AS base:' ../base/Dockerfile > dev/Dockerfile +cp "${base}"/requirements-* . +cp -r "${base}"/libs . +sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Dockerfile # assets -cp -r assets/content dev/ -sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' assets/Dockerfile >> dev/Dockerfile +cp -r "${assets}/content" . +sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile -cat >> dev/Dockerfile <>Dockerfile <> dev/Dockerfile +sed -E '/^#/d;/^(COPY|EXPOSE|HEALTHCHECK|VOLUME|CMD) /d; s:^(.* )[^ ]*pybabel[^\\]*(.*):\1true \2:' "${admin}/Dockerfile" >>Dockerfile -MSG="\\n======================================================================\\nUI can be found here: http://${DEV_LISTEN}/sso/login\\nLog in with: admin@example.com and password admin if this is a new DB.\\n======================================================================\\n" - -cat >> dev/Dockerfile <>Dockerfile </dev/null && flask mailu admin admin example.com admin --mode ifmissing >/dev/null && echo -e '${MSG}' 1>&2 && flask run --host=0.0.0.0 --port=8080"] +CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null && flask run --host=0.0.0.0 --port=8080"] EOF -# TODO: re-compile assets on change? -# TODO: re-run babel on change? - # build -chmod -R u+rwX,go+rX dev/ -"${docker}" build --tag "${DEV_NAME}:latest" dev/ +chmod -R u+rwX,go+rX . +cat Dockerfile +"${docker}" build --tag "${DEV_NAME}:latest" . + +# gather volumes to map into container +volumes=() + +if [[ -n "${DEV_DB}" ]]; then + volumes+=( --volume "${DEV_DB}:/data/main.db" ) +else + mkdir -p "data" + volumes=( --volume "${tmp}/data/:/data/" ) +fi + +for vol in audit.py start.py mailu/ migrations/; do + volumes+=( --volume "${admin}/${vol}:/app/${vol}" ) +done + +for file in "${assets}/content/assets"/*; do + [[ "${file}" == */vendor.js ]] && continue + volumes+=( --volume "${file}:/app/static/${file/*\//}" ) +done + +# show configuration +cat < Date: Fri, 4 Nov 2022 23:39:39 +0100 Subject: [PATCH 131/274] Speed up asset building when developing --- core/admin/assets/Dockerfile | 20 ++++++++++++------ .../admin/assets/{content => }/assets/app.css | 0 core/admin/assets/{content => }/assets/app.js | 0 .../assets/{content => }/assets/mailu.png | Bin .../assets/{content => }/assets/vendor.js | 0 core/admin/assets/{content => }/package.json | 0 .../assets/{content => }/webpack.config.js | 0 core/admin/run_dev.sh | 13 +++++------- 8 files changed, 18 insertions(+), 15 deletions(-) rename core/admin/assets/{content => }/assets/app.css (100%) rename core/admin/assets/{content => }/assets/app.js (100%) rename core/admin/assets/{content => }/assets/mailu.png (100%) rename core/admin/assets/{content => }/assets/vendor.js (100%) rename core/admin/assets/{content => }/package.json (100%) rename core/admin/assets/{content => }/webpack.config.js (100%) diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile index c8556f47..613aa6c0 100644 --- a/core/admin/assets/Dockerfile +++ b/core/admin/assets/Dockerfile @@ -4,13 +4,19 @@ FROM node:16-alpine3.16 WORKDIR /work -COPY content/ ./ +COPY package.json ./ +COPY webpack.config.js ./ RUN set -euxo pipefail \ - && npm config set update-notifier false \ - && npm install --no-audit --no-fund \ - && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ - && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ + ; npm config set update-notifier false \ + ; npm install --no-audit --no-fund \ + ; sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ + ; mkdir assets \ + ; for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ - done \ - && node_modules/.bin/webpack-cli --color + done + +COPY assets/ ./assets/ + +RUN set -euxo pipefail \ + ; node_modules/.bin/webpack-cli --color diff --git a/core/admin/assets/content/assets/app.css b/core/admin/assets/assets/app.css similarity index 100% rename from core/admin/assets/content/assets/app.css rename to core/admin/assets/assets/app.css diff --git a/core/admin/assets/content/assets/app.js b/core/admin/assets/assets/app.js similarity index 100% rename from core/admin/assets/content/assets/app.js rename to core/admin/assets/assets/app.js diff --git a/core/admin/assets/content/assets/mailu.png b/core/admin/assets/assets/mailu.png similarity index 100% rename from core/admin/assets/content/assets/mailu.png rename to core/admin/assets/assets/mailu.png diff --git a/core/admin/assets/content/assets/vendor.js b/core/admin/assets/assets/vendor.js similarity index 100% rename from core/admin/assets/content/assets/vendor.js rename to core/admin/assets/assets/vendor.js diff --git a/core/admin/assets/content/package.json b/core/admin/assets/package.json similarity index 100% rename from core/admin/assets/content/package.json rename to core/admin/assets/package.json diff --git a/core/admin/assets/content/webpack.config.js b/core/admin/assets/webpack.config.js similarity index 100% rename from core/admin/assets/content/webpack.config.js rename to core/admin/assets/webpack.config.js diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index a2e32912..a682252c 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -47,14 +47,11 @@ cp -r "${base}"/libs . sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Dockerfile # assets -cp -r "${assets}/content" . +cp "${assets}/package.json" . +cp -r "${assets}/assets/" . +awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile -cat >>Dockerfile <>Dockerfile @@ -107,8 +104,8 @@ for vol in audit.py start.py mailu/ migrations/; do volumes+=( --volume "${admin}/${vol}:/app/${vol}" ) done -for file in "${assets}/content/assets"/*; do - [[ "${file}" == */vendor.js ]] && continue +for file in "${assets}/assets"/*; do + [[ ! -f "${file}" || "${file}" == */vendor.js ]] && continue volumes+=( --volume "${file}:/app/static/${file/*\//}" ) done From db87a0f3a101ba3b4aaed5237824e82f95fc14c1 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 23:51:32 +0100 Subject: [PATCH 132/274] Move temporary db into container and show docker run command --- core/admin/run_dev.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index a682252c..4a0120c9 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -93,12 +93,7 @@ cat Dockerfile # gather volumes to map into container volumes=() -if [[ -n "${DEV_DB}" ]]; then - volumes+=( --volume "${DEV_DB}:/data/main.db" ) -else - mkdir -p "data" - volumes=( --volume "${tmp}/data/:/data/" ) -fi +[[ -n "${DEV_DB}" ]] && volumes+=( --volume "${DEV_DB}:/data/main.db" ) for vol in audit.py start.py mailu/ migrations/; do volumes+=( --volume "${admin}/${vol}:/app/${vol}" ) @@ -113,6 +108,8 @@ done cat < Date: Mon, 7 Nov 2022 16:16:09 +0100 Subject: [PATCH 133/274] Don't duplicate hidden fields --- core/admin/mailu/ui/templates/macros.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/admin/mailu/ui/templates/macros.html b/core/admin/mailu/ui/templates/macros.html index 0da069a9..46a76991 100644 --- a/core/admin/mailu/ui/templates/macros.html +++ b/core/admin/mailu/ui/templates/macros.html @@ -60,9 +60,7 @@ {{ form.hidden_tag() }} {%- for field in form %} - {%- if bootstrap_is_hidden_field(field) %} - {{ field() }} - {%- else %} + {%- if not bootstrap_is_hidden_field(field) %} {{ form_field(field) }} {%- endif %} {%- endfor %} From 91e12d510df8e43b85b280d2c6d514ae4db5200f Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 7 Nov 2022 16:35:01 +0100 Subject: [PATCH 134/274] Use default password used everywhere else --- core/admin/run_dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 4a0120c9..372c7390 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -10,7 +10,7 @@ DEV_PROFILER="${DEV_PROFILER:-false}" DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}" [[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}" DEV_ADMIN="${DEV_ADMIN:-admin@example.com}" -DEV_PASSWORD="${DEV_PASSWORD:-admin}" +DEV_PASSWORD="${DEV_PASSWORD:-letmein}" ### MAIN From dd3cd1263e76559ac1418252b0e797eb6beffd18 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 7 Nov 2022 16:47:13 +0100 Subject: [PATCH 135/274] Add development documentation again --- docs/contributors/environment.rst | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index 1b53afe6..5949fbe5 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -313,6 +313,48 @@ If git opens a editor for a commit message just save and exit as-is. If you have see above and do the complete procedure from ``git fetch`` onward again. +Web administration development +------------------------------ + +The administration web interface requires a proper dev environment that can easily +be setup using the ``run_dev.sh`` shell script. You need ``docker`` or ``podman`` +to run it. It will create a local webserver listening at port 8080: + +.. code-block:: bash + + cd core/admin + ./run_dev.sh + pip install -r requirements.txt + [...] + ============================================================================= + The "mailu-dev" container was built using this configuration: + + DEV_NAME="mailu-dev" + DEV_DB="" + DEV_PROFILER="false" + DEV_LISTEN="127.0.0.1:8080" + DEV_ADMIN="admin@example.com" + DEV_PASSWORD="letmein" + ============================================================================= + [...] + ============================================================================= + The Mailu UI can be found here: http://127.0.0.1:8080/sso/login + You can log in with user admin@example.com and password letmein + ============================================================================= + +The container will use an empty database and a default user/password unless you +specify a database file to use by setting ``$DEV_DB``. + +.. code-block:: bash + + DEV_DB="/path/to/dev.db" ./run_dev.sh + +Any change to the files will automatically restart the Web server and reload the files. + +When using the development environment, a debugging toolbar is displayed on the right +side of the screen, where you can access query details, internal variables, etc. + + Documentation ------------- From 36019a8ce9b5bad62f176fb9ab5cf0d3afc65298 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 7 Nov 2022 16:48:58 +0100 Subject: [PATCH 136/274] Don't show Dockerfile before building --- core/admin/run_dev.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 372c7390..4ab76e74 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -87,7 +87,6 @@ EOF # build chmod -R u+rwX,go+rX . -cat Dockerfile "${docker}" build --tag "${DEV_NAME}:latest" . # gather volumes to map into container From 6901b0f05e8fb5167fa7c1d0c8cec6ea002bef3f Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Tue, 8 Nov 2022 11:47:20 +0100 Subject: [PATCH 137/274] Implement FETCHMAIL_ENABLED in fetchmail.py --- optional/fetchmail/fetchmail.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 5459de59..1889d808 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 - +import distutils.util import time import os import tempfile @@ -96,5 +96,9 @@ def run(debug): if __name__ == "__main__": while True: time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60))) + + if not os.environ.get("FETCHMAIL_ENABLED", 'True') in ('True', 'true'): + continue + run(os.environ.get("DEBUG", None) == "True") sys.stdout.flush() From 4a74cd9afeb5c75a1f3ceaadd5cdfdb07d0b8918 Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Tue, 8 Nov 2022 11:56:08 +0100 Subject: [PATCH 138/274] Resolve conflict --- optional/fetchmail/fetchmail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 1889d808..2afd103b 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -import distutils.util + import time import os import tempfile From 7e36694b648de79d1f6d3c0f2ae13d9c4c152b9e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 31 Oct 2022 23:54:45 +0100 Subject: [PATCH 139/274] Update python deps --- core/base/requirements-prod.txt | 106 ++++++++++++++------------------ 1 file changed, 47 insertions(+), 59 deletions(-) diff --git a/core/base/requirements-prod.txt b/core/base/requirements-prod.txt index fbbce5d5..ce822cc7 100644 --- a/core/base/requirements-prod.txt +++ b/core/base/requirements-prod.txt @@ -1,90 +1,78 @@ aiodns==3.0.0 aiohttp==3.8.3 aiosignal==1.2.0 -alembic==1.7.4 -appdirs==1.4.4 +alembic==1.8.1 async-timeout==4.0.2 attrs==22.1.0 -Babel==2.9.1 -bcrypt==3.2.0 -blinker==1.4 -CacheControl==0.12.9 -certifi==2021.10.8 +Babel==2.10.3 +blinker==1.5 +certifi==2022.9.24 cffi==1.15.1 -chardet==4.0.0 -charset-normalizer==2.0.12 -click==8.0.3 -colorama==0.4.4 -contextlib2==21.6.0 -cryptography==35.0.0 -decorator==5.1.0 +charset-normalizer==2.1.1 +click==8.1.3 +cryptography==38.0.1 +decorator==5.1.1 defusedxml==0.7.1 -dnspython==2.1.0 -dominate==2.6.0 -email-validator==1.1.3 -Flask==2.0.2 +Deprecated==1.2.13 +dnspython==2.2.1 +dominate==2.7.0 +email-validator==1.3.0 +Flask==2.2.2 Flask-Babel==2.0.0 Flask-Bootstrap==3.3.7.1 -Flask-DebugToolbar==0.11.0 -Flask-Limiter==1.4 -Flask-Login==0.5.0 +Flask-DebugToolbar==0.13.1 +Flask-Login==0.6.2 flask-marshmallow==0.14.0 Flask-Migrate==3.1.0 -Flask-Script==2.0.6 -Flask-SQLAlchemy==2.5.1 -Flask-WTF==0.15.1 +Flask-SQLAlchemy==3.0.2 +Flask-WTF==1.0.1 frozenlist==1.3.1 -greenlet==1.1.2 +greenlet==2.0.0 gunicorn==20.1.0 -html5lib==1.1 -idna==3.3 +idna==3.4 infinity==1.5 intervals==0.9.2 -itsdangerous==2.0.1 -Jinja2==3.0.2 -limits==1.5.1 -lockfile==0.12.2 +itsdangerous==2.1.2 +Jinja2==3.1.2 +limits==2.7.1 Mako==1.2.3 -MarkupSafe==2.0.1 -marshmallow==3.14.0 -marshmallow-sqlalchemy==0.26.1 -msgpack==1.0.2 +MarkupSafe==2.1.1 +marshmallow==3.18.0 +marshmallow-sqlalchemy==0.28.1 multidict==6.0.2 -mysql-connector-python==8.0.25 -ordered-set==4.0.2 +mysql-connector-python==8.0.31 +packaging==21.3 passlib==1.7.4 podop @ file:///app/libs/podop -postfix-mta-sts-resolver==1.0.1 -progress==1.6 -protobuf==4.21.7 -psycopg2-binary==2.9.3 +postfix-mta-sts-resolver==1.1.4 +protobuf==3.20.1 +psycopg2-binary==2.9.5 pycares==4.2.2 -pycparser==2.20 -Pygments==2.10.0 -pyOpenSSL==21.0.0 -pyparsing==3.0.4 +pycparser==2.21 +Pygments==2.13.0 +pyOpenSSL==22.1.0 +pyparsing==3.0.9 python-dateutil==2.8.2 -pytz==2021.3 +pytz==2022.5 PyYAML==6.0 Radicale==3.1.8 -redis==3.5.3 -requests==2.26.0 -retrying==1.3.3 +redis==4.3.4 +requests==2.28.1 six==1.16.0 socrate @ file:///app/libs/socrate -SQLAlchemy==1.4.26 +SQLAlchemy==1.4.42 srslib==0.1.4 -tabulate==0.8.9 -tenacity==8.0.1 -toml==0.10.2 -urllib3==1.26.7 -validators==0.18.2 +tabulate==0.9.0 +tenacity==8.1.0 +typing_extensions==4.4.0 +urllib3==1.26.12 +validators==0.20.0 visitor==0.1.3 vobject==0.9.6.1 watchdog==2.1.9 -webencodings==0.5.1 -Werkzeug==2.0.2 -WTForms==2.3.3 +Werkzeug==2.2.2 +wrapt==1.14.1 +WTForms==3.0.1 WTForms-Components==0.10.5 -xmltodict==0.12.0 +xmltodict==0.13.0 yarl==1.8.1 From 047413185e3032b2c1764d27cbaffe6a5383c181 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 2 Nov 2022 17:46:13 +0100 Subject: [PATCH 140/274] Mask Flask-SQLAlchemy >= 3.0.0 for now as it breaks mailu --- core/base/requirements-dev.txt | 2 +- core/base/requirements-prod.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/requirements-dev.txt b/core/base/requirements-dev.txt index 2fb4352f..be2e2e29 100644 --- a/core/base/requirements-dev.txt +++ b/core/base/requirements-dev.txt @@ -14,7 +14,7 @@ Flask-DebugToolbar Flask-Login flask-marshmallow Flask-Migrate -Flask-SQLAlchemy +Flask-SQLAlchemy<3 Flask-WTF gunicorn idna diff --git a/core/base/requirements-prod.txt b/core/base/requirements-prod.txt index ce822cc7..8040b44a 100644 --- a/core/base/requirements-prod.txt +++ b/core/base/requirements-prod.txt @@ -24,7 +24,7 @@ Flask-DebugToolbar==0.13.1 Flask-Login==0.6.2 flask-marshmallow==0.14.0 Flask-Migrate==3.1.0 -Flask-SQLAlchemy==3.0.2 +Flask-SQLAlchemy==2.5.1 Flask-WTF==1.0.1 frozenlist==1.3.1 greenlet==2.0.0 From a74396a9ef81dd8f69f2088e3fee80773f2c4dde Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 2 Nov 2022 17:52:38 +0100 Subject: [PATCH 141/274] Fix wtforms usage --- core/admin/mailu/ui/forms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index 59097c71..beb44092 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -123,8 +123,8 @@ class UserReplyForm(flask_wtf.FlaskForm): reply_subject = fields.StringField(_('Reply subject')) reply_body = fields.StringField(_('Reply body'), widget=widgets.TextArea()) - reply_startdate = fields.html5.DateField(_('Start of vacation')) - reply_enddate = fields.html5.DateField(_('End of vacation')) + reply_startdate = fields.DateField(_('Start of vacation')) + reply_enddate = fields.DateField(_('End of vacation')) submit = fields.SubmitField(_('Update')) From 1ae91567563f3c312c84f5928800facbc374ecda Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 2 Nov 2022 22:21:32 +0100 Subject: [PATCH 142/274] Add bcyrpt as direct dependency for better crypto. Also some updates --- core/base/requirements-dev.txt | 1 + core/base/requirements-prod.txt | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/base/requirements-dev.txt b/core/base/requirements-dev.txt index be2e2e29..92d34fdb 100644 --- a/core/base/requirements-dev.txt +++ b/core/base/requirements-dev.txt @@ -5,6 +5,7 @@ libs/socrate # core/admin alembic Babel +bcrypt click dnspython Flask diff --git a/core/base/requirements-prod.txt b/core/base/requirements-prod.txt index 8040b44a..716f848e 100644 --- a/core/base/requirements-prod.txt +++ b/core/base/requirements-prod.txt @@ -4,13 +4,14 @@ aiosignal==1.2.0 alembic==1.8.1 async-timeout==4.0.2 attrs==22.1.0 -Babel==2.10.3 +Babel==2.11.0 +bcrypt==4.0.1 blinker==1.5 certifi==2022.9.24 cffi==1.15.1 charset-normalizer==2.1.1 click==8.1.3 -cryptography==38.0.1 +cryptography==38.0.3 decorator==5.1.1 defusedxml==0.7.1 Deprecated==1.2.13 @@ -53,7 +54,7 @@ Pygments==2.13.0 pyOpenSSL==22.1.0 pyparsing==3.0.9 python-dateutil==2.8.2 -pytz==2022.5 +pytz==2022.6 PyYAML==6.0 Radicale==3.1.8 redis==4.3.4 From 6218b363726c3340769a2eb125a88572b1908045 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 11 Jan 2022 16:49:39 +0100 Subject: [PATCH 143/274] configure datatables via html5 data attributes --- core/admin/mailu/ui/templates/admin/list.html | 2 +- core/admin/mailu/ui/templates/alias/list.html | 6 +++--- .../admin/mailu/ui/templates/alternative/list.html | 6 +++--- core/admin/mailu/ui/templates/domain/list.html | 14 +++++++------- core/admin/mailu/ui/templates/fetch/list.html | 6 +++--- core/admin/mailu/ui/templates/macros.html | 4 ++-- core/admin/mailu/ui/templates/relay/list.html | 8 ++++---- core/admin/mailu/ui/templates/token/list.html | 6 +++--- core/admin/mailu/ui/templates/user/list.html | 12 ++++++------ 9 files changed, 32 insertions(+), 32 deletions(-) diff --git a/core/admin/mailu/ui/templates/admin/list.html b/core/admin/mailu/ui/templates/admin/list.html index 84d954a0..e50c0ee6 100644 --- a/core/admin/mailu/ui/templates/admin/list.html +++ b/core/admin/mailu/ui/templates/admin/list.html @@ -14,7 +14,7 @@ {%- call macros.table() %} - {% trans %}Actions{% endtrans %} + {% trans %}Actions{% endtrans %} {% trans %}Email{% endtrans %} diff --git a/core/admin/mailu/ui/templates/alias/list.html b/core/admin/mailu/ui/templates/alias/list.html index 1e66668e..93b6fa73 100644 --- a/core/admin/mailu/ui/templates/alias/list.html +++ b/core/admin/mailu/ui/templates/alias/list.html @@ -16,7 +16,7 @@ {%- call macros.table() %} - {% trans %}Actions{% endtrans %} + {% trans %}Actions{% endtrans %} {% trans %}Email{% endtrans %} {% trans %}Destination{% endtrans %} {% trans %}Comment{% endtrans %} @@ -34,8 +34,8 @@ {{ alias }} {{ alias.destination|join(', ') or '-' }} {{ alias.comment or '' }} - {{ alias.created_at | format_date }} - {{ alias.updated_at | format_date }} + {{ alias.created_at | format_date }} + {{ alias.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/alternative/list.html b/core/admin/mailu/ui/templates/alternative/list.html index 4ca9f3c8..bc018c3d 100644 --- a/core/admin/mailu/ui/templates/alternative/list.html +++ b/core/admin/mailu/ui/templates/alternative/list.html @@ -16,7 +16,7 @@ {%- call macros.table() %} - {% trans %}Actions{% endtrans %} + {% trans %}Actions{% endtrans %} {% trans %}Name{% endtrans %} {% trans %}Created{% endtrans %} {% trans %}Last edit{% endtrans %} @@ -29,8 +29,8 @@ {{ alternative }} - {{ alternative.created_at | format_date }} - {{ alternative.updated_at | format_date }} + {{ alternative.created_at | format_date }} + {{ alternative.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/domain/list.html b/core/admin/mailu/ui/templates/domain/list.html index 61c09151..a6cefe9e 100644 --- a/core/admin/mailu/ui/templates/domain/list.html +++ b/core/admin/mailu/ui/templates/domain/list.html @@ -11,11 +11,11 @@ {%- endblock %} {%- block content %} -{%- call macros.table() %} +{%- call macros.table(order='[[2,"asc"]]') %} - {% trans %}Actions{% endtrans %} - {% trans %}Manage{% endtrans %} + {% trans %}Actions{% endtrans %} + {% trans %}Manage{% endtrans %} {% trans %}Domain name{% endtrans %} {% trans %}Mailbox count{% endtrans %} {% trans %}Alias count{% endtrans %} @@ -43,11 +43,11 @@ {%- endif %} {{ domain.name }} - {{ domain.users | count }} / {{ '∞' if domain.max_users == -1 else domain.max_users }} - {{ domain.aliases | count }} / {{ '∞' if domain.max_aliases == -1 else domain.max_aliases }} + {{ domain.users | count }} / {{ '∞' if domain.max_users == -1 else domain.max_users }} + {{ domain.aliases | count }} / {{ '∞' if domain.max_aliases == -1 else domain.max_aliases }} {{ domain.comment or '' }} - {{ domain.created_at | format_date }} - {{ domain.updated_at | format_date }} + {{ domain.created_at | format_date }} + {{ domain.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html index a504b7a2..9f0570fd 100644 --- a/core/admin/mailu/ui/templates/fetch/list.html +++ b/core/admin/mailu/ui/templates/fetch/list.html @@ -16,7 +16,7 @@ {%- call macros.table() %} - {% trans %}Actions{% endtrans %} + {% trans %}Actions{% endtrans %} {% trans %}Endpoint{% endtrans %} {% trans %}Username{% endtrans %} {% trans %}Keep emails{% endtrans %} @@ -38,8 +38,8 @@ {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} {{ fetch.last_check | format_datetime or '-' }} {{ fetch.error or '-' }} - {{ fetch.created_at | format_date }} - {{ fetch.updated_at | format_date }} + {{ fetch.created_at | format_date }} + {{ fetch.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/macros.html b/core/admin/mailu/ui/templates/macros.html index 46a76991..90084246 100644 --- a/core/admin/mailu/ui/templates/macros.html +++ b/core/admin/mailu/ui/templates/macros.html @@ -86,7 +86,7 @@ {%- endmacro %} -{%- macro table(title=None, theme="primary", datatable=True) %} +{%- macro table(title=None, theme="primary", datatable=True, order=None) %}
    @@ -96,7 +96,7 @@
    {%- endif %}
    - +
    {{- caller() }}
    diff --git a/core/admin/mailu/ui/templates/relay/list.html b/core/admin/mailu/ui/templates/relay/list.html index 1a23ee12..e1184769 100644 --- a/core/admin/mailu/ui/templates/relay/list.html +++ b/core/admin/mailu/ui/templates/relay/list.html @@ -11,10 +11,10 @@ {%- endblock %} {%- block content %} -{%- call macros.table() %} +{%- call macros.table(order='[[1,"asc"]]') %} - {% trans %}Actions{% endtrans %} + {% trans %}Actions{% endtrans %} {% trans %}Domain name{% endtrans %} {% trans %}Remote host{% endtrans %} {% trans %}Comment{% endtrans %} @@ -32,8 +32,8 @@ {{ relay.name }} {{ relay.smtp or '-' }} {{ relay.comment or '' }} - {{ relay.created_at | format_date }} - {{ relay.updated_at | format_date }} + {{ relay.created_at | format_date }} + {{ relay.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/token/list.html b/core/admin/mailu/ui/templates/token/list.html index d7c48737..21cd3df7 100644 --- a/core/admin/mailu/ui/templates/token/list.html +++ b/core/admin/mailu/ui/templates/token/list.html @@ -16,7 +16,7 @@ {%- call macros.table() %} - {% trans %}Actions{% endtrans %} + {% trans %}Actions{% endtrans %} {% trans %}Comment{% endtrans %} {% trans %}Authorized IP{% endtrans %} {% trans %}Created{% endtrans %} @@ -31,8 +31,8 @@ {{ token.comment }} {{ token.ip or "any" }} - {{ token.created_at | format_date }} - {{ token.updated_at | format_date }} + {{ token.created_at | format_date }} + {{ token.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/user/list.html b/core/admin/mailu/ui/templates/user/list.html index 7faddab5..96e917cc 100644 --- a/core/admin/mailu/ui/templates/user/list.html +++ b/core/admin/mailu/ui/templates/user/list.html @@ -16,8 +16,8 @@ {%- call macros.table() %} - {% trans %}Actions{% endtrans %} - {% trans %}User settings{% endtrans %} + {% trans %}Actions{% endtrans %} + {% trans %}User settings{% endtrans %} {% trans %}Email{% endtrans %} {% trans %}Features{% endtrans %} {% trans %}Quota{% endtrans %} @@ -39,14 +39,14 @@   {{ user }} - + {% if user.enable_imap %}imap{% endif %} {% if user.enable_pop %}pop3{% endif %} - {{ user.quota_bytes_used | filesizeformat }} / {{ (user.quota_bytes | filesizeformat) if user.quota_bytes else '∞' }} + {{ user.quota_bytes_used | filesizeformat }} / {{ (user.quota_bytes | filesizeformat) if user.quota_bytes else '∞' }} {{ user.comment or '-' }} - {{ user.created_at | format_date }} - {{ user.updated_at | format_date }} + {{ user.created_at | format_date }} + {{ user.updated_at | format_date }} {%- endfor %} From 4776094ea7058273e524fd8acd00e2c400fe08d6 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 14:04:43 +0100 Subject: [PATCH 144/274] Configure datatables on missing tables, add sign in button to sso page. --- core/admin/mailu/sso/templates/sidebar_sso.html | 6 ++++++ core/admin/mailu/ui/templates/client.html | 5 ++++- core/admin/mailu/ui/templates/domain/list.html | 2 ++ core/admin/mailu/ui/templates/fetch/list.html | 2 +- core/admin/mailu/ui/templates/manager/list.html | 4 ++-- core/admin/mailu/ui/templates/user/signup_domain.html | 6 +++--- 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/core/admin/mailu/sso/templates/sidebar_sso.html b/core/admin/mailu/sso/templates/sidebar_sso.html index 86db3333..0f75a1de 100644 --- a/core/admin/mailu/sso/templates/sidebar_sso.html +++ b/core/admin/mailu/sso/templates/sidebar_sso.html @@ -36,6 +36,12 @@
    • {%- endif %} +
  • + + +

    {% trans %}Sign in{% endtrans %}

    +     +
    • {#- User self-registration is only available when - Admin is available diff --git a/core/admin/mailu/ui/templates/client.html b/core/admin/mailu/ui/templates/client.html index bf0ba64e..fddbe0d2 100644 --- a/core/admin/mailu/ui/templates/client.html +++ b/core/admin/mailu/ui/templates/client.html @@ -9,7 +9,6 @@ {%- endblock %} {%- block content %} -
    If you use an Apple device, click here to autoconfigure it.
    {%- call macros.table(title=_("Incoming mail"), datatable=False) %} @@ -59,4 +58,8 @@ {%- endcall %} +
    + {% trans %}If you use an Apple device,{% endtrans %} + {% trans %}click here to autoconfigure it.{% endtrans %} +
    {%- endblock %} diff --git a/core/admin/mailu/ui/templates/domain/list.html b/core/admin/mailu/ui/templates/domain/list.html index a6cefe9e..4363e2b8 100644 --- a/core/admin/mailu/ui/templates/domain/list.html +++ b/core/admin/mailu/ui/templates/domain/list.html @@ -20,6 +20,7 @@ {% trans %}Mailbox count{% endtrans %} {% trans %}Alias count{% endtrans %} {% trans %}Comment{% endtrans %} + {% trans %}Enable sign-up{% endtrans %} {% trans %}Created{% endtrans %} {% trans %}Last edit{% endtrans %} @@ -46,6 +47,7 @@ {{ domain.users | count }} / {{ '∞' if domain.max_users == -1 else domain.max_users }} {{ domain.aliases | count }} / {{ '∞' if domain.max_aliases == -1 else domain.max_aliases }} {{ domain.comment or '' }} + {% if domain.signup_enabled %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} {{ domain.created_at | format_date }} {{ domain.updated_at | format_date }} diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html index 9f0570fd..05a2f095 100644 --- a/core/admin/mailu/ui/templates/fetch/list.html +++ b/core/admin/mailu/ui/templates/fetch/list.html @@ -35,7 +35,7 @@ {{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }} {{ fetch.username }} - {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} + {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} {{ fetch.last_check | format_datetime or '-' }} {{ fetch.error or '-' }} {{ fetch.created_at | format_date }} diff --git a/core/admin/mailu/ui/templates/manager/list.html b/core/admin/mailu/ui/templates/manager/list.html index 706594c4..95dc9f4a 100644 --- a/core/admin/mailu/ui/templates/manager/list.html +++ b/core/admin/mailu/ui/templates/manager/list.html @@ -13,10 +13,10 @@ {%- endblock %} {%- block content %} -{%- call macros.table() %} +{%- call macros.table(order='[[2,"asc"]]') %} - {% trans %}Actions{% endtrans %} + {% trans %}Actions{% endtrans %} {% trans %}Email{% endtrans %} diff --git a/core/admin/mailu/ui/templates/user/signup_domain.html b/core/admin/mailu/ui/templates/user/signup_domain.html index a7db4c97..433d1b3f 100644 --- a/core/admin/mailu/ui/templates/user/signup_domain.html +++ b/core/admin/mailu/ui/templates/user/signup_domain.html @@ -9,7 +9,7 @@ {%- endblock %} {%- block content %} -{%- call macros.table() %} +{%- call macros.table(order='[[1,"asc"]]') %} {% trans %}Domain{% endtrans %} {% trans %}Available slots{% endtrans %} @@ -18,8 +18,8 @@ {%- for domain_name, domain in available_domains.items() %} {{ domain_name }} - {{ '∞' if domain.max_users == -1 else domain.max_users - (domain.users | count)}} - {{ domain.max_quota_bytes or config['DEFAULT_QUOTA'] | filesizeformat }} + {{ '∞' if domain.max_users == -1 else domain.max_users - (domain.users | count)}} + {{ domain.max_quota_bytes or config['DEFAULT_QUOTA'] | filesizeformat }} {%- endfor %} {%- endcall %} From 2778641e7803f7f8aaa8d17a42a2f5ae73011c1d Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 14:43:45 +0100 Subject: [PATCH 145/274] Fix screen reader title of language selector --- core/admin/mailu/sso/templates/base_sso.html | 2 +- core/admin/mailu/ui/templates/base.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/sso/templates/base_sso.html b/core/admin/mailu/sso/templates/base_sso.html index 9dfb25a5..259e5841 100644 --- a/core/admin/mailu/sso/templates/base_sso.html +++ b/core/admin/mailu/sso/templates/base_sso.html @@ -34,7 +34,7 @@