From e5ab9821f9c7b3d80224222d9928526f5d3743bb Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 18 Nov 2022 13:25:02 +0100
Subject: [PATCH 01/54] Add snuffleupagus

This seems to work in my limited testing.
---
 core/base/Dockerfile                     |  18 +++-
 webmails/Dockerfile                      |   6 +-
 webmails/php.ini                         |   2 +
 webmails/roundcube/config/config.inc.php |   2 +-
 webmails/snuffleupagus.rules             | 126 +++++++++++++++++++++++
 webmails/start.py                        |   6 +-
 6 files changed, 155 insertions(+), 5 deletions(-)
 create mode 100644 webmails/snuffleupagus.rules

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index d5be6a90..ca62e773 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -52,16 +52,32 @@ RUN set -euxo pipefail \
         mkdir -p /root/.cargo/registry/index && \
         git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \
     ; pip install -r requirements-${MAILU_DEPS}.txt \
-    ; apk del -r .build-deps \
     ; rm -rf /root/.cargo /tmp/*.pem \
   ; } \
   ; rm -rf /root/.cache
 
+ARG SNUFFLEUPAGUS_VERSION=0.8.3
+ENV SNUFFLEUPAGUS_URL https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v$SNUFFLEUPAGUS_VERSION.tar.gz
+
+RUN set -euxo pipefail \
+  ; curl -sL  ${SNUFFLEUPAGUS_URL} | tar xz \
+  ; cd snuffleupagus-$SNUFFLEUPAGUS_VERSION \
+  ; rm -rf src/tests/*php7*/ src/tests/*session*/ src/tests/broken_configuration/ src/tests/*cookie* src/tests/upload_validation/ \
+  ; apk add --virtual .build-deps php81-dev php81-cgi php81-simplexml php81-xml pcre-dev build-base php81-pear php81-openssl re2c \
+  ; ln -s /usr/bin/phpize81 /usr/bin/phpize \
+  ; ln -s /usr/bin/pecl81 /usr/bin/pecl \
+  ; ln -s /usr/bin/php-config81 /usr/bin/php-config \
+  ; ln -s /usr/bin/php81 /usr/bin/php \
+  ; pecl install vld-beta \
+  ; make -j $(grep -c processor /proc/cpuinfo) release \
+  ; cp src/.libs/snuffleupagus.so /app \
+  ; apk del -r .build-deps
 
 # base mailu image
 FROM system
 
 COPY --from=build /app/venv/ /app/venv/
+COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/
 
 ENV VIRTUAL_ENV=/app/venv
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index 376399bf..0ec12384 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -21,6 +21,8 @@ RUN set -euxo pipefail \
   ; ln -s /usr/bin/php81 /usr/bin/php \
   ; gpg --import /tmp/snappymail.asc \
   ; gpg --import /tmp/roundcube.asc \
+  ; echo extension=snuffleupagus > /etc/php81/conf.d/snuffleupagus.ini \
+  ; rm -f /tmp/*asc \
   ; mkdir -p /run/nginx \
   ; mkdir -p /conf
 
@@ -41,7 +43,8 @@ RUN set -euxo pipefail \
   ; cd roundcube \
   ; rm -rf CHANGELOG.md SECURITY.md INSTALL LICENSE README.md UPGRADING composer.json-dist installer composer.* \
   ; ln -sf index.php /var/www/roundcube/public_html/sso.php \
-  ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query}
+  ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query} \
+  ; sed -i '/suhosin.session.encrypt/d;/mbstring\.func_overload/d' program/lib/Roundcube/bootstrap.php
 
 COPY roundcube/config/config.inc.php /conf/
 COPY roundcube/login/mailu.php /var/www/roundcube/plugins/mailu/
@@ -81,6 +84,7 @@ COPY start.py /
 COPY php.ini /defaults/
 COPY php-webmail.conf /etc/php81/php-fpm.d/php-webmail.conf
 COPY nginx-webmail.conf /conf/
+COPY snuffleupagus.rules /etc/snuffleupagus.rules.tpl
 
 EXPOSE 80/tcp
 VOLUME /data
diff --git a/webmails/php.ini b/webmails/php.ini
index d9ba892c..87660288 100644
--- a/webmails/php.ini
+++ b/webmails/php.ini
@@ -11,3 +11,5 @@ log_errors=On
 zlib.output_compression=Off
 access.log = /dev/fd/2
 error_log = /dev/fd/2
+module=snuffleupagus.so
+sp.configuration_file=/etc/snuffleupagus.rules
diff --git a/webmails/roundcube/config/config.inc.php b/webmails/roundcube/config/config.inc.php
index 6e5ea0bd..e8aedeff 100644
--- a/webmails/roundcube/config/config.inc.php
+++ b/webmails/roundcube/config/config.inc.php
@@ -5,7 +5,7 @@ $config = array();
 // Generals
 $config['db_dsnw'] = '{{ DB_DSNW }}';
 $config['temp_dir'] = '/dev/shm/';
-$config['des_key'] = '{{ SECRET_KEY }}';
+$config['des_key'] = '{{ ROUNDCUBE_KEY }}';
 $config['cipher_method'] = 'AES-256-CBC';
 $config['identities_level'] = 0;
 $config['reply_all_mode'] = 1;
diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules
new file mode 100644
index 00000000..35f0b7ce
--- /dev/null
+++ b/webmails/snuffleupagus.rules
@@ -0,0 +1,126 @@
+# This is based on default configuration file for Snuffleupagus (https://snuffleupagus.rtfd.io),
+# for php8.
+# It contains "reasonable" defaults that won't break your websites,
+# and a lot of commented directives that you can enable if you want to
+# have a better protection.
+
+# Harden the PRNG
+sp.harden_random.enable();
+
+# Disabled XXE
+sp.xxe_protection.enable();
+
+# Global configuration variables
+sp.global.secret_key("{{ SNUFFLEPAGUS_KEY }}");
+
+# Globally activate strict mode
+# https://www.php.net/manual/en/language.types.declarations.php#language.types.declarations.strict
+sp.global_strict.enable();
+
+# Prevent unserialize-related exploits
+# sp.unserialize_hmac.enable();
+
+# Only allow execution of read-only files. This is a low-hanging fruit that you should enable.
+sp.readonly_exec.enable();
+
+# PHP has a lot of wrappers, most of them aren't usually useful, you should
+# only enable the ones you're using.
+sp.wrappers_whitelist.list("file,php,phar,mailsosubstreams");
+
+# Prevent sloppy comparisons.
+sp.sloppy_comparison.enable();
+
+# Use SameSite on session cookie
+# https://snuffleupagus.readthedocs.io/features.html#protection-against-cross-site-request-forgery
+sp.cookie.name("PHPSESSID").samesite("lax");
+
+# Harden the `chmod` function (0777 (oct = 511, 0666 = 438)
+sp.disable_function.function("chmod").param("permissions").value("438").drop();
+sp.disable_function.function("chmod").param("permissions").value("511").drop();
+
+# Prevent various `mail`-related vulnerabilities
+sp.disable_function.function("mail").param("additional_parameters").value_r("\\-").drop();
+
+# Since it's now burned, me might as well mitigate it publicly
+sp.disable_function.function("putenv").param("assignment").value_r("LD_").drop()
+
+# This one was burned in Nov 2019 - https://gist.github.com/LoadLow/90b60bd5535d6c3927bb24d5f9955b80
+sp.disable_function.function("putenv").param("assignment").value_r("GCONV_").drop()
+
+# Since people are stupid enough to use `extract` on things like $_GET or $_POST, we might as well mitigate this vector
+sp.disable_function.function("extract").param("array").value_r("^_").drop()
+sp.disable_function.function("extract").param("flags").value("0").drop()
+
+# This is also burned:
+# ini_set('open_basedir','..');chdir('..');…;chdir('..');ini_set('open_basedir','/');echo(file_get_contents('/etc/passwd'));
+# Since we have no way of matching on two parameters at the same time, we're
+# blocking calls to open_basedir altogether: nobody is using it via ini_set anyway.
+# Moreover, there are non-public bypasses that are also using this vector ;)
+sp.disable_function.function("ini_set").param("option").value_r("open_basedir").drop()
+
+# Prevent various `include`-related vulnerabilities
+sp.disable_function.function("require_once").value_r("\.(inc|phtml|php)$").allow();
+sp.disable_function.function("include_once").value_r("\.(inc|phtml|php)$").allow();
+sp.disable_function.function("require").value_r("\.(inc|phtml|php)$").allow();
+sp.disable_function.function("include").value_r("\.(inc|phtml|php)$").allow();
+sp.disable_function.function("require_once").drop()
+sp.disable_function.function("include_once").drop()
+sp.disable_function.function("require").drop()
+sp.disable_function.function("include").drop()
+
+# Prevent `system`-related injections
+sp.disable_function.function("system").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop();
+sp.disable_function.function("shell_exec").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop();
+sp.disable_function.function("exec").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop();
+sp.disable_function.function("proc_open").param("command").value_r("^gpg ").allow();
+sp.disable_function.function("proc_open").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop();
+
+# Prevent runtime modification of interesting things
+sp.disable_function.function("ini_set").param("option").value("assert.active").drop();
+sp.disable_function.function("ini_set").param("option").value("zend.assertions").drop();
+sp.disable_function.function("ini_set").param("option").value("memory_limit").drop();
+sp.disable_function.function("ini_set").param("option").value("include_path").drop();
+sp.disable_function.function("ini_set").param("option").value("open_basedir").drop();
+
+# Detect some backdoors via environment recon
+sp.disable_function.function("ini_get").param("option").value("allow_url_fopen").drop();
+sp.disable_function.function("ini_get").param("option").value("open_basedir").drop();
+sp.disable_function.function("ini_get").param("option").value_r("suhosin").drop();
+sp.disable_function.function("function_exists").param("function").value("eval").drop();
+sp.disable_function.function("function_exists").param("function").value("exec").drop();
+sp.disable_function.function("function_exists").param("function").value("system").drop();
+sp.disable_function.function("function_exists").param("function").value("shell_exec").drop();
+sp.disable_function.function("function_exists").param("function").value("proc_open").drop();
+sp.disable_function.function("function_exists").param("function").value("passthru").drop();
+sp.disable_function.function("is_callable").param("value").value("eval").drop();
+sp.disable_function.function("is_callable").param("value").value("exec").drop();
+sp.disable_function.function("is_callable").param("value").value("system").drop();
+sp.disable_function.function("is_callable").param("value").value("shell_exec").drop();
+sp.disable_function.function("is_callable").filename_r("/app/libraries/snappymail/pgp/gpg\.php$").param("value").value("proc_open").allow();
+sp.disable_function.function("is_callable").param("value").value("proc_open").drop();
+sp.disable_function.function("is_callable").param("value").value("passthru").drop();
+
+# Ghetto error-based sqli detection
+#sp.disable_function.function("mysql_query").ret("FALSE").drop();
+#sp.disable_function.function("mysqli_query").ret("FALSE").drop();
+#sp.disable_function.function("PDO::query").ret("FALSE").drop();
+
+# Ensure that certificates are properly verified
+sp.disable_function.function("curl_setopt").param("value").value("1").allow();
+sp.disable_function.function("curl_setopt").param("value").value("2").allow();
+# `81` is SSL_VERIFYHOST and `64` SSL_VERIFYPEER
+sp.disable_function.function("curl_setopt").param("option").value("64").drop().alias("Please don't turn CURLOPT_SSL_VERIFYCLIENT off.");
+sp.disable_function.function("curl_setopt").param("option").value("81").drop().alias("Please don't turn CURLOPT_SSL_VERIFYHOST off.");
+
+# File upload
+sp.disable_function.function("move_uploaded_file").param("to").value_r("\\.ph").drop();
+sp.disable_function.function("move_uploaded_file").param("to").value_r("\\.ht").drop();
+
+# Logging lockdown
+sp.disable_function.function("ini_set").param("option").value_r("error_log").drop()
+sp.disable_function.function("ini_set").param("option").value_r("display_errors").drop()
+
+sp.auto_cookie_secure.enable();
+sp.cookie.name("roundcube_sessauth").samesite("strict");
+sp.cookie.name("roundcube_sessid").samesite("strict");
+sp.ini_protection.policy_silent_fail();
diff --git a/webmails/start.py b/webmails/start.py
index 06b90351..bf4472cd 100755
--- a/webmails/start.py
+++ b/webmails/start.py
@@ -51,7 +51,9 @@ if not secret_key:
         print(f"Can't read SECRET_KEY from file: {exc}", file=sys.stderr)
         exit(2)
 
-context['SECRET_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest()
+context['ROUNDCUBE_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest()
+context['SNUFFLEPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEPAGUS_KEY', 'utf-8'), 'sha256').hexdigest()
+conf.jinja("/etc/snuffleupagus.rules.tpl", context, "/etc/snuffleupagus.rules")
 
 # roundcube plugins
 # (using "dict" because it is ordered and "set" is not)
@@ -118,7 +120,7 @@ if os.path.exists("/var/run/nginx.pid"):
     os.system("nginx -s reload")
 
 # clean env
-[env.pop(key, None) for key in env.keys() if key == "SECRET_KEY" or key.startswith("ROUNDCUBE_")]
+[env.pop(key, None) for key in env.keys() if key == "SECRET_KEY" or key.endswith("_KEY")]
 
 # run nginx
 os.system("php-fpm81")

From 2a4f6836cfa66257c9c2aad4c52a7d9984bfeffe Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 18 Nov 2022 15:39:32 +0100
Subject: [PATCH 02/54] protect unserialize()

---
 webmails/snuffleupagus.rules | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules
index 35f0b7ce..f85b7fe4 100644
--- a/webmails/snuffleupagus.rules
+++ b/webmails/snuffleupagus.rules
@@ -124,3 +124,5 @@ sp.auto_cookie_secure.enable();
 sp.cookie.name("roundcube_sessauth").samesite("strict");
 sp.cookie.name("roundcube_sessid").samesite("strict");
 sp.ini_protection.policy_silent_fail();
+
+sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:\"").drop();

From 017ea5298e75749a97c394201ba9ebeb2f468a40 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 18 Nov 2022 15:52:05 +0100
Subject: [PATCH 03/54] typo

---
 webmails/snuffleupagus.rules | 2 +-
 webmails/start.py            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules
index f85b7fe4..df9dadc4 100644
--- a/webmails/snuffleupagus.rules
+++ b/webmails/snuffleupagus.rules
@@ -11,7 +11,7 @@ sp.harden_random.enable();
 sp.xxe_protection.enable();
 
 # Global configuration variables
-sp.global.secret_key("{{ SNUFFLEPAGUS_KEY }}");
+sp.global.secret_key("{{ SNUFFLEUPAGUS_KEY }}");
 
 # Globally activate strict mode
 # https://www.php.net/manual/en/language.types.declarations.php#language.types.declarations.strict
diff --git a/webmails/start.py b/webmails/start.py
index bf4472cd..f87ac55f 100755
--- a/webmails/start.py
+++ b/webmails/start.py
@@ -52,7 +52,7 @@ if not secret_key:
         exit(2)
 
 context['ROUNDCUBE_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest()
-context['SNUFFLEPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEPAGUS_KEY', 'utf-8'), 'sha256').hexdigest()
+context['SNUFFLEUPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEUPAGUS_KEY', 'utf-8'), 'sha256').hexdigest()
 conf.jinja("/etc/snuffleupagus.rules.tpl", context, "/etc/snuffleupagus.rules")
 
 # roundcube plugins

From 840b2bd9df8ac45c5fee36f449eebac8435a50d6 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 18 Nov 2022 16:00:31 +0100
Subject: [PATCH 04/54] block o:0:{} too

---
 webmails/snuffleupagus.rules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules
index df9dadc4..baa5ecf8 100644
--- a/webmails/snuffleupagus.rules
+++ b/webmails/snuffleupagus.rules
@@ -125,4 +125,4 @@ sp.cookie.name("roundcube_sessauth").samesite("strict");
 sp.cookie.name("roundcube_sessid").samesite("strict");
 sp.ini_protection.policy_silent_fail();
 
-sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:\"").drop();
+sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:[\"{]").drop();

From e2d4e3eb2ef3499da2e994541e9b10ed074b6db3 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sat, 19 Nov 2022 17:59:31 +0100
Subject: [PATCH 05/54] Implement header authentication via external proxy

---
 core/admin/mailu/configuration.py    |  4 ++++
 core/admin/mailu/sso/views/base.py   | 35 ++++++++++++++++++++++++++++
 docs/configuration.rst               | 12 ++++++++++
 towncrier/newsfragments/1972.feature |  1 +
 4 files changed, 52 insertions(+)
 create mode 100644 towncrier/newsfragments/1972.feature

diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py
index d282bfc2..d447e570 100644
--- a/core/admin/mailu/configuration.py
+++ b/core/admin/mailu/configuration.py
@@ -80,6 +80,9 @@ DEFAULT_CONFIG = {
     'TLS_PERMISSIVE': True,
     'TZ': 'Etc/UTC',
     'DEFAULT_SPAM_THRESHOLD': 80,
+    'PROXY_AUTH_WHITELIST': '',
+    'PROXY_AUTH_HEADER': 'X-Auth-Email',
+    'PROXY_AUTH_CREATE': False,
     # Host settings
     'HOST_IMAP': 'imap',
     'HOST_LMTP': 'imap:2525',
@@ -171,6 +174,7 @@ class ConfigManager:
         self.config['HOSTNAMES'] = ','.join(hostnames)
         self.config['HOSTNAME'] = hostnames[0]
         self.config['DEFAULT_SPAM_THRESHOLD'] = int(self.config['DEFAULT_SPAM_THRESHOLD'])
+        self.config['PROXY_AUTH_WHITELIST'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['PROXY_AUTH_WHITELIST'].split(',')) if cidr)
 
         # update the app config
         app.config.update(self.config)
diff --git a/core/admin/mailu/sso/views/base.py b/core/admin/mailu/sso/views/base.py
index 6fa9403f..600f62f4 100644
--- a/core/admin/mailu/sso/views/base.py
+++ b/core/admin/mailu/sso/views/base.py
@@ -6,6 +6,8 @@ from mailu.ui import access
 from flask import current_app as app
 import flask
 import flask_login
+import secrets
+import ipaddress
 
 @sso.route('/login', methods=['GET', 'POST'])
 def login():
@@ -57,3 +59,36 @@ def logout():
     flask.session.destroy()
     return flask.redirect(flask.url_for('.login'))
 
+
+@sso.route('/proxy', methods=['GET'])
+@sso.route('/proxy/<target>', methods=['GET'])
+def proxy(target='webmail'):
+    ip = ipaddress.ip_address(flask.request.remote_addr)
+    if not any(ip in cidr for cidr in app.config['PROXY_AUTH_WHITELIST']):
+        return flask.abort(500, '%s is not on PROXY_AUTH_WHITELIST' % flask.request.remote_addr)
+
+    email = flask.request.headers.get(app.config['PROXY_AUTH_HEADER'], None)
+    if not email:
+        return flask.abort(500, 'No %s header' % app.config['PROXY_AUTH_HEADER'])
+
+    user = models.User.get(email)
+    if user:
+        flask.session.regenerate()
+        flask_login.login_user(user)
+        return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])
+    elif app.config['PROXY_AUTH_CREATE']:
+        localpart, desireddomain = email.split('@')
+        domain = models.Domain.query.get(desireddomain) or flask.abort(500, 'You don\'t exist. Go away! (domain=%s)' % desireddomain)
+        if not domain.max_users == -1 and len(domain.users) >= domain.max_users:
+            flask.current_app.logger.warning('Too many users for domain %s' % domain)
+            return flask.abort(500, 'Too many users in (domain=%s)' % domain)
+        user = models.User(localpart=localpart, domain=domain)
+        user.set_password(secrets.token_urlsafe())
+        models.db.session.add(user)
+        models.db.session.commit()
+        user.send_welcome()
+        client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr)
+        flask.current_app.logger.info(f'Login succeeded by proxy created user: {user} from {client_ip} through {flask.request.remote_addr}.')
+        return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])
+    else:
+        return flask.abort(500, 'You don\'t exist. Go away! (%s)' % email)
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 1a40bf65..b5affad6 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -362,3 +362,15 @@ It can be configured with the following option:
 When ``POSTFIX_LOG_FILE`` is enabled, the logrotate program will automatically rotate the 
 logs every week and keep 52 logs. To override the logrotate configuration, create the file logrotate.conf 
 with the desired configuration in the :ref:`Postfix overrides folder<override-label>`.
+
+
+Header authentication using an external proxy
+---------------------------------------------
+
+The ``PROXY_AUTH_WHITELIST`` (default: unset/disabled) option allows you to configure a comma separated list of CIDRs of proxies to trust for authentication. This list is separate from ``REAL_IP_FROM`` and any entry in ``PROXY_AUTH_WHITELIST`` should also appear in ``REAL_IP_FROM``.
+
+Use ``PROXY_AUTH_HEADER`` (default: 'X-Auth-Email') to customize which HTTP header the email address of the user to authenticate as should be and ``PROXY_AUTH_CREATE`` (default: False) to control whether non-existing accounts should be auto-created. Please note that Mailu doesn't currently support creating new users for non-existing domains; you do need to create all the domains that may be used manually.
+
+Once configured, any request to /sso/proxy will be redirected to the webmail and /sso/proxy/admin to the admin panel. Please check issue `1972` for more details.
+
+.. _`1972`: https://github.com/Mailu/Mailu/issues/1972
diff --git a/towncrier/newsfragments/1972.feature b/towncrier/newsfragments/1972.feature
new file mode 100644
index 00000000..4efe45c9
--- /dev/null
+++ b/towncrier/newsfragments/1972.feature
@@ -0,0 +1 @@
+Implement Header authentication via external proxy

From cf7404e26c8357df43e0a72988497244c2e65734 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sat, 19 Nov 2022 20:33:20 +0100
Subject: [PATCH 06/54] Fix #2242: Make quotas adjustable in 50MiB increments

---
 core/admin/mailu/ui/templates/user/create.html | 2 +-
 towncrier/newsfragments/2242.misc              | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 towncrier/newsfragments/2242.misc

diff --git a/core/admin/mailu/ui/templates/user/create.html b/core/admin/mailu/ui/templates/user/create.html
index 9a32243d..1d597179 100644
--- a/core/admin/mailu/ui/templates/user/create.html
+++ b/core/admin/mailu/ui/templates/user/create.html
@@ -21,7 +21,7 @@
   {%- endcall %}
 
   {%- call macros.card(_("Features and quotas"), theme="success") %}
-  {{ macros.form_field(form.quota_bytes, step=1000000000, max=(max_quota_bytes or domain.max_quota_bytes or 50*10**9), data_infinity="true",
+  {{ macros.form_field(form.quota_bytes, step=50000000, max=(max_quota_bytes or domain.max_quota_bytes or 50*10**9), data_infinity="true",
       prepend='<span class="input-group-text"><span id="quota_bytes_value"></span>&nbsp;GB</span>') }}
   {{ macros.form_field(form.enable_imap) }}
   {{ macros.form_field(form.enable_pop) }}
diff --git a/towncrier/newsfragments/2242.misc b/towncrier/newsfragments/2242.misc
new file mode 100644
index 00000000..cc03e55e
--- /dev/null
+++ b/towncrier/newsfragments/2242.misc
@@ -0,0 +1 @@
+Make quotas adjustable in 50MiB increments

From 6a22c82c022ccf7b8c4e8b0355590db8e4f34cef Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 20 Nov 2022 10:17:19 +0100
Subject: [PATCH 07/54] Fix run_dev

---
 core/admin/run_dev.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh
index cf05fba3..05bf6548 100755
--- a/core/admin/run_dev.sh
+++ b/core/admin/run_dev.sh
@@ -87,7 +87,7 @@ EOF
 
 # build
 chmod -R u+rwX,go+rX .
-"${docker}" build --tag "${DEV_NAME}:latest" .
+"${docker}" build --build-arg TARGETPLATFORM=linux/amd64 --tag "${DEV_NAME}:latest" .
 
 # gather volumes to map into container
 volumes=()

From 38507b2e1be890f2d7603b5401d245f64613bfed Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 20 Nov 2022 10:19:28 +0100
Subject: [PATCH 08/54] Close #2372: Implement a GUI for WILDCARD_SENDERS

---
 core/admin/mailu/internal/views/postfix.py    |  3 ++-
 core/admin/mailu/models.py                    |  1 +
 core/admin/mailu/ui/forms.py                  |  1 +
 .../admin/mailu/ui/templates/user/create.html |  1 +
 .../migrations/versions/7ac252f2bbbf_.py      | 22 +++++++++++++++++++
 docs/webadministration.rst                    |  4 +++-
 towncrier/newsfragments/2372.feature          |  1 +
 7 files changed, 31 insertions(+), 2 deletions(-)
 create mode 100644 core/admin/migrations/versions/7ac252f2bbbf_.py
 create mode 100644 towncrier/newsfragments/2372.feature

diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py
index 8188270c..c0a17319 100644
--- a/core/admin/mailu/internal/views/postfix.py
+++ b/core/admin/mailu/internal/views/postfix.py
@@ -145,8 +145,9 @@ def postfix_sender_login(sender):
     localpart = localpart[:next((i for i, ch in enumerate(localpart) if ch in flask.current_app.config.get('RECIPIENT_DELIMITER')), None)]
     destinations = models.Email.resolve_destination(localpart, domain_name, True) or []
     destinations.extend(wildcard_senders)
+    destinations.extend(i[0] for i in models.User.query.filter_by(allow_spoofing=True).with_entities(models.User.email).all())
     if destinations:
-        return flask.jsonify(",".join(idna_encode(destinations)))
+        return flask.jsonify(",".join(idna_encode(list(set(destinations)))))
     return flask.abort(404)
 
 @internal.route("/postfix/sender/rate/<path:sender>")
diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index 48ce8b33..1c57c8be 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -501,6 +501,7 @@ class User(Base, Email):
     # Features
     enable_imap = db.Column(db.Boolean, nullable=False, default=True)
     enable_pop = db.Column(db.Boolean, nullable=False, default=True)
+    allow_spoofing = db.Column(db.Boolean, nullable=False, default=False)
 
     # Filters
     forward_enabled = db.Column(db.Boolean, nullable=False, default=False)
diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py
index beb44092..3882064d 100644
--- a/core/admin/mailu/ui/forms.py
+++ b/core/admin/mailu/ui/forms.py
@@ -84,6 +84,7 @@ class UserForm(flask_wtf.FlaskForm):
     quota_bytes = fields_.IntegerSliderField(_('Quota'), default=10**9)
     enable_imap = fields.BooleanField(_('Allow IMAP access'), default=True)
     enable_pop = fields.BooleanField(_('Allow POP3 access'), default=True)
+    allow_spoofing = fields.BooleanField(_('Allow the user to spoof the sender (send email as anyone)'), default=False)
     displayed_name = fields.StringField(_('Displayed name'))
     comment = fields.StringField(_('Comment'))
     enabled = fields.BooleanField(_('Enabled'), default=True)
diff --git a/core/admin/mailu/ui/templates/user/create.html b/core/admin/mailu/ui/templates/user/create.html
index 9a32243d..7e1c9122 100644
--- a/core/admin/mailu/ui/templates/user/create.html
+++ b/core/admin/mailu/ui/templates/user/create.html
@@ -25,6 +25,7 @@
       prepend='<span class="input-group-text"><span id="quota_bytes_value"></span>&nbsp;GB</span>') }}
   {{ macros.form_field(form.enable_imap) }}
   {{ macros.form_field(form.enable_pop) }}
+  {{ macros.form_field(form.allow_spoofing) }}
   {%- endcall %}
 
   {{ macros.form_field(form.submit) }}
diff --git a/core/admin/migrations/versions/7ac252f2bbbf_.py b/core/admin/migrations/versions/7ac252f2bbbf_.py
new file mode 100644
index 00000000..0be19d88
--- /dev/null
+++ b/core/admin/migrations/versions/7ac252f2bbbf_.py
@@ -0,0 +1,22 @@
+"""empty message
+
+Revision ID: 7ac252f2bbbf
+Revises: 8f9ea78776f4
+Create Date: 2022-11-20 08:57:16.879152
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '7ac252f2bbbf'
+down_revision = '8f9ea78776f4'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('user', sa.Column('allow_spoofing', sa.Boolean(), nullable=False))
+
+
+def downgrade():
+    op.drop_column('user', 'allow_spoofing')
diff --git a/docs/webadministration.rst b/docs/webadministration.rst
index e17d12f0..2e0de745 100644
--- a/docs/webadministration.rst
+++ b/docs/webadministration.rst
@@ -321,7 +321,7 @@ This page also shows an overview of the following settings of an user:
 
 * Email. The email address of the user.
 
-* Features. Shows if IMAP or POP3 access is enabled.
+* Features. Shows if IMAP or POP3 access is enabled and whether the user should be allowed to spoof emails.
 
 * Storage quota. Shows how much assigned storage has been consumed.
 
@@ -357,6 +357,8 @@ For adding a new user the following options can be configured.
 
 * Allow POP3 access. When ticked, allows email retrieval via the POP3 protocol.
 
+* Allow the user to spoof the sender. When ticked, allows the user to send email as anyone.
+
 
 Aliases
 ```````
diff --git a/towncrier/newsfragments/2372.feature b/towncrier/newsfragments/2372.feature
new file mode 100644
index 00000000..ec2c5bef
--- /dev/null
+++ b/towncrier/newsfragments/2372.feature
@@ -0,0 +1 @@
+Create a GUI for WILDCARD_SENDERS

From ef9cc3c866c64be67391e6c85090cb79dedfcbc5 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 20 Nov 2022 11:09:04 +0100
Subject: [PATCH 09/54] Show spoofing on /admin/user/list too

---
 core/admin/mailu/ui/templates/user/list.html | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/admin/mailu/ui/templates/user/list.html b/core/admin/mailu/ui/templates/user/list.html
index 14626212..ac74a675 100644
--- a/core/admin/mailu/ui/templates/user/list.html
+++ b/core/admin/mailu/ui/templates/user/list.html
@@ -39,9 +39,10 @@
       <a href="{{ url_for('.fetch_list', user_email=user.email) }}" title="{% trans %}Fetched accounts{% endtrans %}"><i class="fa fa-download"></i></a>&nbsp;
     </td>
     <td>{{ user }}</td>
-    <td data-sort="{{ user.enable_imap*2 + user.enable_pop }}">
+    <td data-sort="{{ user.enable_imap*2 + user.enable_pop + user.allow_spoofing }}">
       {% if user.enable_imap %}<span class="badge bg-info">imap</span>{% endif %}
       {% if user.enable_pop %}<span class="badge bg-info">pop3</span>{% endif %}
+      {% if user.allow_spoofing %}<span class="badge bg-info">allow-spoofing</span>{% endif %}
     </td>
     <td data-sort="{{ user.quota_bytes_used }}">{{ user.quota_bytes_used | filesizeformat }} / {{ (user.quota_bytes | filesizeformat) if user.quota_bytes else '∞' }}</td>
     <td>{{ user.comment or '-' }}</td>

From 7822b41048ec2cec8ef0b71080de636879f0aab1 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 20 Nov 2022 08:48:13 +0100
Subject: [PATCH 10/54] same for domains

---
 core/admin/mailu/ui/templates/domain/create.html | 2 +-
 core/admin/mailu/ui/templates/user/create.html   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/admin/mailu/ui/templates/domain/create.html b/core/admin/mailu/ui/templates/domain/create.html
index bc407d27..f0d5308d 100644
--- a/core/admin/mailu/ui/templates/domain/create.html
+++ b/core/admin/mailu/ui/templates/domain/create.html
@@ -10,7 +10,7 @@
   {{ form.hidden_tag() }}
   {{ macros.form_field(form.name) }}
   {{ macros.form_fields((form.max_users, form.max_aliases)) }}
-  {{ macros.form_field(form.max_quota_bytes, step=10**9, max=50*10**9, data_infinity="true",
+  {{ macros.form_field(form.max_quota_bytes, step=50*10**6, max=50*10**9, data_infinity="true",
       prepend='<span class="input-group-text"><span id="max_quota_bytes_value"></span>&nbsp;GB</span>') }}
   {{ macros.form_field(form.signup_enabled) }}
   {{ macros.form_field(form.comment) }}
diff --git a/core/admin/mailu/ui/templates/user/create.html b/core/admin/mailu/ui/templates/user/create.html
index 1d597179..5369621c 100644
--- a/core/admin/mailu/ui/templates/user/create.html
+++ b/core/admin/mailu/ui/templates/user/create.html
@@ -21,7 +21,7 @@
   {%- endcall %}
 
   {%- call macros.card(_("Features and quotas"), theme="success") %}
-  {{ macros.form_field(form.quota_bytes, step=50000000, max=(max_quota_bytes or domain.max_quota_bytes or 50*10**9), data_infinity="true",
+  {{ macros.form_field(form.quota_bytes, step=50*10**6, max=(max_quota_bytes or domain.max_quota_bytes or 50*10**9), data_infinity="true",
       prepend='<span class="input-group-text"><span id="quota_bytes_value"></span>&nbsp;GB</span>') }}
   {{ macros.form_field(form.enable_imap) }}
   {{ macros.form_field(form.enable_pop) }}

From d5ac9199a08f5bfbff81c7268c5c133b5994cf0c Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Sun, 20 Nov 2022 14:59:06 +0100
Subject: [PATCH 11/54] Update 7ac252f2bbbf_.py

---
 core/admin/migrations/versions/7ac252f2bbbf_.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/migrations/versions/7ac252f2bbbf_.py b/core/admin/migrations/versions/7ac252f2bbbf_.py
index 0be19d88..11a30453 100644
--- a/core/admin/migrations/versions/7ac252f2bbbf_.py
+++ b/core/admin/migrations/versions/7ac252f2bbbf_.py
@@ -1,4 +1,4 @@
-"""empty message
+""" Add user.allow_spoofing
 
 Revision ID: 7ac252f2bbbf
 Revises: 8f9ea78776f4

From e94f6eaf33f726e33bfab2571f81f58f409d1331 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 22 Nov 2022 10:11:23 +0100
Subject: [PATCH 12/54] towncrier

---
 towncrier/newsfragments/2550.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 towncrier/newsfragments/2550.misc

diff --git a/towncrier/newsfragments/2550.misc b/towncrier/newsfragments/2550.misc
new file mode 100644
index 00000000..fcd5dacf
--- /dev/null
+++ b/towncrier/newsfragments/2550.misc
@@ -0,0 +1 @@
+Add Snuffleupagus to protect webmails (a Suhosin replacement)

From 9fa3a3e0c7283da2d8fdb6a9e2de4d6decfef9b4 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 22 Nov 2022 10:14:15 +0100
Subject: [PATCH 13/54] doc

---
 README.md                    | 2 +-
 docs/index.rst               | 2 +-
 webmails/snuffleupagus.rules | 5 +++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 0fd737b6..b6ed040b 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Main features include:
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts
 - **Admin features**, global admins, announcements, per-domain delegation, quotas
-- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner
+- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, [Snuffleupagus](https://github.com/jvoisin/snuffleupagus/)
 - **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing
 - **Freedom**, all FOSS components, no tracker included
 
diff --git a/docs/index.rst b/docs/index.rst
index 5c004dc1..0b37cf43 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -28,7 +28,7 @@ Main features include:
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts
 - **Admin features**, global admins, announcements, per-domain delegation, quotas
-- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner
+- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, Snuffleupagus
 - **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing
 - **Freedom**, all FOSS components, no tracker included
 
diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules
index baa5ecf8..ec7bee13 100644
--- a/webmails/snuffleupagus.rules
+++ b/webmails/snuffleupagus.rules
@@ -72,6 +72,7 @@ sp.disable_function.function("include").drop()
 sp.disable_function.function("system").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop();
 sp.disable_function.function("shell_exec").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop();
 sp.disable_function.function("exec").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop();
+# This is **very** broad but doing better is non-straightforward
 sp.disable_function.function("proc_open").param("command").value_r("^gpg ").allow();
 sp.disable_function.function("proc_open").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop();
 
@@ -121,8 +122,12 @@ sp.disable_function.function("ini_set").param("option").value_r("error_log").dro
 sp.disable_function.function("ini_set").param("option").value_r("display_errors").drop()
 
 sp.auto_cookie_secure.enable();
+# TODO: consider encrypting the cookies?
+# TODO: ensure this is up to date
 sp.cookie.name("roundcube_sessauth").samesite("strict");
 sp.cookie.name("roundcube_sessid").samesite("strict");
 sp.ini_protection.policy_silent_fail();
 
+# roundcube uses unserialize() everywhere.
+# This should do the job until https://github.com/jvoisin/snuffleupagus/issues/438 is implemented.
 sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:[\"{]").drop();

From ee512112fb10b62afd9d1861d4da90b6f73c4b85 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Wed, 23 Nov 2022 17:07:19 +0100
Subject: [PATCH 14/54] fix flask db history

---
 core/admin/migrations/versions/7ac252f2bbbf_.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/migrations/versions/7ac252f2bbbf_.py b/core/admin/migrations/versions/7ac252f2bbbf_.py
index 11a30453..c37bf554 100644
--- a/core/admin/migrations/versions/7ac252f2bbbf_.py
+++ b/core/admin/migrations/versions/7ac252f2bbbf_.py
@@ -8,7 +8,7 @@ Create Date: 2022-11-20 08:57:16.879152
 
 # revision identifiers, used by Alembic.
 revision = '7ac252f2bbbf'
-down_revision = '8f9ea78776f4'
+down_revision = 'f4f0f89e0047'
 
 from alembic import op
 import sqlalchemy as sa

From 4d8bd210c542c1859776b9640e55f41da8195af5 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Wed, 23 Nov 2022 17:07:48 +0100
Subject: [PATCH 15/54] Update run_dev.sh

---
 core/admin/run_dev.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh
index 05bf6548..cf05fba3 100755
--- a/core/admin/run_dev.sh
+++ b/core/admin/run_dev.sh
@@ -87,7 +87,7 @@ EOF
 
 # build
 chmod -R u+rwX,go+rX .
-"${docker}" build --build-arg TARGETPLATFORM=linux/amd64 --tag "${DEV_NAME}:latest" .
+"${docker}" build --tag "${DEV_NAME}:latest" .
 
 # gather volumes to map into container
 volumes=()

From c1144612be1d1fd689ad19b28869a5f2837843cd Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Wed, 23 Nov 2022 17:13:15 +0100
Subject: [PATCH 16/54] fix sorting

---
 core/admin/mailu/ui/templates/user/list.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/mailu/ui/templates/user/list.html b/core/admin/mailu/ui/templates/user/list.html
index ac74a675..84c0cd27 100644
--- a/core/admin/mailu/ui/templates/user/list.html
+++ b/core/admin/mailu/ui/templates/user/list.html
@@ -39,7 +39,7 @@
       <a href="{{ url_for('.fetch_list', user_email=user.email) }}" title="{% trans %}Fetched accounts{% endtrans %}"><i class="fa fa-download"></i></a>&nbsp;
     </td>
     <td>{{ user }}</td>
-    <td data-sort="{{ user.enable_imap*2 + user.enable_pop + user.allow_spoofing }}">
+    <td data-sort="{{ user.allow_spoofing*4 + user.enable_imap*2 + user.enable_pop }}">
       {% if user.enable_imap %}<span class="badge bg-info">imap</span>{% endif %}
       {% if user.enable_pop %}<span class="badge bg-info">pop3</span>{% endif %}
       {% if user.allow_spoofing %}<span class="badge bg-info">allow-spoofing</span>{% endif %}

From 546884d10cc5ba995044891051a4c3278f7d686d Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 09:31:27 +0100
Subject: [PATCH 17/54] ghost's requested changes

---
 core/admin/mailu/sso/views/base.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/core/admin/mailu/sso/views/base.py b/core/admin/mailu/sso/views/base.py
index 600f62f4..b7fe9c97 100644
--- a/core/admin/mailu/sso/views/base.py
+++ b/core/admin/mailu/sso/views/base.py
@@ -67,7 +67,7 @@ def proxy(target='webmail'):
     if not any(ip in cidr for cidr in app.config['PROXY_AUTH_WHITELIST']):
         return flask.abort(500, '%s is not on PROXY_AUTH_WHITELIST' % flask.request.remote_addr)
 
-    email = flask.request.headers.get(app.config['PROXY_AUTH_HEADER'], None)
+    email = flask.request.headers.get(app.config['PROXY_AUTH_HEADER'])
     if not email:
         return flask.abort(500, 'No %s header' % app.config['PROXY_AUTH_HEADER'])
 
@@ -76,8 +76,13 @@ def proxy(target='webmail'):
         flask.session.regenerate()
         flask_login.login_user(user)
         return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])
-    elif app.config['PROXY_AUTH_CREATE']:
-        localpart, desireddomain = email.split('@')
+
+    if not app.config['PROXY_AUTH_CREATE']:
+        return flask.abort(500, 'You don\'t exist. Go away! (%s)' % email)
+
+    client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr)
+    try:
+        localpart, desireddomain = email.rsplit('@')
         domain = models.Domain.query.get(desireddomain) or flask.abort(500, 'You don\'t exist. Go away! (domain=%s)' % desireddomain)
         if not domain.max_users == -1 and len(domain.users) >= domain.max_users:
             flask.current_app.logger.warning('Too many users for domain %s' % domain)
@@ -87,8 +92,8 @@ def proxy(target='webmail'):
         models.db.session.add(user)
         models.db.session.commit()
         user.send_welcome()
-        client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr)
         flask.current_app.logger.info(f'Login succeeded by proxy created user: {user} from {client_ip} through {flask.request.remote_addr}.')
         return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])
-    else:
+    except Exception as e:
+        flask.current_app.logger.error('Error creating a new user via proxy for %s from %s: %s' % (email, client_ip, str(e)), e)
         return flask.abort(500, 'You don\'t exist. Go away! (%s)' % email)

From 63a12d985758f824c58d1bce440a69f29f1b0a65 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 10:00:00 +0100
Subject: [PATCH 18/54] changes requested by ghost

---
 core/base/Dockerfile | 27 ++++++++++-----------------
 webmails/Dockerfile  |  5 ++---
 2 files changed, 12 insertions(+), 20 deletions(-)

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 20d17f30..125d576e 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -49,26 +49,19 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
 COPY requirements-${MAILU_DEPS}.txt ./
 COPY libs/ libs/
 
-RUN set -euxo pipefail \
-  ; pip install -r requirements-${MAILU_DEPS}.txt || \
-    { \
-      machine="$(uname -m)" \
-    ; deps="build-base gcc libffi-dev python3-dev" \
-    ; [[ "${machine}" != x86_64 ]] && \
-        deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \
-    ; apk add --virtual .build-deps ${deps} \
-    ; [[ "${machine}" == armv7* ]] && \
-        mkdir -p /root/.cargo/registry/index && \
-        git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \
-    ; pip install -r requirements-${MAILU_DEPS}.txt \
-    ; rm -rf /root/.cargo /tmp/*.pem \
-  ; } \
-  ; rm -rf /root/.cache
-
 ARG SNUFFLEUPAGUS_VERSION=0.8.3
 ENV SNUFFLEUPAGUS_URL https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v$SNUFFLEUPAGUS_VERSION.tar.gz
 
 RUN set -euxo pipefail \
+  ;  machine="$(uname -m)" \
+  ; deps="build-base gcc libffi-dev python3-dev" \
+  ; [[ "${machine}" != x86_64 ]] && \
+      deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \
+  ; apk add --virtual .build-deps ${deps} \
+  ; [[ "${machine}" == armv7* ]] && \
+      mkdir -p /root/.cargo/registry/index && \
+      git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \
+  ; pip install -r requirements-${MAILU_DEPS}.txt \
   ; curl -sL  ${SNUFFLEUPAGUS_URL} | tar xz \
   ; cd snuffleupagus-$SNUFFLEUPAGUS_VERSION \
   ; rm -rf src/tests/*php7*/ src/tests/*session*/ src/tests/broken_configuration/ src/tests/*cookie* src/tests/upload_validation/ \
@@ -80,7 +73,7 @@ RUN set -euxo pipefail \
   ; pecl install vld-beta \
   ; make -j $(grep -c processor /proc/cpuinfo) release \
   ; cp src/.libs/snuffleupagus.so /app \
-  ; apk del -r .build-deps
+  ; rm -rf /root/.cargo /tmp/*.pem /root/.cache
 
 # base mailu image
 FROM system
diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index 2ae8a8f0..77bc65f3 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -22,9 +22,8 @@ RUN set -euxo pipefail \
   ; gpg --import /tmp/snappymail.asc \
   ; gpg --import /tmp/roundcube.asc \
   ; echo extension=snuffleupagus > /etc/php81/conf.d/snuffleupagus.ini \
-  ; rm -f /tmp/*asc \
-  ; mkdir -p /run/nginx \
-  ; mkdir -p /conf
+  ; rm -f /tmp/roundcube.asc /tmp/snappymail.asc \
+  ; mkdir -p /run/nginx /conf
 
 # roundcube
 ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.5.3/roundcubemail-1.5.3-complete.tar.gz

From 9fcff5e7452e53dab8d5f0c5f5db7eab216d8514 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 10:13:04 +0100
Subject: [PATCH 19/54] Pin what we get from edge

---
 core/base/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 125d576e..65e8be94 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -15,7 +15,7 @@ RUN set -euxo pipefail \
   ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \
   ; machine="$(uname -m)" \
   ; ! [[ "${machine}" == x86_64 ]] \
-    || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc
+    || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0
 
 ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so
 ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"

From 12117cef376147c8499c10f7e0c8bc52d911b298 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 12:16:25 +0100
Subject: [PATCH 20/54] Reduce the scope of the try: except

---
 core/admin/mailu/sso/views/base.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/core/admin/mailu/sso/views/base.py b/core/admin/mailu/sso/views/base.py
index b7fe9c97..67f2319a 100644
--- a/core/admin/mailu/sso/views/base.py
+++ b/core/admin/mailu/sso/views/base.py
@@ -83,17 +83,17 @@ def proxy(target='webmail'):
     client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr)
     try:
         localpart, desireddomain = email.rsplit('@')
-        domain = models.Domain.query.get(desireddomain) or flask.abort(500, 'You don\'t exist. Go away! (domain=%s)' % desireddomain)
-        if not domain.max_users == -1 and len(domain.users) >= domain.max_users:
-            flask.current_app.logger.warning('Too many users for domain %s' % domain)
-            return flask.abort(500, 'Too many users in (domain=%s)' % domain)
-        user = models.User(localpart=localpart, domain=domain)
-        user.set_password(secrets.token_urlsafe())
-        models.db.session.add(user)
-        models.db.session.commit()
-        user.send_welcome()
-        flask.current_app.logger.info(f'Login succeeded by proxy created user: {user} from {client_ip} through {flask.request.remote_addr}.')
-        return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])
     except Exception as e:
         flask.current_app.logger.error('Error creating a new user via proxy for %s from %s: %s' % (email, client_ip, str(e)), e)
         return flask.abort(500, 'You don\'t exist. Go away! (%s)' % email)
+    domain = models.Domain.query.get(desireddomain) or flask.abort(500, 'You don\'t exist. Go away! (domain=%s)' % desireddomain)
+    if not domain.max_users == -1 and len(domain.users) >= domain.max_users:
+        flask.current_app.logger.warning('Too many users for domain %s' % domain)
+        return flask.abort(500, 'Too many users in (domain=%s)' % domain)
+    user = models.User(localpart=localpart, domain=domain)
+    user.set_password(secrets.token_urlsafe())
+    models.db.session.add(user)
+    models.db.session.commit()
+    user.send_welcome()
+    flask.current_app.logger.info(f'Login succeeded by proxy created user: {user} from {client_ip} through {flask.request.remote_addr}.')
+    return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])

From 19bd9362d3c0d2c113a894ea3d43a12f725fabbe Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 14:56:26 +0100
Subject: [PATCH 21/54] As suggested by ghost

---
 core/admin/mailu/internal/views/postfix.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py
index c0a17319..62b400d3 100644
--- a/core/admin/mailu/internal/views/postfix.py
+++ b/core/admin/mailu/internal/views/postfix.py
@@ -143,11 +143,11 @@ def postfix_sender_login(sender):
     if localpart is None:
         return flask.jsonify(",".join(wildcard_senders)) if wildcard_senders else flask.abort(404)
     localpart = localpart[:next((i for i, ch in enumerate(localpart) if ch in flask.current_app.config.get('RECIPIENT_DELIMITER')), None)]
-    destinations = models.Email.resolve_destination(localpart, domain_name, True) or []
-    destinations.extend(wildcard_senders)
-    destinations.extend(i[0] for i in models.User.query.filter_by(allow_spoofing=True).with_entities(models.User.email).all())
+    destinations = set(models.Email.resolve_destination(localpart, domain_name, True) or [])
+    destinations.update(wildcard_senders)
+    destinations.update(i[0] for i in models.User.query.filter_by(allow_spoofing=True).with_entities(models.User.email).all())
     if destinations:
-        return flask.jsonify(",".join(idna_encode(list(set(destinations)))))
+        return flask.jsonify(",".join(idna_encode(destinations)))
     return flask.abort(404)
 
 @internal.route("/postfix/sender/rate/<path:sender>")

From d0631558c741f6949087a197a9b4a57e5b20547c Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 16:23:53 +0100
Subject: [PATCH 22/54] Remove Swarm everywhere.

This hasn't been tested
---
 docs/contributors/environment.rst             |   4 +-
 docs/contributors/guidelines.rst              |   2 +-
 docs/faq.rst                                  |   6 +-
 docs/swarm/1.5/README.md                      | 364 ------------------
 docs/swarm/master/README.md                   | 337 ----------------
 docs/swarm/master/README_nfs_example.md       | 357 -----------------
 setup/flavors/stack/docker-compose.yml        | 139 -------
 setup/flavors/stack/mailu.env                 |   1 -
 setup/flavors/stack/setup.html                |  65 ----
 setup/server.py                               |  10 +-
 .../templates/steps/compose/02_services.html  |   2 +-
 setup/templates/steps/compose/03_expose.html  |   2 +-
 setup/templates/steps/config.html             |  13 +-
 setup/templates/steps/flavor.html             |  13 -
 setup/templates/steps/stack/02_services.html  |  62 ---
 setup/templates/steps/stack/03_expose.html    |  24 --
 setup/templates/wizard.html                   |   6 -
 17 files changed, 9 insertions(+), 1398 deletions(-)
 delete mode 100644 docs/swarm/1.5/README.md
 delete mode 100644 docs/swarm/master/README.md
 delete mode 100644 docs/swarm/master/README_nfs_example.md
 delete mode 100644 setup/flavors/stack/docker-compose.yml
 delete mode 120000 setup/flavors/stack/mailu.env
 delete mode 100644 setup/flavors/stack/setup.html
 delete mode 100644 setup/templates/steps/flavor.html
 delete mode 100644 setup/templates/steps/stack/02_services.html
 delete mode 100644 setup/templates/steps/stack/03_expose.html

diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst
index 5949fbe5..25f6fbc0 100644
--- a/docs/contributors/environment.rst
+++ b/docs/contributors/environment.rst
@@ -130,8 +130,8 @@ To re-build only specific containers at a later time.
 
   docker buildx bake -f tests/build.hcl admin webdav
 
-If you have to push the images to Docker Hub for testing in Docker Swarm or a remote
-host, you have to define ``DOCKER_ORG`` (usually your Docker user-name) and login to
+If you have to push the images to Docker Hub for testing, you have to
+define ``DOCKER_ORG`` (usually your Docker user-name) and login to
 the hub.
 
 .. code-block:: bash
diff --git a/docs/contributors/guidelines.rst b/docs/contributors/guidelines.rst
index d55f9adf..8644d83e 100644
--- a/docs/contributors/guidelines.rst
+++ b/docs/contributors/guidelines.rst
@@ -54,7 +54,7 @@ Architecture
 What setups should be supported
 ```````````````````````````````
 
-Mailu supports out-of-the-box Docker Compose, Docker Swarm and Kubernetes
+Mailu supports out-of-the-box Docker Compose and Kubernetes
 environments. In those environments, it consists of many containers and
 supports hosting some of those containers in a separate environment.
 
diff --git a/docs/faq.rst b/docs/faq.rst
index 7782e93c..bd0f4d17 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -195,9 +195,8 @@ Mailu will start to function on IPv6:
 How does Mailu scale up?
 ````````````````````````
 
-Recent works allow Mailu to be deployed in Docker Swarm and Kubernetes.
-This means it can be scaled horizontally. For more information, refer to :ref:`kubernetes`
-or the `Docker swarm howto`_.
+Recent works allow Mailu to be deployed in Docker Kubernetes.
+This means it can be scaled horizontally. For more information, refer to :ref:`kubernetes`.
 
 *Issue reference:* `165`_, `520`_.
 
@@ -360,7 +359,6 @@ How do I use webdav (radicale)?
 .. _`Rspamd`:    https://www.rspamd.com/doc/configuration/index.html
 .. _`Roundcube`: https://github.com/roundcube/roundcubemail/wiki/Configuration#customize-the-look
 
-.. _`Docker swarm howto`: https://github.com/Mailu/Mailu/tree/master/docs/swarm/master
 .. _`125`: https://github.com/Mailu/Mailu/issues/125
 .. _`165`: https://github.com/Mailu/Mailu/issues/165
 .. _`177`: https://github.com/Mailu/Mailu/issues/177
diff --git a/docs/swarm/1.5/README.md b/docs/swarm/1.5/README.md
deleted file mode 100644
index 5af4c9dd..00000000
--- a/docs/swarm/1.5/README.md
+++ /dev/null
@@ -1,364 +0,0 @@
-# Install Mailu on a docker swarm
-
-## Prerequisites
-
-### Swarm
-
-In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
-
-The main command will be:
-```bash
-docker swarm init --advertise-addr <IP_ADDR>
-```
-See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
-
-If you want to add other managers or workers, please use:
-```bash
-docker swarm join --token xxxxx 
-```
-See https://docs.docker.com/engine/swarm/join-nodes/
-
-You have now a working swarm, and you can check its status with:
-```bash
-core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
-ID                            HOSTNAME            STATUS              AVAILABILITY        MANAGER STATUS      ENGINE VERSION
-xhgeekkrlttpmtgmapt5hyxrb     black-pearl         Ready               Active                                  18.06.0-ce
-sczlqjgfhehsfdjhfhhph1nvb *   coreos-01           Ready               Active              Leader              18.03.1-ce
-mzrm9nbdggsfz4sgq6dhs5i6n     flying-dutchman     Ready               Active                                  18.06.0-ce
-```
-
-### Volume definition
-For data persistence (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
-Hereafter we will use a NFS share:
-```bash
-core@coreos-01 ~ $ showmount -e 192.168.0.30
-Export list for 192.168.0.30:
-/mnt/Pool1/pv            192.168.0.0
-```
-
-on the nfs server, I am using the following /etc/exports
-```bash
-$more /etc/exports
-/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 
-```
-on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up)
-```bash
-$mkdir /mnt/Pool1/pv/mailu
-```
-
-On your manager node, mount the nfs share to check that the share is available:
-```bash
-core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/
-```
-If this is ok, you can umount it:
-```bash
-core@coreos-01 ~ $ sudo umount /mnt/local/
-```
-
-
-### Networking mode
-On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) provinding this service.
-With this default networking mode, I cannot get login working properly... As found in https://github.com/Mailu/Mailu/issues/375 ,  a workaround is to use the dnsrr networking mode at least for the front services.
-
-The main consequence/limitation will be that the front services will *not* be available on every node, but only on the node where it will be deployed. In my case, I have only one manager and I choose to deploy the front service to the manager node, so I know on wich IP the front service will be available (aka the IP adress of my manager node).
-
-### Variable substitution and docker-compose.yml
-The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to :
-- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc
-- change the way we define the volumes (nfs share in our case)
-- add a deploy section for every service
-
-### Docker compose 
-An example of docker-compose-stack.yml file is available here:
-
-```yaml
-
-version: '3.2'
-
-services:
-
-  front:
-    image: mailu/nginx:1.5
-    env_file: .env
-    ports:
-      - target: 80
-        published: 80
-        mode: host
-      - target: 443
-        published: 443
-        mode: host
-      - target: 110
-        published: 110
-        mode: host
-      - target: 143
-        published: 143
-        mode: host
-      - target: 993
-        published: 993
-        mode: host
-      - target: 995
-        published: 995
-        mode: host
-      - target: 25
-        published: 25
-        mode: host
-      - target: 465
-        published: 465
-        mode: host
-      - target: 587
-        published: 587
-        mode: host
-    volumes:
-#      - "$ROOT/certs:/certs"
-      - type: volume
-        source: mailu_certs
-        target: /certs
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  redis:
-    image: redis:alpine
-    restart: always
-    volumes:
-#      - "$ROOT/redis:/data"
-      - type: volume
-        source: mailu_redis
-        target: /data
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  imap:
-    image: mailu/dovecot:1.5
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/data:/data"
-      - type: volume
-        source: mailu_data
-        target: /data
-#      - "$ROOT/mail:/mail"
-      - type: volume
-        source: mailu_mail
-        target: /mail
-#      - "$ROOT/overrides:/overrides"
-      - type: volume
-        source: mailu_overrides
-        target: /overrides
-    depends_on:
-      - front
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  smtp:
-    image: mailu/postfix:1.5
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/data:/data"
-      - type: volume
-        source: mailu_data
-        target: /data
-#      - "$ROOT/overrides:/overrides"
-      - type: volume
-        source: mailu_overrides
-        target: /overrides
-    depends_on:
-      - front
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  antispam:
-    image: mailu/rspamd:1.5
-    restart: always
-    env_file: .env
-    depends_on:
-      - front
-    volumes:
-#      - "$ROOT/filter:/var/lib/rspamd"
-      - type: volume
-        source: mailu_filter
-        target: /var/lib/rspamd
-#      - "$ROOT/dkim:/dkim"
-      - type: volume
-        source: mailu_dkim
-        target: /dkim
-#      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
-      - type: volume
-        source: mailu_overrides_rspamd
-        target: /etc/rspamd/override.d
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  antivirus:
-    image: mailu/none:1.5
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/filter:/data"
-      - type: volume
-        source: mailu_filter
-        target: /data
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  webdav:
-    image: mailu/none:1.5
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/dav:/data"
-      - type: volume
-        source: mailu_dav
-        target: /data
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  admin:
-    image: mailu/admin:1.5
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/data:/data"
-      - type: volume
-        source: mailu_data
-        target: /data
-#      - "$ROOT/dkim:/dkim"
-      - type: volume
-        source: mailu_dkim
-        target: /dkim
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    depends_on:
-      - redis
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  webmail:
-    image: "mailu/roundcube:1.5"
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/webmail:/data"
-      - type: volume
-        source: mailu_data
-        target: /data
-    depends_on:
-      - imap
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-  fetchmail:
-    image: mailu/fetchmail:1.5
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/data:/data"
-      - type: volume
-        source: mailu_data
-        target: /data
-    logging:
-      driver: none
-    deploy:
-      endpoint_mode: dnsrr
-      replicas: 1
-      placement:
-        constraints: [node.role == manager]
-
-volumes:
-  mailu_filter:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/filter"
-  mailu_dkim:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/dkim"
-  mailu_overrides_rspamd:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/overrides/rspamd"
-  mailu_data:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/data"
-  mailu_mail:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/mail"
-  mailu_overrides:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/overrides"
-  mailu_dav:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/dav"
-  mailu_certs:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/certs"
-  mailu_redis:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,nolock,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/redis"
-```
-
-### Deploy Mailu on the docker swarm
-Run the following command:
-```bash
-docker stack deploy -c docker-compose-stack.yml mailu
-```
-See how the services are being deployed:
-```bash
-core@coreos-01 ~ $ docker service ls
-ID                  NAME                                 MODE                REPLICAS            IMAGE                                     PORTS
-ywnsetmtkb1l        mailu_antivirus                      replicated          1/1                 mailu/none:1.5
-pqokiaz0q128        mailu_fetchmail                      replicated          1/1                 mailu/fetchmail:1.5
-```
-check a specific service:
-```bash
-core@coreos-01 ~ $ docker service ps mailu_fetchmail
-ID                  NAME                IMAGE                 NODE                DESIRED STATE       CURRENT STATE         ERROR               PORTS
-tbu8ppgsdffj        mailu_fetchmail.1   mailu/fetchmail:1.5   coreos-01           Running             Running 11 days ago
-```
-
-### Remove the stack
-Run the following command:
-```bash
-core@coreos-01 ~ $ docker stack rm mailu
-```
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
deleted file mode 100644
index 56cec36d..00000000
--- a/docs/swarm/master/README.md
+++ /dev/null
@@ -1,337 +0,0 @@
-# Install Mailu on a docker swarm
-
-## Some warnings
-
-### How Docker swarm works
-
-Docker swarm enables replication and fail-over scenarios. As a feature, if a node dies or goes away, Docker will re-schedule it's containers on the remaining nodes.
-In order to take this decisions, docker swarm works on a consensus between managers regarding the state of nodes. Therefore it recommends to always have an uneven amount of manager nodes. This will always give a majority on either halve of a potential network split.
-
-### Storage
-
-On top of this some of Mailu's containers heavily rely on disk storage. As noted below, every host will need the same dataset on every host where related containers are run. So Dovecot IMAP needs `/mailu/mail` replicated to every node it *may* be scheduled to run. There are various solutions for this like NFS and GlusterFS.
-
-### When disaster strikes
-
-So imagine 3 swarm nodes and 3 GlusterFS endpoints:
-
-```
-node-A -> gluster-A --|
-node-B -> gluster-B --|--> Single file system
-node-C -> gluster-C --|
-```
-
-Each node has a connection to the shared file system and maintains connections between the other nodes. Let's say Dovecot is running on `node-A`. Now a network error / outage occurs on the route between `node-A` and the remaining nodes, but stays connected to the `gluster-A` endpoint. `node-B` and `node-C` conclude that `node-A` is down. They reschedule Dovecot to start on either one of them. Dovecot starts reading and writing its indexes to the **shared** filesystem. However, it is possible the Dovecot on `node-A` is still up and handling some client requests. I've seen cases where this situations resulted in:
-
-- Retained locks
-- Corrupted indexes
-- Users no longer able to read any of mail
-- Lost mail
-
-### It gets funkier
-
-Our original deployment also included `main.db` on the GlusterFS. Due to the above we corrupted it once and we decided to move it to local storage and restirct the `admin` container to that host only. This inspired us to put some legwork is supporting different database back-ends like MySQL and PostgreSQL. We highly recommend to use either of them, in favor of sqlite.
-
-### Conclusion
-
-Although the above situation is less-likely to occur on a stable (local) network, it does indicate a failure case where there is a probability of data-loss or downtime. It may help to create redundant networks, but the effort might be too much for the actual results. We will need to look into better and safer methods of replicating mail data. For now, we regret to have to inform you that Docker swarm deployment is **unstable** and should be avoided in production environments.
-
--- @muhlemmer, 17th of January 2019.
-
-## Prerequisites
-
-### Swarm
-
-In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
-
-The main command will be:
-```bash
-docker swarm init --advertise-addr <IP_ADDR>
-```
-See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
-
-If you want to add other managers or workers, please use:
-```bash
-docker swarm join --token xxxxx 
-```
-See https://docs.docker.com/engine/swarm/join-nodes/
-
-You have now a working swarm, and you can check its status with:
-```bash
-core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
-ID                            HOSTNAME            STATUS              AVAILABILITY        MANAGER STATUS      ENGINE VERSION
-xhgeekkrlttpmtgmapt5hyxrb     black-pearl         Ready               Active                                  18.06.0-ce
-sczlqjgfhehsfdjhfhhph1nvb *   coreos-01           Ready               Active              Leader              18.03.1-ce
-mzrm9nbdggsfz4sgq6dhs5i6n     flying-dutchman     Ready               Active                                  18.06.0-ce
-```
-
-### Volume definition
-For data persistence (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
-
-Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT" (GlusterFS and nfs shares have been successfully used).
-
-On this example, we are using:
-- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. 
-- the default ingress mode.
-
-### Allow authentification with the mesh routing
-In order to allow every (front & webmail) container to access the other services, we will use the variable SUBNET.
-
-Let's create the mailu_default network:
-```bash
-core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
-core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
-                    "Subnet": "10.0.1.0/24",
-```
-In the docker-compose.yml file, we will then use SUBNET = 10.0.1.0/24 
-In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of  "mailu_default-endpoint". So it is sufficient to set SUBNET to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
-
-### Limitation with the ingress mode
-With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
-
-This issue is known and discussed here:
-
-https://github.com/moby/moby/issues/25526
-
-A workaround (using network host mode and global deployment) is discussed here:
-
-https://github.com/moby/moby/issues/25526#issuecomment-336363408 
-
-### Don't create an open relay !
-As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
-
-### Ratelimits
-
-When using ingress mode you probably want to disable rate limits, because all requests originate from the same ip address. Otherwise automatic login attempts can easily DoS the legitimate users.
-
-## Scalability
-- smtp and imap are scalable
-- front and webmail are scalable (pending SUBNET is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) 
-- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
-
-## Docker secrets
-There are DB_PW_FILE and SECRET_KEY_FILE environment variables available to specify files for these variables. These can be used to configure Docker secrets instead of writing the values directly into the `docker-compose.yml` or `mailu.env`.
-
-## Variable substitution and docker-compose.yml
-The docker stack deploy command doesn't support variable substitution in the .yml file itself. 
-As a consequence, we cannot simply use ``` docker stack deploy -c docker.compose.yml mailu ```
-Instead, we will use the following work-around:
-``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ```
-
-We need also to:
-- add a deploy section for every service
-- modify the way the ports are defined for the front service
-- add the SUBNET definition for admin (for imap), smtp and antispam services
-
-## Docker compose 
-An example of docker-compose-stack.yml file is available here:
-
-```yaml
-
-version: '3.2'
-
-services:
-
-  front:
-    image: mailu/nginx:$VERSION
-    restart: always
-    env_file: .env
-    ports:
-      - target: 80
-        published: 80
-      - target: 443
-        published: 443
-      - target: 110
-        published: 110
-      - target: 143
-        published: 143
-      - target: 993
-        published: 993
-      - target: 995
-        published: 995
-      - target: 25
-        published: 25
-      - target: 465
-        published: 465
-      - target: 587
-        published: 587
-    volumes:
-      - "$ROOT/certs:/certs"
-    deploy:
-      replicas: 2
-
-  redis:
-    image: redis:alpine
-    restart: always
-    volumes:
-      - "$ROOT/redis:/data"
-    deploy:
-      replicas: 1
-
-  imap:
-    image: mailu/dovecot:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-      - "$ROOT/mail:/mail"
-      - "$ROOT/overrides:/overrides"
-    depends_on:
-      - front
-    deploy:
-      replicas: 2
-
-  smtp:
-    image: mailu/postfix:$VERSION
-    restart: always
-    env_file: .env
-    environment:
-      - SUBNET=10.0.1.0/24
-    volumes:
-      - "$ROOT/overrides:/overrides"
-    depends_on:
-      - front
-    deploy:
-      replicas: 2
-
-  antispam:
-    image: mailu/rspamd:$VERSION
-    restart: always
-    env_file: .env
-    environment:
-      - SUBNET=10.0.1.0/24
-    volumes:
-      - "$ROOT/filter:/var/lib/rspamd"
-      - "$ROOT/dkim:/dkim"
-      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
-    depends_on:
-      - front
-    deploy:
-      replicas: 1
-
-  antivirus:
-    image: mailu/none:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-      - "$ROOT/filter:/data"
-    deploy:
-      replicas: 1
-
-  webdav:
-    image: mailu/none:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-      - "$ROOT/dav:/data"
-    deploy:
-      replicas: 1
-
-  admin:
-    image: mailu/admin:$VERSION
-    restart: always
-    env_file: .env
-    environment:
-      - SUBNET=10.0.1.0/24
-    volumes:
-      - "$ROOT/data:/data"
-      - "$ROOT/dkim:/dkim"
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    depends_on:
-      - redis
-    deploy:
-      replicas: 1
-
-  webmail:
-    image: mailu/roundcube:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-      - "$ROOT/webmail:/data"
-    depends_on:
-      - imap
-    deploy:
-      replicas: 2
-
-  fetchmail:
-    image: mailu/fetchmail:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-    deploy:
-      replicas: 1
-
-networks:
-  default:
-    external:
-      name: mailu_default
-```
-
-## Deploy Mailu on the docker swarm
-Run the following command:
-```bash
-echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu
-```
-See how the services are being deployed:
-```bash
-core@coreos-01 ~ $ docker service ls
-ID                  NAME                                 MODE                REPLICAS            IMAGE                                     PORTS
-ywnsetmtkb1l        mailu_antivirus                      replicated          1/1                 mailu/none:master
-pqokiaz0q128        mailu_fetchmail                      replicated          1/1                 mailu/fetchmail:master
-```
-check a specific service:
-```bash
-core@coreos-01 ~ $ docker service ps mailu_fetchmail
-ID                  NAME                IMAGE                 NODE                DESIRED STATE       CURRENT STATE         ERROR               PORTS
-tbu8ppgsdffj        mailu_fetchmail.1   mailu/fetchmail:master   coreos-01           Running             Running 11 days ago
-```
-You might also have a look on the logs:
-```bash
-core@coreos-01 ~ $ docker service logs -f mailu_fetchmail
-```
-
-## Remove the stack
-Run the following command:
-```bash
-core@coreos-01 ~ $ docker stack rm mailu
-```
-
-## Notes on unbound resolver
-
-In Docker compose flavor we currently have the option to include the unbound DNS resolver. This does not work in Docker Swarm, as it in not possible to configure any static IP addresses. There is an [open issue](https://github.com/moby/moby/issues/24170) for this at Docker. However, this doesn't seem to move anywhere since some time now. For that reasons we've chosen not to include the unbound resolver in the stack flavor.
-
-If you still want to benefit from Unbound as a system resolver, you can install it system-wide. The following procedure was done on a Fedora 28 system and might needs some adjustments for your system. Note that this will need to be done on every swarm node. In this example we will make use of `dnssec-trigger`, which is used to configure unbound. When installing this and running the service, unbound is pulled in as dependency and does not need to be installed, configured or run separately.
-
-Install required packages(unbound will be installed as dependency):
-
-```
-sudo dnf install dnssec-trigger
-```
-
-Enable and start the *dnssec-trigger* daemon:
-
-```
-sudo systemctl enable --now dnssec-triggerd.service
-```
-
-Configure NetworkManager to use unbound, create the file `/etc/NetworkManager/conf.d/unbound.conf` with contents:
-
-```
-[main]
-dns=unbound
-```
-
-You might need to restart NetworkManager for the changes to take effect:
-
-```
-sudo systemctl restart NetworkManager
-```
-
-Verify `resolv.conf`:
-
-```
-$ cat /etc/resolv.conf
-# Generated by dnssec-trigger-script
-nameserver 127.0.0.1
-```
-
-Most of this info was take from this [Fedora Project page](https://fedoraproject.org/wiki/Changes/Default_Local_DNS_Resolver#How_To_Test).
diff --git a/docs/swarm/master/README_nfs_example.md b/docs/swarm/master/README_nfs_example.md
deleted file mode 100644
index 2c4b8145..00000000
--- a/docs/swarm/master/README_nfs_example.md
+++ /dev/null
@@ -1,357 +0,0 @@
-# Install Mailu on a docker swarm
-
-## Prequisites
-
-### Swarm
-
-In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
-
-The main command will be:
-```bash
-docker swarm init --advertise-addr <IP_ADDR>
-```
-See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
-
-If you want to add other managers or workers, please use:
-```bash
-docker swarm join --token xxxxx 
-```
-See https://docs.docker.com/engine/swarm/join-nodes/
-
-You have now a working swarm, and you can check its status with:
-```bash
-core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
-ID                            HOSTNAME            STATUS              AVAILABILITY        MANAGER STATUS      ENGINE VERSION
-xhgeekkrlttpmtgmapt5hyxrb     black-pearl         Ready               Active                                  18.06.0-ce
-sczlqjgfhehsfdjhfhhph1nvb *   coreos-01           Ready               Active              Leader              18.03.1-ce
-mzrm9nbdggsfz4sgq6dhs5i6n     flying-dutchman     Ready               Active                                  18.06.0-ce
-```
-
-### Volume definition
-For data persistence (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
-Hereafter we will use a NFS share:
-```bash
-core@coreos-01 ~ $ showmount -e 192.168.0.30
-Export list for 192.168.0.30:
-/mnt/Pool1/pv            192.168.0.0
-```
-
-on the nfs server, I am using the following /etc/exports
-```bash
-$more /etc/exports
-/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 
-```
-on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up)
-```bash
-$mkdir /mnt/Pool1/pv/mailu
-```
-
-On your manager node, mount the nfs share to check that the share is available:
-```bash
-core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/
-```
-If this is ok, you can umount it:
-```bash
-core@coreos-01 ~ $ sudo umount /mnt/local/
-```
-
-
-## Networking mode
-On this example, we are using:
-- the mesh routing mode (default mode). With this mode, each service is given a virtual IP address and docker manages the routing between this virtual IP and the container(s) providing this service. 
-- the default ingress mode.
-
-### Allow authentification with the mesh routing
-In order to allow every (front & webmail) container to access the other services, we will use the variable SUBNET.
-
-Let's create the mailu_default network:
-```bash
-core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
-core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
-                    "Subnet": "10.0.1.0/24",
-```
-In the docker-compose.yml file, we will then use SUBNET = 10.0.1.0/24 
-In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of  "mailu_default-endpoint". So it is sufficient to set SUBNET to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
-
-### Limitation with the ingress mode
-With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
-
-This issue is known and discussed here:
-
-https://github.com/moby/moby/issues/25526
-
-A workaround (using network host mode and global deployment) is discussed here:
-
-https://github.com/moby/moby/issues/25526#issuecomment-336363408 
-
-### Don't create an open relay !
-As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
-
-
-## Scalability
-- smtp and imap are scalable
-- front and webmail are scalable (pending SUBNET is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) 
-- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
-
-## Variable substitution and docker-compose.yml
-The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around:
-``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ```
-
-We need also to:
-- change the way we define the volumes (nfs share in our case)
-- add a deploy section for every service
-- the way the ports are defined for the front service
-
-## Docker compose 
-An example of docker-compose-stack.yml file is available here:
-
-```yaml
-
-version: '3.2'
-
-services:
-
-  front:
-    image: mailu/nginx:$VERSION
-    restart: always
-    env_file: .env
-    ports:
-      - target: 80
-        published: 80
-      - target: 443
-        published: 443
-      - target: 110
-        published: 110
-      - target: 143
-        published: 143
-      - target: 993
-        published: 993
-      - target: 995
-        published: 995
-      - target: 25
-        published: 25
-      - target: 465
-        published: 465
-      - target: 587
-        published: 587
-    volumes:
-#      - "$ROOT/certs:/certs"
-      - type: volume
-        source: mailu_certs
-        target: /certs
-    deploy:
-      replicas: 2
-
-  redis:
-    image: redis:alpine
-    restart: always
-    volumes:
-#      - "$ROOT/redis:/data"
-      - type: volume
-        source: mailu_redis
-        target: /data
-    deploy:
-      replicas: 1
-
-  imap:
-    image: mailu/dovecot:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/mail:/mail"
-      - type: volume
-        source: mailu_mail
-        target: /mail
-#      - "$ROOT/overrides:/overrides"
-      - type: volume
-        source: mailu_overrides
-        target: /overrides
-    depends_on:
-      - front
-    deploy:
-      replicas: 2
-
-  smtp:
-    image: mailu/postfix:$VERSION
-    restart: always
-    env_file: .env
-    environment:
-      - SUBNET=10.0.1.0/24
-    volumes:
-#      - "$ROOT/overrides:/overrides"
-      - type: volume
-        source: mailu_overrides
-        target: /overrides
-    depends_on:
-      - front
-    deploy:
-      replicas: 2
-
-  antispam:
-    image: mailu/rspamd:$VERSION
-    restart: always
-    env_file: .env
-    environment:
-      - SUBNET=10.0.1.0/24
-    depends_on:
-      - front
-    volumes:
-#      - "$ROOT/filter:/var/lib/rspamd"
-      - type: volume
-        source: mailu_filter
-        target: /var/lib/rspamd
-#      - "$ROOT/dkim:/dkim"
-      - type: volume
-        source: mailu_dkim
-        target: /dkim
-#      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
-      - type: volume
-        source: mailu_overrides_rspamd
-        target: /etc/rspamd/override.d
-    deploy:
-      replicas: 1
-
-  antivirus:
-    image: mailu/none:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/filter:/data"
-      - type: volume
-        source: mailu_filter
-        target: /data
-    deploy:
-      replicas: 1
-
-  webdav:
-    image: mailu/none:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/dav:/data"
-      - type: volume
-        source: mailu_dav
-        target: /data
-    deploy:
-      replicas: 1
-
-  admin:
-    image: mailu/admin:$VERSION
-    restart: always
-    env_file: .env
-    environment:
-      - SUBNET=10.0.1.0/24
-    volumes:
-#      - "$ROOT/data:/data"
-      - type: volume
-        source: mailu_data
-        target: /data
-#      - "$ROOT/dkim:/dkim"
-      - type: volume
-        source: mailu_dkim
-        target: /dkim
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    depends_on:
-      - redis
-    deploy:
-      replicas: 1
-
-  webmail:
-    image: mailu/roundcube:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-#      - "$ROOT/webmail:/data"
-      - type: volume
-        source: mailu_data
-        target: /data
-    depends_on:
-      - imap
-    deploy:
-      replicas: 2
-
-  fetchmail:
-    image: mailu/fetchmail:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-    deploy:
-      replicas: 1
-
-networks:
-  default:
-    external:
-      name: mailu_default
-
-volumes:
-  mailu_filter:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/filter"
-  mailu_dkim:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/dkim"
-  mailu_overrides_rspamd:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/overrides/rspamd"
-  mailu_data:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/data"
-  mailu_mail:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/mail"
-  mailu_overrides:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/overrides"
-  mailu_dav:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/dav"
-  mailu_certs:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/certs"
-  mailu_redis:
-    driver_opts:
-      type: "nfs"
-      o: "addr=192.168.0.30,soft,rw"
-      device: ":/mnt/Pool1/pv/mailu/redis"
-```
-
-## Deploy Mailu on the docker swarm
-Run the following command:
-```bash
-echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu
-```
-See how the services are being deployed:
-```bash
-core@coreos-01 ~ $ docker service ls
-ID                  NAME                                 MODE                REPLICAS            IMAGE                                     PORTS
-ywnsetmtkb1l        mailu_antivirus                      replicated          1/1                 mailu/none:master
-pqokiaz0q128        mailu_fetchmail                      replicated          1/1                 mailu/fetchmail:master
-```
-check a specific service:
-```bash
-core@coreos-01 ~ $ docker service ps mailu_fetchmail
-ID                  NAME                IMAGE                 NODE                DESIRED STATE       CURRENT STATE         ERROR               PORTS
-tbu8ppgsdffj        mailu_fetchmail.1   mailu/fetchmail:master   coreos-01           Running             Running 11 days ago
-```
-
-## Remove the stack
-Run the following command:
-```bash
-core@coreos-01 ~ $ docker stack rm mailu
-```
diff --git a/setup/flavors/stack/docker-compose.yml b/setup/flavors/stack/docker-compose.yml
deleted file mode 100644
index 809362df..00000000
--- a/setup/flavors/stack/docker-compose.yml
+++ /dev/null
@@ -1,139 +0,0 @@
-{% set env='mailu.env' %}
-# This file is auto-generated by the Mailu configuration wizard.
-# Please read the documentation before attempting any change.
-# Generated for {{ flavor }} flavor
-
-version: '3.6'
-
-services:
-
-# External dependencies
-  redis:
-    image: redis:alpine
-    volumes:
-      - "{{ root }}/redis:/data"
-
-# Core services
-  front:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-{{ version }}}
-    env_file: {{ env }}
-    logging:
-      driver: {{ log_driver or 'json-file' }}
-    ports:
-    {% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
-      - target: {{ port }}
-        published: {{ port }}
-        mode: overlay
-    {% endfor %}
-    volumes:
-      - "{{ root }}/certs:/certs"
-      - "{{ root }}/overrides/nginx:/overrides:ro"
-    deploy:
-      replicas: 1
-
-  admin:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-{{ version }}}
-    env_file: {{ env }}
-    {% if not admin_enabled %}
-    ports:
-      - 127.0.0.1:8080:80
-    {% endif %}
-    volumes:
-      - "{{ root }}/data:/data"
-      - "{{ root }}/dkim:/dkim"
-    deploy:
-      replicas: 1
-    healthcheck:
-      disable: true
-
-  imap:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-{{ version }}}
-    env_file: {{ env }}
-    volumes:
-      - "{{ root }}/mail:/mail"
-      - "{{ root }}/overrides/dovecot:/overrides:ro"
-    deploy:
-      replicas: 1
-    healthcheck:
-      disable: true
-
-  smtp:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-{{ version }}}
-    env_file: {{ env }}
-    volumes:
-      - "{{ root }}/mailqueue:/queue"
-      - "{{ root }}/overrides/postfix:/overrides:ro"
-    deploy:
-      replicas: 1
-    healthcheck:
-      disable: true
-
-  antispam:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-{{ version }}}
-    hostname: antispam
-    env_file: {{ env }}
-    volumes:
-      - "{{ root }}/filter:/var/lib/rspamd"
-      - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d:ro"
-    deploy:
-      replicas: 1
-    healthcheck:
-      disable: true
-
-  # Optional services
-  {% if antivirus_enabled %}
-  antivirus:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}clamav:${MAILU_VERSION:-{{ version }}}
-    env_file: {{ env }}
-    volumes:
-      - "{{ root }}/filter:/data"
-    deploy:
-      replicas: 1
-    healthcheck:
-      disable: true
-  {% endif %}
-
-  {% if webdav_enabled %}
-  webdav:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}radicale:${MAILU_VERSION:-{{ version }}}
-    env_file: {{ env }}
-    volumes:
-      - "{{ root }}/dav:/data"
-    deploy:
-      replicas: 1
-    healthcheck:
-      disable: true
-  {% endif %}
-
-  {% if fetchmail_enabled %}
-  fetchmail:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${MAILU_VERSION:-{{ version }}}
-    env_file: {{ env }}
-    volumes:
-      - "{{ root }}/data/fetchmail:/data"
-    deploy:
-      replicas: 1
-    healthcheck:
-      disable: true
-  {% endif %}
-
-  {% if webmail_type != 'none' %}
-  webmail:
-    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}webmail:${MAILU_VERSION:-{{ version }}}
-    env_file: {{ env }}
-    volumes:
-      - "{{ root }}/webmail:/data"
-      - "{{ root }}/overrides/{{ webmail_type }}:/overrides:ro"
-    deploy:
-      replicas: 1
-    healthcheck:
-      disable: true
-  {% endif %}
-
-networks:
-  default:
-    driver: overlay
-    ipam:
-      driver: default
-      config:
-        - subnet: {{ subnet }}
diff --git a/setup/flavors/stack/mailu.env b/setup/flavors/stack/mailu.env
deleted file mode 120000
index 7123102b..00000000
--- a/setup/flavors/stack/mailu.env
+++ /dev/null
@@ -1 +0,0 @@
-../compose/mailu.env
\ No newline at end of file
diff --git a/setup/flavors/stack/setup.html b/setup/flavors/stack/setup.html
deleted file mode 100644
index 2a635562..00000000
--- a/setup/flavors/stack/setup.html
+++ /dev/null
@@ -1,65 +0,0 @@
-{% import "macros.html" as macros %}
-
-{% call macros.panel("info", "Step 1 - Download your configuration files") %}
-<p>Docker Stack expects a project file, named <code>docker-compose.yml</code>
-in a project directory. First create your project directory.</p>
-
-<pre><code>mkdir -p {{ root }}/{redis,certs,data,data/fetchmail,dkim,mail,mailqueue,overrides/rspamd,overrides/postfix,overrides/dovecot,overrides/nginx,filter,dav,webmail}
-</pre></code>
-
-<p>Then download the project file. A side configuration file makes it easier
-to read and check the configuration variables generated by the wizard.</p>
-
-<pre><code>cd {{ root }}
-wget {{ url_for('.file', uid=uid, _scheme='https', filepath='docker-compose.yml', _external=True) }}
-wget {{ url_for('.file', uid=uid, _scheme='https', filepath='mailu.env', _external=True) }}
-</pre></code>
-{% endcall %}
-
-
-{% call macros.panel("info", "Step 2 - Review the configuration") %}
-<p>We did not insert any malicious code on purpose in the configurations we
-distribute, but your download could have been intercepted, or our wizard
-website could have been compromised, so make sure you check the configuration
-files before going any further.</p>
-
-<p>When you are done checking them, check them one last time.</p>
-{% endcall %}
-
-{% call macros.panel("info", "Step 3 - Deploy docker stack") %}
-<p>To deploy the docker stack use the following commands. For more information about setting up docker swarm nodes read the
-	<a href="https://docs.docker.com/get-started">docker documentation</a></p>
-
-<pre><code>cd {{ root }}
-docker swarm init
-docker stack deploy -c docker-compose.yml mailu
-</pre></code>
-
-In the docker stack deploy command, mailu is the app name. Feel free to change it.<br/>
-In order to display the running container you can use<br/>
-<pre><code>docker ps</code></pre>
-or
-<pre><code>docker stack ps --no-trunc mailu</code></pre>
-Command for removing docker stack is
-<pre><code>docker stack rm mailu</code></pre>
-
-Before you can use Mailu, you must create the primary administrator user account. This should be {{ postmaster }}@{{ domain }}. Use the following command, changing PASSWORD to your liking:
-
-<pre><code>docker exec $(docker ps | grep admin | cut -d ' ' -f1) flask mailu admin {{ postmaster }} {{ domain }} PASSWORD
-</pre></code>
-
-<p>Login to the admin interface to change the password for a safe one, at
-{% if admin_enabled %}
-one of the hostnames
-<a href="https://{{ hostnames.split(',')[0] }}{{ admin_path }}">{{ hostnames.split(',')[0] }}{{ admin_path }}</a>.
-{% else %}
-<a href="http://127.0.0.1:8080/ui">http://127.0.0.1:8080/ui</a> (only directly from the host running docker).
-If you run mailu on a remote server, and wish to access the admin interface via a SSH tunnel, you can create a port-forward from your local machine to your server like
-<pre><code>ssh -L 127.0.0.1:8080:127.0.0.1:8080 &lt;user&gt;@&lt;server&gt;
-</code></pre>
-And access the above URL from your local machine.
-<br />
-{% endif %}
-Also, choose the "Update password" option in the left menu.
-</p>
-{% endcall %}
diff --git a/setup/server.py b/setup/server.py
index 5be1fc83..1547a260 100644
--- a/setup/server.py
+++ b/setup/server.py
@@ -78,15 +78,7 @@ def build_app(path):
     @prefix_bp.route("/")
     @root_bp.route("/")
     def wizard():
-        return flask.render_template('wizard.html')
-
-    @prefix_bp.route("/submit_flavor", methods=["POST"])
-    @root_bp.route("/submit_flavor", methods=["POST"])
-    def submit_flavor():
-        data = flask.request.form.copy()
-        subnet6 = random_ipv6_subnet()
-        steps = sorted(os.listdir(os.path.join(path, "templates", "steps", data["flavor"])))
-        return flask.render_template('wizard.html', flavor=data["flavor"], steps=steps, subnet6=subnet6)
+        return flask.render_template('setup.html', subnet6=random_ipv6_subnet())
 
     @prefix_bp.route("/submit", methods=["POST"])
     @root_bp.route("/submit", methods=["POST"])
diff --git a/setup/templates/steps/compose/02_services.html b/setup/templates/steps/compose/02_services.html
index a801f807..b6964bdf 100644
--- a/setup/templates/steps/compose/02_services.html
+++ b/setup/templates/steps/compose/02_services.html
@@ -1,4 +1,4 @@
-{% call macros.panel("info", "Step 3 - Pick some features") %}
+{% call macros.panel("info", "Step 2 - Pick some features") %}
 <p>Mailu comes with multiple base features, including a specific admin
 interface, Web email clients, antispam, antivirus, etc.
 In this section you can enable the services to you liking.</p>
diff --git a/setup/templates/steps/compose/03_expose.html b/setup/templates/steps/compose/03_expose.html
index 201ac3eb..c1d3ca8c 100644
--- a/setup/templates/steps/compose/03_expose.html
+++ b/setup/templates/steps/compose/03_expose.html
@@ -1,4 +1,4 @@
-{% call macros.panel("info", "Step 4 - expose Mailu to the world") %}
+{% call macros.panel("info", "Step 3 - expose Mailu to the world") %}
 <p>A mail server must be exposed to the world to receive emails, send emails,
 and let users access their mailboxes. Mailu has some flexibility in the way
 you expose it to the world.</p>
diff --git a/setup/templates/steps/config.html b/setup/templates/steps/config.html
index 74a45800..ce7ade70 100644
--- a/setup/templates/steps/config.html
+++ b/setup/templates/steps/config.html
@@ -1,15 +1,4 @@
-{% if flavor == "stack" %}
-{% call macros.panel("danger", "Docker stack / swarm is experimental") %}
-Setup is capable of generating a somewhat decent docker-compose.yml,
-for the docker stack flavor. However its usage is for advanced users only and is experimental.
-Expect many challenges such as shared mail storage and fail-over scenarios! Some user experiences
-have been <a href="https://github.com/Mailu/Mailu/blob/master/docs/swarm/master/README.md">shared on GitHub.</a>
-For this reason also think very hard about using a replica count higher than 1. This cannot be used with the default config.
-Manual post-configuration is required for using a replica count higher than 1.
-{% endcall %}
-{% endif %}
-
-{% call macros.panel("info", "Step 2 - Initial configuration") %}
+{% call macros.panel("info", "Step 1 - Initial configuration") %}
 <p>Before starting, some variables must be set.</p>
 
 <div class="form-group">
diff --git a/setup/templates/steps/flavor.html b/setup/templates/steps/flavor.html
deleted file mode 100644
index 9ef16aa0..00000000
--- a/setup/templates/steps/flavor.html
+++ /dev/null
@@ -1,13 +0,0 @@
-{% call macros.panel("info", "Step 1 - Pick a flavor") %}
-<p>Mailu comes in multiple "flavors". It was originally
-designed to run on top of Docker Compose but now offers multiple options
-including Docker Stack, Rancher, Kubernetes.</p>
-<p>Please note that "official" support, that is provided by the most active
-developers will mostly cover Compose and Stack, while other flavors are
-maintained by specific contributors.</p>
-
-<div class="radio">
-  {{ macros.radio("flavor", "compose", "Compose", "simply using Docker Compose manager", flavor) }}
-  {{ macros.radio("flavor", "stack", "Stack", "using stack deployments in a Swarm cluster", flavor) }}
-</div>
-{% endcall %}
diff --git a/setup/templates/steps/stack/02_services.html b/setup/templates/steps/stack/02_services.html
deleted file mode 100644
index 68519fa8..00000000
--- a/setup/templates/steps/stack/02_services.html
+++ /dev/null
@@ -1,62 +0,0 @@
-{% call macros.panel("info", "Step 3 - Pick some features") %}
-<p>Mailu comes with multiple base features, including a specific admin
-interface, Web email clients, antispam, antivirus, etc.
-In this section you can enable the services to you liking.</p>
-
-<!-- 	Switched from radio buttons to dropdown menu in order to remove the checkbox -->
-<p>A Webmail is a Web interface exposing an email client. Mailu webmails are
-bound to the internal IMAP and SMTP server for users to access their mailbox through
-the Web. By exposing a complex application such as a Webmail, you should be aware of
-the security implications caused by such an increase of attack surface.<p>
-<div class="form-group">
-  <label>Enable Web email client (and path to the Web email client)</label>
-  <br/>
-  <select class="btn btn-primary dropdown-toggle" name="webmail_type" id="webmail">
-  	{% for webmailtype in ["none", "roundcube", "snappymail"] %}
-  	<option value="{{ webmailtype }}" >{{ webmailtype }}</option>
-  	{% endfor %}
-  </select>
-  <p></p>
-  <div class="input-group">
-    <input class="form-control" type="text" name="webmail_path" id="webmail_path" style="display: none">
-  </div>
-</div>
-
-
-<div class="form-check form-check-inline">
-  <label class="form-check-label">
-    <input class="form-check-input" type="checkbox" name="antivirus_enabled" value="clamav">
-    Enable the antivirus service
-  </label>
-
-  <i>An antivirus server helps fighting large scale virus spreading campaigns that leverage
-    e-mail for initial infection. Make sure that you have at least 1GB of memory for ClamAV to
-    load its signature database.</i>
-</div>
-
-
-<div class="form-check form-check-inline">
-  <label class="form-check-label">
-    <input class="form-check-input" type="checkbox" name="webdav_enabled" value="radicale">
-    Enable the webdav service
-  </label>
-
-  <i>A Webdav server exposes a Dav interface over HTTP so that clients can store
-    contacts or calendars using the mail account.</i>
-</div>
-
-
-<div class="form-check form-check-inline">
-  <label class="form-check-label">
-    <input class="form-check-input" type="checkbox" name="fetchmail_enabled" value="true">
-    Enable fetchmail
-  </label>
-
-  <i>Fetchmail allows users to retrieve mail from an external mail-server via IMAP/POP3 and puts it in their inbox.</i>
-</div>
-
-<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
-
-<script type="text/javascript" src="{{ url_for('static', filename='render.js') }}"></script>
-
-{% endcall %}
diff --git a/setup/templates/steps/stack/03_expose.html b/setup/templates/steps/stack/03_expose.html
deleted file mode 100644
index 85b5e022..00000000
--- a/setup/templates/steps/stack/03_expose.html
+++ /dev/null
@@ -1,24 +0,0 @@
-{% call macros.panel("info", "Step 4 - expose Mailu to the world") %}
-<p>A mail server must be exposed to the world to receive emails, send emails,
-and let users access their mailboxes. Mailu has some flexibility in the way
-you expose it to the world.</p>
-
-<div class="form-group">
-  <label>Subnet of the docker network. This should not conflict with any networks to which your system is connected. (Internal and external!)</label>
-  <input class="form-control" type="text" name="subnet" required pattern="^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))$"
-    value="192.168.203.0/24">
-</div>
-
-<p>You server will be available under a main hostname but may expose multiple public
-hostnames. Every e-mail domain that points to this server must have one of the
-hostnames in its <code>MX</code> record. Hostnames must be comma-separated. If you're having
-trouble accessing your admin interface, make sure it is the first entry here (and possibly the
-same as your <code>DOMAIN</code> entry from earlier.</p>
-
-<div class="form-group">
-  <label>Public hostnames</label>
-<!--   Validates hostname or list of hostnames -->
-  <input class="form-control" type="text" name="hostnames" placeholder="my.host.name,other.host.name" multiple required
-    	pattern="^(?:(?:\w+(?:-+\w+)*\.)+[a-z]+)*(?:,(?:(?:\w+(?:-+\w+)*\.)+[a-z]+)\s*)*$">
-</div>
-{% endcall %}
diff --git a/setup/templates/wizard.html b/setup/templates/wizard.html
index e618b716..c98672a6 100644
--- a/setup/templates/wizard.html
+++ b/setup/templates/wizard.html
@@ -8,11 +8,6 @@
   ready when using this wizard.
   {% endcall %}
 
-  <form method="post" action="{{ url_for(".submit_flavor") }}">
-  {% include "steps/flavor.html" %}
-  <input class="btn btn-primary" type="submit" value="Next >" style="margin-bottom: 15px">
-  </form>
-  {% if flavor %}
   <form method="post" action="{{ url_for(".submit") }}">
   <input type="hidden" name="flavor" value="{{ flavor }}">
   {% include "steps/config.html" %}
@@ -21,6 +16,5 @@
   {% endfor %}
   {% include "steps/database.html" %}
   <input class="btn btn-primary" type="submit" value="Setup Mailu">
-  {% endif %}
   </form>
 {% endblock %}

From b3f534a6ac3020b350fa481a4be6d66106a33e20 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 16:37:42 +0100
Subject: [PATCH 23/54] Wizard.html should still be the default destination

---
 setup/server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup/server.py b/setup/server.py
index 1547a260..eda0a36f 100644
--- a/setup/server.py
+++ b/setup/server.py
@@ -78,7 +78,7 @@ def build_app(path):
     @prefix_bp.route("/")
     @root_bp.route("/")
     def wizard():
-        return flask.render_template('setup.html', subnet6=random_ipv6_subnet())
+        return flask.render_template('wizard.html', subnet6=random_ipv6_subnet())
 
     @prefix_bp.route("/submit", methods=["POST"])
     @root_bp.route("/submit", methods=["POST"])

From 9566c297d975348257f2c74a71fabee3cbb1a1d1 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 18:40:56 +0100
Subject: [PATCH 24/54] Don't do it as root

---
 webmails/start.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/webmails/start.py b/webmails/start.py
index f87ac55f..bd395e5d 100755
--- a/webmails/start.py
+++ b/webmails/start.py
@@ -2,6 +2,7 @@
 
 import os
 import logging
+from pwd import getpwnam
 import sys
 import subprocess
 import shutil
@@ -77,10 +78,17 @@ conf.jinja("/conf/config.inc.php", context, "/var/www/roundcube/config/config.in
 # create dirs
 os.system("mkdir -p /data/gpg")
 
+def demote(user_uid, user_gid):
+    def result():
+        os.setgid(user_gid)
+        os.setuid(user_uid)
+    return result
+id_mailu = getpwnam('mailu')
+
 print("Initializing database")
 try:
     result = subprocess.check_output(["/var/www/roundcube/bin/initdb.sh", "--dir", "/var/www/roundcube/SQL"],
-                                     stderr=subprocess.STDOUT)
+                                     stderr=subprocess.STDOUT, preexec_fn=demote(id_mailu.pw_uid,id_mailu.pw_gid))
     print(result.decode())
 except subprocess.CalledProcessError as exc:
     err = exc.stdout.decode()
@@ -92,13 +100,13 @@ except subprocess.CalledProcessError as exc:
 
 print("Upgrading database")
 try:
-    subprocess.check_call(["/var/www/roundcube/bin/update.sh", "--version=?", "-y"], stderr=subprocess.STDOUT)
+    subprocess.check_call(["/var/www/roundcube/bin/update.sh", "--version=?", "-y"], stderr=subprocess.STDOUT, preexec_fn=demote(id_mailu.pw_uid,id_mailu.pw_gid))
 except subprocess.CalledProcessError as exc:
     exit(4)
 else:
     print("Cleaning database")
     try:
-        subprocess.check_call(["/var/www/roundcube/bin/cleandb.sh"], stderr=subprocess.STDOUT)
+        subprocess.check_call(["/var/www/roundcube/bin/cleandb.sh"], stderr=subprocess.STDOUT, preexec_fn=demote(id_mailu.pw_uid,id_mailu.pw_gid))
     except subprocess.CalledProcessError as exc:
         exit(5)
 

From c4595fddca5a9a5515266d3b1ed4bf46da806ec4 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 24 Nov 2022 19:08:30 +0100
Subject: [PATCH 25/54] Change perms first

---
 webmails/start.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/webmails/start.py b/webmails/start.py
index bd395e5d..f6dd4d56 100755
--- a/webmails/start.py
+++ b/webmails/start.py
@@ -78,6 +78,18 @@ conf.jinja("/conf/config.inc.php", context, "/var/www/roundcube/config/config.in
 # create dirs
 os.system("mkdir -p /data/gpg")
 
+base = "/data/_data_/_default_/"
+shutil.rmtree(base + "domains/", ignore_errors=True)
+os.makedirs(base + "domains", exist_ok=True)
+os.makedirs(base + "configs", exist_ok=True)
+
+conf.jinja("/defaults/default.json", context, "/data/_data_/_default_/domains/default.json")
+conf.jinja("/defaults/application.ini", context, "/data/_data_/_default_/configs/application.ini")
+conf.jinja("/defaults/php.ini", context, "/etc/php81/php.ini")
+
+# setup permissions
+os.system("chown -R mailu:mailu /data")
+
 def demote(user_uid, user_gid):
     def result():
         os.setgid(user_gid)
@@ -110,18 +122,6 @@ else:
     except subprocess.CalledProcessError as exc:
         exit(5)
 
-base = "/data/_data_/_default_/"
-shutil.rmtree(base + "domains/", ignore_errors=True)
-os.makedirs(base + "domains", exist_ok=True)
-os.makedirs(base + "configs", exist_ok=True)
-
-conf.jinja("/defaults/default.json", context, "/data/_data_/_default_/domains/default.json")
-conf.jinja("/defaults/application.ini", context, "/data/_data_/_default_/configs/application.ini")
-conf.jinja("/defaults/php.ini", context, "/etc/php81/php.ini")
-
-# setup permissions
-os.system("chown -R mailu:mailu /data")
-
 # Configure nginx
 conf.jinja("/conf/nginx-webmail.conf", context, "/etc/nginx/http.d/webmail.conf")
 if os.path.exists("/var/run/nginx.pid"):

From 78281151023508cd1c5ae7a4b01447727764855c Mon Sep 17 00:00:00 2001
From: Alexander Graf <ghostwheel42@users.noreply.github.com>
Date: Fri, 25 Nov 2022 08:29:50 +0100
Subject: [PATCH 26/54] Re-add flavor and steps to wizard.

---
 setup/server.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/setup/server.py b/setup/server.py
index eda0a36f..622905ed 100644
--- a/setup/server.py
+++ b/setup/server.py
@@ -78,7 +78,12 @@ def build_app(path):
     @prefix_bp.route("/")
     @root_bp.route("/")
     def wizard():
-        return flask.render_template('wizard.html', subnet6=random_ipv6_subnet())
+        return flask.render_template(
+            'wizard.html',
+            flavor="compose",
+            steps=sorted(os.listdir(os.path.join(path, "templates", "steps", "compose"))),
+            subnet6=random_ipv6_subnet()
+        )
 
     @prefix_bp.route("/submit", methods=["POST"])
     @root_bp.route("/submit", methods=["POST"])

From e927426dfafd31d407d30acee71c24b0a88ac906 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 25 Nov 2022 09:37:05 +0100
Subject: [PATCH 27/54] Turns out that php81-ctype is required by roundcube

see https://github.com/roundcube/roundcubemail/issues/7049
---
 webmails/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index 77bc65f3..19b739c9 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -12,7 +12,7 @@ RUN set -euxo pipefail \
   ; apk add --no-cache \
     nginx gpg gpg-agent \
     php81 php81-fpm php81-mbstring php81-zip php81-xml php81-simplexml php81-pecl-apcu \
-    php81-dom php81-curl php81-exif gd php81-gd php81-iconv php81-intl php81-openssl \
+    php81-dom php81-curl php81-exif gd php81-gd php81-iconv php81-intl php81-openssl php81-ctype \
     php81-pdo_sqlite php81-pdo_mysql php81-pdo_pgsql php81-pdo php81-sodium libsodium php81-tidy php81-pecl-uuid \
     php81-pspell php81-pecl-imagick php81-opcache php81-session php81-sockets php81-fileinfo \
     aspell-uk aspell-ru aspell-fr aspell-de aspell-en \

From a5eeab37e1210ccf31f6f850790262d9d52d00e6 Mon Sep 17 00:00:00 2001
From: Alexander Graf <ghostwheel42@users.noreply.github.com>
Date: Fri, 25 Nov 2022 10:43:00 +0100
Subject: [PATCH 28/54] Add default for column allow_spoofing

---
 core/admin/migrations/versions/7ac252f2bbbf_.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/migrations/versions/7ac252f2bbbf_.py b/core/admin/migrations/versions/7ac252f2bbbf_.py
index c37bf554..039ab8b6 100644
--- a/core/admin/migrations/versions/7ac252f2bbbf_.py
+++ b/core/admin/migrations/versions/7ac252f2bbbf_.py
@@ -15,7 +15,7 @@ import sqlalchemy as sa
 
 
 def upgrade():
-    op.add_column('user', sa.Column('allow_spoofing', sa.Boolean(), nullable=False))
+    op.add_column('user', sa.Column('allow_spoofing', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
 
 
 def downgrade():

From 53720876b481b4c26b20fd58706e53ef24d8ab5f Mon Sep 17 00:00:00 2001
From: Alexander Graf <ghostwheel42@users.noreply.github.com>
Date: Fri, 25 Nov 2022 10:47:49 +0100
Subject: [PATCH 29/54] Colorize feature badges

---
 core/admin/mailu/ui/templates/user/list.html | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/admin/mailu/ui/templates/user/list.html b/core/admin/mailu/ui/templates/user/list.html
index 84c0cd27..1c845062 100644
--- a/core/admin/mailu/ui/templates/user/list.html
+++ b/core/admin/mailu/ui/templates/user/list.html
@@ -40,9 +40,9 @@
     </td>
     <td>{{ user }}</td>
     <td data-sort="{{ user.allow_spoofing*4 + user.enable_imap*2 + user.enable_pop }}">
-      {% if user.enable_imap %}<span class="badge bg-info">imap</span>{% endif %}
-      {% if user.enable_pop %}<span class="badge bg-info">pop3</span>{% endif %}
-      {% if user.allow_spoofing %}<span class="badge bg-info">allow-spoofing</span>{% endif %}
+      {% if user.enable_imap %}<span class="badge bg-primary">imap</span>{% endif %}
+      {% if user.enable_pop %}<span class="badge bg-secondary">pop3</span>{% endif %}
+      {% if user.allow_spoofing %}<span class="badge bg-danger">allow-spoofing</span>{% endif %}
     </td>
     <td data-sort="{{ user.quota_bytes_used }}">{{ user.quota_bytes_used | filesizeformat }} / {{ (user.quota_bytes | filesizeformat) if user.quota_bytes else '∞' }}</td>
     <td>{{ user.comment or '-' }}</td>

From b0990460a47035cfbf10051c0d9bcd3f5e4d104f Mon Sep 17 00:00:00 2001
From: Alexander Graf <ghostwheel42@users.noreply.github.com>
Date: Fri, 25 Nov 2022 11:32:21 +0100
Subject: [PATCH 30/54] Fix error display

---
 core/admin/mailu/ui/templates/macros.html | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/core/admin/mailu/ui/templates/macros.html b/core/admin/mailu/ui/templates/macros.html
index 90084246..31efd0e4 100644
--- a/core/admin/mailu/ui/templates/macros.html
+++ b/core/admin/mailu/ui/templates/macros.html
@@ -3,7 +3,7 @@
     {%- for fieldname, errors in form.errors.items() %}
       {%- if bootstrap_is_hidden_field(form[fieldname]) %}
         {%- for error in errors %}
-          <p class="error">{{error}}</p>
+          <p class="form-text text-danger">{{error}}</p>
         {%- endfor %}
       {%- endif %}
     {%- endfor %}
@@ -13,7 +13,7 @@
 {%- macro form_field_errors(field) %}
   {%- if field.errors %}
     {%- for error in field.errors %}
-      <p class="help-block inline">{{ error }}</p>
+      <p class="form-text text-danger">{{ error }}</p>
     {%- endfor %}
   {%- endif %}
 {%- endmacro %}
@@ -23,7 +23,7 @@
   <div class="form-group">
     <div class="row">
       {%- for field in fields %}
-      <div class="col-lg-{{ width }} col-xs-12 {{ 'has-error' if field.errors else '' }}">
+      <div class="col-lg-{{ width }} col-xs-12">
         {%- if field.__class__.__name__ == 'list' %}
           {%- for subfield in field %}
             {{ form_individual_field(subfield, prepend=prepend, append=append, label=label, **kwargs) }}
@@ -38,12 +38,13 @@
 {%- endmacro %}
 
 {%- macro form_individual_field(field, prepend='', append='', label=True, class_="") %}
+  {%- set fieldclass=" ".join(["form-control"] + ([class_] if class_ else []) + (["is-invalid"] if field.errors else [])) %}
   {%- if field.type == "BooleanField" %}
     {{ field(**kwargs) }}<span>&nbsp;&nbsp;</span>{{ field.label if label else '' }}
   {%- else %}
     {{ field.label if label else '' }}{{ form_field_errors(field) }}
     {%- if prepend %}<div class="input-group-prepend">{%- elif append %}<div class="input-group-append">{%- endif %}
-      {{ prepend|safe }}{{ field(class_=("form-control " + class_) if class_ else "form-control", **kwargs) }}{{ append|safe }}
+      {{ prepend|safe }}{{ field(class_=fieldclass, **kwargs) }}{{ append|safe }}
     {%- if prepend or append %}</div>{%- endif %}
   {%- endif %}
 {%- endmacro %}

From c1062f3db24a07d104690975adc493ab0957cc4f Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 25 Nov 2022 17:53:25 +0100
Subject: [PATCH 31/54] set the umask

---
 core/admin/mailu/manage.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py
index 098f1283..32619fe3 100644
--- a/core/admin/mailu/manage.py
+++ b/core/admin/mailu/manage.py
@@ -385,6 +385,7 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None
         'dns': dns,
     }
 
+    old_umask = os.umask(0o077)
     try:
         schema = MailuSchema(only=only, context=context)
         if as_json:
@@ -396,6 +397,8 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None
         if msg := log.format_exception(exc):
             raise click.ClickException(msg) from exc
         raise
+    finally:
+        os.umask(old_umask)
 
 
 @mailu.command()

From 86edc3a9191a53dd37c68f1885f27b92d0316b74 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 27 Nov 2022 09:56:04 +0100
Subject: [PATCH 32/54] Close #1483: remove postfix's /queue/pid/master.pid

---
 core/postfix/start.py               | 2 ++
 towncrier/newsfragments/1483.bugfix | 1 +
 2 files changed, 3 insertions(+)
 create mode 100644 towncrier/newsfragments/1483.bugfix

diff --git a/core/postfix/start.py b/core/postfix/start.py
index 509f961a..80c1c4bf 100755
--- a/core/postfix/start.py
+++ b/core/postfix/start.py
@@ -14,6 +14,8 @@ from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
+os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
+
 def start_podop():
     os.setuid(getpwnam('postfix').pw_uid)
     os.makedirs('/dev/shm/postfix',mode=0o700, exist_ok=True)
diff --git a/towncrier/newsfragments/1483.bugfix b/towncrier/newsfragments/1483.bugfix
new file mode 100644
index 00000000..16e28f39
--- /dev/null
+++ b/towncrier/newsfragments/1483.bugfix
@@ -0,0 +1 @@
+Remove postfix's master.pid on startup if there is no other instance running

From bf588d19a4296c5b1f3af9f438a0cf77f8fb1561 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 27 Nov 2022 10:49:31 +0100
Subject: [PATCH 33/54] Fix RECIPIENT_DELIMITER

---
 core/dovecot/conf/dovecot.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index 7a987582..29fbb9a2 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -116,9 +116,9 @@ service imap-login {
 ###############
 # Delivery
 ###############
+recipient_delimiter = {{ RECIPIENT_DELIMITER }}
 protocol lmtp {
   mail_plugins = $mail_plugins sieve
-  recipient_delimiter = {{ RECIPIENT_DELIMITER }}
 }
 
 service lmtp {

From 5da2ab8fd1494f19315522c3b739b328f0c32613 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 27 Nov 2022 10:59:18 +0100
Subject: [PATCH 34/54] drop privs

---
 core/dovecot/conf/dovecot.conf | 9 ++++-----
 core/dovecot/start.py          | 8 ++++++--
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index 29fbb9a2..d9b85172 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -7,6 +7,10 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
 hostname = {{ HOSTNAMES.split(",")[0] }}
 submission_host = {{ FRONT_ADDRESS }}
 
+default_internal_user = dovecot
+default_login_user = mail
+default_internal_group = dovecot
+
 ###############
 # Mailboxes
 ###############
@@ -80,18 +84,13 @@ userdb {
 }
 
 service auth {
-  user = dovecot
   unix_listener auth-userdb {
   }
 }
 
 service auth-worker {
   unix_listener auth-worker {
-    user = dovecot
-    group = mail
-    mode = 0660
   }
-  user = mail
 }
 
 ###############
diff --git a/core/dovecot/start.py b/core/dovecot/start.py
index a8c85ebf..cfa477bc 100755
--- a/core/dovecot/start.py
+++ b/core/dovecot/start.py
@@ -5,6 +5,7 @@ import glob
 import multiprocessing
 import logging as log
 import sys
+from pwd import getpwnam
 
 from podop import run_server
 from socrate import system, conf
@@ -12,7 +13,9 @@ from socrate import system, conf
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 def start_podop():
-    os.setuid(8)
+    id_mail = getpwnam('mail')
+    os.setgid(id_mail.pw_gid)
+    os.setuid(id_mail.pw_uid)
     url = "http://" + os.environ["ADMIN_ADDRESS"] + "/internal/dovecot/§"
     run_server(0, "dovecot", "/tmp/podop.socket", [
 		("quota", "url", url ),
@@ -35,7 +38,8 @@ for script_file in glob.glob("/conf/*.script"):
     os.chmod(out_file, 0o555)
 
 # Run Podop, then postfix
-multiprocessing.Process(target=start_podop).start()
 os.system("chown mail:mail /mail")
 os.system("chown -R mail:mail /var/lib/dovecot /conf")
+
+multiprocessing.Process(target=start_podop).start()
 os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

From 98f16b1d47c4d0c736f3aa76441ef85880f55e18 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 27 Nov 2022 13:57:03 +0100
Subject: [PATCH 35/54] Fix DB downgrade

---
 core/admin/migrations/versions/f4f0f89e0047_.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/admin/migrations/versions/f4f0f89e0047_.py b/core/admin/migrations/versions/f4f0f89e0047_.py
index 8d20063c..da38dffb 100644
--- a/core/admin/migrations/versions/f4f0f89e0047_.py
+++ b/core/admin/migrations/versions/f4f0f89e0047_.py
@@ -21,5 +21,5 @@ def upgrade():
 
 def downgrade():
     with op.batch_alter_table('fetch') as batch:
-        batch.drop_column('fetch', 'folders')
-        batch.drop_column('fetch', 'scan')
+        batch.drop_column('folders')
+        batch.drop_column('scan')

From 3e38e7b89dde858d590c40ebeee3ede9f6f3b5f4 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sun, 27 Nov 2022 15:41:21 +0100
Subject: [PATCH 36/54] Remove the dependency on pyOpenSSL

---
 core/admin/mailu/dkim.py        | 15 ++++++++-------
 core/admin/mailu/schemas.py     |  4 ++--
 core/base/requirements-dev.txt  |  1 -
 core/base/requirements-prod.txt |  1 -
 4 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/core/admin/mailu/dkim.py b/core/admin/mailu/dkim.py
index e682c64c..5719a62e 100644
--- a/core/admin/mailu/dkim.py
+++ b/core/admin/mailu/dkim.py
@@ -2,20 +2,21 @@
 They are thus represented as ASCII armored PEM.
 """
 
-from OpenSSL import crypto
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
 
 
-def gen_key(key_type=crypto.TYPE_RSA, bits=2048):
+def gen_key(bits=2048):
     """ Generate and return a new RSA key.
     """
-    key = crypto.PKey()
-    key.generate_key(key_type, bits)
-    return crypto.dump_privatekey(crypto.FILETYPE_PEM, key)
+    k = rsa.generate_private_key(public_exponent=65537, key_size=bits)
+    return k.private_bytes(encoding=serialization.Encoding.PEM,format=serialization.PrivateFormat.PKCS8,encryption_algorithm=serialization.NoEncryption())
 
 
 def strip_key(pem):
     """ Return only the b64 part of the ASCII armored PEM.
     """
-    key = crypto.load_privatekey(crypto.FILETYPE_PEM, pem)
-    public_pem = crypto.dump_publickey(crypto.FILETYPE_PEM, key)
+
+    priv_key = serialization.load_pem_private_key(pem, password=None)
+    public_pem = priv_key.public_key().public_bytes(encoding=serialization.Encoding.PEM,format=serialization.PublicFormat.SubjectPublicKeyInfo)
     return public_pem.replace(b"\n", b"").split(b"-----")[2]
diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py
index ca3530fa..bae9be16 100644
--- a/core/admin/mailu/schemas.py
+++ b/core/admin/mailu/schemas.py
@@ -19,7 +19,7 @@ from marshmallow_sqlalchemy.fields import RelatedList
 
 from flask_marshmallow import Marshmallow
 
-from OpenSSL import crypto
+from cryptography.hazmat.primitives import serialization
 
 from pygments import highlight
 from pygments.token import Token
@@ -609,7 +609,7 @@ class DkimKeyField(fields.String):
 
         # check key validity
         try:
-            crypto.load_privatekey(crypto.FILETYPE_PEM, value)
+            serialization.load_pem_private_key(value, password=None)
         except crypto.Error as exc:
             raise ValidationError(f'invalid dkim key {bad_key!r}') from exc
         else:
diff --git a/core/base/requirements-dev.txt b/core/base/requirements-dev.txt
index ebcdde92..52874a86 100644
--- a/core/base/requirements-dev.txt
+++ b/core/base/requirements-dev.txt
@@ -27,7 +27,6 @@ mysql-connector-python==8.0.29
 passlib
 psycopg2-binary
 Pygments
-pyOpenSSL
 PyYAML
 redis
 SQLAlchemy
diff --git a/core/base/requirements-prod.txt b/core/base/requirements-prod.txt
index 4cf70cd0..8b861cd5 100644
--- a/core/base/requirements-prod.txt
+++ b/core/base/requirements-prod.txt
@@ -51,7 +51,6 @@ psycopg2-binary==2.9.5
 pycares==4.2.2
 pycparser==2.21
 Pygments==2.13.0
-pyOpenSSL==22.1.0
 pyparsing==3.0.9
 python-dateutil==2.8.2
 pytz==2022.6

From 00f07ef533d6aaa4188f59eecb44132610840f34 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Tue, 29 Nov 2022 13:25:50 +0100
Subject: [PATCH 37/54] close #2451: prevent an auth-loop on webmails

---
 core/admin/mailu/internal/nginx.py  | 6 ++++--
 towncrier/newsfragments/2451.bugfix | 1 +
 2 files changed, 5 insertions(+), 2 deletions(-)
 create mode 100644 towncrier/newsfragments/2451.bugfix

diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py
index 43e4dd6a..5b321ad3 100644
--- a/core/admin/mailu/internal/nginx.py
+++ b/core/admin/mailu/internal/nginx.py
@@ -26,12 +26,14 @@ STATUSES = {
     }),
 }
 
+WEBMAIL_PORTS = ['10143', '10025']
+
 def check_credentials(user, password, ip, protocol=None, auth_port=None):
-    if not user or not user.enabled or (protocol == "imap" and not user.enable_imap) or (protocol == "pop3" and not user.enable_pop):
+    if not user or not user.enabled or (protocol == "imap" and not user.enable_imap and not auth_port in WEBMAIL_PORTS) or (protocol == "pop3" and not user.enable_pop):
         return False
     is_ok = False
     # webmails
-    if auth_port in ['10143', '10025'] and password.startswith('token-'):
+    if auth_port in WEBMAIL_PORTS and password.startswith('token-'):
         if utils.verify_temp_token(user.get_id(), password):
             is_ok = True
     # All tokens are 32 characters hex lowercase
diff --git a/towncrier/newsfragments/2451.bugfix b/towncrier/newsfragments/2451.bugfix
new file mode 100644
index 00000000..d7e821ea
--- /dev/null
+++ b/towncrier/newsfragments/2451.bugfix
@@ -0,0 +1 @@
+Fix a bug preventing users without IMAP access to access the webmails

From b553d025eb029fe4189f0c51fbe60332203540e5 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Tue, 29 Nov 2022 13:32:40 +0100
Subject: [PATCH 38/54] remove newline

---
 core/admin/mailu/dkim.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/admin/mailu/dkim.py b/core/admin/mailu/dkim.py
index 5719a62e..7eda45d7 100644
--- a/core/admin/mailu/dkim.py
+++ b/core/admin/mailu/dkim.py
@@ -16,7 +16,6 @@ def gen_key(bits=2048):
 def strip_key(pem):
     """ Return only the b64 part of the ASCII armored PEM.
     """
-
     priv_key = serialization.load_pem_private_key(pem, password=None)
     public_pem = priv_key.public_key().public_bytes(encoding=serialization.Encoding.PEM,format=serialization.PublicFormat.SubjectPublicKeyInfo)
     return public_pem.replace(b"\n", b"").split(b"-----")[2]

From c565e69a018317ff46802770a3d78fd13ca44ca1 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Tue, 29 Nov 2022 13:34:22 +0100
Subject: [PATCH 39/54] as requested

---
 core/admin/mailu/schemas.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py
index bae9be16..4a9792ec 100644
--- a/core/admin/mailu/schemas.py
+++ b/core/admin/mailu/schemas.py
@@ -609,8 +609,8 @@ class DkimKeyField(fields.String):
 
         # check key validity
         try:
-            serialization.load_pem_private_key(value, password=None)
-        except crypto.Error as exc:
+            serialization.load_pem_private_key(bytes(value, "ascii"), password=None)
+        except (UnicodeEncodeError, ValueError) as exc:
             raise ValidationError(f'invalid dkim key {bad_key!r}') from exc
         else:
             return value

From 619a5fbda20625dc3c99d238e7bfc9ca0130fc90 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 2 Dec 2022 16:44:44 +0100
Subject: [PATCH 40/54] Upgrade to alpine 3.17.0

---
 core/base/Dockerfile              | 2 +-
 towncrier/newsfragments/2570.misc | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 towncrier/newsfragments/2570.misc

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 65e8be94..e5ee50da 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile-upstream:1.4.3
 
 # base system image (intermediate)
-ARG DISTRO=alpine:3.16.3
+ARG DISTRO=alpine:3.17.0
 FROM $DISTRO as system
 
 ENV TZ=Etc/UTC LANG=C.UTF-8
diff --git a/towncrier/newsfragments/2570.misc b/towncrier/newsfragments/2570.misc
new file mode 100644
index 00000000..ec31181e
--- /dev/null
+++ b/towncrier/newsfragments/2570.misc
@@ -0,0 +1 @@
+Upgrade to Alpine 3.17.0

From 73107ba1124b8f3904b7406b6defbb113127907d Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 2 Dec 2022 17:19:11 +0100
Subject: [PATCH 41/54] libressl-dev is broken in the new release

---
 core/base/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index e5ee50da..1b281384 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -56,7 +56,7 @@ RUN set -euxo pipefail \
   ;  machine="$(uname -m)" \
   ; deps="build-base gcc libffi-dev python3-dev" \
   ; [[ "${machine}" != x86_64 ]] && \
-      deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \
+      deps="${deps} cargo git openssl-dev mariadb-connector-c-dev postgresql-dev" \
   ; apk add --virtual .build-deps ${deps} \
   ; [[ "${machine}" == armv7* ]] && \
       mkdir -p /root/.cargo/registry/index && \

From 622e09312262839d683302ceda878c59d274458a Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 2 Dec 2022 17:23:58 +0100
Subject: [PATCH 42/54] not required anymore

---
 core/base/Dockerfile | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 1b281384..f46a38a2 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -66,10 +66,6 @@ RUN set -euxo pipefail \
   ; cd snuffleupagus-$SNUFFLEUPAGUS_VERSION \
   ; rm -rf src/tests/*php7*/ src/tests/*session*/ src/tests/broken_configuration/ src/tests/*cookie* src/tests/upload_validation/ \
   ; apk add --virtual .build-deps php81-dev php81-cgi php81-simplexml php81-xml pcre-dev build-base php81-pear php81-openssl re2c \
-  ; ln -s /usr/bin/phpize81 /usr/bin/phpize \
-  ; ln -s /usr/bin/pecl81 /usr/bin/pecl \
-  ; ln -s /usr/bin/php-config81 /usr/bin/php-config \
-  ; ln -s /usr/bin/php81 /usr/bin/php \
   ; pecl install vld-beta \
   ; make -j $(grep -c processor /proc/cpuinfo) release \
   ; cp src/.libs/snuffleupagus.so /app \

From 8150ca77b28f03f7a813faea550ebaf7fa5157ea Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Fri, 2 Dec 2022 17:29:44 +0100
Subject: [PATCH 43/54] this isn't required anymore either

---
 webmails/Dockerfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index 19b739c9..2421e9ba 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -18,7 +18,6 @@ RUN set -euxo pipefail \
     aspell-uk aspell-ru aspell-fr aspell-de aspell-en \
   ; rm /etc/nginx/http.d/default.conf \
   ; rm /etc/php81/php-fpm.d/www.conf \
-  ; ln -s /usr/bin/php81 /usr/bin/php \
   ; gpg --import /tmp/snappymail.asc \
   ; gpg --import /tmp/roundcube.asc \
   ; echo extension=snuffleupagus > /etc/php81/conf.d/snuffleupagus.ini \

From 020982527707c7eb0d5ac8d39ff8bd098da25f97 Mon Sep 17 00:00:00 2001
From: fastlorenzo <git@bernardi.be>
Date: Wed, 7 Dec 2022 11:42:04 +0100
Subject: [PATCH 44/54] Add net_bind_service capability for python executable

Signed-off-by: fastlorenzo <git@bernardi.be>
---
 core/base/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 65e8be94..9f0b204f 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -80,7 +80,8 @@ FROM system
 
 COPY --from=build /app/venv/ /app/venv/
 COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/
-RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn
+RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn \
+  ; setcap 'cap_net_bind_service=+ep' /usr/bin/python3.10
 
 ENV VIRTUAL_ENV=/app/venv
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

From dfaba5bb17d9e460ce43cf12b9b7033900ac425d Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@users.noreply.github.com>
Date: Wed, 7 Dec 2022 15:51:54 +0100
Subject: [PATCH 45/54] No need for two commands here

---
 core/base/Dockerfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index 9f0b204f..e3e53dc6 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -80,8 +80,7 @@ FROM system
 
 COPY --from=build /app/venv/ /app/venv/
 COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/
-RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn \
-  ; setcap 'cap_net_bind_service=+ep' /usr/bin/python3.10
+RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn 'cap_net_bind_service=+ep' /usr/bin/python3.10
 
 ENV VIRTUAL_ENV=/app/venv
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

From 4e3874b0c147e2c0dbca6a9a85f09d8ea54d734a Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 8 Dec 2022 12:46:31 +0100
Subject: [PATCH 46/54] Enable dynamic resolution of hostnames

---
 core/admin/mailu/configuration.py         | 30 +++-------------
 core/admin/mailu/internal/nginx.py        | 13 +++----
 core/admin/mailu/models.py                |  3 +-
 core/admin/mailu/ui/templates/client.html |  4 +--
 core/admin/run_dev.sh                     | 11 +++---
 core/admin/start.py                       |  2 ++
 core/base/Dockerfile                      | 20 ++++++++---
 core/base/libs/socrate/socrate/system.py  | 42 ++++++++++-------------
 core/dovecot/conf/ham.script              |  5 ++-
 core/dovecot/conf/spam.script             |  5 ++-
 core/dovecot/start.py                     |  5 +--
 core/nginx/conf/nginx.conf                |  4 +--
 core/nginx/config.py                      |  9 +----
 core/postfix/conf/main.cf                 |  4 +--
 core/postfix/start.py                     |  5 +--
 core/rspamd/conf/antivirus.conf           |  2 +-
 core/rspamd/start.py                      |  9 ++---
 docs/configuration.rst                    | 36 +++++++------------
 optional/fetchmail/fetchmail.py           | 22 ++----------
 optional/unbound/start.py                 |  3 +-
 setup/flavors/compose/docker-compose.yml  |  3 ++
 towncrier/newsfragments/1341.misc         |  4 +++
 webmails/start.py                         | 17 ++-------
 23 files changed, 94 insertions(+), 164 deletions(-)
 create mode 100644 towncrier/newsfragments/1341.misc

diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py
index d447e570..89469ea8 100644
--- a/core/admin/mailu/configuration.py
+++ b/core/admin/mailu/configuration.py
@@ -1,7 +1,6 @@
 import os
 
 from datetime import timedelta
-from socrate import system
 import ipaddress
 
 DEFAULT_CONFIG = {
@@ -83,17 +82,6 @@ DEFAULT_CONFIG = {
     'PROXY_AUTH_WHITELIST': '',
     'PROXY_AUTH_HEADER': 'X-Auth-Email',
     'PROXY_AUTH_CREATE': False,
-    # Host settings
-    'HOST_IMAP': 'imap',
-    'HOST_LMTP': 'imap:2525',
-    'HOST_POP3': 'imap',
-    'HOST_SMTP': 'smtp',
-    'HOST_AUTHSMTP': 'smtp',
-    'HOST_ADMIN': 'admin',
-    'HOST_WEBMAIL': 'webmail',
-    'HOST_WEBDAV': 'webdav:5232',
-    'HOST_REDIS': 'redis',
-    'HOST_FRONT': 'front',
     'SUBNET': '192.168.203.0/24',
     'SUBNET6': None
 }
@@ -111,19 +99,6 @@ class ConfigManager:
     def __init__(self):
         self.config = dict()
 
-    def get_host_address(self, name):
-        # if MYSERVICE_ADDRESS is defined, use this
-        if f'{name}_ADDRESS' in os.environ:
-            return os.environ.get(f'{name}_ADDRESS')
-        # otherwise use the host name and resolve it
-        return system.resolve_address(self.config[f'HOST_{name}'])
-
-    def resolve_hosts(self):
-        for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']:
-            self.config[f'{key}_ADDRESS'] = self.get_host_address(key)
-        if self.config['WEBMAIL'] != 'none':
-            self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL')
-
     def __get_env(self, key, value):
         key_file = key + "_FILE"
         if key_file in os.environ:
@@ -144,11 +119,14 @@ class ConfigManager:
         # get current app config
         self.config.update(app.config)
         # get environment variables
+        for key in os.environ:
+            if key.endswith('_ADDRESS'):
+                self.config[key] = os.environ[key]
+
         self.config.update({
             key: self.__coerce_value(self.__get_env(key, value))
             for key, value in DEFAULT_CONFIG.items()
         })
-        self.resolve_hosts()
 
         # automatically set the sqlalchemy string
         if self.config['DB_FLAVOR']:
diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py
index 5b321ad3..577e5a44 100644
--- a/core/admin/mailu/internal/nginx.py
+++ b/core/admin/mailu/internal/nginx.py
@@ -2,7 +2,6 @@ from mailu import models, utils
 from flask import current_app as app
 from socrate import system
 
-import re
 import urllib
 import ipaddress
 import sqlalchemy.exc
@@ -128,20 +127,16 @@ def get_status(protocol, status):
     status, codes = STATUSES[status]
     return status, codes[protocol]
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
 def get_server(protocol, authenticated=False):
     if protocol == "imap":
-        hostname, port = extract_host_port(app.config['IMAP_ADDRESS'], 143)
+        hostname, port = app.config['IMAP_ADDRESS'], 143
     elif protocol == "pop3":
-        hostname, port = extract_host_port(app.config['POP3_ADDRESS'], 110)
+        hostname, port = app.config['IMAP_ADDRESS'], 110
     elif protocol == "smtp":
         if authenticated:
-            hostname, port = extract_host_port(app.config['AUTHSMTP_ADDRESS'], 10025)
+            hostname, port = app.config['SMTP_ADDRESS'], 10025
         else:
-            hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25)
+            hostname, port = app.config['SMTP_ADDRESS'], 25
     try:
         # test if hostname is already resolved to an ip address
         ipaddress.ip_address(hostname)
diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index b33a0776..a7d0d006 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -421,8 +421,7 @@ class Email(object):
         """ send an email to the address """
         try:
             f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}'
-            ip, port = app.config['HOST_LMTP'].rsplit(':')
-            with smtplib.LMTP(ip, port=port) as lmtp:
+            with smtplib.LMTP(ip=app.config['IMAP_ADDRESS'], port=2525) as lmtp:
                 to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}'
                 msg = text.MIMEText(body)
                 msg['Subject'] = subject
diff --git a/core/admin/mailu/ui/templates/client.html b/core/admin/mailu/ui/templates/client.html
index fddbe0d2..593fd258 100644
--- a/core/admin/mailu/ui/templates/client.html
+++ b/core/admin/mailu/ui/templates/client.html
@@ -21,7 +21,7 @@
     </tr>
     <tr>
       <th>{% trans %}Server name{% endtrans %}</th>
-      <td><pre class="pre-config border bg-light">{{ config["HOSTNAMES"] }}</pre></td>
+      <td><pre class="pre-config border bg-light">{{ config["HOSTNAME"] }}</pre></td>
     </tr>
     <tr>
       <th>{% trans %}Username{% endtrans %}</th>
@@ -46,7 +46,7 @@
     </tr>
     <tr>
       <th>{% trans %}Server name{% endtrans %}</th>
-      <td><pre class="pre-config border bg-light">{{ config["HOSTNAMES"] }}</pre></td>
+      <td><pre class="pre-config border bg-light">{{ config["HOSTNAME"] }}</pre></td>
     </tr>
     <tr>
       <th>{% trans %}Username{% endtrans %}</th>
diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh
index cf05fba3..947ad873 100755
--- a/core/admin/run_dev.sh
+++ b/core/admin/run_dev.sh
@@ -75,12 +75,15 @@ ENV \
     DEBUG_ASSETS="/app/static" \
     DEBUG_TB_INTERCEPT_REDIRECTS=False \
     \
-    IMAP_ADDRESS="127.0.0.1" \
-    POP3_ADDRESS="127.0.0.1" \
-    AUTHSMTP_ADDRESS="127.0.0.1" \
+    ADMIN_ADDRESS="127.0.0.1" \
+    FRONT_ADDRESS="127.0.0.1" \
     SMTP_ADDRESS="127.0.0.1" \
+    IMAP_ADDRESS="127.0.0.1" \
     REDIS_ADDRESS="127.0.0.1" \
-    WEBMAIL_ADDRESS="127.0.0.1"
+    ANTIVIRUS_ADDRESS="127.0.0.1" \
+    ANTISPAM_ADDRESS="127.0.0.1" \
+    WEBMAIL_ADDRESS="127.0.0.1" \
+    WEBDAV_ADDRESS="127.0.0.1"
 
 CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"]
 EOF
diff --git a/core/admin/start.py b/core/admin/start.py
index e2163398..6aa0d1a4 100755
--- a/core/admin/start.py
+++ b/core/admin/start.py
@@ -4,6 +4,7 @@ import os
 import logging as log
 from pwd import getpwnam
 import sys
+from socrate import system
 
 os.system("chown mailu:mailu -R /dkim")
 os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
@@ -12,6 +13,7 @@ os.setgid(mailu_id.pw_gid)
 os.setuid(mailu_id.pw_uid)
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
+system.set_env(['SECRET'])
 
 os.system("flask mailu advertise")
 os.system("flask db upgrade")
diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index e3e53dc6..c4ba6b3f 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -17,11 +17,21 @@ RUN set -euxo pipefail \
   ; ! [[ "${machine}" == x86_64 ]] \
     || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0
 
-ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so
-ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
-ENV CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
-ENV CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
-ENV LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now"
+ENV \
+  LD_PRELOAD="/usr/lib/libhardened_malloc.so" \
+  CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \
+  CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \
+  CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" \
+  LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" \
+  ADMIN_ADDRESS="admin" \
+  FRONT_ADDRESS="front" \
+  SMTP_ADDRESS="smtp" \
+  IMAP_ADDRESS="imap" \
+  REDIS_ADDRESS="redis" \
+  ANTIVIRUS_ADDRESS="antivirus" \
+  ANTISPAM_ADDRESS="antispam" \
+  WEBMAIL_ADDRESS="webmail" \
+  WEBDAV_ADDRESS="webdav"
 
 WORKDIR /app
 
diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py
index d4e3802a..319da0a9 100644
--- a/core/base/libs/socrate/socrate/system.py
+++ b/core/base/libs/socrate/socrate/system.py
@@ -1,7 +1,8 @@
+import hmac
+import logging as log
+import os
 import socket
 import tenacity
-from os import environ
-import logging as log
 
 @tenacity.retry(stop=tenacity.stop_after_attempt(100),
                 wait=tenacity.wait_random(min=2, max=5))
@@ -14,25 +15,20 @@ def resolve_hostname(hostname):
     except Exception as e:
         log.warn("Unable to lookup '%s': %s",hostname,e)
         raise e
+def set_env(required_secrets=[]):
+    """ This will set all the environment variables and retains only the secrets we need """
+    secret_key = os.environ.get('SECRET_KEY')
+    if not secret_key:
+        try:
+            secret_key = open(env.get("SECRET_KEY_FILE"), "r").read().strip()
+        except Exception as exc:
+            log.error(f"Can't read SECRET_KEY from file: {exc}")
+            raise exc
+    clean_env()
+    # derive the keys we need
+    for secret in required_secrets:
+        os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest()
 
-
-def resolve_address(address):
-    """ This function is identical to ``resolve_hostname`` but also supports
-    resolving an address, i.e. including a port.
-    """
-    hostname, *rest = address.rsplit(":", 1)
-    ip_address = resolve_hostname(hostname)
-    if ":" in ip_address:
-        ip_address = "[{}]".format(ip_address)
-    return ip_address + "".join(":" + port for port in rest)
-
-
-def get_host_address_from_environment(name, default):
-    """ This function looks up an envionment variable ``{{ name }}_ADDRESS``.
-    If it's defined, it is returned unmodified. If it's undefined, an environment
-    variable ``HOST_{{ name }}`` is looked up and resolved to an ip address.
-    If this is also not defined, the default is resolved to an ip address.
-    """
-    if "{}_ADDRESS".format(name) in environ:
-        return environ.get("{}_ADDRESS".format(name))
-    return resolve_address(environ.get("HOST_{}".format(name), default))
+def clean_env():
+    """ remove all secret keys """
+    [os.environ.pop(key, None) for key in os.environ.keys() if key.endswith("_KEY")]
diff --git a/core/dovecot/conf/ham.script b/core/dovecot/conf/ham.script
index 57112747..7066d170 100755
--- a/core/dovecot/conf/ham.script
+++ b/core/dovecot/conf/ham.script
@@ -1,9 +1,8 @@
 #!/bin/bash
-{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %}
-RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\  -f1):{{ port }}"
+RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\  -f1):11334"
 if [[ $? -ne 0 ]]
 then
-	echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2
+	echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2
 	exit 1
 fi
 
diff --git a/core/dovecot/conf/spam.script b/core/dovecot/conf/spam.script
index 2e3872b0..94d664ae 100755
--- a/core/dovecot/conf/spam.script
+++ b/core/dovecot/conf/spam.script
@@ -1,9 +1,8 @@
 #!/bin/bash
-{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %}
-RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\  -f1):{{ port }}"
+RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\  -f1):11334"
 if [[ $? -ne 0 ]]
 then
-	echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2
+	echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2
 	exit 1
 fi
 
diff --git a/core/dovecot/start.py b/core/dovecot/start.py
index cfa477bc..4da7a09c 100755
--- a/core/dovecot/start.py
+++ b/core/dovecot/start.py
@@ -11,6 +11,7 @@ from podop import run_server
 from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 def start_podop():
     id_mail = getpwnam('mail')
@@ -24,10 +25,6 @@ def start_podop():
     ])
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-os.environ["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334")
-
 for dovecot_file in glob.glob("/conf/*.conf"):
     conf.jinja(dovecot_file, os.environ, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
 
diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf
index f9278f38..b373fb13 100644
--- a/core/nginx/conf/nginx.conf
+++ b/core/nginx/conf/nginx.conf
@@ -77,12 +77,12 @@ http {
       root /static;
       # Variables for proxifying
       set $admin {{ ADMIN_ADDRESS }};
-      set $antispam {{ ANTISPAM_WEBUI_ADDRESS }};
+      set $antispam {{ ANTISPAM_ADDRESS }}:11334;
       {% if WEBMAIL_ADDRESS %}
       set $webmail {{ WEBMAIL_ADDRESS }};
       {% endif %}
       {% if WEBDAV_ADDRESS %}
-      set $webdav {{ WEBDAV_ADDRESS }};
+      set $webdav {{ WEBDAV_ADDRESS }}:5232;
       {% endif %}
       client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }};
 
diff --git a/core/nginx/config.py b/core/nginx/config.py
index 7930ff12..cee8bce4 100755
--- a/core/nginx/config.py
+++ b/core/nginx/config.py
@@ -5,8 +5,8 @@ import logging as log
 import sys
 from socrate import system, conf
 
+system.set_env()
 args = os.environ.copy()
-
 log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING"))
 
 args['TLS_PERMISSIVE'] = str(args.get('TLS_PERMISSIVE')).lower() not in ('false', 'no')
@@ -17,13 +17,6 @@ with open("/etc/resolv.conf") as handle:
     resolver = content[content.index("nameserver") + 1]
     args["RESOLVER"] = f"[{resolver}]" if ":" in resolver else resolver
 
-args["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-args["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334")
-if args["WEBMAIL"] != "none":
-    args["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail")
-if args["WEBDAV"] != "none":
-    args["WEBDAV_ADDRESS"] = system.get_host_address_from_environment("WEBDAV", "webdav:5232")
-
 # TLS configuration
 cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem")
 keypair_name = os.getenv("TLS_KEYPAIR_FILENAME", default="key.pem")
diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf
index f3b789f9..2f0275b7 100644
--- a/core/postfix/conf/main.cf
+++ b/core/postfix/conf/main.cf
@@ -81,7 +81,7 @@ virtual_mailbox_maps = ${podop}mailbox
 # Mails are transported if required, then forwarded to Dovecot for delivery
 relay_domains = ${podop}transport
 transport_maps = lmdb:/etc/postfix/transport.map, ${podop}transport
-virtual_transport = lmtp:inet:{{ LMTP_ADDRESS }}
+virtual_transport = lmtp:inet:{{ IMAP_ADDRESS }}:2525
 
 # Sender and recipient canonical maps, mostly for SRS
 sender_canonical_maps = ${podop}sendermap
@@ -126,7 +126,7 @@ unverified_recipient_reject_reason = Address lookup failure
 # Milter
 ###############
 
-smtpd_milters = inet:{{ ANTISPAM_MILTER_ADDRESS }}
+smtpd_milters = inet:{{ ANTISPAM_ADDRESS }}:11332
 milter_protocol = 6
 milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen}
 milter_default_action = tempfail
diff --git a/core/postfix/start.py b/core/postfix/start.py
index 80c1c4bf..b9a058f7 100755
--- a/core/postfix/start.py
+++ b/core/postfix/start.py
@@ -13,6 +13,7 @@ from pwd import getpwnam
 from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid")
 
@@ -45,10 +46,6 @@ def is_valid_postconf_line(line):
 
 # Actual startup script
 os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True'
-os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_MILTER", "antispam:11332")
-os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
 os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local")
 os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "")
 
diff --git a/core/rspamd/conf/antivirus.conf b/core/rspamd/conf/antivirus.conf
index 1d492850..53da0768 100644
--- a/core/rspamd/conf/antivirus.conf
+++ b/core/rspamd/conf/antivirus.conf
@@ -3,7 +3,7 @@ clamav {
   scan_mime_parts = true;
   symbol = "CLAM_VIRUS";
   type = "clamav";
-  servers = "{{ ANTIVIRUS_ADDRESS }}";
+  servers = "{{ ANTIVIRUS_ADDRESS }}:3310";
   {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %}
   action = "reject"
   {% endif %}
diff --git a/core/rspamd/start.py b/core/rspamd/start.py
index 37de1df9..507da65d 100755
--- a/core/rspamd/start.py
+++ b/core/rspamd/start.py
@@ -6,18 +6,13 @@ import logging as log
 import requests
 import sys
 import time
-from socrate import system, conf
+from socrate import system,conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 # Actual startup script
 
-os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-
-if os.environ.get("ANTIVIRUS") == 'clamav':
-    os.environ["ANTIVIRUS_ADDRESS"] = system.get_host_address_from_environment("ANTIVIRUS", "antivirus:3310")
-
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
 
diff --git a/docs/configuration.rst b/docs/configuration.rst
index b5affad6..f4510626 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -249,32 +249,22 @@ virus mails during SMTP dialogue, so the sender will receive a reject message.
 Infrastructure settings
 -----------------------
 
-Various environment variables ``HOST_*`` can be used to run Mailu containers
+Various environment variables ``*_ADDRESS`` can be used to run Mailu containers
 separately from a supported orchestrator. It is used by the various components
-to find the location of the other containers it depends on. They can contain an
-optional port number. Those variables are:
+to find the location of the other containers it depends on. Those variables are:
 
-- ``HOST_IMAP``: the container that is running the IMAP server (default: ``imap``, port 143)
-- ``HOST_LMTP``: the container that is running the LMTP server (default: ``imap:2525``)
-- ``HOST_HOSTIMAP``: the container that is running the IMAP server for the webmail (default: ``imap``, port 10143)
-- ``HOST_POP3``: the container that is running the POP3 server (default: ``imap``, port 110)
-- ``HOST_SMTP``: the container that is running the SMTP server (default: ``smtp``, port 25)
-- ``HOST_AUTHSMTP``: the container that is running the authenticated SMTP server for the webnmail (default: ``smtp``, port 10025)
-- ``HOST_ADMIN``: the container that is running the admin interface (default: ``admin``)
-- ``HOST_ANTISPAM_MILTER``: the container that is running the antispam milter service (default: ``antispam:11332``)
-- ``HOST_ANTISPAM_WEBUI``: the container that is running the antispam webui service (default: ``antispam:11334``)
-- ``HOST_ANTIVIRUS``: the container that is running the antivirus service (default: ``antivirus:3310``)
-- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
-- ``HOST_WEBDAV``: the container that is running the webdav server (default: ``webdav:5232``)
-- ``HOST_REDIS``: the container that is running the redis daemon (default: ``redis``)
-- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
+- ``ADMIN_ADDRESS``
+- ``ANTISPAM_ADDRESS``
+- ``ANTIVIRUS_ADDRESS``
+- ``FRONT_ADDRESS``
+- ``IMAP_ADDRESS``
+- ``REDIS_ADDRESS``
+- ``SMTP_ADDRESS``
+- ``WEBDAV_ADDRESS``
+- ``WEBMAIL_ADDRESS``
 
-The startup scripts will resolve ``HOST_*`` to their IP addresses and store the result in ``*_ADDRESS`` for further use.
-
-Alternatively, ``*_ADDRESS`` can directly be set. In this case, the values of ``*_ADDRESS`` is kept and not
-resolved. This can be used to rely on DNS based service discovery with changing services IP addresses.
-When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to
-resolve the hostnames.
+These are used for DNS based service discovery with possibly changing services IP addresses.
+``*_ADDRESS`` values must be fully qualified domain names without port numbers.
 
 .. _db_settings:
 
diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py
index 62bd7124..20af1e7f 100755
--- a/optional/fetchmail/fetchmail.py
+++ b/optional/fetchmail/fetchmail.py
@@ -7,7 +7,6 @@ from pwd import getpwnam
 import tempfile
 import shlex
 import subprocess
-import re
 import requests
 from socrate import system
 import sys
@@ -34,11 +33,6 @@ poll "{host}" proto {protocol}  port {port}
 """
 
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
-
 def escape_rc_string(arg):
     return "".join("\\x%2x" % ord(char) for char in arg)
 
@@ -54,20 +48,7 @@ def fetchmail(fetchmailrc):
 
 def run(debug):
     try:
-        os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
-        os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
         fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
-        smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
-        if smtpport is None:
-            smtphostport = smtphost
-        else:
-            smtphostport = "%s/%d" % (smtphost, smtpport)
-        os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
-        lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
-        if lmtpport is None:
-            lmtphostport = lmtphost
-        else:
-            lmtphostport = "%s/%d" % (lmtphost, lmtpport)
         for fetch in fetches:
             fetchmailrc = ""
             options = "options antispam 501, 504, 550, 553, 554"
@@ -79,7 +60,7 @@ def run(debug):
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=smtphostport if fetch['scan'] else lmtphostport,
+                smtphost=f'{os.environ["SMTP_ADDRESS"]}' if fetch['scan'] else f'{os.environ["IMAP_ADDRESS"]}/2525',
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
                 options=options,
@@ -118,6 +99,7 @@ if __name__ == "__main__":
     os.chmod("/data/fetchids", 0o700)
     os.setgid(id_fetchmail.pw_gid)
     os.setuid(id_fetchmail.pw_uid)
+    system.set_env()
     while True:
         delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
         print("Sleeping for {} seconds".format(delay))
diff --git a/optional/unbound/start.py b/optional/unbound/start.py
index f3a5bee7..df768092 100755
--- a/optional/unbound/start.py
+++ b/optional/unbound/start.py
@@ -3,9 +3,10 @@
 import os
 import logging as log
 import sys
-from socrate import conf
+from socrate import conf, system
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf")
 
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index b6c99ca5..cd470098 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -113,6 +113,9 @@ services:
       - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d:ro"
     depends_on:
       - front
+    {% if antivirus_enabled %}
+      - antivirus
+    {% endif %}
     {% if resolver_enabled %}
       - resolver
     dns:
diff --git a/towncrier/newsfragments/1341.misc b/towncrier/newsfragments/1341.misc
new file mode 100644
index 00000000..53f8df91
--- /dev/null
+++ b/towncrier/newsfragments/1341.misc
@@ -0,0 +1,4 @@
+Remove HOST_* variables, use *_ADDRESS everywhere instead. Please note that those should only contain a FQDN (no port number).
+Derive a different key for admin/SECRET_KEY; this will invalidate existing sessions
+Ensure that rspamd starts after clamav
+Only display a single HOSTNAME on the client configuration page
diff --git a/webmails/start.py b/webmails/start.py
index f6dd4d56..954c8407 100755
--- a/webmails/start.py
+++ b/webmails/start.py
@@ -13,14 +13,13 @@ from socrate import conf, system
 env = os.environ
 
 logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING"))
+system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS'])
 
 # jinja context
 context = {}
 context.update(env)
 
 context["MAX_FILESIZE"] = str(int(int(env.get("MESSAGE_SIZE_LIMIT", "50000000")) * 0.66 / 1048576))
-context["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-context["IMAP_ADDRESS"] = system.get_host_address_from_environment("IMAP", "imap")
 
 db_flavor = env.get("ROUNDCUBE_DB_FLAVOR", "sqlite")
 if db_flavor == "sqlite":
@@ -43,17 +42,6 @@ else:
     print(f"Unknown ROUNDCUBE_DB_FLAVOR: {db_flavor}", file=sys.stderr)
     exit(1)
 
-# derive roundcube secret key
-secret_key = env.get("SECRET_KEY")
-if not secret_key:
-    try:
-        secret_key = open(env.get("SECRET_KEY_FILE"), "r").read().strip()
-    except Exception as exc:
-        print(f"Can't read SECRET_KEY from file: {exc}", file=sys.stderr)
-        exit(2)
-
-context['ROUNDCUBE_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest()
-context['SNUFFLEUPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEUPAGUS_KEY', 'utf-8'), 'sha256').hexdigest()
 conf.jinja("/etc/snuffleupagus.rules.tpl", context, "/etc/snuffleupagus.rules")
 
 # roundcube plugins
@@ -127,8 +115,7 @@ conf.jinja("/conf/nginx-webmail.conf", context, "/etc/nginx/http.d/webmail.conf"
 if os.path.exists("/var/run/nginx.pid"):
     os.system("nginx -s reload")
 
-# clean env
-[env.pop(key, None) for key in env.keys() if key == "SECRET_KEY" or key.endswith("_KEY")]
+system.clean_env()
 
 # run nginx
 os.system("php-fpm81")

From b630355d03faef808a1063bd45f196a949e01819 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 8 Dec 2022 15:17:58 +0100
Subject: [PATCH 47/54] Autofocus the login form on /sso/login

---
 core/admin/mailu/sso/forms.py     | 2 +-
 towncrier/newsfragments/2577.misc | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 towncrier/newsfragments/2577.misc

diff --git a/core/admin/mailu/sso/forms.py b/core/admin/mailu/sso/forms.py
index ca124c02..c01ef572 100644
--- a/core/admin/mailu/sso/forms.py
+++ b/core/admin/mailu/sso/forms.py
@@ -5,7 +5,7 @@ import flask_wtf
 class LoginForm(flask_wtf.FlaskForm):
     class Meta:
         csrf = False
-    email = fields.StringField(_('E-mail'), [validators.Email(), validators.DataRequired()])
+    email = fields.StringField(_('E-mail'), [validators.Email(), validators.DataRequired()], render_kw={'autofocus': True})
     pw = fields.PasswordField(_('Password'), [validators.DataRequired()])
     pwned = fields.HiddenField(label='', default=-1)
     submitWebmail = fields.SubmitField(_('Sign in'))
diff --git a/towncrier/newsfragments/2577.misc b/towncrier/newsfragments/2577.misc
new file mode 100644
index 00000000..a9c467cf
--- /dev/null
+++ b/towncrier/newsfragments/2577.misc
@@ -0,0 +1 @@
+Autofocus the login form on /sso/login

From ae6af92b1de85f56b0ecb2dfb253f1bf520858a6 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 8 Dec 2022 16:38:06 +0100
Subject: [PATCH 48/54] it's called libretls!

---
 core/base/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/base/Dockerfile b/core/base/Dockerfile
index f46a38a2..dbb353f7 100644
--- a/core/base/Dockerfile
+++ b/core/base/Dockerfile
@@ -56,7 +56,7 @@ RUN set -euxo pipefail \
   ;  machine="$(uname -m)" \
   ; deps="build-base gcc libffi-dev python3-dev" \
   ; [[ "${machine}" != x86_64 ]] && \
-      deps="${deps} cargo git openssl-dev mariadb-connector-c-dev postgresql-dev" \
+      deps="${deps} cargo git libretls-dev mariadb-connector-c-dev postgresql-dev" \
   ; apk add --virtual .build-deps ${deps} \
   ; [[ "${machine}" == armv7* ]] && \
       mkdir -p /root/.cargo/registry/index && \

From e42d029c2539a6717fde8324f38afafe67dce5cb Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Thu, 8 Dec 2022 17:41:33 +0100
Subject: [PATCH 49/54] normalize booleans

---
 core/base/libs/socrate/socrate/system.py | 13 +++++++++++++
 optional/fetchmail/fetchmail.py          |  8 ++++----
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py
index 319da0a9..b6ee7761 100644
--- a/core/base/libs/socrate/socrate/system.py
+++ b/core/base/libs/socrate/socrate/system.py
@@ -15,6 +15,14 @@ def resolve_hostname(hostname):
     except Exception as e:
         log.warn("Unable to lookup '%s': %s",hostname,e)
         raise e
+
+def _coerce_value(value):
+    if isinstance(value, str) and value.lower() in ('true','yes'):
+        return True
+    elif isinstance(value, str) and value.lower() in ('false', 'no'):
+        return False
+    return value
+
 def set_env(required_secrets=[]):
     """ This will set all the environment variables and retains only the secrets we need """
     secret_key = os.environ.get('SECRET_KEY')
@@ -29,6 +37,11 @@ def set_env(required_secrets=[]):
     for secret in required_secrets:
         os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest()
 
+    return {
+            key: _coerce_value(os.environ.get(key, value))
+            for key, value in os.environ.items()
+           }
+
 def clean_env():
     """ remove all secret keys """
     [os.environ.pop(key, None) for key in os.environ.keys() if key.endswith("_KEY")]
diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py
index 20af1e7f..97622feb 100755
--- a/optional/fetchmail/fetchmail.py
+++ b/optional/fetchmail/fetchmail.py
@@ -99,15 +99,15 @@ if __name__ == "__main__":
     os.chmod("/data/fetchids", 0o700)
     os.setgid(id_fetchmail.pw_gid)
     os.setuid(id_fetchmail.pw_uid)
-    system.set_env()
+    config = system.set_env()
     while True:
-        delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
+        delay = int(os.environ.get('FETCHMAIL_DELAY', 60))
         print("Sleeping for {} seconds".format(delay))
         time.sleep(delay)
 
-        if not os.environ.get("FETCHMAIL_ENABLED", 'True') in ('True', 'true'):
+        if not config.get('FETCHMAIL_ENABLED', True):
             print("Fetchmail disabled, skipping...")
             continue
 
-        run(os.environ.get("DEBUG", None) == "True")
+        run(config.get('DEBUG', False))
         sys.stdout.flush()

From 2fa8dcb51d9a31614eb76df2d6b1652ec1ed92b4 Mon Sep 17 00:00:00 2001
From: fastlorenzo <git@bernardi.be>
Date: Mon, 12 Dec 2022 13:32:06 +0100
Subject: [PATCH 50/54] Fixed roundcube carddav module

Signed-off-by: fastlorenzo <git@bernardi.be>
---
 webmails/Dockerfile          | 2 +-
 webmails/snuffleupagus.rules | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index 2421e9ba..106edc19 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -14,7 +14,7 @@ RUN set -euxo pipefail \
     php81 php81-fpm php81-mbstring php81-zip php81-xml php81-simplexml php81-pecl-apcu \
     php81-dom php81-curl php81-exif gd php81-gd php81-iconv php81-intl php81-openssl php81-ctype \
     php81-pdo_sqlite php81-pdo_mysql php81-pdo_pgsql php81-pdo php81-sodium libsodium php81-tidy php81-pecl-uuid \
-    php81-pspell php81-pecl-imagick php81-opcache php81-session php81-sockets php81-fileinfo \
+    php81-pspell php81-pecl-imagick php81-opcache php81-session php81-sockets php81-fileinfo php81-xmlreader php81-xmlwriter \
     aspell-uk aspell-ru aspell-fr aspell-de aspell-en \
   ; rm /etc/nginx/http.d/default.conf \
   ; rm /etc/php81/php-fpm.d/www.conf \
diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules
index ec7bee13..fbb8a776 100644
--- a/webmails/snuffleupagus.rules
+++ b/webmails/snuffleupagus.rules
@@ -84,6 +84,7 @@ sp.disable_function.function("ini_set").param("option").value("include_path").dr
 sp.disable_function.function("ini_set").param("option").value("open_basedir").drop();
 
 # Detect some backdoors via environment recon
+sp.disable_function.function("ini_get").filename("/var/www/roundcube/vendor/guzzlehttp/guzzle/src/functions.php").param("option").value("allow_url_fopen").allow();
 sp.disable_function.function("ini_get").param("option").value("allow_url_fopen").drop();
 sp.disable_function.function("ini_get").param("option").value("open_basedir").drop();
 sp.disable_function.function("ini_get").param("option").value_r("suhosin").drop();
@@ -97,7 +98,7 @@ sp.disable_function.function("is_callable").param("value").value("eval").drop();
 sp.disable_function.function("is_callable").param("value").value("exec").drop();
 sp.disable_function.function("is_callable").param("value").value("system").drop();
 sp.disable_function.function("is_callable").param("value").value("shell_exec").drop();
-sp.disable_function.function("is_callable").filename_r("/app/libraries/snappymail/pgp/gpg\.php$").param("value").value("proc_open").allow();
+sp.disable_function.function("is_callable").filename_r("^/var/www/snappymail/snappymail/v/\d+\.\d+\.\d+/app/libraries/snappymail/pgp/gpg\.php$").param("value").value("proc_open").allow();
 sp.disable_function.function("is_callable").param("value").value("proc_open").drop();
 sp.disable_function.function("is_callable").param("value").value("passthru").drop();
 

From 135207db3e4637b8bb57373159328bb310ef6d6c Mon Sep 17 00:00:00 2001
From: fastlorenzo <git@bernardi.be>
Date: Wed, 14 Dec 2022 01:00:23 +0100
Subject: [PATCH 51/54] fix missing casting to int for SESSION_KEY_BITS

Signed-off-by: fastlorenzo <git@bernardi.be>
---
 core/admin/mailu/configuration.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py
index d447e570..a4957da1 100644
--- a/core/admin/mailu/configuration.py
+++ b/core/admin/mailu/configuration.py
@@ -165,6 +165,7 @@ class ConfigManager:
             self.config['SESSION_COOKIE_SECURE'] = self.config['TLS_FLAVOR'] != 'notls'
         self.config['SESSION_PERMANENT'] = True
         self.config['SESSION_TIMEOUT'] = int(self.config['SESSION_TIMEOUT'])
+        self.config['SESSION_KEY_BITS'] = int(self.config['SESSION_KEY_BITS'])
         self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME'])
         self.config['AUTH_RATELIMIT_IP_V4_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V4_MASK'])
         self.config['AUTH_RATELIMIT_IP_V6_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V6_MASK'])

From 170b12baf0ad32631ac4da021644ed31c0b55cef Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Sat, 17 Dec 2022 15:24:30 +0100
Subject: [PATCH 52/54] fix sieve

---
 webmails/Dockerfile               |  6 ++--
 webmails/roundcube/roundcube.diff | 47 +++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 webmails/roundcube/roundcube.diff

diff --git a/webmails/Dockerfile b/webmails/Dockerfile
index 106edc19..cf70abe8 100644
--- a/webmails/Dockerfile
+++ b/webmails/Dockerfile
@@ -7,10 +7,11 @@ LABEL version=$VERSION
 
 COPY snappymail/pubkey.asc /tmp/snappymail.asc
 COPY roundcube/pubkey.asc /tmp/roundcube.asc
+COPY roundcube/roundcube.diff /tmp/roundcube.diff
 
 RUN set -euxo pipefail \
   ; apk add --no-cache \
-    nginx gpg gpg-agent \
+    nginx gpg gpg-agent patch \
     php81 php81-fpm php81-mbstring php81-zip php81-xml php81-simplexml php81-pecl-apcu \
     php81-dom php81-curl php81-exif gd php81-gd php81-iconv php81-intl php81-openssl php81-ctype \
     php81-pdo_sqlite php81-pdo_mysql php81-pdo_pgsql php81-pdo php81-sodium libsodium php81-tidy php81-pecl-uuid \
@@ -42,7 +43,8 @@ RUN set -euxo pipefail \
   ; rm -rf CHANGELOG.md SECURITY.md INSTALL LICENSE README.md UPGRADING composer.json-dist installer composer.* \
   ; ln -sf index.php /var/www/roundcube/public_html/sso.php \
   ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query} \
-  ; sed -i '/suhosin.session.encrypt/d;/mbstring\.func_overload/d' program/lib/Roundcube/bootstrap.php
+  ; patch -p0 < /tmp/roundcube.diff \
+  ; rm /tmp/roundcube.diff
 
 COPY roundcube/config/config.inc.php /conf/
 COPY roundcube/login/mailu.php /var/www/roundcube/plugins/mailu/
diff --git a/webmails/roundcube/roundcube.diff b/webmails/roundcube/roundcube.diff
new file mode 100644
index 00000000..1aece369
--- /dev/null
+++ b/webmails/roundcube/roundcube.diff
@@ -0,0 +1,47 @@
+--- plugins/managesieve/lib/Roundcube/rcube_sieve_engine.php
++++ plugins/managesieve/lib/Roundcube/rcube_sieve_engine.php
+@@ -529,28 +529,13 @@
+         // get request size limits (#1488648)
+         $max_post = max([
+                 ini_get('max_input_vars'),
+-                ini_get('suhosin.request.max_vars'),
+-                ini_get('suhosin.post.max_vars'),
+         ]);
+-        $max_depth = max([
+-                ini_get('suhosin.request.max_array_depth'),
+-                ini_get('suhosin.post.max_array_depth'),
+-        ]);
+ 
+         // check request size limit
+         if ($max_post && count($_POST, COUNT_RECURSIVE) >= $max_post) {
+             rcube::raise_error([
+                     'code' => 500, 'file' => __FILE__, 'line' => __LINE__,
+                     'message' => "Request size limit exceeded (one of max_input_vars/suhosin.request.max_vars/suhosin.post.max_vars)"
+-                ], true, false
+-            );
+-            $this->rc->output->show_message('managesieve.filtersaveerror', 'error');
+-        }
+-        // check request depth limits
+-        else if ($max_depth && count($_POST['_header']) > $max_depth) {
+-            rcube::raise_error([
+-                    'code' => 500, 'file' => __FILE__, 'line' => __LINE__,
+-                    'message' => "Request size limit exceeded (one of suhosin.request.max_array_depth/suhosin.post.max_array_depth)"
+                 ], true, false
+             );
+             $this->rc->output->show_message('managesieve.filtersaveerror', 'error');
+--- program/lib/Roundcube/bootstrap.php
++++ program/lib/Roundcube/bootstrap.php
+@@ -32,13 +32,11 @@
+     // Some users are not using Installer, so we'll check some
+     // critical PHP settings here. Only these, which doesn't provide
+     // an error/warning in the logs later. See (#1486307).
+-    'mbstring.func_overload' => 0,
+ ];
+ 
+ // check these additional ini settings if not called via CLI
+ if (php_sapi_name() != 'cli') {
+     $config += [
+-        'suhosin.session.encrypt' => false,
+         'file_uploads'            => true,
+         'session.auto_start'      => false,
+         'zlib.output_compression' => false,

From 0fa239da110a10c69266a21e821472955f00a34e Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 19 Dec 2022 10:42:31 +0100
Subject: [PATCH 53/54] These tests are not required anymore

---
 core/base/libs/socrate/test.py | 35 ----------------------------------
 1 file changed, 35 deletions(-)

diff --git a/core/base/libs/socrate/test.py b/core/base/libs/socrate/test.py
index f6088345..03977685 100644
--- a/core/base/libs/socrate/test.py
+++ b/core/base/libs/socrate/test.py
@@ -78,40 +78,5 @@ class TestSystem(unittest.TestCase):
             "2001:db8::f00"
         )
 
-
-    def test_resolve_address(self):
-        self.assertEqual(
-            system.resolve_address("1.2.3.4.sslip.io:80"),
-            "1.2.3.4:80"
-        )
-        self.assertEqual(
-            system.resolve_address("2001-db8--f00.sslip.io:80"),
-            "[2001:db8::f00]:80"
-        )
-
-    def test_get_host_address_from_environment(self):
-        if "TEST_ADDRESS" in os.environ:
-            del os.environ["TEST_ADDRESS"]
-        if "HOST_TEST" in os.environ:
-            del os.environ["HOST_TEST"]
-        # if nothing is set, the default must be resolved
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.4:80"
-        )
-        # if HOST is set, the HOST must be resolved
-        os.environ['HOST_TEST']="1.2.3.5.sslip.io:80"
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.5:80"
-        )
-        # if ADDRESS is set, the ADDRESS must be returned unresolved
-        os.environ['TEST_ADDRESS']="1.2.3.6.sslip.io:80"
-        self.assertEqual(
-            system.get_host_address_from_environment("TEST", "1.2.3.4.sslip.io:80"),
-            "1.2.3.6.sslip.io:80"
-        )
-
-
 if __name__ == "__main__":
     unittest.main()

From df924b0864fb6832a7041a75c647ed0cb3d5fbf2 Mon Sep 17 00:00:00 2001
From: Florent Daigniere <nextgens@freenetproject.org>
Date: Mon, 19 Dec 2022 11:04:25 +0100
Subject: [PATCH 54/54] doh

---
 core/base/libs/socrate/socrate/system.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py
index b6ee7761..96247158 100644
--- a/core/base/libs/socrate/socrate/system.py
+++ b/core/base/libs/socrate/socrate/system.py
@@ -28,7 +28,7 @@ def set_env(required_secrets=[]):
     secret_key = os.environ.get('SECRET_KEY')
     if not secret_key:
         try:
-            secret_key = open(env.get("SECRET_KEY_FILE"), "r").read().strip()
+            secret_key = open(os.environ.get("SECRET_KEY_FILE"), "r").read().strip()
         except Exception as exc:
             log.error(f"Can't read SECRET_KEY from file: {exc}")
             raise exc