Merge branch 'master' into ratelimits

core/admin/assets/app.js

@@ -66,5 +66,12 @@ $('document').ready(function() {
     // init clipboard.js
     new ClipboardJS('.btn-clip');
 
+    // disable login if not possible
+    var l = $('#login_needs_https');
+    if (l.length && window.location.protocol != 'https:') {
+        l.removeClass("d-none");
+        $('form :input').prop('disabled', true);
+    }
+
 });
 

core/admin/mailu/__init__.py

@@ -30,6 +30,7 @@ def create_app_from_config(config):
 
     app.device_cookie_key = hmac.new(bytearray(app.secret_key, 'utf-8'), bytearray('DEVICE_COOKIE_KEY', 'utf-8'), 'sha256').digest()
     app.temp_token_key = hmac.new(bytearray(app.secret_key, 'utf-8'), bytearray('WEBMAIL_TEMP_TOKEN_KEY', 'utf-8'), 'sha256').digest()
+    app.srs_key = hmac.new(bytearray(app.secret_key, 'utf-8'), bytearray('SRS_KEY', 'utf-8'), 'sha256').digest()
 
     # Initialize list of translations
     config.translations = {

core/admin/mailu/configuration.py

@@ -52,6 +52,7 @@ DEFAULT_CONFIG = {
     'DKIM_PATH': '/dkim/{domain}.{selector}.key',
     'DEFAULT_QUOTA': 1000000000,
     'MESSAGE_RATELIMIT': '200/day',
+    'RECIPIENT_DELIMITER': '',
     # Web settings
     'SITENAME': 'Mailu',
     'WEBSITE': 'https://mailu.io',

core/admin/mailu/internal/nginx.py

@@ -5,6 +5,7 @@ import re
 import urllib
 import ipaddress
 import socket
+import sqlalchemy.exc
 import tenacity
 
 SUPPORTED_AUTH_METHODS = ["none", "plain"]
@@ -77,7 +78,7 @@ def handle_authentication(headers):
     # Authenticated user
     elif method == "plain":
         is_valid_user = False
-        if 'Auth-Port' in headers and int(urllib.parse.unquote(headers["Auth-Port"])) == 25:
+        if 'Auth-Port' in headers and urllib.parse.unquote(headers["Auth-Port"]) == '25':
             return {
                 "Auth-Status": "AUTH not supported",
                 "Auth-Error-Code": "502 5.5.1",
@@ -92,21 +93,26 @@ def handle_authentication(headers):
         try:
             user_email = raw_user_email.encode("iso8859-1").decode("utf8")
             password = raw_password.encode("iso8859-1").decode("utf8")
+            ip = urllib.parse.unquote(headers["Client-Ip"])
         except:
             app.logger.warn(f'Received undecodable user/password from nginx: {raw_user_email!r}/{raw_password!r}')
         else:
-            user = models.User.query.get(user_email)
-            is_valid_user = True
-            ip = urllib.parse.unquote(headers["Client-Ip"])
-            if check_credentials(user, password, ip, protocol):
-                server, port = get_server(headers["Auth-Protocol"], True)
-                return {
-                    "Auth-Status": "OK",
-                    "Auth-Server": server,
-                    "Auth-User": user_email,
-                    "Auth-User-Exists": is_valid_user,
-                    "Auth-Port": port
-                }
+            try:
+                user = models.User.query.get(user_email)
+                is_valid_user = True
+            except sqlalchemy.exc.StatementError as exc:
+                exc = str(exc).split('\n', 1)[0]
+                app.logger.warn(f'Invalid user {user_email!r}: {exc}')
+            else:
+                if check_credentials(user, password, ip, protocol):
+                    server, port = get_server(headers["Auth-Protocol"], True)
+                    return {
+                        "Auth-Status": "OK",
+                        "Auth-Server": server,
+                        "Auth-User": user_email,
+                        "Auth-User-Exists": is_valid_user,
+                        "Auth-Port": port
+                    }
         status, code = get_status(protocol, "authentication")
         return {
             "Auth-Status": status,

core/admin/mailu/internal/views/postfix.py

@@ -108,7 +108,7 @@ def postfix_recipient_map(recipient):
 
     This is meant for bounces to go back to the original sender.
     """
-    srs = srslib.SRS(flask.current_app.config["SECRET_KEY"])
+    srs = srslib.SRS(flask.current_app.srs_key)
     if srslib.SRS.is_srs_address(recipient):
         try:
             return flask.jsonify(srs.reverse(recipient))
@@ -123,7 +123,7 @@ def postfix_sender_map(sender):
 
     This is for bounces to come back the reverse path properly.
     """
-    srs = srslib.SRS(flask.current_app.config["SECRET_KEY"])
+    srs = srslib.SRS(flask.current_app.srs_key)
     domain = flask.current_app.config["DOMAIN"]
     try:
         localpart, domain_name = models.Email.resolve_domain(sender)
@@ -140,6 +140,7 @@ def postfix_sender_login(sender):
     localpart, domain_name = models.Email.resolve_domain(sender)
     if localpart is None:
         return flask.jsonify(",".join(wildcard_senders)) if wildcard_senders else flask.abort(404)
+    localpart = localpart[:next((i for i, ch in enumerate(localpart) if ch in flask.current_app.config.get('RECIPIENT_DELIMITER')), None)]
     destination = models.Email.resolve_destination(localpart, domain_name, True)
     destination = [*destination, *wildcard_senders] if destination else [*wildcard_senders]
     return flask.jsonify(",".join(destination)) if destination else flask.abort(404)

core/admin/mailu/models.py

@@ -57,6 +57,8 @@ class IdnaEmail(db.TypeDecorator):
 
     def process_bind_param(self, value, dialect):
         """ encode unicode domain part of email address to punycode """
+        if not '@' in value:
+            raise ValueError('invalid email address (no "@")')
         localpart, domain_name = value.lower().rsplit('@', 1)
         if '@' in localpart:
             raise ValueError('email local part must not contain "@"')

core/admin/mailu/ui/templates/login.html

@@ -7,3 +7,12 @@
 {%- block subtitle %}
 {% trans %}to access the administration tools{% endtrans %}
 {%- endblock %}
+
+{%- block content %}
+{% if config["SESSION_COOKIE_SECURE"] %}
+<div id="login_needs_https" class="alert alert-danger d-none" role="alert">
+{% trans %}The login form has been disabled as <b>SESSION_COOKIE_SECURE</b> is on but you are accessing Mailu over HTTP.{% endtrans %}
+</div>
+{% endif %}
+{{ super() }}
+{%- endblock %}

core/admin/mailu/utils.py

@@ -60,7 +60,7 @@ def has_dane_record(domain, timeout=10):
         # we will receive this non-specific exception. The safe behaviour is to
         # accept to defer the email.
         flask.current_app.logger.warn(f'Unable to lookup the TLSA record for {domain}. Is the DNSSEC zone okay on https://dnsviz.net/d/{domain}/dnssec/?')
-        return app.config['DEFER_ON_TLS_ERROR']
+        return flask.current_app.config['DEFER_ON_TLS_ERROR']
     except dns.exception.Timeout:
         flask.current_app.logger.warn(f'Timeout while resolving the TLSA record for {domain} ({timeout}s).')
     except dns.resolver.NXDOMAIN:

core/postfix/conf/outclean_header_filter.cf

@@ -1,17 +1,8 @@
 # This configuration was copied from Mailinabox. The original version is available at:
 # https://raw.githubusercontent.com/mail-in-a-box/mailinabox/master/conf/postfix_outgoing_mail_header_filters
 
-# Remove the first line of the Received: header. Note that we cannot fully remove the Received: header
-# because OpenDKIM requires that a header be present when signing outbound mail. The first line is
-# where the user's home IP address would be.
-/^\s*Received:[^\n]*(.*)/         REPLACE Received: from authenticated-user ({{OUTCLEAN}} [{{OUTCLEAN_ADDRESS}}])$1
-
-# Remove other typically private information.
-/^\s*User-Agent:/        IGNORE
-/^\s*X-Enigmail:/        IGNORE
-/^\s*X-Mailer:/          IGNORE
-/^\s*X-Originating-IP:/  IGNORE
-/^\s*X-Pgp-Agent:/       IGNORE
+# Remove typically private information.
+/^\s*(Received|User-Agent|X-(Enigmail|Mailer|Originating-IP|Pgp-Agent)):/	IGNORE
 
 # The Mime-Version header can leak the user agent too, e.g. in Mime-Version: 1.0 (Mac OS X Mail 8.1 \(2010.6\)).
 /^\s*(Mime-Version:\s*[0-9\.]+)\s.+/  REPLACE $1

core/postfix/start.py

@@ -46,15 +46,6 @@ os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT",
 os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
 os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_MILTER", "antispam:11332")
 os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
-os.environ["OUTCLEAN"] = os.environ["HOSTNAMES"].split(",")[0]
-try:
-    _to_lookup = os.environ["OUTCLEAN"]
-    # Ensure we lookup a FQDN: @see #1884
-    if not _to_lookup.endswith('.'):
-        _to_lookup += '.'
-    os.environ["OUTCLEAN_ADDRESS"] = system.resolve_hostname(_to_lookup)
-except:
-    os.environ["OUTCLEAN_ADDRESS"] = "10.10.10.10"
 
 for postfix_file in glob.glob("/conf/*.cf"):
     conf.jinja(postfix_file, os.environ, os.path.join("/etc/postfix", os.path.basename(postfix_file)))

docs/configuration.rst

@@ -100,9 +100,10 @@ go and fetch new email if available. Do not use too short delays if you do not
 want to be blacklisted by external services, but not too long delays if you
 want to receive your email in time.
 
-The ``RECIPIENT_DELIMITER`` is a character used to delimit localpart from a
-custom address part. For instance, if set to ``+``, users can use addresses
-like ``localpart+custom@domain.tld`` to deliver mail to ``localpart@domain.tld``.
+The ``RECIPIENT_DELIMITER`` is a list of characters used to delimit localpart
+from a custom address part. For instance, if set to ``+-``, users can use
+addresses like ``localpart+custom@example.com`` or ``localpart-custom@example.com``
+to deliver mail to ``localpart@example.com``.
 This is useful to provide external parties with different email addresses and
 later classify incoming mail based on the custom part.
 

optional/unbound/unbound.conf

@@ -1,19 +1,20 @@
 server:
   verbosity: 1
   interface: 0.0.0.0
-  interface: ::0
+  {{ 'interface: ::0' if SUBNET6 }}
   logfile: ""
   do-ip4: yes
-  do-ip6: yes
+  do-ip6: {{ 'yes' if SUBNET6 else 'no' }}
   do-udp: yes
   do-tcp: yes
   do-daemonize: no
   access-control: {{ SUBNET }} allow
+  {{ 'access-control: {{ SUBNET6 }} allow' if SUBNET6 }}
   directory: "/etc/unbound"
   username: unbound
   auto-trust-anchor-file: trusted-key.key
   root-hints: "/etc/unbound/root.hints"
   hide-identity: yes
   hide-version: yes
-  max-udp-size: 4096
-  msg-buffer-size: 65552
+  cache-min-ttl: 300
+

towncrier/newsfragments/1990.bugfix

@@ -0,0 +1 @@
+Fixed roundcube sso login not working.

towncrier/newsfragments/1992.enhancement

@@ -0,0 +1,3 @@
+Make unbound work with ipv6
+Add a cache-min-ttl of 5minutes
+Enable qname minimisation (privacy)

towncrier/newsfragments/1996.enhancement

@@ -0,0 +1 @@
+Disable the login page if SESSION_COOKIE_SECURE is incompatible with how Mailu is accessed as this seems to be a common misconfiguration.

towncrier/newsfragments/2002.enhancement

@@ -0,0 +1 @@
+Derive a new subkey (from SECRET_KEY) for SRS

towncrier/newsfragments/2007.enhancement

@@ -0,0 +1 @@
+allow sending emails as user+detail@domain.tld

towncrier/newsfragments/466.feature

@@ -0,0 +1 @@
+Remove the Received header with PRIMARY_HOSTNAME [PUBLIC_IP]

webmails/roundcube/Dockerfile

@@ -11,7 +11,8 @@ FROM build_${QEMU}
 RUN apt-get update && apt-get install -y \
   python3 curl python3-pip git python3-multidict \
   && rm -rf /var/lib/apt/lists \
-  && echo "ServerSignature Off" >> /etc/apache2/apache2.conf
+  && echo "ServerSignature Off\nServerName roundcube" >> /etc/apache2/apache2.conf \
+  && sed -i 's,CustomLog.*combined$,\0 "'"expr=!(%{HTTP_USER_AGENT}=='health'\&\&(-R '127.0.0.1/8' || -R '::1'))"'",' /etc/apache2/sites-available/000-default.conf
 
 # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
 RUN pip3 install socrate
@@ -33,11 +34,15 @@ RUN apt-get update && apt-get install -y \
  && mv roundcubemail-* html \
  && mv carddav html/plugins/ \
  && cd html \
- && rm -rf CHANGELOG INSTALL LICENSE README.md UPGRADING composer.json-dist installer \
+ && rm -rf CHANGELOG INSTALL LICENSE README.md UPGRADING composer.json-dist installer composer.* \
  && sed -i 's,mod_php5.c,mod_php7.c,g' .htaccess \
  && sed -i 's,^php_value.*post_max_size,#&,g' .htaccess \
  && sed -i 's,^php_value.*upload_max_filesize,#&,g' .htaccess \
- && chown -R www-data: logs temp \
+ && ln -sf index.php /var/www/html/sso.php \
+ && ln -sf /dev/stderr /var/www/html/logs/errors.log \
+ && chown -R root:root . \
+ && chown www-data:www-data logs temp \
+ && chmod -R a+rX . \
  && rm -rf /var/lib/apt/lists \
  && a2enmod rewrite deflate expires headers
 
@@ -51,4 +56,4 @@ VOLUME ["/data"]
 
 CMD /start.py
 
-HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1
+HEALTHCHECK CMD curl -f -L -H 'User-Agent: health' http://localhost/ || exit 1

webmails/roundcube/mailu.php

@@ -52,6 +52,12 @@ class mailu extends rcube_plugin
   }
   function login_failed($args)
   {
+      $ua = $_SERVER['HTTP_USER_AGENT'];
+      $ra = $_SERVER['REMOTE_ADDR'];
+      if ($ua == 'health' and ($ra == '127.0.0.1' or $ra == '::1')) {
+        echo "OK";
+        exit;
+      }
       header('Location: sso.php');
       exit();
   }

webmails/roundcube/start.py

@@ -34,11 +34,7 @@ else:
 conf.jinja("/php.ini", os.environ, "/usr/local/etc/php/conf.d/roundcube.ini")
 
 # Create dirs, setup permissions
-os.system("mkdir -p /data/gpg /var/www/html/logs")
-os.system("touch /var/www/html/logs/errors.log")
-os.system("chown -R www-data:www-data /var/www/html/logs")
-os.system("chmod -R a+rX /var/www/html/")
-os.system("ln -sf /var/www/html/index.php /var/www/html/sso.php")
+os.system("mkdir -p /data/gpg")
 
 try:
     print("Initializing database")
@@ -61,8 +57,5 @@ except subprocess.CalledProcessError as e:
 # Setup database permissions
 os.system("chown -R www-data:www-data /data")
 
-# Tail roundcube logs
-subprocess.Popen(["tail", "-f", "-n", "0", "/var/www/html/logs/errors.log"])
-
 # Run apache
 os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"])