Merge remote-tracking branch 'upstream/master' into update_roundcube

core/admin/Dockerfile

@@ -27,7 +27,7 @@ ENV TZ Etc/UTC
 
 # python3 shared with most images
 RUN set -eu \
- && apk add --no-cache python3 py3-pip git bash tzdata \
+ && apk add --no-cache python3 py3-pip py3-wheel git bash tzdata \
  && pip3 install --upgrade pip
 
 RUN mkdir -p /app
@@ -37,13 +37,15 @@ COPY requirements-prod.txt requirements.txt
 RUN set -eu \
  && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \
  && apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \
- && pip3 install -r requirements.txt \
+ && pip install --upgrade pip \
+ && pip install -r requirements.txt \
  && apk del --no-cache build-dep
 
 COPY --from=assets static ./mailu/static
 COPY mailu ./mailu
 COPY migrations ./migrations
 COPY start.py /start.py
+COPY audit.py /audit.py
 
 RUN pybabel compile -d mailu/translations
 

core/admin/assets/app.css

@@ -52,3 +52,8 @@ fieldset:disabled .form-control:disabled {
 .select2-container--default .select2-selection--multiple .select2-selection__choice {
   color: black;
 }
+
+/* range input spacing */
+.input-group-text {
+  margin-right: 1em;
+}

core/admin/assets/app.js

@@ -18,7 +18,7 @@ $('document').ready(function() {
         $.post({
             url: $(this).attr('href'),
             success: function() {
-                location.reload();
+                window.location = window.location.href;
             },
         });
     });
@@ -28,10 +28,10 @@ $('document').ready(function() {
         var fieldset = $(this).parents('fieldset');
         if (this.checked) {
             fieldset.removeAttr('disabled');
-            fieldset.find('input').not(this).removeAttr('disabled');
+            fieldset.find('input,textarea').not(this).removeAttr('disabled');
         } else {
             fieldset.attr('disabled', '');
-            fieldset.find('input').not(this).attr('disabled', '');
+            fieldset.find('input,textarea').not(this).attr('disabled', '');
         }
     });
 
@@ -43,7 +43,9 @@ $('document').ready(function() {
             var infinity = $(this).data('infinity');
             var step = $(this).attr('step');
             $(this).on('input', function() {
-                value_element.text((infinity && this.value == 0) ? '∞' : (this.value/step).toFixed(2));
+                var num = (infinity && this.value == 0) ? '∞' : (this.value/step).toFixed(2);
+                if (num.endsWith('.00')) num = num.substr(0, num.length - 3);
+                value_element.text(num);
             }).trigger('input');
         }
     });

core/admin/audit.py

@@ -1,14 +1,19 @@
-from mailu import app
+#!/usr/bin/python3
 
 import sys
 import tabulate
 
+sys.path[0:0] = ['/app']
+
+import mailu
+app = mailu.create_app()
+
 
 # Known endpoints without permissions
 known_missing_permissions = [
-    "index",
-    "static", "bootstrap.static",
-    "admin.static", "admin.login"
+    'index',
+    'static', 'bootstrap.static',
+    'admin.static', 'admin.login'
 ]
 
 
@@ -16,7 +21,7 @@ known_missing_permissions = [
 missing_permissions = []
 permissions = {}
 for endpoint, function in app.view_functions.items():
-    audit = function.__dict__.get("_audit_permissions")
+    audit = function.__dict__.get('_audit_permissions')
     if audit:
         handler, args = audit
         if args:
@@ -28,16 +33,15 @@ for endpoint, function in app.view_functions.items():
     elif endpoint not in known_missing_permissions:
         missing_permissions.append(endpoint)
 
-
-# Fail if any endpoint is missing a permission check
-if missing_permissions:
-    print("The following endpoints are missing permission checks:")
-    print(missing_permissions.join(","))
-    sys.exit(1)
-
-
 # Display the permissions table
 print(tabulate.tabulate([
     [route, *permissions[route.endpoint]]
     for route in app.url_map.iter_rules() if route.endpoint in permissions
 ]))
+
+# Warn if any endpoint is missing a permission check
+if missing_permissions:
+    print()
+    print('The following endpoints are missing permission checks:')
+    print(','.join(missing_permissions))
+

core/admin/mailu/__init__.py

@@ -33,7 +33,7 @@ def create_app_from_config(config):
     app.srs_key = hmac.new(bytearray(app.secret_key, 'utf-8'), bytearray('SRS_KEY', 'utf-8'), 'sha256').digest()
 
     # Initialize list of translations
-    config.translations = {
+    app.config.translations = {
         str(locale): locale
         for locale in sorted(
             utils.babel.list_translations(),
@@ -57,6 +57,15 @@ def create_app_from_config(config):
             config        = app.config,
         )
 
+    # Jinja filters
+    @app.template_filter()
+    def format_date(value):
+        return utils.flask_babel.format_date(value) if value else ''
+
+    @app.template_filter()
+    def format_datetime(value):
+        return utils.flask_babel.format_datetime(value) if value else ''
+
     # Import views
     from mailu import ui, internal, sso
     app.register_blueprint(ui.ui, url_prefix=app.config['WEB_ADMIN'])

core/admin/mailu/configuration.py

@@ -54,6 +54,7 @@ DEFAULT_CONFIG = {
     'DKIM_PATH': '/dkim/{domain}.{selector}.key',
     'DEFAULT_QUOTA': 1000000000,
     'MESSAGE_RATELIMIT': '200/day',
+    'MESSAGE_RATELIMIT_EXEMPTION': '',
     'RECIPIENT_DELIMITER': '',
     # Web settings
     'SITENAME': 'Mailu',
@@ -89,7 +90,7 @@ DEFAULT_CONFIG = {
     'POD_ADDRESS_RANGE': None
 }
 
-class ConfigManager(dict):
+class ConfigManager:
     """ Naive configuration manager that uses environment only
     """
 
@@ -104,19 +105,16 @@ class ConfigManager(dict):
 
     def get_host_address(self, name):
         # if MYSERVICE_ADDRESS is defined, use this
-        if '{}_ADDRESS'.format(name) in os.environ:
-            return os.environ.get('{}_ADDRESS'.format(name))
+        if f'{name}_ADDRESS' in os.environ:
+            return os.environ.get(f'{name}_ADDRESS')
         # otherwise use the host name and resolve it
-        return system.resolve_address(self.config['HOST_{}'.format(name)])
+        return system.resolve_address(self.config[f'HOST_{name}'])
 
     def resolve_hosts(self):
-        self.config["IMAP_ADDRESS"] = self.get_host_address("IMAP")
-        self.config["POP3_ADDRESS"] = self.get_host_address("POP3")
-        self.config["AUTHSMTP_ADDRESS"] = self.get_host_address("AUTHSMTP")
-        self.config["SMTP_ADDRESS"] = self.get_host_address("SMTP")
-        self.config["REDIS_ADDRESS"] = self.get_host_address("REDIS")
-        if self.config["WEBMAIL"] != "none":
-            self.config["WEBMAIL_ADDRESS"] = self.get_host_address("WEBMAIL")
+        for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']:
+            self.config[f'{key}_ADDRESS'] = self.get_host_address(key)
+        if self.config['WEBMAIL'] != 'none':
+            self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL')
 
     def __get_env(self, key, value):
         key_file = key + "_FILE"
@@ -135,6 +133,7 @@ class ConfigManager(dict):
         return value
 
     def init_app(self, app):
+        # get current app config
         self.config.update(app.config)
         # get environment variables
         self.config.update({
@@ -148,35 +147,18 @@ class ConfigManager(dict):
             template = self.DB_TEMPLATES[self.config['DB_FLAVOR']]
             self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config)
 
-        self.config['RATELIMIT_STORAGE_URL'] = 'redis://{0}/2'.format(self.config['REDIS_ADDRESS'])
-        self.config['QUOTA_STORAGE_URL'] = 'redis://{0}/1'.format(self.config['REDIS_ADDRESS'])
-        self.config['SESSION_STORAGE_URL'] = 'redis://{0}/3'.format(self.config['REDIS_ADDRESS'])
+        self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2'
+        self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1'
+        self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3'
         self.config['SESSION_COOKIE_SAMESITE'] = 'Strict'
         self.config['SESSION_COOKIE_HTTPONLY'] = True
         self.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=int(self.config['SESSION_LIFETIME']))
         hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')]
         self.config['AUTH_RATELIMIT_EXEMPTION'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['AUTH_RATELIMIT_EXEMPTION'].split(',')) if cidr)
+        self.config['MESSAGE_RATELIMIT_EXEMPTION'] = set([s for s in self.config['MESSAGE_RATELIMIT_EXEMPTION'].lower().replace(' ', '').split(',') if s])
         self.config['HOSTNAMES'] = ','.join(hostnames)
         self.config['HOSTNAME'] = hostnames[0]
-        # update the app config itself
-        app.config = self
 
-    def setdefault(self, key, value):
-        if key not in self.config:
-            self.config[key] = value
-        return self.config[key]
+        # update the app config
+        app.config.update(self.config)
 
-    def get(self, *args):
-        return self.config.get(*args)
-
-    def keys(self):
-        return self.config.keys()
-
-    def __getitem__(self, key):
-        return self.config.get(key)
-
-    def __setitem__(self, key, value):
-        self.config[key] = value
-
-    def __contains__(self, key):
-        return key in self.config

core/admin/mailu/debug.py

@@ -1,6 +1,6 @@
 import flask_debugtoolbar
 
-from werkzeug.contrib import profiler as werkzeug_profiler
+from werkzeug.middleware.profiler import ProfilerMiddleware
 
 
 # Debugging toolbar
@@ -10,7 +10,7 @@ toolbar = flask_debugtoolbar.DebugToolbarExtension()
 # Profiler
 class Profiler(object):
     def init_app(self, app):
-        app.wsgi_app = werkzeug_profiler.ProfilerMiddleware(
+        app.wsgi_app = ProfilerMiddleware(
             app.wsgi_app, restrictions=[30]
         )
 

core/admin/mailu/internal/views/postfix.py

@@ -149,6 +149,8 @@ def postfix_sender_login(sender):
 def postfix_sender_rate(sender):
     """ Rate limit outbound emails per sender login
     """
+    if sender in flask.current_app.config['MESSAGE_RATELIMIT_EXEMPTION']:
+        flask.abort(404)
     user = models.User.get(sender) or flask.abort(404)
     return flask.abort(404) if user.sender_limiter.hit() else flask.jsonify("450 4.2.1 You are sending too many emails too fast.")
 

core/admin/mailu/internal/views/rspamd.py

@@ -14,17 +14,14 @@ def vault_error(*messages, status=404):
 
 @internal.route("/rspamd/vault/v1/dkim/<domain_name>", methods=['GET'])
 def rspamd_dkim_key(domain_name):
-    domain = models.Domain.query.get(domain_name) or flask.abort(vault_error('unknown domain'))
-    key = domain.dkim_key or flask.abort(vault_error('no dkim key', status=400))
-    return flask.jsonify({
-        'data': {
-            'selectors': [
+    selectors = []
+    if domain := models.Domain.query.get(domain_name):
+        if key := domain.dkim_key:
+            selectors.append(
                 {
                     'domain'  : domain.name,
                     'key'     : key.decode('utf8'),
                     'selector': flask.current_app.config.get('DKIM_SELECTOR', 'dkim'),
                 }
-            ]
-        }
-    })
-
+            )
+    return flask.jsonify({'data': {'selectors': selectors}})

core/admin/mailu/models.py

@@ -19,7 +19,8 @@ import os
 import hmac
 import smtplib
 import idna
-import dns
+import dns.resolver
+import dns.exception
 
 from flask import current_app as app
 from sqlalchemy.ext import declarative
@@ -38,6 +39,8 @@ class IdnaDomain(db.TypeDecorator):
     """
 
     impl = db.String(80)
+    cache_ok = True
+    python_type = str
 
     def process_bind_param(self, value, dialect):
         """ encode unicode domain name to punycode """
@@ -47,13 +50,13 @@ class IdnaDomain(db.TypeDecorator):
         """ decode punycode domain name to unicode """
         return idna.decode(value)
 
-    python_type = str
-
 class IdnaEmail(db.TypeDecorator):
     """ Stores a Unicode string in it's IDNA representation (ASCII only)
     """
 
     impl = db.String(255)
+    cache_ok = True
+    python_type = str
 
     def process_bind_param(self, value, dialect):
         """ encode unicode domain part of email address to punycode """
@@ -69,13 +72,13 @@ class IdnaEmail(db.TypeDecorator):
         localpart, domain_name = value.rsplit('@', 1)
         return f'{localpart}@{idna.decode(domain_name)}'
 
-    python_type = str
-
 class CommaSeparatedList(db.TypeDecorator):
     """ Stores a list as a comma-separated string, compatible with Postfix.
     """
 
     impl = db.String
+    cache_ok = True
+    python_type = list
 
     def process_bind_param(self, value, dialect):
         """ join list of items to comma separated string """
@@ -90,13 +93,13 @@ class CommaSeparatedList(db.TypeDecorator):
         """ split comma separated string to list """
         return list(filter(bool, (item.strip() for item in value.split(',')))) if value else []
 
-    python_type = list
-
 class JSONEncoded(db.TypeDecorator):
     """ Represents an immutable structure as a json-encoded string.
     """
 
     impl = db.String
+    cache_ok = True
+    python_type = str
 
     def process_bind_param(self, value, dialect):
         """ encode data as json """
@@ -106,8 +109,6 @@ class JSONEncoded(db.TypeDecorator):
         """ decode json to data """
         return json.loads(value) if value else None
 
-    python_type = str
-
 class Base(db.Model):
     """ Base class for all models
     """

core/admin/mailu/schemas.py

@@ -145,6 +145,11 @@ class Logger:
             if history.has_changes() and history.deleted:
                 before = history.deleted[-1]
                 after = getattr(target, attr.key)
+                # we don't have ordered lists
+                if isinstance(before, list):
+                    before = set(before)
+                if isinstance(after, list):
+                    after = set(after)
                 # TODO: this can be removed when comment is not nullable in model
                 if attr.key == 'comment' and not before and not after:
                     pass

core/admin/mailu/sso/templates/form_sso.html

@@ -5,7 +5,7 @@
 <form class="form" method="post" role="form">
   {{ macros.form_field(form.email) }}
   {{ macros.form_field(form.pw) }}
-  {{ macros.form_fields(fields, label=False, class="btn btn-default", spacing=False) }}
+  {{ macros.form_fields(fields, label=False, class="btn btn-default") }}
 </form>
 {%- endcall %}
 {%- endblock %}

core/admin/mailu/sso/views/base.py

@@ -19,6 +19,7 @@ def login():
         fields.append(form.submitAdmin)
     if str(app.config["WEBMAIL"]).upper() != "NONE":
         fields.append(form.submitWebmail)
+    fields = [fields]
 
     if form.validate_on_submit():
         if form.submitAdmin.data:
@@ -38,7 +39,7 @@ def login():
             flask.session.regenerate()
             flask_login.login_user(user)
             response = flask.redirect(destination)
-            response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login'))
+            response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login'), secure=app.config['SESSION_COOKIE_SECURE'], httponly=True)
             flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip}.')
             return response
         else:

core/admin/mailu/ui/templates/alias/list.html

@@ -34,8 +34,8 @@
     <td>{{ alias }}</td>
     <td>{{ alias.destination|join(', ') or '-' }}</td>
     <td>{{ alias.comment or '' }}</td>
-    <td>{{ alias.created_at }}</td>
-    <td>{{ alias.updated_at or '' }}</td>
+    <td>{{ alias.created_at | format_date }}</td>
+    <td>{{ alias.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/alternative/list.html

@@ -19,6 +19,7 @@
     <th>{% trans %}Actions{% endtrans %}</th>
     <th>{% trans %}Name{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
+    <th>{% trans %}Last edit{% endtrans %}</th>
   </tr>
 </thead>
 <tbody>
@@ -28,7 +29,8 @@
       <a href="{{ url_for('.alternative_delete', alternative=alternative.name) }}" title="{% trans %}Delete{% endtrans %}"><i class="fa fa-trash"></i></a>
     </td>
     <td>{{ alternative }}</td>
-    <td>{{ alternative.created_at }}</td>
+    <td>{{ alternative.created_at | format_date }}</td>
+    <td>{{ alternative.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/domain/list.html

@@ -46,8 +46,8 @@
     <td>{{ domain.users | count }} / {{ '∞' if domain.max_users == -1 else domain.max_users }}</td>
     <td>{{ domain.aliases | count }} / {{ '∞' if domain.max_aliases == -1 else domain.max_aliases }}</td>
     <td>{{ domain.comment or '' }}</td>
-    <td>{{ domain.created_at }}</td>
-    <td>{{ domain.updated_at or '' }}</td>
+    <td>{{ domain.created_at | format_date }}</td>
+    <td>{{ domain.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/fetch/list.html

@@ -36,10 +36,10 @@
     <td>{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }}</td>
     <td>{{ fetch.username }}</td>
     <td>{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
-    <td>{{ fetch.last_check or '-' }}</td>
+    <td>{{ fetch.last_check | format_datetime or '-' }}</td>
     <td>{{ fetch.error or '-' }}</td>
-    <td>{{ fetch.created_at }}</td>
-    <td>{{ fetch.updated_at or '' }}</td>
+    <td>{{ fetch.created_at | format_date }}</td>
+    <td>{{ fetch.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/macros.html

@@ -18,17 +18,19 @@
   {%- endif %}
 {%- endmacro %}
 
-{%- macro form_fields(fields, prepend='', append='', label=True, spacing=True) %}
-  {%- if spacing %}
+{%- macro form_fields(fields, prepend='', append='', label=True) %}
   {%- set width = (12 / fields|length)|int %}
-  {%- else %}
-  {%- set width = 0 %}
-  {% endif %}
   <div class="form-group">
     <div class="row">
       {%- for field in fields %}
       <div class="col-lg-{{ width }} col-xs-12 {{ 'has-error' if field.errors else '' }}">
-        {{ form_individual_field(field, prepend=prepend, append=append, label=label, **kwargs) }}
+        {%- if field.__class__.__name__ == 'list' %}
+          {%- for subfield in field %}
+            {{ form_individual_field(subfield, prepend=prepend, append=append, label=label, **kwargs) }}
+          {%- endfor %}
+        {%- else %}
+          {{ form_individual_field(field, prepend=prepend, append=append, label=label, **kwargs) }}
+        {%- endif %}
       </div>
       {%- endfor %}
     </div>

core/admin/mailu/ui/templates/relay/list.html

@@ -32,8 +32,8 @@
     <td>{{ relay.name }}</td>
     <td>{{ relay.smtp or '-' }}</td>
     <td>{{ relay.comment or '' }}</td>
-    <td>{{ relay.created_at }}</td>
-    <td>{{ relay.updated_at or '' }}</td>
+    <td>{{ relay.created_at | format_date }}</td>
+    <td>{{ relay.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/token/list.html

@@ -20,6 +20,7 @@
     <th>{% trans %}Comment{% endtrans %}</th>
     <th>{% trans %}Authorized IP{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
+    <th>{% trans %}Last edit{% endtrans %}</th>
   </tr>
 </thead>
 <tbody>
@@ -30,7 +31,8 @@
     </td>
     <td>{{ token.comment }}</td>
     <td>{{ token.ip or "any" }}</td>
-    <td>{{ token.created_at }}</td>
+    <td>{{ token.created_at | format_date }}</td>
+    <td>{{ token.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/user/list.html

@@ -45,8 +45,8 @@
     </td>
     <td>{{ user.quota_bytes_used | filesizeformat }} / {{ (user.quota_bytes | filesizeformat) if user.quota_bytes else '∞' }}</td>
     <td>{{ user.comment or '-' }}</td>
-    <td>{{ user.created_at }}</td>
-    <td>{{ user.updated_at or '' }}</td>
+    <td>{{ user.created_at | format_date }}</td>
+    <td>{{ user.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/views/domains.py

@@ -5,7 +5,6 @@ from flask import current_app as app
 import flask
 import flask_login
 import wtforms_components
-import dns.resolver
 
 
 @ui.route('/domain', methods=['GET'])

core/admin/mailu/utils.py

@@ -6,18 +6,21 @@ try:
 except ImportError:
     import pickle
 
-import dns
 import dns.resolver
+import dns.exception
+import dns.flags
+import dns.rdtypes
+import dns.rdatatype
+import dns.rdataclass
 
 import hmac
 import secrets
 import time
 
 from multiprocessing import Value
-
 from mailu import limiter
-
 from flask import current_app as app
+
 import flask
 import flask_login
 import flask_migrate
@@ -28,7 +31,7 @@ import redis
 from flask.sessions import SessionMixin, SessionInterface
 from itsdangerous.encoding import want_bytes
 from werkzeug.datastructures import CallbackDict
-from werkzeug.contrib import fixers
+from werkzeug.middleware.proxy_fix import ProxyFix
 
 # Login configuration
 login = flask_login.LoginManager()
@@ -106,7 +109,7 @@ class PrefixMiddleware(object):
         return self.app(environ, start_response)
 
     def init_app(self, app):
-        self.app = fixers.ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
+        self.app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1)
         app.wsgi_app = self
 
 proxy = PrefixMiddleware()
@@ -265,7 +268,7 @@ class MailuSession(CallbackDict, SessionMixin):
 
         # set uid from dict data
         if self._uid is None:
-            self._uid = self.app.session_config.gen_uid(self.get('user_id', ''))
+            self._uid = self.app.session_config.gen_uid(self.get('_user_id', ''))
 
         # create new session id for new or regenerated sessions and force setting the cookie
         if self._sid is None:

core/admin/migrations/env.py

@@ -1,10 +1,12 @@
-from __future__ import with_statement
+import logging
+import tenacity
+
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig
-import logging
-import tenacity
-from tenacity import retry
+
+from flask import current_app
+from mailu import models
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -17,20 +19,12 @@ logger = logging.getLogger('alembic.env')
 
 # add your model's MetaData object here
 # for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-from flask import current_app
-config.set_main_option('sqlalchemy.url',
-                       current_app.config.get('SQLALCHEMY_DATABASE_URI'))
-#target_metadata = current_app.extensions['migrate'].db.metadata
-from mailu import models
+config.set_main_option(
+    'sqlalchemy.url',
+    current_app.config.get('SQLALCHEMY_DATABASE_URI')
+)
 target_metadata = models.Base.metadata
 
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
-
 
 def run_migrations_offline():
     """Run migrations in 'offline' mode.
@@ -44,7 +38,7 @@ def run_migrations_offline():
     script output.
 
     """
-    url = config.get_main_option("sqlalchemy.url")
+    url = config.get_main_option('sqlalchemy.url')
     context.configure(url=url)
 
     with context.begin_transaction():
@@ -69,28 +63,35 @@ def run_migrations_online():
                 directives[:] = []
                 logger.info('No changes in schema detected.')
 
-    engine = engine_from_config(config.get_section(config.config_ini_section),
-                                prefix='sqlalchemy.',
-                                poolclass=pool.NullPool)
+    engine = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix    = 'sqlalchemy.',
+        poolclass = pool.NullPool
+    )
 
-    connection = tenacity.Retrying(
-        stop=tenacity.stop_after_attempt(100),
-        wait=tenacity.wait_random(min=2, max=5),
-        before=tenacity.before_log(logging.getLogger("tenacity.retry"), logging.DEBUG),
-        before_sleep=tenacity.before_sleep_log(logging.getLogger("tenacity.retry"), logging.INFO),
-        after=tenacity.after_log(logging.getLogger("tenacity.retry"), logging.DEBUG)
-        ).call(engine.connect)
+    @tenacity.retry(
+        stop   = tenacity.stop_after_attempt(100),
+        wait   = tenacity.wait_random(min=2, max=5),
+        before = tenacity.before_log(logging.getLogger('tenacity.retry'), logging.DEBUG),
+        before_sleep = tenacity.before_sleep_log(logging.getLogger('tenacity.retry'), logging.INFO),
+        after  = tenacity.after_log(logging.getLogger('tenacity.retry'), logging.DEBUG)
+    )
+    def try_connect(db):
+        return db.connect()
 
-    context.configure(connection=connection,
-                      target_metadata=target_metadata,
-                      process_revision_directives=process_revision_directives,
-                      **current_app.extensions['migrate'].configure_args)
+    with try_connect(engine) as connection:
+
+        context.configure(
+            connection      = connection,
+            target_metadata = target_metadata,
+            process_revision_directives = process_revision_directives,
+            **current_app.extensions['migrate'].configure_args
+        )
 
-    try:
         with context.begin_transaction():
             context.run_migrations()
-    finally:
-        connection.close()
+
+    connection.close()
 
 if context.is_offline_mode():
     run_migrations_offline()

core/admin/requirements-prod.txt

@@ -1,56 +1,75 @@
-alembic==1.0.10
-asn1crypto==0.24.0
-Babel==2.6.0
-bcrypt==3.1.6
+alembic==1.7.4
+appdirs==1.4.4
+Babel==2.9.1
+bcrypt==3.2.0
 blinker==1.4
-cffi==1.12.3
-Click==7.0
-cryptography==3.4.7
-decorator==4.4.0
-dnspython==1.16.0
-dominate==2.3.5
-Flask==1.0.2
-Flask-Babel==0.12.2
+CacheControl==0.12.9
+certifi==2021.10.8
+cffi==1.15.0
+chardet==4.0.0
+click==8.0.3
+colorama==0.4.4
+contextlib2==21.6.0
+cryptography==35.0.0
+decorator==5.1.0
+# distlib==0.3.1
+# distro==1.5.0
+dnspython==2.1.0
+dominate==2.6.0
+email-validator==1.1.3
+Flask==2.0.2
+Flask-Babel==2.0.0
 Flask-Bootstrap==3.3.7.1
-Flask-DebugToolbar==0.10.1
-Flask-Limiter==1.0.1
-Flask-Login==0.4.1
+Flask-DebugToolbar==0.11.0
+Flask-Limiter==1.4
+Flask-Login==0.5.0
 flask-marshmallow==0.14.0
-Flask-Migrate==2.4.0
+Flask-Migrate==3.1.0
 Flask-Script==2.0.6
-Flask-SQLAlchemy==2.4.0
-Flask-WTF==0.14.2
+Flask-SQLAlchemy==2.5.1
+Flask-WTF==0.15.1
+greenlet==1.1.2
 gunicorn==20.1.0
-idna==2.8
-infinity==1.4
-intervals==0.8.1
-itsdangerous==1.1.0
-Jinja2==2.11.3
-limits==1.3
-Mako==1.0.9
-MarkupSafe==1.1.1
-mysqlclient==1.4.2.post1
-marshmallow==3.10.0
-marshmallow-sqlalchemy==0.24.1
+html5lib==1.1
+idna==3.3
+infinity==1.5
+intervals==0.9.2
+itsdangerous==2.0.1
+Jinja2==3.0.2
+limits==1.5.1
+lockfile==0.12.2
+Mako==1.1.5
+MarkupSafe==2.0.1
+marshmallow==3.14.0
+marshmallow-sqlalchemy==0.26.1
+msgpack==1.0.2
+mysqlclient==2.0.3
+ordered-set==4.0.2
+# packaging==20.9
 passlib==1.7.4
-psycopg2==2.8.2
-pycparser==2.19
-Pygments==2.8.1
-pyOpenSSL==20.0.1
-python-dateutil==2.8.0
-python-editor==1.0.4
-pytz==2019.1
-PyYAML==5.4.1
-redis==3.2.1
-#alpine3:12 provides six==1.15.0
-#six==1.12.0
-socrate==0.1.1
-SQLAlchemy==1.3.3
+# pep517==0.10.0
+progress==1.6
+psycopg2==2.9.1
+pycparser==2.20
+Pygments==2.10.0
+pyOpenSSL==21.0.0
+pyparsing==3.0.4
+pytz==2021.3
+PyYAML==6.0
+redis==3.5.3
+requests==2.26.0
+retrying==1.3.3
+# six==1.15.0
+socrate==0.2.0
+SQLAlchemy==1.4.26
 srslib==0.1.4
-tabulate==0.8.3
-tenacity==5.0.4
-validators==0.12.6
+tabulate==0.8.9
+tenacity==8.0.1
+toml==0.10.2
+urllib3==1.26.7
+validators==0.18.2
 visitor==0.1.3
-Werkzeug==0.15.5
-WTForms==2.2.1
-WTForms-Components==0.10.4
+webencodings==0.5.1
+Werkzeug==2.0.2
+WTForms==2.3.3
+WTForms-Components==0.10.5

core/admin/requirements.txt

@@ -18,10 +18,8 @@ PyYAML
 PyOpenSSL
 Pygments
 dnspython
-bcrypt
 tenacity
 mysqlclient
-psycopg2
 idna
 srslib
 marshmallow

core/postfix/conf/main.cf

@@ -17,7 +17,7 @@ queue_directory = /queue
 message_size_limit = {{ MESSAGE_SIZE_LIMIT }}
 
 # Relayed networks
-mynetworks = 127.0.0.1/32 [::1]/128 {{ SUBNET }} {{ RELAYNETS.split(",") | join(' ') }}
+mynetworks = 127.0.0.1/32 [::1]/128 {{ SUBNET }}  {% if RELAYNETS %}{{ RELAYNETS.split(",") | join(' ') }}{% endif %}
 
 # Empty alias list to override the configuration variable and disable NIS
 alias_maps =

design/mailu-directory-structure.md

@@ -0,0 +1,199 @@
+# RFC: Mailu directory structure
+The current layout of the Mailu directory structure can be improved to allow for easier replicated deployments, like Docker Swarm and Kubernetes. Please read https://usrpro.com/publications/mailu-persistent-storage/ for the background motivation of this RFC.
+
+## Scope
+This document only describes the re-arrangement of the `$ROOT` Mailu filesystem as-is. This means moving around of current files and directories. The linked article also proposes more advanced improvements. Those are not included in this RFC and need to be evaluated and implemented independently. However, the changes proposed in this document should make such improvements easier.
+
+Currently some services are wrongfully sharing mountpoints or have unused volumes declared. Those are also taken care of in this RFC.
+
+## Compatibility
+If these changes were to be accepted, it will break compatibility with previous Mailu version `<=1.7`. As such, we will propably increment to version `2.0`.
+
+## Root
+The root of the Mailu filesystem is located at `/mailu` by default. For simplicity we will assume this location throughout the document. Within `/mailu` we will aim to define 3 main sub-directories:
+
+### Config
+
+- Path: `/mailu/config/`
+
+Small config bearing files, sometimes shared between multiple services. The performance and storage needs for this filesystem are low. Availability is important for correct functioning of the mail server. No file locking issues are expected from concurrent access. A basic (and redundant) filesystem should suffice.
+
+#### Dovecot
+
+- Old path: `/mailu/overrides` (shared with postfix, nginx and rspamd)
+- New Path `/mailu/config/dovecot`
+
+Dovecot configuration overrides.
+
+#### Postfix
+
+- Old path: `/mailu/overrides` (shared with dovecot, nginx and rspamd)
+- New Path `/mailu/config/posfix`
+
+Postfix configuration overrides.
+
+#### Rspamd
+
+- Old path: `/mailu/overrides/rspamd`
+- New path: `/mailu/config/rspamd`
+
+RSpamD configuration overrides.
+
+#### Rainloop
+
+- Old path: `/mailu/webmail/_data_/_default_/storage` (part of `/mailu/webmail` mountpoint, shared with Roundcube)
+- New path: `/mailu/config/rainloop`
+
+User specific configs. The remaining files under the old `/mailu/webmail` don't need to be persistent. Except for `AddressBook.sqlite`, see `/mailu/data`.
+
+#### Roundcube
+
+- Old path: `/mailu/webmail/gpg` (part of `/mailu/webmail` mountpoint, shared with Rainloop)
+- New path: `/mailu/config/roundcube/gpg`
+
+User configured GPG keys.
+
+#### Redis
+
+- Old path: `/mailu/redis`
+- New path: `/mailu/config/redis`
+
+Holds `dump.rdb` for data restoration. Although technically a database, Redis works from memory. The dump file is only written to every minute and read from during start. Hence it fits better in the replicated config directory filesystem.
+
+#### Share
+
+- Path: `/mailu/config/share/`
+
+Shared configuration between different services
+
+##### DKIM
+
+- Old path: `/mailu/dkim/`
+- New path: `/mailu/config/share/dkim`
+
+DKIM private keys store. Read/write access by Admin. Read only access by rSpamD.
+
+##### Certs
+
+- Old path: `/mailu/certs`
+- New path: `/mailu/config/share/certs` (Proposal in anticipation of the RFC outcome at: https://github.com/Mailu/Mailu/issues/1222.)
+
+TLS certificates. Write access from `nginx` in case `TLS_FLAVOR=letsencrypt`. Or write access from `treafik-certdumper` or any other tool obtaining certificates.
+
+`letsencrypt` setting is not compatible with replicated setups. Multiple instances would disrupt the ACME challenge verification and cause race conditions on requesting certificates. In such cases certificates will need to be provided by other tools.
+
+If RFC issue #1222 is accepted, Dovecot will need read-only access to the certificates.
+
+### Data
+
+- Path: `/mailu/data/`
+
+Database files, like SQLite or PostgreSQL files. Databases don't perform well on network filesystems as they depend heavily on file locking and full controll on the database files. Making it unfit for concurrent access from multiple hosts. This directory should always live on a local filesystem. This makes it only usable in `docker-compose` deployments. Usage of this directory should be avoided in Kubernetes and Docker Swarm deployments. Some services will need to be improved to allow for this.
+
+#### admin data
+
+- Old path: `/mailu/data/`
+- New path: `/mailu/data/admin/` (mount point on `admin` directory)
+
+Holds `main.db` SQLite database file holding domains, users, aliases etcetera. Read/write access only by admin. Can be avoided by using a remote DB server like PostgreSQL, MySQL or MariaDB.
+
+Also holds `instance` for unique statistics transmission. Removing of this file is proposed in RFC [issue 129](https://github.com/Mailu/Mailu/issues/1219).
+
+This move is needed in order to be able to mount the directory without exposing data files from other services into admin.
+
+#### rspamd
+
+- Old path: `/mailu/filter` (shared with ClamAV)
+- New path: `/mailu/data/rspamd`
+
+Storage of Bayes and Fuzzy learning SQLite databases and caches. As future optimization we should look into moving all this into Redis.
+
+#### Rainloop
+
+- Old path: `/mailu/webmail/_data_/_default_/AddressBook.sqlite` (part of `/mailu/webmail` mountpoint, shared with Roundcube)
+- New path: `/mailu/data/rainloop/AddressBook.sqlite` (mount on `rainloop` directory)
+
+Addressbook SQLite file. For future replicated deployments this might better be configured to use an external DB.
+
+For this modification, the `AddressBook.sqlite` will need to be moved to a different directory inside the container.
+
+#### Roundcube
+
+- Old path: `/mailu/webmail/roundcube.db` (part of `/mailu/webmail` mountpoint, shared with Rainloop)
+- New path: `/mailu/data/roundcube/roundcube.db` (mount on `roundcube` directory)
+
+User settings SQLite database file for roundcube. For future replicated deployments this might better be configured to use an external DB.
+
+For this modification, the `rouncube.db` file will need to be moved to a different directory inside the container.
+
+### Mail
+
+- Path: `/mailu/mail` (unmodified)
+
+User mail, managed my Dovecot IMAP server. In replicated deployments, this filesystem needs to be shared over all IMAP server instances. It should be high performant and capable of propgating file locks. Storage size is proportional to the users and their quotas. Old versions of NFS are known to be buggy with file locking. Also Samaba or CIFS should be avoided.
+
+In the old situation, Maildir indexes are stored on the same volume. However, they need not to be persistent and should be located on a voletile filesystem instead. This allows better performance on network filesystems.
+
+### Local
+
+- Path: `/mailu/local` (new)
+
+Persistent storage not suitable for replication. In `docker-compose` deployments it lives inside `/mailu` and in replicated deployments it should live somewhere on the local host machine.
+
+#### Mailqueue
+
+- Old path: `/mailu/mailqueue`
+- New path: `/mailu/local/mailqueue`
+
+The SMTP mailqueue should be persistant, as per SMTP spec it is not allowed to loose mail. However, persistance should be local only for performance reasons and the possibility to replicate Postfix servers. In setups like Docker Swarm and Kubernetes, admins should take care that Postfix is always restarted on same hosts in order to empty any remaining queue after a crash.
+
+#### ClamAV
+
+- Old path: `/mailu/filters` (shared with rSpamD)
+- New path: `/mailu/local/clamav`
+
+Virus definitions do not need to be replicated, as they can be easily pulled in when ClamAV instances migrate to other nodes. Persistance does allow for some bandwith and time saving if ClamAV would be restarted on a previously used node (in case of updates or similar cases). Local only storage also prevents `freshclam` race conditions.
+
+## Conclusion
+
+The final layout of the Mailu filesystem will look like:
+
+````
+/mailu
+├── config
+│   ├── dovecot
+│   ├── postfix
+│   ├── rainloop
+│   ├── redis
+│   ├── roundcube
+│   │   └── gpg
+│   ├── rspamd
+│   └── share
+│       ├── certs
+│       └── dkim
+├── data
+│   ├── admin
+│   ├── rainloop
+│   ├── roundcube
+│   └── rspamd
+├── local
+│   ├── clamav
+│   └── mailqueue
+└── mail
+````
+
+Where in replicated environments:
+
+- `/mailu/config/`: should be a small, low performant and shared filesystem.
+- `/mailu/data`: should be avoided. More work will need to be done to configure external DB servers for relevant services. Ideally, this directory should only exist on docker-compose deployments.
+- `/mailu/local/`: Should exist only on local file systems of worker nodes.
+- `/mailu/mail`: A distributed filesystem with sufficient performance and storage requirements to hold and process all user mailboxes. Ideally only Maildir without indexes.
+
+### Implementing
+
+The works to implement this changes should happen outside the `master` branch. Inclusion into `master` can only be accepted if:
+
+1. `docker-compose.yml` from setup reflects this changes correctly.
+2. Kubernetes documentation is updated.
+3. Legacy `docker-compose.yml` is either updated or deleted.
+4. A clear data migration guide is written.

docs/configuration.rst

@@ -69,9 +69,11 @@ The ``MESSAGE_SIZE_LIMIT`` is the maximum size of a single email. It should not
 be too low to avoid dropping legitimate emails and should not be too high to
 avoid filling the disks with large junk emails.
 
-The ``MESSAGE_RATELIMIT`` is the limit of messages a single user can send. This is
-meant to fight outbound spam in case of compromised or malicious account on the
-server.
+The ``MESSAGE_RATELIMIT`` (default: 200/day) is the maximum number of messages
+a single user can send. ``MESSAGE_RATELIMIT_EXEMPTION`` contains a comma delimited
+list of user email addresses that are exempted from any restriction.  Those
+settings are meant to reduce outbound spam in case of compromised or malicious
+account on the server.
 
 The ``RELAYNETS`` (default: unset) is a comma delimited list of network addresses
 for which mail is relayed for with no authentication required. This should be
@@ -192,7 +194,9 @@ The ``LETSENCRYPT_SHORTCHAIN`` (default: False) setting controls whether we send
 
 .. _`android handsets older than 7.1.1`: https://community.letsencrypt.org/t/production-chain-changes/150739
 
-The ``REAL_IP_HEADER`` (default: unset) and ``REAL_IP_FROM`` (default: unset) settings controls whether HTTP headers such as ``X-Forwarded-For`` or ``X-Real-IP`` should be trusted. The former should be the name of the HTTP header to extract the client IP address from and the later a comma separated list of IP addresses designing which proxies to trust. If you are using Mailu behind a reverse proxy, you should set both. Setting the former without the later introduces a security vulnerability allowing a potential attacker to spoof his source address.
+.. _reverse_proxy_headers:
+
+The ``REAL_IP_HEADER`` (default: unset) and ``REAL_IP_FROM`` (default: unset) settings controls whether HTTP headers such as ``X-Forwarded-For`` or ``X-Real-IP`` should be trusted. The former should be the name of the HTTP header to extract the client IP address from and the later a comma separated list of IP addresses designating which proxies to trust. If you are using Mailu behind a reverse proxy, you should set both. Setting the former without the later introduces a security vulnerability allowing a potential attacker to spoof his source address.
 
 The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usually uses a ``Region/City`` format. See `TZ database name`_  for a list of valid timezones This defaults to ``Etc/UTC``. Warning: if you are observing different timestamps in your log files you should change your hosts timezone to UTC instead of changing TZ to your local timezone. Using UTC allows easy log correlation with remote MTAs.
 

docs/reverse.rst

@@ -11,7 +11,10 @@ There are basically three options, from the most to the least recommended one:
 - `use Traefik in another container as central system-reverse-proxy`_
 - `override Mailu Web frontend configuration`_
 
-All options will require that you modify the ``docker-compose.yml`` file.
+All options will require that you modify the ``docker-compose.yml`` and ``mailu.env`` file.
+
+Mailu must also be configured with the information what header is used by the reverse proxy for passing the remote client IP. 
+This is configured in the mailu.env file. See the :ref:`configuration reference <reverse_proxy_headers>` for more information.
 
 Have Mailu Web frontend listen locally
 --------------------------------------
@@ -43,10 +46,19 @@ Then on your own frontend, point to these local ports. In practice, you only nee
     # [...] here goes your standard configuration
 
     location / {
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr
       proxy_pass https://localhost:8443;
     }
   }
 
+.. code-block:: docker
+
+  #mailu.env file
+  REAL_IP_HEADER=X-Real-IP
+  REAL_IP_FROM=x.x.x.x,y.y.y.y.y
+  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
+  
 Because the admin interface is served as ``/admin``, the Webmail as ``/webmail``, the single sign on page as ``/sso``, webdav as ``/webdav`` and the static files endpoint as ``/static``, you may also want to use a single virtual host and serve other applications (still Nginx):
 
 .. code-block:: nginx
@@ -55,8 +67,9 @@ Because the admin interface is served as ``/admin``, the Webmail as ``/webmail``
     # [...] here goes your standard configuration
 
     location ~ ^/(admin|sso|static|webdav|webmail)/ {
-      proxy_pass https://localhost:8443;
-      proxy_set_header Host $http_host;
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr
+      proxy_pass https://localhost:8443;     
     }
 
     location /main_app {
@@ -76,6 +89,13 @@ Because the admin interface is served as ``/admin``, the Webmail as ``/webmail``
     }
   }
 
+.. code-block:: docker
+  
+  #mailu.env file
+  REAL_IP_HEADER=X-Real-IP
+  REAL_IP_FROM=x.x.x.x,y.y.y.y.y
+  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
+
 Finally, you might want to serve the admin interface on a separate virtual host but not expose the admin container directly (have your own HTTPS virtual hosts on top of Mailu, one public for the Webmail and one internal for administration for instance).
 
 Here is an example configuration :
@@ -88,6 +108,8 @@ Here is an example configuration :
     # [...] here goes your standard configuration
 
     location /webmail {
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr
       proxy_pass https://localhost:8443/webmail;
     }
   }
@@ -98,12 +120,21 @@ Here is an example configuration :
     # [...] here goes your standard configuration
 
     location /admin {
+      proxy_set_header Host $host;
+      proxy_set_header X-Real-IP $remote_addr
       proxy_pass https://localhost:8443/admin;
       proxy_set_header Host $http_host;
     }
 
   }
 
+.. code-block:: docker
+
+  #mailu.env file
+  REAL_IP_HEADER=X-Real-IP
+  REAL_IP_FROM=x.x.x.x,y.y.y.y.y
+  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
+
 Depending on how you access the front server, you might want to add a ``proxy_redirect`` directive to your ``location`` blocks:
 
 .. code-block:: nginx
@@ -151,7 +182,16 @@ If your Traefik is configured to automatically request certificates from *letsen
 and this is the ``DOMAIN`` in your ``.env``?
 To support that use-case, Traefik can request ``SANs`` for your domain. The configuration for this will depend on your Traefik version.
 
-----
+Mailu must also be configured with the information what header is used by the reverse proxy for passing the remote client IP.  This is configured in mailu.env:
+
+.. code-block:: docker
+  
+  #mailu.env file
+  REAL_IP_HEADER=X-Real-IP
+  REAL_IP_FROM=x.x.x.x,y.y.y.y.y
+  #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. 
+
+For more information see the :ref:`configuration reference <reverse_proxy_headers>` for more information.
 
 Traefik 2.x using labels configuration
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -179,53 +219,6 @@ Of course, be sure to define the Certificate Resolver ``foo`` in the static conf
 
 Alternatively, you can define SANs in the Traefik static configuration using routers, or in the static configuration using entrypoints. Refer to the Traefik documentation for more details.
 
-Traefik 1.x with TOML configuration
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Lets add something like
-
-.. code-block:: yaml
-
-  [acme]
-    [[acme.domains]]
-      main = "your.example.com" # this is the same as $TRAEFIK_DOMAIN!
-      sans = ["mail.your.example.com", "webmail.your.example.com", "smtp.your.example.com"]
-
-to your ``traefik.toml``.
-
-----
-
-You might need to clear your ``acme.json``, if a certificate for one of these domains already exists.
-
-You will need some solution which dumps the certificates in ``acme.json``, so you can include them in the ``mailu/front`` container.
-One such example is ``mailu/traefik-certdumper``, which has been adapted for use in Mailu. You can add it to your ``docker-compose.yml`` like:
-
-.. code-block:: yaml
-
-  certdumper:
-    restart: always
-    image: mailu/traefik-certdumper:$VERSION
-    environment:
-    # Make sure this is the same as the main=-domain in traefik.toml
-    # !!! Also don’t forget to add "TRAEFIK_DOMAIN=[...]" to your .env!
-      - DOMAIN=$TRAEFIK_DOMAIN
-    volumes:
-      # Folder, which contains the acme.json
-      - "/data/traefik:/traefik"
-      # Folder, where cert.pem and key.pem will be written
-      - "/data/mailu/certs:/output"
-
-
-Assuming you have ``volume-mounted`` your ``acme.json`` put to ``/data/traefik`` on your host. The dumper will then write out ``/data/mailu/certs/cert.pem`` and ``/data/mailu/certs/key.pem`` whenever ``acme.json`` is updated.
-Yay! Now let’s mount this to our ``front`` container like:
-
-.. code-block:: yaml
-
-    volumes:
-      - /data/mailu/certs:/certs
-
-This works, because we set ``TLS_FLAVOR=mail``, which picks up the key-certificate pair (e.g., ``cert.pem`` and ``key.pem``) from the certs folder in the root path (``/certs/``).
-
 .. _`Traefik`: https://traefik.io/
 
 Override Mailu configuration

optional/fetchmail/Dockerfile

@@ -12,8 +12,8 @@ RUN apk add --no-cache \
 RUN apk add --no-cache fetchmail ca-certificates openssl \
  && pip3 install requests
 
+RUN mkdir -p /data
+
 COPY fetchmail.py /fetchmail.py
 
-USER fetchmail
-
-CMD ["/fetchmail.py"]
+CMD ["/fetchmail.py"]

optional/fetchmail/fetchmail.py

@@ -13,6 +13,7 @@ import traceback
 
 FETCHMAIL = """
 fetchmail -N \
+    --idfile /data/fetchids --uidl \
     --sslcertck --sslcertpath /etc/ssl/certs \
     -f {}
 """

setup/flavors/compose/docker-compose.yml

@@ -129,6 +129,8 @@ services:
     image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
+    volumes:
+      - "{{ root }}/data/fetchmail:/data"
     {% if resolver_enabled %}
     depends_on:
       - resolver

setup/flavors/stack/docker-compose.yml

@@ -110,7 +110,7 @@ services:
     image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     volumes:
-      - "{{ root }}/data:/data"
+      - "{{ root }}/data/fetchmail:/data"
     deploy:
       replicas: 1
     healthcheck:

towncrier/newsfragments/1223.bugfix

@@ -0,0 +1,4 @@
+Fixed fetchmail losing track of fetched emails upon container recreation. 
+The relevant fetchmail files are now retained in the /data folder (in the fetchmail image).
+See the docker-compose.yml file for the relevant volume mapping.
+If you already had your own mapping, you must double check the volume mapping and take action.

towncrier/newsfragments/1962.bugfix

@@ -0,0 +1,5 @@
+Reverse proxy documentation has been updated to reflect new security hardening from PR#1959.
+If you do not set the configuration parameters in Mailu what reverse proxy header to trust,
+then Mailu will not have access to the real ip address of the connecting client.
+This means that rate limiting will not properly work. You can also not use fail2ban. 
+It is very important to configure this when using a reverse proxy.