diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 8dda76d2..1958ae61 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -27,7 +27,7 @@ ENV TZ Etc/UTC # python3 shared with most images RUN set -eu \ - && apk add --no-cache python3 py3-pip git bash tzdata \ + && apk add --no-cache python3 py3-pip py3-wheel git bash tzdata \ && pip3 install --upgrade pip RUN mkdir -p /app @@ -37,13 +37,15 @@ COPY requirements-prod.txt requirements.txt RUN set -eu \ && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ && apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ - && pip3 install -r requirements.txt \ + && pip install --upgrade pip \ + && pip install -r requirements.txt \ && apk del --no-cache build-dep COPY --from=assets static ./mailu/static COPY mailu ./mailu COPY migrations ./migrations COPY start.py /start.py +COPY audit.py /audit.py RUN pybabel compile -d mailu/translations diff --git a/core/admin/assets/app.css b/core/admin/assets/app.css index 3886b5c1..84644900 100644 --- a/core/admin/assets/app.css +++ b/core/admin/assets/app.css @@ -52,3 +52,8 @@ fieldset:disabled .form-control:disabled { .select2-container--default .select2-selection--multiple .select2-selection__choice { color: black; } + +/* range input spacing */ +.input-group-text { + margin-right: 1em; +} diff --git a/core/admin/assets/app.js b/core/admin/assets/app.js index 54602d1f..03ea6215 100644 --- a/core/admin/assets/app.js +++ b/core/admin/assets/app.js @@ -18,7 +18,7 @@ $('document').ready(function() { $.post({ url: $(this).attr('href'), success: function() { - location.reload(); + window.location = window.location.href; }, }); }); @@ -28,10 +28,10 @@ $('document').ready(function() { var fieldset = $(this).parents('fieldset'); if (this.checked) { fieldset.removeAttr('disabled'); - fieldset.find('input').not(this).removeAttr('disabled'); + fieldset.find('input,textarea').not(this).removeAttr('disabled'); } else { fieldset.attr('disabled', ''); - fieldset.find('input').not(this).attr('disabled', ''); + fieldset.find('input,textarea').not(this).attr('disabled', ''); } }); @@ -43,7 +43,9 @@ $('document').ready(function() { var infinity = $(this).data('infinity'); var step = $(this).attr('step'); $(this).on('input', function() { - value_element.text((infinity && this.value == 0) ? '∞' : (this.value/step).toFixed(2)); + var num = (infinity && this.value == 0) ? '∞' : (this.value/step).toFixed(2); + if (num.endsWith('.00')) num = num.substr(0, num.length - 3); + value_element.text(num); }).trigger('input'); } }); diff --git a/core/admin/audit.py b/core/admin/audit.py old mode 100644 new mode 100755 index db105ff4..60583f83 --- a/core/admin/audit.py +++ b/core/admin/audit.py @@ -1,14 +1,19 @@ -from mailu import app +#!/usr/bin/python3 import sys import tabulate +sys.path[0:0] = ['/app'] + +import mailu +app = mailu.create_app() + # Known endpoints without permissions known_missing_permissions = [ - "index", - "static", "bootstrap.static", - "admin.static", "admin.login" + 'index', + 'static', 'bootstrap.static', + 'admin.static', 'admin.login' ] @@ -16,7 +21,7 @@ known_missing_permissions = [ missing_permissions = [] permissions = {} for endpoint, function in app.view_functions.items(): - audit = function.__dict__.get("_audit_permissions") + audit = function.__dict__.get('_audit_permissions') if audit: handler, args = audit if args: @@ -28,16 +33,15 @@ for endpoint, function in app.view_functions.items(): elif endpoint not in known_missing_permissions: missing_permissions.append(endpoint) - -# Fail if any endpoint is missing a permission check -if missing_permissions: - print("The following endpoints are missing permission checks:") - print(missing_permissions.join(",")) - sys.exit(1) - - # Display the permissions table print(tabulate.tabulate([ [route, *permissions[route.endpoint]] for route in app.url_map.iter_rules() if route.endpoint in permissions ])) + +# Warn if any endpoint is missing a permission check +if missing_permissions: + print() + print('The following endpoints are missing permission checks:') + print(','.join(missing_permissions)) + diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index e4024e47..fe1f376c 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -33,7 +33,7 @@ def create_app_from_config(config): app.srs_key = hmac.new(bytearray(app.secret_key, 'utf-8'), bytearray('SRS_KEY', 'utf-8'), 'sha256').digest() # Initialize list of translations - config.translations = { + app.config.translations = { str(locale): locale for locale in sorted( utils.babel.list_translations(), @@ -57,6 +57,15 @@ def create_app_from_config(config): config = app.config, ) + # Jinja filters + @app.template_filter() + def format_date(value): + return utils.flask_babel.format_date(value) if value else '' + + @app.template_filter() + def format_datetime(value): + return utils.flask_babel.format_datetime(value) if value else '' + # Import views from mailu import ui, internal, sso app.register_blueprint(ui.ui, url_prefix=app.config['WEB_ADMIN']) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 54856c09..b60b8a3e 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -54,6 +54,7 @@ DEFAULT_CONFIG = { 'DKIM_PATH': '/dkim/{domain}.{selector}.key', 'DEFAULT_QUOTA': 1000000000, 'MESSAGE_RATELIMIT': '200/day', + 'MESSAGE_RATELIMIT_EXEMPTION': '', 'RECIPIENT_DELIMITER': '', # Web settings 'SITENAME': 'Mailu', @@ -89,7 +90,7 @@ DEFAULT_CONFIG = { 'POD_ADDRESS_RANGE': None } -class ConfigManager(dict): +class ConfigManager: """ Naive configuration manager that uses environment only """ @@ -104,19 +105,16 @@ class ConfigManager(dict): def get_host_address(self, name): # if MYSERVICE_ADDRESS is defined, use this - if '{}_ADDRESS'.format(name) in os.environ: - return os.environ.get('{}_ADDRESS'.format(name)) + if f'{name}_ADDRESS' in os.environ: + return os.environ.get(f'{name}_ADDRESS') # otherwise use the host name and resolve it - return system.resolve_address(self.config['HOST_{}'.format(name)]) + return system.resolve_address(self.config[f'HOST_{name}']) def resolve_hosts(self): - self.config["IMAP_ADDRESS"] = self.get_host_address("IMAP") - self.config["POP3_ADDRESS"] = self.get_host_address("POP3") - self.config["AUTHSMTP_ADDRESS"] = self.get_host_address("AUTHSMTP") - self.config["SMTP_ADDRESS"] = self.get_host_address("SMTP") - self.config["REDIS_ADDRESS"] = self.get_host_address("REDIS") - if self.config["WEBMAIL"] != "none": - self.config["WEBMAIL_ADDRESS"] = self.get_host_address("WEBMAIL") + for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']: + self.config[f'{key}_ADDRESS'] = self.get_host_address(key) + if self.config['WEBMAIL'] != 'none': + self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL') def __get_env(self, key, value): key_file = key + "_FILE" @@ -135,6 +133,7 @@ class ConfigManager(dict): return value def init_app(self, app): + # get current app config self.config.update(app.config) # get environment variables self.config.update({ @@ -148,35 +147,18 @@ class ConfigManager(dict): template = self.DB_TEMPLATES[self.config['DB_FLAVOR']] self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config) - self.config['RATELIMIT_STORAGE_URL'] = 'redis://{0}/2'.format(self.config['REDIS_ADDRESS']) - self.config['QUOTA_STORAGE_URL'] = 'redis://{0}/1'.format(self.config['REDIS_ADDRESS']) - self.config['SESSION_STORAGE_URL'] = 'redis://{0}/3'.format(self.config['REDIS_ADDRESS']) + self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' + self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' + self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3' self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True self.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=int(self.config['SESSION_LIFETIME'])) hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')] self.config['AUTH_RATELIMIT_EXEMPTION'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['AUTH_RATELIMIT_EXEMPTION'].split(',')) if cidr) + self.config['MESSAGE_RATELIMIT_EXEMPTION'] = set([s for s in self.config['MESSAGE_RATELIMIT_EXEMPTION'].lower().replace(' ', '').split(',') if s]) self.config['HOSTNAMES'] = ','.join(hostnames) self.config['HOSTNAME'] = hostnames[0] - # update the app config itself - app.config = self - def setdefault(self, key, value): - if key not in self.config: - self.config[key] = value - return self.config[key] + # update the app config + app.config.update(self.config) - def get(self, *args): - return self.config.get(*args) - - def keys(self): - return self.config.keys() - - def __getitem__(self, key): - return self.config.get(key) - - def __setitem__(self, key, value): - self.config[key] = value - - def __contains__(self, key): - return key in self.config diff --git a/core/admin/mailu/debug.py b/core/admin/mailu/debug.py index 7677901b..4d63f3c5 100644 --- a/core/admin/mailu/debug.py +++ b/core/admin/mailu/debug.py @@ -1,6 +1,6 @@ import flask_debugtoolbar -from werkzeug.contrib import profiler as werkzeug_profiler +from werkzeug.middleware.profiler import ProfilerMiddleware # Debugging toolbar @@ -10,7 +10,7 @@ toolbar = flask_debugtoolbar.DebugToolbarExtension() # Profiler class Profiler(object): def init_app(self, app): - app.wsgi_app = werkzeug_profiler.ProfilerMiddleware( + app.wsgi_app = ProfilerMiddleware( app.wsgi_app, restrictions=[30] ) diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index ab965967..ed951943 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -149,6 +149,8 @@ def postfix_sender_login(sender): def postfix_sender_rate(sender): """ Rate limit outbound emails per sender login """ + if sender in flask.current_app.config['MESSAGE_RATELIMIT_EXEMPTION']: + flask.abort(404) user = models.User.get(sender) or flask.abort(404) return flask.abort(404) if user.sender_limiter.hit() else flask.jsonify("450 4.2.1 You are sending too many emails too fast.") diff --git a/core/admin/mailu/internal/views/rspamd.py b/core/admin/mailu/internal/views/rspamd.py index 8551eb8f..458dbb81 100644 --- a/core/admin/mailu/internal/views/rspamd.py +++ b/core/admin/mailu/internal/views/rspamd.py @@ -14,17 +14,14 @@ def vault_error(*messages, status=404): @internal.route("/rspamd/vault/v1/dkim/", methods=['GET']) def rspamd_dkim_key(domain_name): - domain = models.Domain.query.get(domain_name) or flask.abort(vault_error('unknown domain')) - key = domain.dkim_key or flask.abort(vault_error('no dkim key', status=400)) - return flask.jsonify({ - 'data': { - 'selectors': [ + selectors = [] + if domain := models.Domain.query.get(domain_name): + if key := domain.dkim_key: + selectors.append( { 'domain' : domain.name, 'key' : key.decode('utf8'), 'selector': flask.current_app.config.get('DKIM_SELECTOR', 'dkim'), } - ] - } - }) - + ) + return flask.jsonify({'data': {'selectors': selectors}}) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index f5fe3b5e..697e4df7 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -19,7 +19,8 @@ import os import hmac import smtplib import idna -import dns +import dns.resolver +import dns.exception from flask import current_app as app from sqlalchemy.ext import declarative @@ -38,6 +39,8 @@ class IdnaDomain(db.TypeDecorator): """ impl = db.String(80) + cache_ok = True + python_type = str def process_bind_param(self, value, dialect): """ encode unicode domain name to punycode """ @@ -47,13 +50,13 @@ class IdnaDomain(db.TypeDecorator): """ decode punycode domain name to unicode """ return idna.decode(value) - python_type = str - class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ impl = db.String(255) + cache_ok = True + python_type = str def process_bind_param(self, value, dialect): """ encode unicode domain part of email address to punycode """ @@ -69,13 +72,13 @@ class IdnaEmail(db.TypeDecorator): localpart, domain_name = value.rsplit('@', 1) return f'{localpart}@{idna.decode(domain_name)}' - python_type = str - class CommaSeparatedList(db.TypeDecorator): """ Stores a list as a comma-separated string, compatible with Postfix. """ impl = db.String + cache_ok = True + python_type = list def process_bind_param(self, value, dialect): """ join list of items to comma separated string """ @@ -90,13 +93,13 @@ class CommaSeparatedList(db.TypeDecorator): """ split comma separated string to list """ return list(filter(bool, (item.strip() for item in value.split(',')))) if value else [] - python_type = list - class JSONEncoded(db.TypeDecorator): """ Represents an immutable structure as a json-encoded string. """ impl = db.String + cache_ok = True + python_type = str def process_bind_param(self, value, dialect): """ encode data as json """ @@ -106,8 +109,6 @@ class JSONEncoded(db.TypeDecorator): """ decode json to data """ return json.loads(value) if value else None - python_type = str - class Base(db.Model): """ Base class for all models """ diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 191d01ac..00cbf464 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -145,6 +145,11 @@ class Logger: if history.has_changes() and history.deleted: before = history.deleted[-1] after = getattr(target, attr.key) + # we don't have ordered lists + if isinstance(before, list): + before = set(before) + if isinstance(after, list): + after = set(after) # TODO: this can be removed when comment is not nullable in model if attr.key == 'comment' and not before and not after: pass diff --git a/core/admin/mailu/sso/templates/form_sso.html b/core/admin/mailu/sso/templates/form_sso.html index b14e7600..d2451597 100644 --- a/core/admin/mailu/sso/templates/form_sso.html +++ b/core/admin/mailu/sso/templates/form_sso.html @@ -5,7 +5,7 @@
{{ macros.form_field(form.email) }} {{ macros.form_field(form.pw) }} - {{ macros.form_fields(fields, label=False, class="btn btn-default", spacing=False) }} + {{ macros.form_fields(fields, label=False, class="btn btn-default") }}
{%- endcall %} {%- endblock %} diff --git a/core/admin/mailu/sso/views/base.py b/core/admin/mailu/sso/views/base.py index fbee52a7..390d5bbf 100644 --- a/core/admin/mailu/sso/views/base.py +++ b/core/admin/mailu/sso/views/base.py @@ -19,6 +19,7 @@ def login(): fields.append(form.submitAdmin) if str(app.config["WEBMAIL"]).upper() != "NONE": fields.append(form.submitWebmail) + fields = [fields] if form.validate_on_submit(): if form.submitAdmin.data: @@ -38,7 +39,7 @@ def login(): flask.session.regenerate() flask_login.login_user(user) response = flask.redirect(destination) - response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login')) + response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login'), secure=app.config['SESSION_COOKIE_SECURE'], httponly=True) flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip}.') return response else: diff --git a/core/admin/mailu/ui/templates/alias/list.html b/core/admin/mailu/ui/templates/alias/list.html index 0b784d52..6b52165e 100644 --- a/core/admin/mailu/ui/templates/alias/list.html +++ b/core/admin/mailu/ui/templates/alias/list.html @@ -34,8 +34,8 @@ {{ alias }} {{ alias.destination|join(', ') or '-' }} {{ alias.comment or '' }} - {{ alias.created_at }} - {{ alias.updated_at or '' }} + {{ alias.created_at | format_date }} + {{ alias.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/alternative/list.html b/core/admin/mailu/ui/templates/alternative/list.html index b56cd751..4ca9f3c8 100644 --- a/core/admin/mailu/ui/templates/alternative/list.html +++ b/core/admin/mailu/ui/templates/alternative/list.html @@ -19,6 +19,7 @@ {% trans %}Actions{% endtrans %} {% trans %}Name{% endtrans %} {% trans %}Created{% endtrans %} + {% trans %}Last edit{% endtrans %} @@ -28,7 +29,8 @@ {{ alternative }} - {{ alternative.created_at }} + {{ alternative.created_at | format_date }} + {{ alternative.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/domain/list.html b/core/admin/mailu/ui/templates/domain/list.html index 6f6bc467..61c09151 100644 --- a/core/admin/mailu/ui/templates/domain/list.html +++ b/core/admin/mailu/ui/templates/domain/list.html @@ -46,8 +46,8 @@ {{ domain.users | count }} / {{ '∞' if domain.max_users == -1 else domain.max_users }} {{ domain.aliases | count }} / {{ '∞' if domain.max_aliases == -1 else domain.max_aliases }} {{ domain.comment or '' }} - {{ domain.created_at }} - {{ domain.updated_at or '' }} + {{ domain.created_at | format_date }} + {{ domain.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html index d9374fc6..60b214de 100644 --- a/core/admin/mailu/ui/templates/fetch/list.html +++ b/core/admin/mailu/ui/templates/fetch/list.html @@ -36,10 +36,10 @@ {{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }} {{ fetch.username }} {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} - {{ fetch.last_check or '-' }} + {{ fetch.last_check | format_datetime or '-' }} {{ fetch.error or '-' }} - {{ fetch.created_at }} - {{ fetch.updated_at or '' }} + {{ fetch.created_at | format_date }} + {{ fetch.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/macros.html b/core/admin/mailu/ui/templates/macros.html index 5143b697..0da069a9 100644 --- a/core/admin/mailu/ui/templates/macros.html +++ b/core/admin/mailu/ui/templates/macros.html @@ -18,17 +18,19 @@ {%- endif %} {%- endmacro %} -{%- macro form_fields(fields, prepend='', append='', label=True, spacing=True) %} - {%- if spacing %} +{%- macro form_fields(fields, prepend='', append='', label=True) %} {%- set width = (12 / fields|length)|int %} - {%- else %} - {%- set width = 0 %} - {% endif %}
{%- for field in fields %}
- {{ form_individual_field(field, prepend=prepend, append=append, label=label, **kwargs) }} + {%- if field.__class__.__name__ == 'list' %} + {%- for subfield in field %} + {{ form_individual_field(subfield, prepend=prepend, append=append, label=label, **kwargs) }} + {%- endfor %} + {%- else %} + {{ form_individual_field(field, prepend=prepend, append=append, label=label, **kwargs) }} + {%- endif %}
{%- endfor %}
diff --git a/core/admin/mailu/ui/templates/relay/list.html b/core/admin/mailu/ui/templates/relay/list.html index 07838273..80bc5338 100644 --- a/core/admin/mailu/ui/templates/relay/list.html +++ b/core/admin/mailu/ui/templates/relay/list.html @@ -32,8 +32,8 @@ {{ relay.name }} {{ relay.smtp or '-' }} {{ relay.comment or '' }} - {{ relay.created_at }} - {{ relay.updated_at or '' }} + {{ relay.created_at | format_date }} + {{ relay.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/token/list.html b/core/admin/mailu/ui/templates/token/list.html index c3cc9b5c..d7c48737 100644 --- a/core/admin/mailu/ui/templates/token/list.html +++ b/core/admin/mailu/ui/templates/token/list.html @@ -20,6 +20,7 @@ {% trans %}Comment{% endtrans %} {% trans %}Authorized IP{% endtrans %} {% trans %}Created{% endtrans %} + {% trans %}Last edit{% endtrans %} @@ -30,7 +31,8 @@ {{ token.comment }} {{ token.ip or "any" }} - {{ token.created_at }} + {{ token.created_at | format_date }} + {{ token.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/templates/user/list.html b/core/admin/mailu/ui/templates/user/list.html index 59a06ea7..7faddab5 100644 --- a/core/admin/mailu/ui/templates/user/list.html +++ b/core/admin/mailu/ui/templates/user/list.html @@ -45,8 +45,8 @@ {{ user.quota_bytes_used | filesizeformat }} / {{ (user.quota_bytes | filesizeformat) if user.quota_bytes else '∞' }} {{ user.comment or '-' }} - {{ user.created_at }} - {{ user.updated_at or '' }} + {{ user.created_at | format_date }} + {{ user.updated_at | format_date }} {%- endfor %} diff --git a/core/admin/mailu/ui/views/domains.py b/core/admin/mailu/ui/views/domains.py index f394ce7d..a48bb154 100644 --- a/core/admin/mailu/ui/views/domains.py +++ b/core/admin/mailu/ui/views/domains.py @@ -5,7 +5,6 @@ from flask import current_app as app import flask import flask_login import wtforms_components -import dns.resolver @ui.route('/domain', methods=['GET']) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index e46ad7d9..024c487f 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -6,18 +6,21 @@ try: except ImportError: import pickle -import dns import dns.resolver +import dns.exception +import dns.flags +import dns.rdtypes +import dns.rdatatype +import dns.rdataclass import hmac import secrets import time from multiprocessing import Value - from mailu import limiter - from flask import current_app as app + import flask import flask_login import flask_migrate @@ -28,7 +31,7 @@ import redis from flask.sessions import SessionMixin, SessionInterface from itsdangerous.encoding import want_bytes from werkzeug.datastructures import CallbackDict -from werkzeug.contrib import fixers +from werkzeug.middleware.proxy_fix import ProxyFix # Login configuration login = flask_login.LoginManager() @@ -106,7 +109,7 @@ class PrefixMiddleware(object): return self.app(environ, start_response) def init_app(self, app): - self.app = fixers.ProxyFix(app.wsgi_app, x_for=1, x_proto=1) + self.app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1) app.wsgi_app = self proxy = PrefixMiddleware() @@ -265,7 +268,7 @@ class MailuSession(CallbackDict, SessionMixin): # set uid from dict data if self._uid is None: - self._uid = self.app.session_config.gen_uid(self.get('user_id', '')) + self._uid = self.app.session_config.gen_uid(self.get('_user_id', '')) # create new session id for new or regenerated sessions and force setting the cookie if self._sid is None: diff --git a/core/admin/migrations/env.py b/core/admin/migrations/env.py index 3e45bb18..65a4e471 100755 --- a/core/admin/migrations/env.py +++ b/core/admin/migrations/env.py @@ -1,10 +1,12 @@ -from __future__ import with_statement +import logging +import tenacity + from alembic import context from sqlalchemy import engine_from_config, pool from logging.config import fileConfig -import logging -import tenacity -from tenacity import retry + +from flask import current_app +from mailu import models # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -17,20 +19,12 @@ logger = logging.getLogger('alembic.env') # add your model's MetaData object here # for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -from flask import current_app -config.set_main_option('sqlalchemy.url', - current_app.config.get('SQLALCHEMY_DATABASE_URI')) -#target_metadata = current_app.extensions['migrate'].db.metadata -from mailu import models +config.set_main_option( + 'sqlalchemy.url', + current_app.config.get('SQLALCHEMY_DATABASE_URI') +) target_metadata = models.Base.metadata -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - def run_migrations_offline(): """Run migrations in 'offline' mode. @@ -44,7 +38,7 @@ def run_migrations_offline(): script output. """ - url = config.get_main_option("sqlalchemy.url") + url = config.get_main_option('sqlalchemy.url') context.configure(url=url) with context.begin_transaction(): @@ -69,28 +63,35 @@ def run_migrations_online(): directives[:] = [] logger.info('No changes in schema detected.') - engine = engine_from_config(config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) + engine = engine_from_config( + config.get_section(config.config_ini_section), + prefix = 'sqlalchemy.', + poolclass = pool.NullPool + ) - connection = tenacity.Retrying( - stop=tenacity.stop_after_attempt(100), - wait=tenacity.wait_random(min=2, max=5), - before=tenacity.before_log(logging.getLogger("tenacity.retry"), logging.DEBUG), - before_sleep=tenacity.before_sleep_log(logging.getLogger("tenacity.retry"), logging.INFO), - after=tenacity.after_log(logging.getLogger("tenacity.retry"), logging.DEBUG) - ).call(engine.connect) + @tenacity.retry( + stop = tenacity.stop_after_attempt(100), + wait = tenacity.wait_random(min=2, max=5), + before = tenacity.before_log(logging.getLogger('tenacity.retry'), logging.DEBUG), + before_sleep = tenacity.before_sleep_log(logging.getLogger('tenacity.retry'), logging.INFO), + after = tenacity.after_log(logging.getLogger('tenacity.retry'), logging.DEBUG) + ) + def try_connect(db): + return db.connect() - context.configure(connection=connection, - target_metadata=target_metadata, - process_revision_directives=process_revision_directives, - **current_app.extensions['migrate'].configure_args) + with try_connect(engine) as connection: + + context.configure( + connection = connection, + target_metadata = target_metadata, + process_revision_directives = process_revision_directives, + **current_app.extensions['migrate'].configure_args + ) - try: with context.begin_transaction(): context.run_migrations() - finally: - connection.close() + + connection.close() if context.is_offline_mode(): run_migrations_offline() diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index 3406122d..d6c7aca3 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -1,56 +1,75 @@ -alembic==1.0.10 -asn1crypto==0.24.0 -Babel==2.6.0 -bcrypt==3.1.6 +alembic==1.7.4 +appdirs==1.4.4 +Babel==2.9.1 +bcrypt==3.2.0 blinker==1.4 -cffi==1.12.3 -Click==7.0 -cryptography==3.4.7 -decorator==4.4.0 -dnspython==1.16.0 -dominate==2.3.5 -Flask==1.0.2 -Flask-Babel==0.12.2 +CacheControl==0.12.9 +certifi==2021.10.8 +cffi==1.15.0 +chardet==4.0.0 +click==8.0.3 +colorama==0.4.4 +contextlib2==21.6.0 +cryptography==35.0.0 +decorator==5.1.0 +# distlib==0.3.1 +# distro==1.5.0 +dnspython==2.1.0 +dominate==2.6.0 +email-validator==1.1.3 +Flask==2.0.2 +Flask-Babel==2.0.0 Flask-Bootstrap==3.3.7.1 -Flask-DebugToolbar==0.10.1 -Flask-Limiter==1.0.1 -Flask-Login==0.4.1 +Flask-DebugToolbar==0.11.0 +Flask-Limiter==1.4 +Flask-Login==0.5.0 flask-marshmallow==0.14.0 -Flask-Migrate==2.4.0 +Flask-Migrate==3.1.0 Flask-Script==2.0.6 -Flask-SQLAlchemy==2.4.0 -Flask-WTF==0.14.2 +Flask-SQLAlchemy==2.5.1 +Flask-WTF==0.15.1 +greenlet==1.1.2 gunicorn==20.1.0 -idna==2.8 -infinity==1.4 -intervals==0.8.1 -itsdangerous==1.1.0 -Jinja2==2.11.3 -limits==1.3 -Mako==1.0.9 -MarkupSafe==1.1.1 -mysqlclient==1.4.2.post1 -marshmallow==3.10.0 -marshmallow-sqlalchemy==0.24.1 +html5lib==1.1 +idna==3.3 +infinity==1.5 +intervals==0.9.2 +itsdangerous==2.0.1 +Jinja2==3.0.2 +limits==1.5.1 +lockfile==0.12.2 +Mako==1.1.5 +MarkupSafe==2.0.1 +marshmallow==3.14.0 +marshmallow-sqlalchemy==0.26.1 +msgpack==1.0.2 +mysqlclient==2.0.3 +ordered-set==4.0.2 +# packaging==20.9 passlib==1.7.4 -psycopg2==2.8.2 -pycparser==2.19 -Pygments==2.8.1 -pyOpenSSL==20.0.1 -python-dateutil==2.8.0 -python-editor==1.0.4 -pytz==2019.1 -PyYAML==5.4.1 -redis==3.2.1 -#alpine3:12 provides six==1.15.0 -#six==1.12.0 -socrate==0.1.1 -SQLAlchemy==1.3.3 +# pep517==0.10.0 +progress==1.6 +psycopg2==2.9.1 +pycparser==2.20 +Pygments==2.10.0 +pyOpenSSL==21.0.0 +pyparsing==3.0.4 +pytz==2021.3 +PyYAML==6.0 +redis==3.5.3 +requests==2.26.0 +retrying==1.3.3 +# six==1.15.0 +socrate==0.2.0 +SQLAlchemy==1.4.26 srslib==0.1.4 -tabulate==0.8.3 -tenacity==5.0.4 -validators==0.12.6 +tabulate==0.8.9 +tenacity==8.0.1 +toml==0.10.2 +urllib3==1.26.7 +validators==0.18.2 visitor==0.1.3 -Werkzeug==0.15.5 -WTForms==2.2.1 -WTForms-Components==0.10.4 +webencodings==0.5.1 +Werkzeug==2.0.2 +WTForms==2.3.3 +WTForms-Components==0.10.5 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index e1de3b01..65130a8c 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -18,10 +18,8 @@ PyYAML PyOpenSSL Pygments dnspython -bcrypt tenacity mysqlclient -psycopg2 idna srslib marshmallow diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 5fffc330..8ac646da 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -17,7 +17,7 @@ queue_directory = /queue message_size_limit = {{ MESSAGE_SIZE_LIMIT }} # Relayed networks -mynetworks = 127.0.0.1/32 [::1]/128 {{ SUBNET }} {{ RELAYNETS.split(",") | join(' ') }} +mynetworks = 127.0.0.1/32 [::1]/128 {{ SUBNET }} {% if RELAYNETS %}{{ RELAYNETS.split(",") | join(' ') }}{% endif %} # Empty alias list to override the configuration variable and disable NIS alias_maps = diff --git a/design/mailu-directory-structure.md b/design/mailu-directory-structure.md new file mode 100644 index 00000000..ad75eeaa --- /dev/null +++ b/design/mailu-directory-structure.md @@ -0,0 +1,199 @@ +# RFC: Mailu directory structure +The current layout of the Mailu directory structure can be improved to allow for easier replicated deployments, like Docker Swarm and Kubernetes. Please read https://usrpro.com/publications/mailu-persistent-storage/ for the background motivation of this RFC. + +## Scope +This document only describes the re-arrangement of the `$ROOT` Mailu filesystem as-is. This means moving around of current files and directories. The linked article also proposes more advanced improvements. Those are not included in this RFC and need to be evaluated and implemented independently. However, the changes proposed in this document should make such improvements easier. + +Currently some services are wrongfully sharing mountpoints or have unused volumes declared. Those are also taken care of in this RFC. + +## Compatibility +If these changes were to be accepted, it will break compatibility with previous Mailu version `<=1.7`. As such, we will propably increment to version `2.0`. + +## Root +The root of the Mailu filesystem is located at `/mailu` by default. For simplicity we will assume this location throughout the document. Within `/mailu` we will aim to define 3 main sub-directories: + +### Config + +- Path: `/mailu/config/` + +Small config bearing files, sometimes shared between multiple services. The performance and storage needs for this filesystem are low. Availability is important for correct functioning of the mail server. No file locking issues are expected from concurrent access. A basic (and redundant) filesystem should suffice. + +#### Dovecot + +- Old path: `/mailu/overrides` (shared with postfix, nginx and rspamd) +- New Path `/mailu/config/dovecot` + +Dovecot configuration overrides. + +#### Postfix + +- Old path: `/mailu/overrides` (shared with dovecot, nginx and rspamd) +- New Path `/mailu/config/posfix` + +Postfix configuration overrides. + +#### Rspamd + +- Old path: `/mailu/overrides/rspamd` +- New path: `/mailu/config/rspamd` + +RSpamD configuration overrides. + +#### Rainloop + +- Old path: `/mailu/webmail/_data_/_default_/storage` (part of `/mailu/webmail` mountpoint, shared with Roundcube) +- New path: `/mailu/config/rainloop` + +User specific configs. The remaining files under the old `/mailu/webmail` don't need to be persistent. Except for `AddressBook.sqlite`, see `/mailu/data`. + +#### Roundcube + +- Old path: `/mailu/webmail/gpg` (part of `/mailu/webmail` mountpoint, shared with Rainloop) +- New path: `/mailu/config/roundcube/gpg` + +User configured GPG keys. + +#### Redis + +- Old path: `/mailu/redis` +- New path: `/mailu/config/redis` + +Holds `dump.rdb` for data restoration. Although technically a database, Redis works from memory. The dump file is only written to every minute and read from during start. Hence it fits better in the replicated config directory filesystem. + +#### Share + +- Path: `/mailu/config/share/` + +Shared configuration between different services + +##### DKIM + +- Old path: `/mailu/dkim/` +- New path: `/mailu/config/share/dkim` + +DKIM private keys store. Read/write access by Admin. Read only access by rSpamD. + +##### Certs + +- Old path: `/mailu/certs` +- New path: `/mailu/config/share/certs` (Proposal in anticipation of the RFC outcome at: https://github.com/Mailu/Mailu/issues/1222.) + +TLS certificates. Write access from `nginx` in case `TLS_FLAVOR=letsencrypt`. Or write access from `treafik-certdumper` or any other tool obtaining certificates. + +`letsencrypt` setting is not compatible with replicated setups. Multiple instances would disrupt the ACME challenge verification and cause race conditions on requesting certificates. In such cases certificates will need to be provided by other tools. + +If RFC issue #1222 is accepted, Dovecot will need read-only access to the certificates. + +### Data + +- Path: `/mailu/data/` + +Database files, like SQLite or PostgreSQL files. Databases don't perform well on network filesystems as they depend heavily on file locking and full controll on the database files. Making it unfit for concurrent access from multiple hosts. This directory should always live on a local filesystem. This makes it only usable in `docker-compose` deployments. Usage of this directory should be avoided in Kubernetes and Docker Swarm deployments. Some services will need to be improved to allow for this. + +#### admin data + +- Old path: `/mailu/data/` +- New path: `/mailu/data/admin/` (mount point on `admin` directory) + +Holds `main.db` SQLite database file holding domains, users, aliases etcetera. Read/write access only by admin. Can be avoided by using a remote DB server like PostgreSQL, MySQL or MariaDB. + +Also holds `instance` for unique statistics transmission. Removing of this file is proposed in RFC [issue 129](https://github.com/Mailu/Mailu/issues/1219). + +This move is needed in order to be able to mount the directory without exposing data files from other services into admin. + +#### rspamd + +- Old path: `/mailu/filter` (shared with ClamAV) +- New path: `/mailu/data/rspamd` + +Storage of Bayes and Fuzzy learning SQLite databases and caches. As future optimization we should look into moving all this into Redis. + +#### Rainloop + +- Old path: `/mailu/webmail/_data_/_default_/AddressBook.sqlite` (part of `/mailu/webmail` mountpoint, shared with Roundcube) +- New path: `/mailu/data/rainloop/AddressBook.sqlite` (mount on `rainloop` directory) + +Addressbook SQLite file. For future replicated deployments this might better be configured to use an external DB. + +For this modification, the `AddressBook.sqlite` will need to be moved to a different directory inside the container. + +#### Roundcube + +- Old path: `/mailu/webmail/roundcube.db` (part of `/mailu/webmail` mountpoint, shared with Rainloop) +- New path: `/mailu/data/roundcube/roundcube.db` (mount on `roundcube` directory) + +User settings SQLite database file for roundcube. For future replicated deployments this might better be configured to use an external DB. + +For this modification, the `rouncube.db` file will need to be moved to a different directory inside the container. + +### Mail + +- Path: `/mailu/mail` (unmodified) + +User mail, managed my Dovecot IMAP server. In replicated deployments, this filesystem needs to be shared over all IMAP server instances. It should be high performant and capable of propgating file locks. Storage size is proportional to the users and their quotas. Old versions of NFS are known to be buggy with file locking. Also Samaba or CIFS should be avoided. + +In the old situation, Maildir indexes are stored on the same volume. However, they need not to be persistent and should be located on a voletile filesystem instead. This allows better performance on network filesystems. + +### Local + +- Path: `/mailu/local` (new) + +Persistent storage not suitable for replication. In `docker-compose` deployments it lives inside `/mailu` and in replicated deployments it should live somewhere on the local host machine. + +#### Mailqueue + +- Old path: `/mailu/mailqueue` +- New path: `/mailu/local/mailqueue` + +The SMTP mailqueue should be persistant, as per SMTP spec it is not allowed to loose mail. However, persistance should be local only for performance reasons and the possibility to replicate Postfix servers. In setups like Docker Swarm and Kubernetes, admins should take care that Postfix is always restarted on same hosts in order to empty any remaining queue after a crash. + +#### ClamAV + +- Old path: `/mailu/filters` (shared with rSpamD) +- New path: `/mailu/local/clamav` + +Virus definitions do not need to be replicated, as they can be easily pulled in when ClamAV instances migrate to other nodes. Persistance does allow for some bandwith and time saving if ClamAV would be restarted on a previously used node (in case of updates or similar cases). Local only storage also prevents `freshclam` race conditions. + +## Conclusion + +The final layout of the Mailu filesystem will look like: + +```` +/mailu +├── config +│   ├── dovecot +│   ├── postfix +│   ├── rainloop +│   ├── redis +│   ├── roundcube +│   │   └── gpg +│   ├── rspamd +│   └── share +│   ├── certs +│   └── dkim +├── data +│   ├── admin +│   ├── rainloop +│   ├── roundcube +│   └── rspamd +├── local +│   ├── clamav +│   └── mailqueue +└── mail +```` + +Where in replicated environments: + +- `/mailu/config/`: should be a small, low performant and shared filesystem. +- `/mailu/data`: should be avoided. More work will need to be done to configure external DB servers for relevant services. Ideally, this directory should only exist on docker-compose deployments. +- `/mailu/local/`: Should exist only on local file systems of worker nodes. +- `/mailu/mail`: A distributed filesystem with sufficient performance and storage requirements to hold and process all user mailboxes. Ideally only Maildir without indexes. + +### Implementing + +The works to implement this changes should happen outside the `master` branch. Inclusion into `master` can only be accepted if: + +1. `docker-compose.yml` from setup reflects this changes correctly. +2. Kubernetes documentation is updated. +3. Legacy `docker-compose.yml` is either updated or deleted. +4. A clear data migration guide is written. \ No newline at end of file diff --git a/docs/configuration.rst b/docs/configuration.rst index 03817988..8f363a63 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -69,9 +69,11 @@ The ``MESSAGE_SIZE_LIMIT`` is the maximum size of a single email. It should not be too low to avoid dropping legitimate emails and should not be too high to avoid filling the disks with large junk emails. -The ``MESSAGE_RATELIMIT`` is the limit of messages a single user can send. This is -meant to fight outbound spam in case of compromised or malicious account on the -server. +The ``MESSAGE_RATELIMIT`` (default: 200/day) is the maximum number of messages +a single user can send. ``MESSAGE_RATELIMIT_EXEMPTION`` contains a comma delimited +list of user email addresses that are exempted from any restriction. Those +settings are meant to reduce outbound spam in case of compromised or malicious +account on the server. The ``RELAYNETS`` (default: unset) is a comma delimited list of network addresses for which mail is relayed for with no authentication required. This should be @@ -192,7 +194,9 @@ The ``LETSENCRYPT_SHORTCHAIN`` (default: False) setting controls whether we send .. _`android handsets older than 7.1.1`: https://community.letsencrypt.org/t/production-chain-changes/150739 -The ``REAL_IP_HEADER`` (default: unset) and ``REAL_IP_FROM`` (default: unset) settings controls whether HTTP headers such as ``X-Forwarded-For`` or ``X-Real-IP`` should be trusted. The former should be the name of the HTTP header to extract the client IP address from and the later a comma separated list of IP addresses designing which proxies to trust. If you are using Mailu behind a reverse proxy, you should set both. Setting the former without the later introduces a security vulnerability allowing a potential attacker to spoof his source address. +.. _reverse_proxy_headers: + +The ``REAL_IP_HEADER`` (default: unset) and ``REAL_IP_FROM`` (default: unset) settings controls whether HTTP headers such as ``X-Forwarded-For`` or ``X-Real-IP`` should be trusted. The former should be the name of the HTTP header to extract the client IP address from and the later a comma separated list of IP addresses designating which proxies to trust. If you are using Mailu behind a reverse proxy, you should set both. Setting the former without the later introduces a security vulnerability allowing a potential attacker to spoof his source address. The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usually uses a ``Region/City`` format. See `TZ database name`_ for a list of valid timezones This defaults to ``Etc/UTC``. Warning: if you are observing different timestamps in your log files you should change your hosts timezone to UTC instead of changing TZ to your local timezone. Using UTC allows easy log correlation with remote MTAs. diff --git a/docs/reverse.rst b/docs/reverse.rst index c6b98e4a..6cde55f2 100644 --- a/docs/reverse.rst +++ b/docs/reverse.rst @@ -11,7 +11,10 @@ There are basically three options, from the most to the least recommended one: - `use Traefik in another container as central system-reverse-proxy`_ - `override Mailu Web frontend configuration`_ -All options will require that you modify the ``docker-compose.yml`` file. +All options will require that you modify the ``docker-compose.yml`` and ``mailu.env`` file. + +Mailu must also be configured with the information what header is used by the reverse proxy for passing the remote client IP. +This is configured in the mailu.env file. See the :ref:`configuration reference ` for more information. Have Mailu Web frontend listen locally -------------------------------------- @@ -43,10 +46,19 @@ Then on your own frontend, point to these local ports. In practice, you only nee # [...] here goes your standard configuration location / { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr proxy_pass https://localhost:8443; } } +.. code-block:: docker + + #mailu.env file + REAL_IP_HEADER=X-Real-IP + REAL_IP_FROM=x.x.x.x,y.y.y.y.y + #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. + Because the admin interface is served as ``/admin``, the Webmail as ``/webmail``, the single sign on page as ``/sso``, webdav as ``/webdav`` and the static files endpoint as ``/static``, you may also want to use a single virtual host and serve other applications (still Nginx): .. code-block:: nginx @@ -55,8 +67,9 @@ Because the admin interface is served as ``/admin``, the Webmail as ``/webmail`` # [...] here goes your standard configuration location ~ ^/(admin|sso|static|webdav|webmail)/ { - proxy_pass https://localhost:8443; - proxy_set_header Host $http_host; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr + proxy_pass https://localhost:8443; } location /main_app { @@ -76,6 +89,13 @@ Because the admin interface is served as ``/admin``, the Webmail as ``/webmail`` } } +.. code-block:: docker + + #mailu.env file + REAL_IP_HEADER=X-Real-IP + REAL_IP_FROM=x.x.x.x,y.y.y.y.y + #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. + Finally, you might want to serve the admin interface on a separate virtual host but not expose the admin container directly (have your own HTTPS virtual hosts on top of Mailu, one public for the Webmail and one internal for administration for instance). Here is an example configuration : @@ -88,6 +108,8 @@ Here is an example configuration : # [...] here goes your standard configuration location /webmail { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr proxy_pass https://localhost:8443/webmail; } } @@ -98,12 +120,21 @@ Here is an example configuration : # [...] here goes your standard configuration location /admin { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr proxy_pass https://localhost:8443/admin; proxy_set_header Host $http_host; } } +.. code-block:: docker + + #mailu.env file + REAL_IP_HEADER=X-Real-IP + REAL_IP_FROM=x.x.x.x,y.y.y.y.y + #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. + Depending on how you access the front server, you might want to add a ``proxy_redirect`` directive to your ``location`` blocks: .. code-block:: nginx @@ -151,7 +182,16 @@ If your Traefik is configured to automatically request certificates from *letsen and this is the ``DOMAIN`` in your ``.env``? To support that use-case, Traefik can request ``SANs`` for your domain. The configuration for this will depend on your Traefik version. ----- +Mailu must also be configured with the information what header is used by the reverse proxy for passing the remote client IP. This is configured in mailu.env: + +.. code-block:: docker + + #mailu.env file + REAL_IP_HEADER=X-Real-IP + REAL_IP_FROM=x.x.x.x,y.y.y.y.y + #x.x.x.x,y.y.y.y.y is the static IP address your reverse proxy uses for connecting to Mailu. + +For more information see the :ref:`configuration reference ` for more information. Traefik 2.x using labels configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -179,53 +219,6 @@ Of course, be sure to define the Certificate Resolver ``foo`` in the static conf Alternatively, you can define SANs in the Traefik static configuration using routers, or in the static configuration using entrypoints. Refer to the Traefik documentation for more details. -Traefik 1.x with TOML configuration -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Lets add something like - -.. code-block:: yaml - - [acme] - [[acme.domains]] - main = "your.example.com" # this is the same as $TRAEFIK_DOMAIN! - sans = ["mail.your.example.com", "webmail.your.example.com", "smtp.your.example.com"] - -to your ``traefik.toml``. - ----- - -You might need to clear your ``acme.json``, if a certificate for one of these domains already exists. - -You will need some solution which dumps the certificates in ``acme.json``, so you can include them in the ``mailu/front`` container. -One such example is ``mailu/traefik-certdumper``, which has been adapted for use in Mailu. You can add it to your ``docker-compose.yml`` like: - -.. code-block:: yaml - - certdumper: - restart: always - image: mailu/traefik-certdumper:$VERSION - environment: - # Make sure this is the same as the main=-domain in traefik.toml - # !!! Also don’t forget to add "TRAEFIK_DOMAIN=[...]" to your .env! - - DOMAIN=$TRAEFIK_DOMAIN - volumes: - # Folder, which contains the acme.json - - "/data/traefik:/traefik" - # Folder, where cert.pem and key.pem will be written - - "/data/mailu/certs:/output" - - -Assuming you have ``volume-mounted`` your ``acme.json`` put to ``/data/traefik`` on your host. The dumper will then write out ``/data/mailu/certs/cert.pem`` and ``/data/mailu/certs/key.pem`` whenever ``acme.json`` is updated. -Yay! Now let’s mount this to our ``front`` container like: - -.. code-block:: yaml - - volumes: - - /data/mailu/certs:/certs - -This works, because we set ``TLS_FLAVOR=mail``, which picks up the key-certificate pair (e.g., ``cert.pem`` and ``key.pem``) from the certs folder in the root path (``/certs/``). - .. _`Traefik`: https://traefik.io/ Override Mailu configuration diff --git a/optional/fetchmail/Dockerfile b/optional/fetchmail/Dockerfile index 995ec48f..068a5dce 100644 --- a/optional/fetchmail/Dockerfile +++ b/optional/fetchmail/Dockerfile @@ -12,8 +12,8 @@ RUN apk add --no-cache \ RUN apk add --no-cache fetchmail ca-certificates openssl \ && pip3 install requests +RUN mkdir -p /data + COPY fetchmail.py /fetchmail.py -USER fetchmail - -CMD ["/fetchmail.py"] +CMD ["/fetchmail.py"] \ No newline at end of file diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 4be3c2bd..5459de59 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -13,6 +13,7 @@ import traceback FETCHMAIL = """ fetchmail -N \ + --idfile /data/fetchids --uidl \ --sslcertck --sslcertpath /etc/ssl/certs \ -f {} """ diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 2675a2ab..18a881b8 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -129,6 +129,8 @@ services: image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${MAILU_VERSION:-{{ version }}} restart: always env_file: {{ env }} + volumes: + - "{{ root }}/data/fetchmail:/data" {% if resolver_enabled %} depends_on: - resolver diff --git a/setup/flavors/stack/docker-compose.yml b/setup/flavors/stack/docker-compose.yml index 24afa9f3..0c744d7e 100644 --- a/setup/flavors/stack/docker-compose.yml +++ b/setup/flavors/stack/docker-compose.yml @@ -110,7 +110,7 @@ services: image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}fetchmail:${MAILU_VERSION:-{{ version }}} env_file: {{ env }} volumes: - - "{{ root }}/data:/data" + - "{{ root }}/data/fetchmail:/data" deploy: replicas: 1 healthcheck: diff --git a/towncrier/newsfragments/1223.bugfix b/towncrier/newsfragments/1223.bugfix new file mode 100644 index 00000000..3c23d1a4 --- /dev/null +++ b/towncrier/newsfragments/1223.bugfix @@ -0,0 +1,4 @@ +Fixed fetchmail losing track of fetched emails upon container recreation. +The relevant fetchmail files are now retained in the /data folder (in the fetchmail image). +See the docker-compose.yml file for the relevant volume mapping. +If you already had your own mapping, you must double check the volume mapping and take action. diff --git a/towncrier/newsfragments/1962.bugfix b/towncrier/newsfragments/1962.bugfix new file mode 100644 index 00000000..70b2aa72 --- /dev/null +++ b/towncrier/newsfragments/1962.bugfix @@ -0,0 +1,5 @@ +Reverse proxy documentation has been updated to reflect new security hardening from PR#1959. +If you do not set the configuration parameters in Mailu what reverse proxy header to trust, +then Mailu will not have access to the real ip address of the connecting client. +This means that rate limiting will not properly work. You can also not use fail2ban. +It is very important to configure this when using a reverse proxy.