From bd20ef04cc4de16af66befb3c0f09997a2da7de4 Mon Sep 17 00:00:00 2001 From: Johnson Thiang Date: Thu, 22 Dec 2022 18:10:13 +0800 Subject: [PATCH 01/47] change field type to db.text --- core/admin/mailu/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a7d0d006..9b308b32 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -75,7 +75,7 @@ class CommaSeparatedList(db.TypeDecorator): """ Stores a list as a comma-separated string, compatible with Postfix. """ - impl = db.String + impl = db.Text cache_ok = True python_type = list @@ -96,7 +96,7 @@ class JSONEncoded(db.TypeDecorator): """ Represents an immutable structure as a json-encoded string. """ - impl = db.String + impl = db.Text cache_ok = True python_type = str From c30944404d4317db08bd7150c4dcbe1e4965de3e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 16:31:03 +0100 Subject: [PATCH 02/47] Add "API" flag to config (default: disabled) --- core/admin/mailu/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index b2864b57..739d0189 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -69,6 +69,7 @@ DEFAULT_CONFIG = { 'RECAPTCHA_PRIVATE_KEY': '', 'LOGO_URL': None, 'LOGO_BACKGROUND': None, + 'API': False, # Advanced settings 'LOG_LEVEL': 'WARNING', 'SESSION_KEY_BITS': 128, @@ -157,4 +158,3 @@ class ConfigManager: # update the app config app.config.update(self.config) - From 866ad89dfcbe3503b7004b721c7173e129217ff6 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 17:05:21 +0100 Subject: [PATCH 03/47] first try at api using flask-restx & marshmallow --- core/admin/mailu/__init__.py | 4 +- core/admin/mailu/api/__init__.py | 38 ++++++ core/admin/mailu/api/common.py | 8 ++ core/admin/mailu/api/v1/__init__.py | 27 ++++ core/admin/mailu/api/v1/domains.py | 183 ++++++++++++++++++++++++++++ 5 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 core/admin/mailu/api/__init__.py create mode 100644 core/admin/mailu/api/common.py create mode 100644 core/admin/mailu/api/v1/__init__.py create mode 100644 core/admin/mailu/api/v1/domains.py diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 3b88024f..d17c8734 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -70,10 +70,12 @@ def create_app_from_config(config): return utils.flask_babel.format_datetime(value) if value else '' # Import views - from mailu import ui, internal, sso + from mailu import ui, internal, sso, api app.register_blueprint(ui.ui, url_prefix=app.config['WEB_ADMIN']) app.register_blueprint(internal.internal, url_prefix='/internal') app.register_blueprint(sso.sso, url_prefix='/sso') + if app.config.get('API'): + api.register(app) return app diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py new file mode 100644 index 00000000..6e7d6386 --- /dev/null +++ b/core/admin/mailu/api/__init__.py @@ -0,0 +1,38 @@ +from flask import redirect, url_for + +# import api version(s) +from . import v1 + +# api +ROOT='/api' +ACTIVE=v1 + +# patch url for swaggerui static assets +from flask_restx.apidoc import apidoc +apidoc.static_url_path = f'{ROOT}/swaggerui' + +def register(app): + + # register api bluprint(s) + app.register_blueprint(v1.blueprint, url_prefix=f'{ROOT}/v{int(v1.VERSION)}') + + # add redirect to current api version + @app.route(f'{ROOT}/') + def redir(): + return redirect(url_for(f'{ACTIVE.blueprint.name}.root')) + + # swagger ui config + app.config.SWAGGER_UI_DOC_EXPANSION = 'list' + app.config.SWAGGER_UI_OPERATION_ID = True + app.config.SWAGGER_UI_REQUEST_DURATION = True + + # TODO: remove patch of static assets for debugging + import os + if 'STATIC_ASSETS' in os.environ: + app.blueprints['ui'].static_folder = os.environ['STATIC_ASSETS'] + +# TODO: authentication via username + password +# TODO: authentication via api token +# TODO: api access for all users (via token) +# TODO: use permissions from "manager_of" +# TODO: switch to marshmallow, as parser is deprecated. use flask_accepts? diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py new file mode 100644 index 00000000..700835ac --- /dev/null +++ b/core/admin/mailu/api/common.py @@ -0,0 +1,8 @@ +from .. import models + +def fqdn_in_use(*names): + for name in names: + for model in models.Domain, models.Alternative, models.Relay: + if model.query.get(name): + return model + return None diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py new file mode 100644 index 00000000..c6de6fa4 --- /dev/null +++ b/core/admin/mailu/api/v1/__init__.py @@ -0,0 +1,27 @@ +from flask import Blueprint +from flask_restx import Api, fields + +VERSION = 1.0 + +blueprint = Blueprint(f'api_v{int(VERSION)}', __name__) + +api = Api( + blueprint, version=f'{VERSION:.1f}', + title='Mailu API', default_label='Mailu', + validate=True +) + +response_fields = api.model('Response', { + 'code': fields.Integer, + 'message': fields.String, +}) + +error_fields = api.model('Error', { + 'errors': fields.Nested(api.model('Error_Key', { + 'key': fields.String, + 'message':fields.String + })), + 'message': fields.String, +}) + +from . import domains diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py new file mode 100644 index 00000000..a0d37186 --- /dev/null +++ b/core/admin/mailu/api/v1/domains.py @@ -0,0 +1,183 @@ +from flask_restx import Resource, fields, abort + +from . import api, response_fields, error_fields +from .. import common +from ... import models + +db = models.db + +dom = api.namespace('domain', description='Domain operations') +alt = api.namespace('alternative', description='Alternative operations') + +domain_fields = api.model('Domain', { + 'name': fields.String(description='FQDN', example='example.com', required=True), + 'comment': fields.String(description='a comment'), + 'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1), + 'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1), + 'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0), + 'signup_enabled': fields.Boolean(description='allow signup'), +# 'dkim_key': fields.String, + 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example.com')), +}) +# TODO - name ist required on creation but immutable on change +# TODO - name and alteranatives need to be checked to be a fqdn (regex) + +domain_parser = api.parser() +domain_parser.add_argument('max_users', type=int, help='maximum number of users') +# TODO ... add more (or use marshmallow) + +alternative_fields = api.model('Domain', { + 'name': fields.String(description='alternative FQDN', example='example.com', required=True), + 'domain': fields.String(description='domain FQDN', example='example.com', required=True), + 'dkim_key': fields.String, +}) +# TODO: domain and name are not always required and can't be changed + + +@dom.route('') +class Domains(Resource): + + @dom.doc('list_domain') + @dom.marshal_with(domain_fields, as_list=True, skip_none=True, mask=['dkim_key']) + def get(self): + """ List domains """ + return models.Domain.query.all() + + @dom.doc('create_domain') + @dom.expect(domain_fields) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(409, 'Duplicate domain name', error_fields) + def post(self): + """ Create a new domain """ + data = api.payload + if common.fqdn_in_use(data['name']): + abort(409, f'Duplicate domain name {data["name"]!r}', errors={ + 'name': data['name'], + }) + for item, created in models.Domain.from_dict(data): + if not created: + abort(409, f'Duplicate domain name {item.name!r}', errors={ + 'alternatives': item.name, + }) + db.session.add(item) + db.session.commit() + +@dom.route('/') +class Domain(Resource): + + @dom.doc('get_domain') + @dom.response(200, 'Success', domain_fields) + @dom.response(404, 'Domain not found') + @dom.marshal_with(domain_fields) + def get(self, name): + """ Find domain by name """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + return domain + + @dom.doc('update_domain') + @dom.expect(domain_fields) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(404, 'Domain not found') + def put(self, name): + """ Update an existing domain """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + data = api.payload + data['name'] = name + for item, created in models.Domain.from_dict(data): + if created is True: + db.session.add(item) + db.session.commit() + + @dom.doc('modify_domain') + @dom.expect(domain_parser) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(404, 'Domain not found') + def post(self, name=None): + """ Updates domain with form data """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + data = dict(domain_parser.parse_args()) + data['name'] = name + for item, created in models.Domain.from_dict(data): + if created is True: + db.session.add(item) + # TODO: flush? + db.session.commit() + + @dom.doc('delete_domain') + @dom.response(200, 'Success', response_fields) + @dom.response(404, 'Domain not found') + def delete(self, name=None): + """ Delete domain """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + db.session.delete(domain) + db.session.commit() + + +# @dom.route('//alternative') +# @alt.route('') +# class Alternatives(Resource): + +# @alt.doc('alternatives_list') +# @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=['dkim_key']) +# def get(self, name=None): +# """ List alternatives (of domain) """ +# if name is None: +# return models.Alternative.query.all() +# else: +# return models.Alternative.query.filter_by(domain_name = name).all() + +# @alt.doc('alternative_create') +# @alt.expect(alternative_fields) +# @alt.response(200, 'Success', response_fields) +# @alt.response(400, 'Input validation exception', error_fields) +# @alt.response(404, 'Domain not found') +# @alt.response(409, 'Duplicate domain name', error_fields) +# def post(self, name=None): +# """ Create new alternative (for domain) """ +# # abort(501) +# data = api.payload +# if name is not None: +# data['name'] = name +# domain = models.Domain.query.get(name) +# if not domain: +# abort(404) +# if common.fqdn_in_use(data['name']): +# abort(409, f'Duplicate domain name {data["name"]!r}', errors={ +# 'name': data['name'], +# }) +# for item, created in models.Alternative.from_dict(data): +# # TODO: handle creation of domain +# if not created: +# abort(409, f'Duplicate domain name {item.name!r}', errors={ +# 'alternatives': item.name, +# }) +# # db.session.add(item) +# # db.session.commit() + +# @dom.route('//alternative/') +# @alt.route('/') +# class Alternative(Resource): +# def get(self, name, alt=None): +# """ Find alternative (of domain) """ +# abort(501) +# def put(self, name, alt=None): +# """ Update alternative (of domain) """ +# abort(501) +# def post(self, name, alt=None): +# """ Update alternative (of domain) with form data """ +# abort(501) +# def delete(self, name, alt=None): +# """ Delete alternative (for domain) """ +# abort(501) + From 5c9cdfe1de387af9e8a3516a54472e0150af424b Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sun, 25 Sep 2022 08:18:09 +0000 Subject: [PATCH 04/47] Introduction of the Mailu RESTful API. Anything that can be configured in the web administration interface, can also be configured via the Mailu RESTful API. See the section Advanced configuration in the configuration reference for the relevant settings in mailu.env for enabling the API. (API, WEB_API, API_TOKEN). --- .gitignore | 1 + core/admin/mailu/__init__.py | 4 +- core/admin/mailu/api/__init__.py | 31 +- core/admin/mailu/api/common.py | 23 +- core/admin/mailu/api/v1/__init__.py | 19 +- core/admin/mailu/api/v1/alias.py | 128 ++++++++ core/admin/mailu/api/v1/domains.py | 475 ++++++++++++++++++++-------- core/admin/mailu/api/v1/relay.py | 119 +++++++ core/admin/mailu/api/v1/user.py | 255 +++++++++++++++ core/admin/mailu/configuration.py | 3 + core/nginx/conf/nginx.conf | 7 + docs/api.rst | 29 ++ docs/cli.rst | 2 +- docs/configuration.rst | 65 ++-- docs/faq.rst | 79 +++-- docs/index.rst | 1 + docs/requirements.txt | 8 +- towncrier/newsfragments/445.feature | 2 + 18 files changed, 1042 insertions(+), 209 deletions(-) create mode 100644 core/admin/mailu/api/v1/alias.py create mode 100644 core/admin/mailu/api/v1/relay.py create mode 100644 core/admin/mailu/api/v1/user.py create mode 100644 docs/api.rst create mode 100644 towncrier/newsfragments/445.feature diff --git a/.gitignore b/.gitignore index f5e9f8ee..6c48a270 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ pip-selfcheck.json /docs/include /docs/_build /.env +/.venv /docker-compose.yml /.idea /.vscode diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index d17c8734..89e06c9b 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -74,8 +74,8 @@ def create_app_from_config(config): app.register_blueprint(ui.ui, url_prefix=app.config['WEB_ADMIN']) app.register_blueprint(internal.internal, url_prefix='/internal') app.register_blueprint(sso.sso, url_prefix='/sso') - if app.config.get('API'): - api.register(app) + if app.config.get('API_TOKEN'): + api.register(app, web_api=app.config.get('WEB_API')) return app diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py index 6e7d6386..44cda962 100644 --- a/core/admin/mailu/api/__init__.py +++ b/core/admin/mailu/api/__init__.py @@ -1,21 +1,20 @@ from flask import redirect, url_for - -# import api version(s) +from flask_restx import apidoc from . import v1 -# api -ROOT='/api' -ACTIVE=v1 -# patch url for swaggerui static assets -from flask_restx.apidoc import apidoc -apidoc.static_url_path = f'{ROOT}/swaggerui' - -def register(app): +def register(app, web_api): + ACTIVE=v1 + ROOT=web_api + v1.app = app # register api bluprint(s) + apidoc.apidoc.url_prefix = f'{ROOT}/v{int(v1.VERSION)}' + v1.api_token = app.config['API_TOKEN'] app.register_blueprint(v1.blueprint, url_prefix=f'{ROOT}/v{int(v1.VERSION)}') + + # add redirect to current api version @app.route(f'{ROOT}/') def redir(): @@ -25,14 +24,4 @@ def register(app): app.config.SWAGGER_UI_DOC_EXPANSION = 'list' app.config.SWAGGER_UI_OPERATION_ID = True app.config.SWAGGER_UI_REQUEST_DURATION = True - - # TODO: remove patch of static assets for debugging - import os - if 'STATIC_ASSETS' in os.environ: - app.blueprints['ui'].static_folder = os.environ['STATIC_ASSETS'] - -# TODO: authentication via username + password -# TODO: authentication via api token -# TODO: api access for all users (via token) -# TODO: use permissions from "manager_of" -# TODO: switch to marshmallow, as parser is deprecated. use flask_accepts? + app.config.RESTX_MASK_SWAGGER = False diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py index 700835ac..0805c5a3 100644 --- a/core/admin/mailu/api/common.py +++ b/core/admin/mailu/api/common.py @@ -1,4 +1,9 @@ -from .. import models +from .. import models, utils +from . import v1 +from flask import request +import flask +from functools import wraps +from flask_restx import abort def fqdn_in_use(*names): for name in names: @@ -6,3 +11,19 @@ def fqdn_in_use(*names): if model.query.get(name): return model return None + +""" Decorator for validating api token for authentication """ +def api_token_authorization(func): + @wraps(func) + def decorated_function(*args, **kwds): + client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr) + if utils.limiter.should_rate_limit_ip(client_ip): + abort(429, 'Too many attempts from your IP (rate-limit)' ) + if request.args.get('api_token') != v1.api_token: + utils.limiter.rate_limit_ip(client_ip) + flask.current_app.logger.warn(f'Invalid API token provided by {client_ip}.') + abort(401, 'A valid API token is expected as query string parameter') + else: + flask.current_app.logger.info(f'Valid API token provided by {client_ip}.') + return func(*args, **kwds) + return decorated_function diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py index c6de6fa4..5cc878c9 100644 --- a/core/admin/mailu/api/v1/__init__.py +++ b/core/admin/mailu/api/v1/__init__.py @@ -1,14 +1,28 @@ from flask import Blueprint from flask_restx import Api, fields + VERSION = 1.0 +api_token = None blueprint = Blueprint(f'api_v{int(VERSION)}', __name__) +authorization = { + 'apikey': { + 'type': 'apiKey', + 'in': 'query', + 'name': 'api_token' + } +} + + api = Api( blueprint, version=f'{VERSION:.1f}', title='Mailu API', default_label='Mailu', - validate=True + validate=True, + authorizations=authorization, + security='apikey', + doc='/swaggerui/' ) response_fields = api.model('Response', { @@ -25,3 +39,6 @@ error_fields = api.model('Error', { }) from . import domains +from . import alias +from . import relay +from . import user diff --git a/core/admin/mailu/api/v1/alias.py b/core/admin/mailu/api/v1/alias.py new file mode 100644 index 00000000..65cbd25c --- /dev/null +++ b/core/admin/mailu/api/v1/alias.py @@ -0,0 +1,128 @@ +from flask_restx import Resource, fields, marshal +from . import api, response_fields +from .. import common +from ... import models + +db = models.db + +alias = api.namespace('alias', description='Alias operations') + +alias_fields = api.model('Alias', { + 'email': fields.String(description='the alias email address', example='user@example.com', required=True), + 'comment': fields.String(description='a comment'), + 'destination': fields.List(fields.String(description='alias email address', example='user@example.com', required=True)), + 'wildcard': fields.Boolean(description='enable SQL Like wildcard syntax') +}) + +alias_fields_update = api.model('AliasUpdate', { + 'comment': fields.String(description='a comment'), + 'destination': fields.List(fields.String(description='alias email address', example='user@example.com')), + 'wildcard': fields.Boolean(description='enable SQL Like wildcard syntax') +}) + + +@alias.route('') +class Aliases(Resource): + @alias.doc('list_alias') + @alias.marshal_with(alias_fields, as_list=True, skip_none=True, mask=None) + @alias.doc(security='apikey') + @common.api_token_authorization + def get(self): + """ List aliases """ + return models.Alias.query.all() + + @alias.doc('create_alias') + @alias.expect(alias_fields) + @alias.response(200, 'Success', response_fields) + @alias.response(400, 'Input validation exception', response_fields) + @alias.response(409, 'Duplicate alias', response_fields) + @alias.doc(security='apikey') + @common.api_token_authorization + def post(self): + """ Create a new alias """ + data = api.payload + + alias_found = models.Alias.query.filter_by(email = data['email']).all() + if alias_found: + return { 'code': 409, 'message': f'Duplicate alias {data["email"]}'}, 409 + + alias_model = models.Alias(email=data["email"],destination=data['destination']) + if 'comment' in data: + alias_model.comment = data['comment'] + if 'wildcard' in data: + alias_model.wildcard = data['wildcard'] + db.session.add(alias_model) + db.session.commit() + + return {'code': 200, 'message': f'Alias {data["email"]} to destination {data["destination"]} has been created'}, 200 + +@alias.route('/') +class Alias(Resource): + @alias.doc('find_alias') + @alias.response(200, 'Success', alias_fields) + @alias.response(404, 'Alias not found', response_fields) + @alias.doc(security='apikey') + @common.api_token_authorization + def get(self, alias): + """ Find alias """ + alias_found = models.Alias.query.filter_by(email = alias).all() + if alias_found is None: + return { 'code': 404, 'message': f'Alias {alias} cannot be found'}, 404 + else: + return marshal(alias_found,alias_fields), 200 + + @alias.doc('update_alias') + @alias.expect(alias_fields_update) + @alias.response(200, 'Success', response_fields) + @alias.response(404, 'Alias not found', response_fields) + @alias.response(400, 'Input validation exception', response_fields) + @alias.doc(security='apikey') + @common.api_token_authorization + def put(self, alias): + """ Update alias """ + data = api.payload + alias_found = models.Alias.query.filter_by(email = alias).first() + if alias_found is None: + return { 'code': 404, 'message': f'Alias {alias} cannot be found'}, 404 + if 'comment' in data: + alias_found.comment = data['comment'] + if 'destination' in data: + destination_csl = ",".join(data['destination']) + alias_found.destination = destination_csl + if 'wildcard' in data: + alias_found.wildcard = data['wildcard'] + db.session.add(alias_found) + db.session.commit() + return {'code': 200, 'message': f'Alias {alias} has been updated'} + + @alias.doc('delete_alias') + @alias.response(200, 'Success', response_fields) + @alias.response(404, 'Alias not found', response_fields) + @alias.doc(security='apikey') + @common.api_token_authorization + def delete(self, alias): + """ Delete alias """ + alias_found = models.Alias.query.filter_by(email = alias).first() + if alias_found is None: + return { 'code': 404, 'message': f'Alias {alias} cannot be found'}, 404 + db.session.delete(alias_found) + db.session.commit() + return {'code': 200, 'message': f'Alias {alias} has been deleted'}, 200 + +@alias.route('/destination/') +class AliasWithDest(Resource): + @alias.doc('find_alias_filter_domain') + @alias.response(200, 'Success', alias_fields) + @alias.response(404, 'Alias or domain not found', response_fields) + @alias.doc(security='apikey') + @common.api_token_authorization + def get(self, domain): + """ Find aliases of domain """ + domain_found = models.Domain.query.filter_by(name=domain).first() + if domain_found is None: + return { 'code': 404, 'message': f'Domain {domain} cannot be found'}, 404 + aliases_found = domain_found.aliases + if aliases_found.count == 0: + return { 'code': 404, 'message': f'No alias can be found for domain {domain}'}, 404 + else: + return marshal(aliases_found, alias_fields), 200 diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py index a0d37186..35393b81 100644 --- a/core/admin/mailu/api/v1/domains.py +++ b/core/admin/mailu/api/v1/domains.py @@ -1,6 +1,6 @@ -from flask_restx import Resource, fields, abort - -from . import api, response_fields, error_fields +import validators +from flask_restx import Resource, fields, marshal +from . import api, response_fields, user from .. import common from ... import models @@ -10,35 +10,76 @@ dom = api.namespace('domain', description='Domain operations') alt = api.namespace('alternative', description='Alternative operations') domain_fields = api.model('Domain', { - 'name': fields.String(description='FQDN', example='example.com', required=True), + 'name': fields.String(description='FQDN (e.g. example.com)', example='example.com', required=True), 'comment': fields.String(description='a comment'), 'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1), 'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1), 'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0), 'signup_enabled': fields.Boolean(description='allow signup'), -# 'dkim_key': fields.String, - 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example.com')), + 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example2.com')), }) -# TODO - name ist required on creation but immutable on change -# TODO - name and alteranatives need to be checked to be a fqdn (regex) -domain_parser = api.parser() -domain_parser.add_argument('max_users', type=int, help='maximum number of users') -# TODO ... add more (or use marshmallow) +domain_fields_update = api.model('DomainUpdate', { + 'comment': fields.String(description='a comment'), + 'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1), + 'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1), + 'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0), + 'signup_enabled': fields.Boolean(description='allow signup'), + 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example2.com')), +}) -alternative_fields = api.model('Domain', { - 'name': fields.String(description='alternative FQDN', example='example.com', required=True), +domain_fields_get = api.model('DomainGet', { + 'name': fields.String(description='FQDN (e.g. example.com)', example='example.com', required=True), + 'comment': fields.String(description='a comment'), + 'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1), + 'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1), + 'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0), + 'signup_enabled': fields.Boolean(description='allow signup'), + 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example2.com')), + 'dns_autoconfig': fields.List(fields.String(description='DNS client auto-configuration entry')), + 'dns_mx': fields.String(Description='MX record for domain'), + 'dns_spf': fields.String(Description='SPF record for domain'), + 'dns_dkim': fields.String(Description='DKIM record for domain'), + 'dns_dmarc': fields.String(Description='DMARC record for domain'), + 'dns_dmarc_report': fields.String(Description='DMARC report record for domain'), + 'dns_tlsa': fields.String(Description='TLSA record for domain'), +}) + +domain_fields_dns = api.model('DomainDNS', { + 'dns_autoconfig': fields.List(fields.String(description='DNS client auto-configuration entry')), + 'dns_mx': fields.String(Description='MX record for domain'), + 'dns_spf': fields.String(Description='SPF record for domain'), + 'dns_dkim': fields.String(Description='DKIM record for domain'), + 'dns_dmarc': fields.String(Description='DMARC record for domain'), + 'dns_dmarc_report': fields.String(Description='DMARC report record for domain'), + 'dns_tlsa': fields.String(Description='TLSA record for domain'), +}) + +manager_fields = api.model('Manager', { + 'domain_name': fields.String(description='domain managed by manager'), + 'user_email': fields.String(description='email address of manager'), +}) + +manager_fields_create = api.model('ManagerCreate', { + 'user_email': fields.String(description='email address of manager', required=True), +}) + +alternative_fields_update = api.model('AlternativeDomainUpdate', { + 'domain': fields.String(description='domain FQDN', example='example.com', required=False), +}) + +alternative_fields = api.model('AlternativeDomain', { + 'name': fields.String(description='alternative FQDN', example='example2.com', required=True), 'domain': fields.String(description='domain FQDN', example='example.com', required=True), - 'dkim_key': fields.String, }) -# TODO: domain and name are not always required and can't be changed @dom.route('') class Domains(Resource): - @dom.doc('list_domain') - @dom.marshal_with(domain_fields, as_list=True, skip_none=True, mask=['dkim_key']) + @dom.marshal_with(domain_fields_get, as_list=True, skip_none=True, mask=None) + @dom.doc(security='apikey') + @common.api_token_authorization def get(self): """ List domains """ return models.Domain.query.all() @@ -46,138 +87,324 @@ class Domains(Resource): @dom.doc('create_domain') @dom.expect(domain_fields) @dom.response(200, 'Success', response_fields) - @dom.response(400, 'Input validation exception', error_fields) - @dom.response(409, 'Duplicate domain name', error_fields) + @dom.response(400, 'Input validation exception', response_fields) + @dom.response(409, 'Duplicate domain/alternative name', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization def post(self): """ Create a new domain """ data = api.payload - if common.fqdn_in_use(data['name']): - abort(409, f'Duplicate domain name {data["name"]!r}', errors={ - 'name': data['name'], - }) - for item, created in models.Domain.from_dict(data): - if not created: - abort(409, f'Duplicate domain name {item.name!r}', errors={ - 'alternatives': item.name, - }) - db.session.add(item) - db.session.commit() + if not validators.domain(data['name']): + return { 'code': 400, 'message': f'Domain {data["name"]} is not a valid domain'}, 400 -@dom.route('/') + if common.fqdn_in_use(data['name']): + return { 'code': 409, 'message': f'Duplicate domain name {data["name"]}'}, 409 + if 'alternatives' in data: + #check if duplicate alternatives are supplied + if [x for x in data['alternatives'] if data['alternatives'].count(x) >= 2]: + return { 'code': 409, 'message': f'Duplicate alternative domain names in request' }, 409 + for item in data['alternatives']: + if common.fqdn_in_use(item): + return { 'code': 409, 'message': f'Duplicate alternative domain name {item}' }, 409 + if not validators.domain(item): + return { 'code': 400, 'message': f'Alternative domain {item} is not a valid domain'}, 400 + for item in data['alternatives']: + alternative = models.Alternative(name=item, domain_name=data['name']) + models.db.session.add(alternative) + domain_new = models.Domain(name=data['name']) + if 'comment' in data: + domain_new.comment = data['comment'] + if 'max_users' in data: + domain_new.comment = data['max_users'] + if 'max_aliases' in data: + domain_new.comment = data['max_aliases'] + if 'max_quota_bytes' in data: + domain_new.comment = data['max_quota_bytes'] + if 'signup_enabled' in data: + domain_new.comment = data['signup_enabled'] + models.db.session.add(domain_new) + #apply the changes + db.session.commit() + return {'code': 200, 'message': f'Domain {data["name"]} has been created'}, 200 + +@dom.route('/') class Domain(Resource): - @dom.doc('get_domain') + @dom.doc('find_domain') @dom.response(200, 'Success', domain_fields) - @dom.response(404, 'Domain not found') - @dom.marshal_with(domain_fields) - def get(self, name): + @dom.response(404, 'Domain not found', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def get(self, domain): """ Find domain by name """ - domain = models.Domain.query.get(name) - if not domain: - abort(404) - return domain + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + domain_found = models.Domain.query.get(domain) + if not domain_found: + return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404 + return marshal(domain_found, domain_fields_get), 200 @dom.doc('update_domain') - @dom.expect(domain_fields) + @dom.expect(domain_fields_update) @dom.response(200, 'Success', response_fields) - @dom.response(400, 'Input validation exception', error_fields) - @dom.response(404, 'Domain not found') - def put(self, name): + @dom.response(400, 'Input validation exception', response_fields) + @dom.response(404, 'Domain not found', response_fields) + @dom.response(409, 'Duplicate domain/alternative name', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def put(self, domain): """ Update an existing domain """ - domain = models.Domain.query.get(name) + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + domain_found = models.Domain.query.get(domain) if not domain: - abort(404) + return { 'code': 404, 'message': f'Domain {data["name"]} does not exist'}, 404 data = api.payload - data['name'] = name - for item, created in models.Domain.from_dict(data): - if created is True: - db.session.add(item) - db.session.commit() - @dom.doc('modify_domain') - @dom.expect(domain_parser) - @dom.response(200, 'Success', response_fields) - @dom.response(400, 'Input validation exception', error_fields) - @dom.response(404, 'Domain not found') - def post(self, name=None): - """ Updates domain with form data """ - domain = models.Domain.query.get(name) - if not domain: - abort(404) - data = dict(domain_parser.parse_args()) - data['name'] = name - for item, created in models.Domain.from_dict(data): - if created is True: - db.session.add(item) - # TODO: flush? + if 'alternatives' in data: + #check if duplicate alternatives are supplied + if [x for x in data['alternatives'] if data['alternatives'].count(x) >= 2]: + return { 'code': 409, 'message': f'Duplicate alternative domain names in request' }, 409 + for item in data['alternatives']: + if common.fqdn_in_use(item): + return { 'code': 409, 'message': f'Duplicate alternative domain name {item}' }, 409 + if not validators.domain(item): + return { 'code': 400, 'message': f'Alternative domain {item} is not a valid domain'}, 400 + for item in data['alternatives']: + alternative = models.Alternative(name=item, domain_name=data['name']) + models.db.session.add(alternative) + + if 'comment' in data: + domain_found.comment = data['comment'] + if 'max_users' in data: + domain_found.comment = data['max_users'] + if 'max_aliases' in data: + domain_found.comment = data['max_aliases'] + if 'max_quota_bytes' in data: + domain_found.comment = data['max_quota_bytes'] + if 'signup_enabled' in data: + domain_found.comment = data['signup_enabled'] + models.db.session.add(domain_found) + + #apply the changes db.session.commit() + return {'code': 200, 'message': f'Domain {domain} has been updated'}, 200 @dom.doc('delete_domain') @dom.response(200, 'Success', response_fields) - @dom.response(404, 'Domain not found') - def delete(self, name=None): + @dom.response(400, 'Input validation exception', response_fields) + @dom.response(404, 'Domain not found', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def delete(self, domain): """ Delete domain """ - domain = models.Domain.query.get(name) + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + domain_found = models.Domain.query.get(domain) if not domain: - abort(404) - db.session.delete(domain) + return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404 + db.session.delete(domain_found) db.session.commit() + return {'code': 200, 'message': f'Domain {domain} has been deleted'}, 200 + +@dom.route('//dkim') +class Domain(Resource): + @dom.doc('generate_dkim') + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', response_fields) + @dom.response(404, 'Domain not found', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def post(self, domain): + """ Generate new DKIM/DMARC keys for domain """ + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + domain_found = models.Domain.query.get(domain) + if not domain_found: + return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404 + domain_found.generate_dkim_key() + domain_found.save_dkim_key() + return {'code': 200, 'message': f'DKIM/DMARC keys have been generated for domain {domain}'}, 200 + +@dom.route('//manager') +class Manager(Resource): + @dom.doc('list_managers') + @dom.marshal_with(manager_fields, as_list=True, skip_none=True, mask=None) + @dom.response(400, 'Input validation exception', response_fields) + @dom.response(404, 'domain not found', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def get(self, domain): + """ List managers of domain """ + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + if not domain: + return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404 + domain = models.Domain.query.filter_by(name=domain) + return domain.managers + + @dom.doc('create_manager') + @dom.expect(manager_fields_create) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', response_fields) + @dom.response(404, 'User or domain not found', response_fields) + @dom.response(409, 'Duplicate domain manager', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def post(self, domain): + """ Create a new domain manager """ + data = api.payload + if not validators.email(data['user_email']): + return {'code': 400, 'message': f'Invalid email address {data["user_email"]}'}, 400 + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + domain = models.Domain.query.get(domain) + if not domain: + return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404 + user = models.User.query.get(data['user_email']) + if not user: + return { 'code': 404, 'message': f'User {data["user_email"]} does not exist'}, 404 + if user in domain.managers: + return {'code': 409, 'message': f'User {data["user_email"]} is already a manager of the domain {domain} '}, 409 + domain.managers.append(user) + models.db.session.commit() + return {'code': 200, 'message': f'User {data["user_email"]} has been added as manager of the domain {domain} '},200 + +@dom.route('//manager/') +class Domain(Resource): + @dom.doc('find_manager') + @dom.response(200, 'Success', manager_fields) + @dom.response(404, 'Manager not found', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def get(self, domain, email): + """ Find manager by email address """ + if not validators.email(email): + return {'code': 400, 'message': f'Invalid email address {email}'}, 400 + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + domain = models.Domain.query.get(domain) + if not domain: + return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404 + user = models.User.query.get(email) + if not user: + return { 'code': 404, 'message': f'User {email} does not exist'}, 404 + if user in domain.managers: + for manager in domain.managers: + if manager.email == email: + return marshal(manager, manager_fields),200 + else: + return { 'code': 404, 'message': f'User {email} is not a manager of the domain {domain}'}, 404 -# @dom.route('//alternative') -# @alt.route('') -# class Alternatives(Resource): + @dom.doc('delete_manager') + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', response_fields) + @dom.response(404, 'Manager not found', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def delete(self, domain, email): + if not validators.email(email): + return {'code': 400, 'message': f'Invalid email address {email}'}, 400 + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + domain = models.Domain.query.get(domain) + if not domain: + return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404 + user = models.User.query.get(email) + if not user: + return { 'code': 404, 'message': f'User {email} does not exist'}, 404 + if user in domain.managers: + domain.managers.remove(user) + models.db.session.commit() + return {'code': 200, 'message': f'User {email} has been removed as a manager of the domain {domain} '},200 + else: + return { 'code': 404, 'message': f'User {email} is not a manager of the domain {domain}'}, 404 -# @alt.doc('alternatives_list') -# @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=['dkim_key']) -# def get(self, name=None): -# """ List alternatives (of domain) """ -# if name is None: -# return models.Alternative.query.all() -# else: -# return models.Alternative.query.filter_by(domain_name = name).all() +@dom.route('//users') +class User(Resource): + @dom.doc('list_user_domain') + @dom.marshal_with(user.user_fields_get, as_list=True, skip_none=True, mask=None) + @dom.response(400, 'Input validation exception', response_fields) + @dom.response(404, 'Domain not found', response_fields) + @dom.doc(security='apikey') + @common.api_token_authorization + def get(self, domain): + """ List users from domain """ + if not validators.domain(domain): + return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 + domain_found = models.Domain.query.get(domain) + if not domain_found: + return { 'code': 404, 'message': f'Domain {domain} does not exist'}, 404 + return models.User.query.filter_by(domain=domain_found).all() -# @alt.doc('alternative_create') -# @alt.expect(alternative_fields) -# @alt.response(200, 'Success', response_fields) -# @alt.response(400, 'Input validation exception', error_fields) -# @alt.response(404, 'Domain not found') -# @alt.response(409, 'Duplicate domain name', error_fields) -# def post(self, name=None): -# """ Create new alternative (for domain) """ -# # abort(501) -# data = api.payload -# if name is not None: -# data['name'] = name -# domain = models.Domain.query.get(name) -# if not domain: -# abort(404) -# if common.fqdn_in_use(data['name']): -# abort(409, f'Duplicate domain name {data["name"]!r}', errors={ -# 'name': data['name'], -# }) -# for item, created in models.Alternative.from_dict(data): -# # TODO: handle creation of domain -# if not created: -# abort(409, f'Duplicate domain name {item.name!r}', errors={ -# 'alternatives': item.name, -# }) -# # db.session.add(item) -# # db.session.commit() +@alt.route('') +class Alternatives(Resource): -# @dom.route('//alternative/') -# @alt.route('/') -# class Alternative(Resource): -# def get(self, name, alt=None): -# """ Find alternative (of domain) """ -# abort(501) -# def put(self, name, alt=None): -# """ Update alternative (of domain) """ -# abort(501) -# def post(self, name, alt=None): -# """ Update alternative (of domain) with form data """ -# abort(501) -# def delete(self, name, alt=None): -# """ Delete alternative (for domain) """ -# abort(501) + @alt.doc('list_alternative') + @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=None) + @alt.doc(security='apikey') + @common.api_token_authorization + def get(self): + """ List alternatives """ + return models.Alternative.query.all() + + @alt.doc('create_alternative') + @alt.expect(alternative_fields) + @alt.response(200, 'Success', response_fields) + @alt.response(400, 'Input validation exception', response_fields) + @alt.response(404, 'Domain not found or missing', response_fields) + @alt.response(409, 'Duplicate alternative domain name', response_fields) + @alt.doc(security='apikey') + @common.api_token_authorization + def post(self): + """ Create new alternative (for domain) """ + data = api.payload + if not validators.domain(data['name']): + return { 'code': 400, 'message': f'Alternative domain {data["name"]} is not a valid domain'}, 400 + if not validators.domain(data['domain']): + return { 'code': 400, 'message': f'Domain {data["domain"]} is not a valid domain'}, 400 + domain = models.Domain.query.get(data['domain']) + if not domain: + return { 'code': 404, 'message': f'Domain {data["domain"]} does not exist'}, 404 + if common.fqdn_in_use(data['name']): + return { 'code': 409, 'message': f'Duplicate alternative domain name {data["name"]}'}, 409 + + alternative = models.Alternative(name=data['name'], domain_name=data['domain']) + models.db.session.add(alternative) + db.session.commit() + return {'code': 200, 'message': f'Alternative {data["name"]} for domain {data["domain"]} has been created'}, 200 + +@alt.route('/') +class Alternative(Resource): + @alt.doc('find_alternative') + @alt.doc(security='apikey') + @common.api_token_authorization + def get(self, alt): + """ Find alternative (of domain) """ + if not validators.domain(alt): + return { 'code': 400, 'message': f'Alternative domain {alt} is not a valid domain'}, 400 + alternative = models.Alternative.query.filter_by(name=alt).first() + if not alternative: + return{ 'code': 404, 'message': f'Alternative domain {alt} does not exist'}, 404 + return marshal(alternative, alternative_fields), 200 + + @alt.doc('delete_alternative') + @alt.response(200, 'Success', response_fields) + @alt.response(400, 'Input validation exception', response_fields) + @alt.response(404, 'Alternative/Domain not found or missing', response_fields) + @alt.response(409, 'Duplicate domain name', response_fields) + @alt.doc(security='apikey') + @common.api_token_authorization + def delete(self, alt): + """ Delete alternative (for domain) """ + if not validators.domain(alt): + return { 'code': 400, 'message': f'Alternative domain {alt} is not a valid domain'}, 400 + alternative = models.Alternative.query.filter_by(name=alt).first + if not alternative: + return { 'code': 404, 'message': f'Alternative domain {alt} does not exist'}, 404 + domain = alternative.domain + db.session.delete(alternative) + db.session.commit() + return {'code': 200, 'message': f'Alternative {alt} for domain {domain} has been deleted'}, 200 diff --git a/core/admin/mailu/api/v1/relay.py b/core/admin/mailu/api/v1/relay.py new file mode 100644 index 00000000..67089d9a --- /dev/null +++ b/core/admin/mailu/api/v1/relay.py @@ -0,0 +1,119 @@ +from flask_restx import Resource, fields, marshal +import validators + +from . import api, response_fields +from .. import common +from ... import models + +db = models.db + +relay = api.namespace('relay', description='Relay operations') + +relay_fields = api.model('Relay', { + 'name': fields.String(description='relayed domain name', example='example.com', required=True), + 'smtp': fields.String(description='remote host', example='example.com', required=False), + 'comment': fields.String(description='a comment', required=False) +}) + +relay_fields_update = api.model('RelayUpdate', { + 'smtp': fields.String(description='remote host', example='example.com', required=False), + 'comment': fields.String(description='a comment', required=False) +}) + +@relay.route('') +class Relays(Resource): + @relay.doc('list_relays') + @relay.marshal_with(relay_fields, as_list=True, skip_none=True, mask=None) + @relay.doc(security='apikey') + @common.api_token_authorization + def get(self): + "List relays" + return models.Relay.query.all() + + @relay.doc('create_relay') + @relay.expect(relay_fields) + @relay.response(200, 'Success', response_fields) + @relay.response(400, 'Input validation exception') + @relay.response(409, 'Duplicate relay', response_fields) + @relay.doc(security='apikey') + @common.api_token_authorization + def post(self): + """ Create relay """ + data = api.payload + + if not validators.domain(name): + return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400 + + relay_found = models.Relay.query.filter_by(name=data['name']).all() + if common.fqdn_in_use(data['name']): + return { 'code': 409, 'message': f'Duplicate domain {data["name"]}'}, 409 + relay_model = models.Relay(name=data['name']) + if 'smtp' in data: + relay_model.smtp = data['smtp'] + if 'comment' in data: + relay_model.comment = data['comment'] + db.session.add(relay_model) + db.session.commit() + return {'code': 200, 'message': f'Relayed domain {data["name"]} has been created'}, 200 + +@relay.route('/') +class Relay(Resource): + @relay.doc('find_relay') + @relay.response(400, 'Input validation exception', response_fields) + @relay.response(404, 'Relay not found', response_fields) + @relay.doc(security='apikey') + @common.api_token_authorization + def get(self, name): + """ Find relay """ + if not validators.domain(name): + return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400 + + relay_found = models.Relay.query.filter_by(name=name).first() + if relay_found is None: + return { 'code': 404, 'message': f'Relayed domain {name} cannot be found'}, 404 + return marshal(relay_found, relay_fields), 200 + + @relay.doc('update_relay') + @relay.expect(relay_fields_update) + @relay.response(200, 'Success', response_fields) + @relay.response(400, 'Input validation exception', response_fields) + @relay.response(404, 'Relay not found', response_fields) + @relay.response(409, 'Duplicate relay', response_fields) + @relay.doc(security='apikey') + @common.api_token_authorization + def put(self, name): + """ Update relay """ + data = api.payload + + if not validators.domain(name): + return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400 + + relay_found = models.Relay.query.filter_by(name=name).first() + if relay_found is None: + return { 'code': 404, 'message': f'Relayed domain {name} cannot be found'}, 404 + + if 'smtp' in data: + relay_found.smtp = data['smtp'] + if 'comment' in data: + relay_found.comment = data['comment'] + db.session.add(relay_found) + db.session.commit() + return { 'code': 200, 'message': f'Relayed domain {name} has been updated'}, 200 + + + @relay.doc('delete_relay') + @relay.response(200, 'Success', response_fields) + @relay.response(400, 'Input validation exception', response_fields) + @relay.response(404, 'Relay not found', response_fields) + @relay.doc(security='apikey') + @common.api_token_authorization + def delete(self, name): + """ Delete relay """ + if not validators.domain(name): + return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400 + relay_found = models.Relay.query.filter_by(name=name).first() + if relay_found is None: + return { 'code': 404, 'message': f'Relayed domain {name} cannot be found'}, 404 + db.session.delete(relay_found) + db.session.commit() + return { 'code': 200, 'message': f'Relayed domain {name} has been deleted'}, 200 diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py new file mode 100644 index 00000000..5345e007 --- /dev/null +++ b/core/admin/mailu/api/v1/user.py @@ -0,0 +1,255 @@ +from flask_restx import Resource, fields, marshal +import validators, datetime + +from . import api, response_fields +from .. import common +from ... import models + +db = models.db + +user = api.namespace('user', description='User operations') + +user_fields_get = api.model('UserGet', { + 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'), + 'password': fields.String(description='PBKDF2-HMAC-SHA256 based password of the user. For more info see passlib.hash.pbkdf2_sha256', example='$pbkdf2-sha256$1$.6UI/S.nXIk8jcbdHx3Fhg$98jZicV16ODfEsEZeYPGHU3kbrUrvUEXOPimVSQDD44'), + 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), + 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), + 'global_admin': fields.Boolean(description='Make the user a global administrator'), + 'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'), + 'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'), + 'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'), + 'forward_enabled': fields.Boolean(description='Enable auto forwarding'), + 'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'), + 'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'), + 'reply_enabled': fields.Boolean(description='Enable automatic replies. This is also known as out of office (ooo) or out of facility (oof) replies'), + 'reply_subject': fields.String(description='Optional subject for the automatic reply', example='Out of office'), + 'reply_body': fields.String(description='The body of the automatic reply email', example='Hello, I am out of office. I will respond when I am back.'), + 'reply_startdate': fields.Date(description='Start date for automatic replies in YYYY-MM-DD format.', example='2022-02-10'), + 'reply_enddate': fields.Date(description='End date for automatic replies in YYYY-MM-DD format.', example='2022-02-22'), + 'displayed_name': fields.String(description='The display name of the user within the Admin GUI', example='John Doe'), + 'spam_enabled': fields.Boolean(description='Enable the spam filter'), + 'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'), + 'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'), +}) + +user_fields_post = api.model('UserCreate', { + 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email', required=True), + 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using PBKDF2-HMAC-SHA256', example='secret', required=True), + 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), + 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), + 'global_admin': fields.Boolean(description='Make the user a global administrator'), + 'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'), + 'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'), + 'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'), + 'forward_enabled': fields.Boolean(description='Enable auto forwarding'), + 'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'), + 'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'), + 'reply_enabled': fields.Boolean(description='Enable automatic replies. This is also known as out of office (ooo) or out of facility (oof) replies'), + 'reply_subject': fields.String(description='Optional subject for the automatic reply', example='Out of office'), + 'reply_body': fields.String(description='The body of the automatic reply email', example='Hello, I am out of office. I will respond when I am back.'), + 'reply_startdate': fields.Date(description='Start date for automatic replies in YYYY-MM-DD format.', example='2022-02-10'), + 'reply_enddate': fields.Date(description='End date for automatic replies in YYYY-MM-DD format.', example='2022-02-22'), + 'displayed_name': fields.String(description='The display name of the user within the Admin GUI', example='John Doe'), + 'spam_enabled': fields.Boolean(description='Enable the spam filter'), + 'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'), + 'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'), +}) + +user_fields_put = api.model('UserUpdate', { + 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using PBKDF2-HMAC-SHA256', example='secret'), + 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), + 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), + 'global_admin': fields.Boolean(description='Make the user a global administrator'), + 'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'), + 'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'), + 'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'), + 'forward_enabled': fields.Boolean(description='Enable auto forwarding'), + 'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'), + 'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'), + 'reply_enabled': fields.Boolean(description='Enable automatic replies. This is also known as out of office (ooo) or out of facility (oof) replies'), + 'reply_subject': fields.String(description='Optional subject for the automatic reply', example='Out of office'), + 'reply_body': fields.String(description='The body of the automatic reply email', example='Hello, I am out of office. I will respond when I am back.'), + 'reply_startdate': fields.Date(description='Start date for automatic replies in YYYY-MM-DD format.', example='2022-02-10'), + 'reply_enddate': fields.Date(description='End date for automatic replies in YYYY-MM-DD format.', example='2022-02-22'), + 'displayed_name': fields.String(description='The display name of the user within the Admin GUI', example='John Doe'), + 'spam_enabled': fields.Boolean(description='Enable the spam filter'), + 'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'), + 'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'), +}) + + +@user.route('') +class Users(Resource): + @user.doc('list_users') + @user.marshal_with(user_fields_get, as_list=True, skip_none=True, mask=None) + @user.doc(security='apikey') + @common.api_token_authorization + def get(self): + "List users" + return models.User.query.all() + + @user.doc('create_user') + @user.expect(user_fields_post) + @user.response(200, 'Success', response_fields) + @user.response(400, 'Input validation exception') + @user.response(409, 'Duplicate user', response_fields) + @user.doc(security='apikey') + @common.api_token_authorization + def post(self): + """ Create user """ + data = api.payload + if not validators.email(data['email']): + return { 'code': 400, 'message': f'Provided email address {data["email"]} is not a valid email address'}, 400 + localpart, domain_name = data['email'].lower().rsplit('@', 1) + domain_found = models.Domain.query.get(domain_name) + if not domain_found: + return { 'code': 404, 'message': f'Domain {domain_name} does not exist'}, 404 + + user_new = models.User(email=data['email']) + if 'raw_password' in data: + user_new.set_password(data['raw_password']) + if 'comment' in data: + user_new.comment = data['comment'] + if 'quota_bytes' in data: + user_new.quota_bytes = data['quota_bytes'] + if 'global_admin' in data: + user_new.global_admin = data['global_admin'] + if 'enabled' in data: + user_new.enabled = data['enabled'] + if 'enable_imap' in data: + user_new.enable_imap = data['enable_imap'] + if 'enable_pop' in data: + user_new.enable_pop = data['enable_pop'] + if 'forward_enabled' in data: + user_new.forward_enabled = data['forward_enabled'] + if 'forward_destination' in data: + user_new.forward_destination = data['forward_destination'] + if 'forward_keep' in data: + user_new.forward_keep = data['forward_keep'] + if 'reply_enabled' in data: + user_new.reply_enabled = data['reply_enabled'] + if 'reply_subject' in data: + user_new.reply_subject = data['reply_subject'] + if 'reply_body' in data: + user_new.reply_body = data['reply_body'] + if 'reply_startdate' in data: + year, month, day = data['reply_startdate'].split('-') + date = datetime.datetime(int(year), int(month), int(day)) + user_new.reply_startdate = date + if 'reply_enddate' in data: + year, month, day = data['reply_enddate'].split('-') + date = datetime.datetime(int(year), int(month), int(day)) + user_new.reply_enddate = date + if 'displayed_name' in data: + user_new.displayed_name = data['displayed_name'] + if 'spam_enabled' in data: + user_new.spam_enabled = data['spam_enabled'] + if 'spam_mark_as_read' in data: + user_new.spam_mark_as_read = data['spam_mark_as_read'] + if 'spam_threshold' in data: + user_new.spam_threshold = data['spam_threshold'] + db.session.add(user_new) + db.session.commit() + + return {'code': 200,'message': f'User {data["email"]} has been created'}, 200 + + +@user.route('/') +class User(Resource): + @user.doc('find_user') + @user.response(400, 'Input validation exception', response_fields) + @user.response(404, 'User not found', response_fields) + @user.doc(security='apikey') + @common.api_token_authorization + def get(self, email): + """ Find user """ + if not validators.email(email): + return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400 + + email_found = models.User.query.filter_by(email=email).first() + if email_found is None: + return { 'code': 404, 'message': f'User {email} cannot be found'}, 404 + return marshal(email_found, user_fields_get), 200 + + @user.doc('update_user') + @user.expect(user_fields_put) + @user.response(200, 'Success', response_fields) + @user.response(400, 'Input validation exception', response_fields) + @user.response(404, 'User not found', response_fields) + @user.response(409, 'Duplicate user', response_fields) + @user.doc(security='apikey') + @common.api_token_authorization + def put(self, email): + """ Update user """ + data = api.payload + if not validators.email(email): + return { 'code': 400, 'message': f'Provided email address {data["email"]} is not a valid email address'}, 400 + user_found = models.User.query.filter_by(email=email).first() + if not user_found: + return {'code': 404, 'message': f'User {email} cannot be found'}, 404 + + if 'raw_password' in data: + user_found.set_password(data['raw_password']) + if 'comment' in data: + user_found.comment = data['comment'] + if 'quota_bytes' in data: + user_found.quota_bytes = data['quota_bytes'] + if 'global_admin' in data: + user_found.global_admin = data['global_admin'] + if 'enabled' in data: + user_found.enabled = data['enabled'] + if 'enable_imap' in data: + user_found.enable_imap = data['enable_imap'] + if 'enable_pop' in data: + user_found.enable_pop = data['enable_pop'] + if 'forward_enabled' in data: + user_found.forward_enabled = data['forward_enabled'] + if 'forward_destination' in data: + user_found.forward_destination = data['forward_destination'] + if 'forward_keep' in data: + user_found.forward_keep = data['forward_keep'] + if 'reply_enabled' in data: + user_found.reply_enabled = data['reply_enabled'] + if 'reply_subject' in data: + user_found.reply_subject = data['reply_subject'] + if 'reply_body' in data: + user_found.reply_body = data['reply_body'] + if 'reply_startdate' in data: + year, month, day = data['reply_startdate'].split('-') + date = datetime.datetime(int(year), int(month), int(day)) + user_found.reply_startdate = date + if 'reply_enddate' in data: + year, month, day = data['reply_enddate'].split('-') + date = datetime.datetime(int(year), int(month), int(day)) + user_found.reply_enddate = date + if 'displayed_name' in data: + user_found.displayed_name = data['displayed_name'] + if 'spam_enabled' in data: + user_found.spam_enabled = data['spam_enabled'] + if 'spam_mark_as_read' in data: + user_found.spam_mark_as_read = data['spam_mark_as_read'] + if 'spam_threshold' in data: + user_found.spam_threshold = data['spam_threshold'] + db.session.add(user_found) + db.session.commit() + + return {'code': 200,'message': f'User {email} has been updated'}, 200 + + + @user.doc('delete_user') + @user.response(200, 'Success', response_fields) + @user.response(400, 'Input validation exception', response_fields) + @user.response(404, 'User not found', response_fields) + @user.doc(security='apikey') + @common.api_token_authorization + def delete(self, email): + """ Delete user """ + if not validators.email(email): + return { 'code': 400, 'message': f'Provided email address {email} is not a valid email address'}, 400 + + email_found = models.User.query.filter_by(email=email).first() + if email_found is None: + return { 'code': 404, 'message': f'User {email} cannot be found'}, 404 + db.session.delete(email_found) + db.session.commit() + return { 'code': 200, 'message': f'User {email} has been deleted'}, 200 diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 739d0189..612bcb18 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -71,6 +71,9 @@ DEFAULT_CONFIG = { 'LOGO_BACKGROUND': None, 'API': False, # Advanced settings + 'API' : 'false', + 'WEB_API' : '/api', + 'API_TOKEN': None, 'LOG_LEVEL': 'WARNING', 'SESSION_KEY_BITS': 128, 'SESSION_TIMEOUT': 3600, diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index b373fb13..722e5e50 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -244,6 +244,13 @@ http { {% endif %} {% endif %} + {% if API == 'true' %} + location ~ {{ WEB_API }} { + include /etc/nginx/proxy.conf; + proxy_pass http://$admin; + } + {% endif %} + location /internal { internal; diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..66ccc03d --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,29 @@ +Mailu RESTful API +================= + +Mailu offers a RESTful API for changing the Mailu configuration. +Anything that can be configured via the Mailu web administration interface, +can also be configured via the API. + +The Mailu API is disabled by default. It can be enabled and configured via +the settings: + +* `API` +* `WEB_API` +* `API_TOKEN` + +For more information see the section :ref:`Advanced configuration ` +in the configuration reference. + + +Swagger.json +------------ + +The swagger.json file can be retrieved via: https://myserver/api/v1/swagger.json. +The swagger.json file can be consumed in programs such as Postman for generating all API calls. + + +In-built SwaggerUI +------------------ +The Mailu API comes with an in-built SwaggerUI. It is a web client that allows +anyone to visualize and interact with the Mailu API. \ No newline at end of file diff --git a/docs/cli.rst b/docs/cli.rst index 36815fd0..5513e41f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1,7 +1,7 @@ Mailu command line ================== -Managing users and aliases can be done from CLI using commands: +Managing domains, users and aliases can be done from CLI using the commands: * alias * alias-delete diff --git a/docs/configuration.rst b/docs/configuration.rst index 5ecfa347..3a7d8c66 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -37,7 +37,7 @@ The ``POSTMASTER`` is the local part of the postmaster email address. It is recommended to setup a generic value and later configure a mail alias for that address. -The ``WILDCARD_SENDERS`` setting is a comma delimited list of user email addresses +The ``WILDCARD_SENDERS`` setting is a comma delimited list of user email addresses that are allowed to send emails from any existing address (spoofing the sender). The ``AUTH_RATELIMIT_IP`` (default: 60/hour) holds a security setting for fighting @@ -141,9 +141,9 @@ Web settings - ``WEB_WEBMAIL`` contains the path to the Web email client. - ``WEBROOT_REDIRECT`` redirects all non-found queries to the set path. - An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables + An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables classic behavior of a 404 result when not found. - Alternatively, ``WEBROOT_REDIRECT`` can be set to ``none`` if you + Alternatively, ``WEBROOT_REDIRECT`` can be set to ``none`` if you are using an Nginx override for ``location /``. All three options need a leading slash (``/``) to work. @@ -156,11 +156,11 @@ Both ``SITENAME`` and ``WEBSITE`` are customization options for the panel menu in the admin interface, while ``SITENAME`` is a customization option for every Web interface. -- ``LOGO_BACKGROUND`` sets a custom background colour for the brand logo - in the top left of the main admin interface. +- ``LOGO_BACKGROUND`` sets a custom background colour for the brand logo + in the topleft of the main admin interface. For a list of colour codes refer to this page of `w3schools`_. -- ``LOGO_URL`` sets a URL for a custom logo. This logo replaces the Mailu +- ``LOGO_URL`` sets a URL for a custom logo. This logo replaces the Mailu logo in the topleft of the main admin interface. .. _`w3schools`: https://www.w3schools.com/cssref/css_colors.asp @@ -184,7 +184,7 @@ To have the account created automatically, you just need to define a few environ - ``ifmissing``: creates a new admin account when the admin account does not exist. - ``update``: creates a new admin account when it does not exist, or update the password of an existing admin account. -Note: It is recommended to set ``INITIAL_ADMIN_MODE`` to either ``update`` or ``ifmissing``. Leaving it with the +Note: It is recommended to set ``INITIAL_ADMIN_MODE`` to either ``update`` or ``ifmissing``. Leaving it with the default value will cause an error when the system is restarted. An example: @@ -198,23 +198,32 @@ An example: Depending on your particular deployment you most probably will want to change the default. -.. _advanced_cfg: +.. _advanced_settings: Advanced settings ----------------- -The ``CREDENTIAL_ROUNDS`` (default: 12) setting is the number of rounds used by the -password hashing scheme. The number of rounds can be reduced in case faster -authentication is needed or increased when additional protection is desired. -Keep in mind that this is a mitigation against offline attacks on password hashes, +The ``API`` (default: False) setting controls if the API endpoint is publicly +reachable. + +The ``WEB_API`` (default: /api) setting configures the endpoint that the API +listens on publicly&interally. The path must always start with a leading slash. + +The ``API_TOKEN`` (default: None) enables the API endpoint. This token must be +passed as query parameter with requests to the API as authentication token. + +The ``CREDENTIAL_ROUNDS`` (default: 12) setting is the number of rounds used by the +password hashing scheme. The number of rounds can be reduced in case faster +authentication is needed or increased when additional protection is desired. +Keep in mind that this is a mitigation against offline attacks on password hashes, aiming to prevent credential stuffing (due to password re-use) on other systems. -The ``SESSION_COOKIE_SECURE`` (default: True) setting controls the secure flag on -the cookies of the administrative interface. It should only be turned off if you +The ``SESSION_COOKIE_SECURE`` (default: True) setting controls the secure flag on +the cookies of the administrative interface. It should only be turned off if you intend to access it over plain HTTP. -``SESSION_TIMEOUT`` (default: 3600) is the maximum amount of time in seconds between -requests before a session is invalidated. ``PERMANENT_SESSION_LIFETIME`` (default: 108000) +``SESSION_TIMEOUT`` (default: 3600) is the maximum amount of time in seconds between +requests before a session is invalidated. ``PERMANENT_SESSION_LIFETIME`` (default: 108000) is the maximum amount of time in seconds a session can be kept alive for if it hasn't timed-out. The ``LOG_LEVEL`` setting is used by the python start-up scripts as a logging threshold. @@ -224,8 +233,8 @@ See the `python docs`_ for more information. .. _`python docs`: https://docs.python.org/3.6/library/logging.html#logging-levels -The ``LETSENCRYPT_SHORTCHAIN`` (default: False) setting controls whether we send the -ISRG Root X1 certificate in TLS handshakes. This is required for `android handsets older than 7.1.1` +The ``LETSENCRYPT_SHORTCHAIN`` (default: False) setting controls whether we send the +ISRG Root X1 certificate in TLS handshakes. This is required for `android handsets older than 7.1.1` but slows down the performance of modern devices. .. _`android handsets older than 7.1.1`: https://community.letsencrypt.org/t/production-chain-changes/150739 @@ -234,11 +243,11 @@ The ``TLS_PERMISSIVE`` (default: true) setting controls whether ciphers and prot .. _reverse_proxy_headers: -The ``REAL_IP_HEADER`` (default: unset) and ``REAL_IP_FROM`` (default: unset) settings -controls whether HTTP headers such as ``X-Forwarded-For`` or ``X-Real-IP`` should be trusted. -The former should be the name of the HTTP header to extract the client IP address from and the -later a comma separated list of IP addresses designating which proxies to trust. -If you are using Mailu behind a reverse proxy, you should set both. Setting the former without +The ``REAL_IP_HEADER`` (default: unset) and ``REAL_IP_FROM`` (default: unset) settings +controls whether HTTP headers such as ``X-Forwarded-For`` or ``X-Real-IP`` should be trusted. +The former should be the name of the HTTP header to extract the client IP address from and the +later a comma separated list of IP addresses designating which proxies to trust. +If you are using Mailu behind a reverse proxy, you should set both. Setting the former without the later introduces a security vulnerability allowing a potential attacker to spoof his source address. The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usually uses a ``Region/City`` format. See `TZ database name`_ for a list of valid timezones This defaults to ``Etc/UTC``. Warning: if you are observing different timestamps in your log files you should change your hosts timezone to UTC instead of changing TZ to your local timezone. Using UTC allows easy log correlation with remote MTAs. @@ -348,15 +357,15 @@ Mail log settings By default, all services log directly to stdout/stderr. Logs can be collected by any docker log processing solution. -Postfix writes the logs to a syslog server which logs to stdout. This is used to filter -out messages from the healthcheck. In some situations, a separate mail log is required -(e.g. for legal reasons). The syslog server can be configured to write log files to a volume. +Postfix writes the logs to a syslog server which logs to stdout. This is used to filter +out messages from the healthcheck. In some situations, a separate mail log is required +(e.g. for legal reasons). The syslog server can be configured to write log files to a volume. It can be configured with the following option: - ``POSTFIX_LOG_FILE``: The file to log the mail log to. When enabled, the syslog server will also log to stdout. -When ``POSTFIX_LOG_FILE`` is enabled, the logrotate program will automatically rotate the -logs every week and keep 52 logs. To override the logrotate configuration, create the file logrotate.conf +When ``POSTFIX_LOG_FILE`` is enabled, the logrotate program will automatically rotate the +logs every week and keep 52 logs. To override the logrotate configuration, create the file logrotate.conf with the desired configuration in the :ref:`Postfix overrides folder`. diff --git a/docs/faq.rst b/docs/faq.rst index bd0f4d17..7ea2068e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -24,7 +24,7 @@ advice in the `Technical issues`_ section of this page. I think I found a bug! `````````````````````` -If you did not manage to solve the issue using this FAQ and there are not any +If you did not manage to solve the issue using this FAQ and there are not any `open issues`_ describing the same problem, you can open a `new issue`_ on GitHub. @@ -64,7 +64,7 @@ We currently maintain a strict work flow: #. We use Github actions for some very basic building and testing; #. The pull request needs to be code-reviewed and tested by at least two members from the contributors team. - + Please consider that this project is mostly developed in people their free time. We thank you for your understanding and patience. @@ -152,7 +152,7 @@ Lets start with quoting everything that's wrong: It was added later and, while it has come a long way, is still not as usable as one would want. Much discussion is still going on as to how IPv6 should be used in a containerized world; See the various GitHub issues linked below: - + - Giving each container a publicly routable address means all ports (even unexposed / unpublished ports) are suddenly reachable by everyone, if no additional filtering is done (`docker/docker#21614 `_) @@ -163,14 +163,14 @@ Lets start with quoting everything that's wrong: (which, for now, is enabled by default in Docker) - The userland proxy, however, seems to be on its way out (`docker/docker#14856 `_) and has various issues, like: - + - It can use a lot of RAM (`docker/docker#11185 `_) - - Source IP addresses are rewritten, making it completely unusable for many purposes, e.g. mail servers + - Source IP addresses are rewritten, making it completely unusable for many purposes, e.g. mail servers (`docker/docker#17666 `_), (`docker/libnetwork#1099 `_). - + -- `Robbert Klarenbeek `_ (docker-ipv6nat author) - + Okay, but I still want to use IPv6! Can I just use the installers IPv6 checkbox? **NO, YOU SHOULD NOT DO THAT!** Why you ask? Mailu has its own trusted IPv4 network, every container inside this network can use e.g. the SMTP container without further authentication. If you enabled IPv6 inside the setup assistant (and fixed the ports to also be exposed on IPv6) Docker will @@ -223,7 +223,7 @@ For **service** HA, please see: `How does Mailu scale up?`_ *Issue reference:* `177`_, `591`_. -.. _`spam magnet`: https://web.archive.org/web/20130131032707/https://blog.zensoftware.co.uk/2012/07/02/why-we-tend-to-recommend-not-having-a-secondary-mx-these-days/ +.. _`spam magnet`: https://web.archive.org/web/20130131032707/https://blog.zensoftware.co.uk/2012/07/02/why-we-tend-to-recommend-not-having-a-secondary-mx-these-days/ Does Mailu run on Rancher? `````````````````````````` @@ -292,7 +292,7 @@ I want to integrate Nextcloud 15 (and newer) with Mailu ), ), ), - + If a domain name (e.g. example.com) is specified, then this makes sure that only users from this domain will be allowed to login. After successfull login the domain part will be stripped and the rest used as username in Nextcloud. e.g. 'username@example.com' will be 'username' in Nextcloud. Disable this behaviour by changing true (the fifth parameter) to false. @@ -346,7 +346,7 @@ How do I use webdav (radicale)? | | Subsequently to use webdav (radicale), you can configure your carddav/caldav client to use the following url: | `https://mail.example.com/webdav/user@example.com` -| As username you must provide the complete email address (user@example.com). +| As username you must provide the complete email address (user@example.com). | As password you must provide the password of the email address. | The user must be an existing Mailu user. @@ -545,14 +545,14 @@ inside a container. The ``front`` container does use authentication rate limitin down brute force attacks. The same applies to login attempts via the single sign on page. We *do* provide a possibility to export the logs from the ``front`` service and ``Admin`` service to the host. -The ``front`` container logs failed logon attempts on SMTP, IMAP and POP3. +The ``front`` container logs failed logon attempts on SMTP, IMAP and POP3. The ``Admin``container logs failed logon attempt on the single sign on page. For this you need to set ``LOG_DRIVER=journald`` or ``syslog``, depending on the log manager of the host. You will need to setup the proper Regex in the Fail2Ban configuration. -Below an example how to do so. +Below an example how to do so. If you use a reverse proxy in front of Mailu, it is vital to set the environment variables REAL_IP_HEADER and REAL_IP_FROM. -Without these environment variables, Mailu will not trust the remote client IP passed on by the reverse proxy and as a result your reverse proxy will be banned. +Without these environment variables, Mailu will not trust the remote client IP passed on by the reverse proxy and as a result your reverse proxy will be banned. See the :ref:`[configuration reference ` for more information. @@ -591,12 +591,12 @@ follow these steps: maxretry = 10 action = docker-action -The above will block flagged IPs for a week, you can of course change it to you needs. +The above will block flagged IPs for a week, you can of course change it to your needs. 4. In the mailu docker-compose set the logging driver of the Admin container to journald; and set the tag to mailu-admin .. code-block:: bash - + logging: driver: journald options: @@ -625,28 +625,53 @@ The above will block flagged IPs for a week, you can of course change it to you maxretry = 10 action = docker-action -The above will block flagged IPs for a week, you can of course change it to you needs. +The above will block flagged IPs for a week, you can of course change it to your needs. + +7. Add the /etc/fail2ban/filter.d/bad-auth-api.conf + +.. code-block:: bash + + # Fail2Ban configuration file + [Definition] + failregex = .* Invalid API token provided by . + ignoreregex = + journalmatch = CONTAINER_TAG=mailu-admin + +8. Add the /etc/fail2ban/jail.d/bad-auth-api.conf + +.. code-block:: bash + + [bad-auth-api] + enabled = true + backend = systemd + filter = bad-auth-api + bantime = 604800 + findtime = 300 + maxretry = 10 + action = docker-action + +The above will block flagged IPs for a week, you can of course change it to your needs. + +9. Add the /etc/fail2ban/action.d/docker-action.conf -7. Add the /etc/fail2ban/action.d/docker-action.conf - Option 1: Use plain iptables .. code-block:: bash [Definition] - + actionstart = iptables -N f2b-bad-auth iptables -A f2b-bad-auth -j RETURN iptables -I DOCKER-USER -j f2b-bad-auth - + actionstop = iptables -D DOCKER-USER -j f2b-bad-auth iptables -F f2b-bad-auth iptables -X f2b-bad-auth - + actioncheck = iptables -n -L DOCKER-USER | grep -q 'f2b-bad-auth[ \t]' - + actionban = iptables -I f2b-bad-auth 1 -s -j DROP - + actionunban = iptables -D f2b-bad-auth -s -j DROP Using DOCKER-USER chain ensures that the blocked IPs are processed in the correct order with Docker. See more in: https://docs.docker.com/network/iptables/ @@ -657,7 +682,7 @@ IMPORTANT: You have to install ipset on the host system, eg. `apt-get install ip See ipset homepage for details on ipset, https://ipset.netfilter.org/. ipset and iptables provide one big advantage over just using iptables: This setup reduces the overall iptable rules. -There is just one rule for the bad authentications and the IPs are within the ipset. +There is just one rule for the bad authentications and the IPs are within the ipset. Specially in larger setups with a high amount of brute force attacks this comes in handy. Using iptables with ipset might reduce the system load in such attacks significantly. @@ -678,7 +703,7 @@ Using iptables with ipset might reduce the system load in such attacks significa Using DOCKER-USER chain ensures that the blocked IPs are processed in the correct order with Docker. See more in: https://docs.docker.com/network/iptables/ -1. Configure and restart the Fail2Ban service +10. Configure and restart the Fail2Ban service Make sure Fail2Ban is started after the Docker service by adding a partial override which appends this to the existing configuration. @@ -727,7 +752,7 @@ In any case, using a dedicated DNS server will improve the performance of your m Can I learn ham/spam messages from an already existing mailbox? ``````````````````````````````````````````````````````````````` -Mailu supports automatic spam learning for messages moved to the Junk mailbox. Any email moved from the Junk Folder will learnt as ham. +Mailu supports automatic spam learning for messages moved to the Junk mailbox. Any email moved from the Junk Folder will learnt as ham. If you already have an existing mailbox and want Mailu to learn them all as ham messages, you might run rspamc from within the dovecot container: @@ -736,7 +761,7 @@ If you already have an existing mailbox and want Mailu to learn them all as ham rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /mail/user\@example.com/.Ham_Learn/cur/ rspamc -h antispam:11334 -P mailu learn_ham /mail/user\@example.com/.Ham_Learn/cur/ -This should learn every file located in the ``Ham_Learn`` folder from user@example.com +This should learn every file located in the ``Ham_Learn`` folder from user@example.com Likewise, to lean all messages within the folder ``Spam_Learn`` as spam messages : diff --git a/docs/index.rst b/docs/index.rst index b73bf529..ae148f71 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -70,6 +70,7 @@ the version of Mailu that you are running. webadministration antispam cli + api .. toctree:: :maxdepth: 2 diff --git a/docs/requirements.txt b/docs/requirements.txt index f49e26d5..2c3169b7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ -recommonmark -Sphinx -sphinx-autobuild -sphinx-rtd-theme +recommonmark==0.7.1 +Sphinx==5.2.0 +sphinx-autobuild==2021.3.14 +sphinx-rtd-theme==1.0.0 docutils==0.16 diff --git a/towncrier/newsfragments/445.feature b/towncrier/newsfragments/445.feature new file mode 100644 index 00000000..7cb94079 --- /dev/null +++ b/towncrier/newsfragments/445.feature @@ -0,0 +1,2 @@ +Introduction of the Mailu RESTful API. The full Mailu config can be changed via the Mailu API. +See the section Mailu RESTful API & the section configuration reference in the documentation for more information. \ No newline at end of file From 7a36f6bbb9a0bb6e57b21f0adcd47cbb1cb43338 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 27 Sep 2022 06:46:32 +0000 Subject: [PATCH 05/47] Use hmac.compare_digest to prevent timing attacks. --- core/admin/mailu/api/common.py | 8 ++++++-- core/admin/mailu/api/v1/alias.py | 15 +++++++-------- core/admin/mailu/api/v1/user.py | 27 ++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py index 0805c5a3..26372bd9 100644 --- a/core/admin/mailu/api/common.py +++ b/core/admin/mailu/api/common.py @@ -2,6 +2,7 @@ from .. import models, utils from . import v1 from flask import request import flask +import hmac from functools import wraps from flask_restx import abort @@ -19,10 +20,13 @@ def api_token_authorization(func): client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr) if utils.limiter.should_rate_limit_ip(client_ip): abort(429, 'Too many attempts from your IP (rate-limit)' ) - if request.args.get('api_token') != v1.api_token: + if (request.args.get('api_token') == '' or + request.args.get('api_token') == None): + abort(401, 'A valid API token is expected as query string parameter') + if not hmac.compare_digest(request.args.get('api_token'), v1.api_token): utils.limiter.rate_limit_ip(client_ip) flask.current_app.logger.warn(f'Invalid API token provided by {client_ip}.') - abort(401, 'A valid API token is expected as query string parameter') + abort(403, 'A valid API token is expected as query string parameter') else: flask.current_app.logger.info(f'Valid API token provided by {client_ip}.') return func(*args, **kwds) diff --git a/core/admin/mailu/api/v1/alias.py b/core/admin/mailu/api/v1/alias.py index 65cbd25c..29c03195 100644 --- a/core/admin/mailu/api/v1/alias.py +++ b/core/admin/mailu/api/v1/alias.py @@ -7,19 +7,18 @@ db = models.db alias = api.namespace('alias', description='Alias operations') -alias_fields = api.model('Alias', { - 'email': fields.String(description='the alias email address', example='user@example.com', required=True), - 'comment': fields.String(description='a comment'), - 'destination': fields.List(fields.String(description='alias email address', example='user@example.com', required=True)), - 'wildcard': fields.Boolean(description='enable SQL Like wildcard syntax') -}) - -alias_fields_update = api.model('AliasUpdate', { +alias_fields_update = alias.model('AliasUpdate', { 'comment': fields.String(description='a comment'), 'destination': fields.List(fields.String(description='alias email address', example='user@example.com')), 'wildcard': fields.Boolean(description='enable SQL Like wildcard syntax') }) +alias_fields = alias.inherit('Alias',alias_fields_update, { + 'email': fields.String(description='the alias email address', example='user@example.com', required=True), + 'destination': fields.List(fields.String(description='alias email address', example='user@example.com', required=True)), + +}) + @alias.route('') class Aliases(Resource): diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py index 5345e007..1826963d 100644 --- a/core/admin/mailu/api/v1/user.py +++ b/core/admin/mailu/api/v1/user.py @@ -9,9 +9,8 @@ db = models.db user = api.namespace('user', description='User operations') -user_fields_get = api.model('UserGet', { - 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'), - 'password': fields.String(description='PBKDF2-HMAC-SHA256 based password of the user. For more info see passlib.hash.pbkdf2_sha256', example='$pbkdf2-sha256$1$.6UI/S.nXIk8jcbdHx3Fhg$98jZicV16ODfEsEZeYPGHU3kbrUrvUEXOPimVSQDD44'), +""" +base_model = { 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), 'global_admin': fields.Boolean(description='Make the user a global administrator'), @@ -30,6 +29,28 @@ user_fields_get = api.model('UserGet', { 'spam_enabled': fields.Boolean(description='Enable the spam filter'), 'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'), 'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'), +} + +user_fields_get = api.model('UserGet', { + 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'), + 'password': fields.String(description='PBKDF2-HMAC-SHA256 based password of the user. For more info see passlib.hash.pbkdf2_sha256', example='$pbkdf2-sha256$1$.6UI/S.nXIk8jcbdHx3Fhg$98jZicV16ODfEsEZeYPGHU3kbrUrvUEXOPimVSQDD44'), + +}.update(base_model)) + +user_fields_post = api.model('UserCreate', { + 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email', required=True), + 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using PBKDF2-HMAC-SHA256', example='secret', required=True), +}.update(base_model)) + +user_fields_put = api.model('UserUpdate', { + 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using PBKDF2-HMAC-SHA256', example='secret'), +}.update(base_model)) +""" + +user_fields_get = api.model('UserGet', { + 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'), + 'password': fields.String(description='PBKDF2-HMAC-SHA256 based password of the user. For more info see passlib.hash.pbkdf2_sha256', example='$pbkdf2-sha256$1$.6UI/S.nXIk8jcbdHx3Fhg$98jZicV16ODfEsEZeYPGHU3kbrUrvUEXOPimVSQDD44'), + }) user_fields_post = api.model('UserCreate', { From 67c423d61f62261966df52a90f7a00a32af8b08c Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 5 Oct 2022 08:28:57 +0000 Subject: [PATCH 06/47] Add URL for accessing swaggerui to documentation --- docs/api.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 66ccc03d..8f3acbc6 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,9 +8,9 @@ can also be configured via the API. The Mailu API is disabled by default. It can be enabled and configured via the settings: -* `API` -* `WEB_API` -* `API_TOKEN` +* ```API``` +* ``WEB_API`` +* ```API_TOKEN``` For more information see the section :ref:`Advanced configuration ` in the configuration reference. @@ -26,4 +26,6 @@ The swagger.json file can be consumed in programs such as Postman for generating In-built SwaggerUI ------------------ The Mailu API comes with an in-built SwaggerUI. It is a web client that allows -anyone to visualize and interact with the Mailu API. \ No newline at end of file +anyone to visualize and interact with the Mailu API. + +It is accessible via the URL: https://myserver/api/v1/swaggerui From 46d07ec23641586f55baeefbfa2718ed79275768 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sun, 13 Nov 2022 08:38:24 +0000 Subject: [PATCH 07/47] Fix syntax styling api documentation. --- docs/api.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 8f3acbc6..b18398ac 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -8,9 +8,9 @@ can also be configured via the API. The Mailu API is disabled by default. It can be enabled and configured via the settings: -* ```API``` +* ``API`` * ``WEB_API`` -* ```API_TOKEN``` +* ``API_TOKEN`` For more information see the section :ref:`Advanced configuration ` in the configuration reference. From d4e5db508406fab0020d9b3e18cb9ddb06187398 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Mon, 14 Nov 2022 09:45:36 +0000 Subject: [PATCH 08/47] Remove unneeded comment --- core/admin/mailu/api/v1/user.py | 38 --------------------------------- 1 file changed, 38 deletions(-) diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py index 1826963d..fa7d5d19 100644 --- a/core/admin/mailu/api/v1/user.py +++ b/core/admin/mailu/api/v1/user.py @@ -9,44 +9,6 @@ db = models.db user = api.namespace('user', description='User operations') -""" -base_model = { - 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), - 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), - 'global_admin': fields.Boolean(description='Make the user a global administrator'), - 'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'), - 'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'), - 'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'), - 'forward_enabled': fields.Boolean(description='Enable auto forwarding'), - 'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'), - 'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'), - 'reply_enabled': fields.Boolean(description='Enable automatic replies. This is also known as out of office (ooo) or out of facility (oof) replies'), - 'reply_subject': fields.String(description='Optional subject for the automatic reply', example='Out of office'), - 'reply_body': fields.String(description='The body of the automatic reply email', example='Hello, I am out of office. I will respond when I am back.'), - 'reply_startdate': fields.Date(description='Start date for automatic replies in YYYY-MM-DD format.', example='2022-02-10'), - 'reply_enddate': fields.Date(description='End date for automatic replies in YYYY-MM-DD format.', example='2022-02-22'), - 'displayed_name': fields.String(description='The display name of the user within the Admin GUI', example='John Doe'), - 'spam_enabled': fields.Boolean(description='Enable the spam filter'), - 'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'), - 'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'), -} - -user_fields_get = api.model('UserGet', { - 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'), - 'password': fields.String(description='PBKDF2-HMAC-SHA256 based password of the user. For more info see passlib.hash.pbkdf2_sha256', example='$pbkdf2-sha256$1$.6UI/S.nXIk8jcbdHx3Fhg$98jZicV16ODfEsEZeYPGHU3kbrUrvUEXOPimVSQDD44'), - -}.update(base_model)) - -user_fields_post = api.model('UserCreate', { - 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email', required=True), - 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using PBKDF2-HMAC-SHA256', example='secret', required=True), -}.update(base_model)) - -user_fields_put = api.model('UserUpdate', { - 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using PBKDF2-HMAC-SHA256', example='secret'), -}.update(base_model)) -""" - user_fields_get = api.model('UserGet', { 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'), 'password': fields.String(description='PBKDF2-HMAC-SHA256 based password of the user. For more info see passlib.hash.pbkdf2_sha256', example='$pbkdf2-sha256$1$.6UI/S.nXIk8jcbdHx3Fhg$98jZicV16ODfEsEZeYPGHU3kbrUrvUEXOPimVSQDD44'), From afb224e79697338c234f1346a7e9db278ea637aa Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 16 Nov 2022 12:05:41 +0000 Subject: [PATCH 09/47] Update password hash description for user API endpoint --- core/admin/mailu/api/v1/user.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py index fa7d5d19..01dac786 100644 --- a/core/admin/mailu/api/v1/user.py +++ b/core/admin/mailu/api/v1/user.py @@ -11,13 +11,12 @@ user = api.namespace('user', description='User operations') user_fields_get = api.model('UserGet', { 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'), - 'password': fields.String(description='PBKDF2-HMAC-SHA256 based password of the user. For more info see passlib.hash.pbkdf2_sha256', example='$pbkdf2-sha256$1$.6UI/S.nXIk8jcbdHx3Fhg$98jZicV16ODfEsEZeYPGHU3kbrUrvUEXOPimVSQDD44'), - + 'password': fields.String(description="Hash of the user's password; Example='$bcrypt-sha256$v=2,t=2b,r=12$fmsAdJbYAD1gGQIE5nfJq.$zLkQUEs2XZfTpAEpcix/1k5UTNPm0jO'"), }) user_fields_post = api.model('UserCreate', { 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email', required=True), - 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using PBKDF2-HMAC-SHA256', example='secret', required=True), + 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using BCRYPT-SHA256', example='secret', required=True), 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), 'global_admin': fields.Boolean(description='Make the user a global administrator'), @@ -39,7 +38,7 @@ user_fields_post = api.model('UserCreate', { }) user_fields_put = api.model('UserUpdate', { - 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using PBKDF2-HMAC-SHA256', example='secret'), + 'raw_password': fields.String(description='The raw (plain text) password of the user. Mailu will hash the password using BCRYPT-SHA256', example='secret'), 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), 'global_admin': fields.Boolean(description='Make the user a global administrator'), From 61d092922cc2b8da4ede8c98f35b9b0b7bde98d3 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 25 Nov 2022 11:21:33 +0000 Subject: [PATCH 10/47] Process review comments (PR2464) --- core/admin/mailu/__init__.py | 2 +- core/admin/mailu/api/__init__.py | 20 ++++++++----------- core/admin/mailu/api/common.py | 26 ++++++++++++------------- core/admin/mailu/api/v1/__init__.py | 9 ++++----- core/admin/mailu/api/v1/alias.py | 12 ++++++------ core/admin/mailu/api/v1/domains.py | 30 ++++++++++++++--------------- core/admin/mailu/api/v1/relay.py | 10 +++++----- core/admin/mailu/api/v1/user.py | 10 +++++----- core/admin/mailu/configuration.py | 2 +- docs/configuration.rst | 5 ++--- 10 files changed, 60 insertions(+), 66 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 89e06c9b..de3503d0 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -75,7 +75,7 @@ def create_app_from_config(config): app.register_blueprint(internal.internal, url_prefix='/internal') app.register_blueprint(sso.sso, url_prefix='/sso') if app.config.get('API_TOKEN'): - api.register(app, web_api=app.config.get('WEB_API')) + api.register(app, web_api_root=app.config.get('WEB_API')) return app diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py index 44cda962..3fd9d495 100644 --- a/core/admin/mailu/api/__init__.py +++ b/core/admin/mailu/api/__init__.py @@ -1,24 +1,20 @@ from flask import redirect, url_for from flask_restx import apidoc -from . import v1 +from . import v1 as APIv1 -def register(app, web_api): +def register(app, web_api_root): - ACTIVE=v1 - ROOT=web_api - v1.app = app + APIv1.app = app # register api bluprint(s) - apidoc.apidoc.url_prefix = f'{ROOT}/v{int(v1.VERSION)}' - v1.api_token = app.config['API_TOKEN'] - app.register_blueprint(v1.blueprint, url_prefix=f'{ROOT}/v{int(v1.VERSION)}') - - + apidoc.apidoc.url_prefix = f'{web_api_root}/v{int(APIv1.VERSION)}' + APIv1.api_token = app.config['API_TOKEN'] + app.register_blueprint(APIv1.blueprint, url_prefix=f'{web_api_root}/v{int(APIv1.VERSION)}') # add redirect to current api version - @app.route(f'{ROOT}/') + @app.route(f'{web_api_root}/') def redir(): - return redirect(url_for(f'{ACTIVE.blueprint.name}.root')) + return redirect(url_for(f'{APIv1.blueprint.name}.root')) # swagger ui config app.config.SWAGGER_UI_DOC_EXPANSION = 'list' diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py index 26372bd9..1b69f4d7 100644 --- a/core/admin/mailu/api/common.py +++ b/core/admin/mailu/api/common.py @@ -5,13 +5,15 @@ import flask import hmac from functools import wraps from flask_restx import abort +from sqlalchemy.sql.expression import label -def fqdn_in_use(*names): - for name in names: - for model in models.Domain, models.Alternative, models.Relay: - if model.query.get(name): - return model - return None +def fqdn_in_use(name): + d = models.db.session.query(label('name', models.Domain.name)) + a = models.db.session.query(label('name', models.Alternative.name)) + r = models.db.session.query(label('name', models.Relay.name)) + if d.union_all(a).union_all(r).filter_by(name=name).count() > 0: + return True + return False """ Decorator for validating api token for authentication """ def api_token_authorization(func): @@ -20,14 +22,12 @@ def api_token_authorization(func): client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr) if utils.limiter.should_rate_limit_ip(client_ip): abort(429, 'Too many attempts from your IP (rate-limit)' ) - if (request.args.get('api_token') == '' or - request.args.get('api_token') == None): - abort(401, 'A valid API token is expected as query string parameter') - if not hmac.compare_digest(request.args.get('api_token'), v1.api_token): + if not request.headers.get('Authorization'): + abort(401, 'A valid API token is expected which is provided as request header') + if not hmac.compare_digest(request.headers.get('Authorization'), v1.api_token): utils.limiter.rate_limit_ip(client_ip) flask.current_app.logger.warn(f'Invalid API token provided by {client_ip}.') - abort(403, 'A valid API token is expected as query string parameter') - else: - flask.current_app.logger.info(f'Valid API token provided by {client_ip}.') + abort(403, 'A valid API token is expected which is provided as request header') + flask.current_app.logger.info(f'Valid API token provided by {client_ip}.') return func(*args, **kwds) return decorated_function diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py index 5cc878c9..5cb1fc82 100644 --- a/core/admin/mailu/api/v1/__init__.py +++ b/core/admin/mailu/api/v1/__init__.py @@ -8,20 +8,19 @@ api_token = None blueprint = Blueprint(f'api_v{int(VERSION)}', __name__) authorization = { - 'apikey': { + 'Bearer': { 'type': 'apiKey', - 'in': 'query', - 'name': 'api_token' + 'in': 'header', + 'name': 'Authorization' } } - api = Api( blueprint, version=f'{VERSION:.1f}', title='Mailu API', default_label='Mailu', validate=True, authorizations=authorization, - security='apikey', + security='Bearer', doc='/swaggerui/' ) diff --git a/core/admin/mailu/api/v1/alias.py b/core/admin/mailu/api/v1/alias.py index 29c03195..7635aa19 100644 --- a/core/admin/mailu/api/v1/alias.py +++ b/core/admin/mailu/api/v1/alias.py @@ -24,7 +24,7 @@ alias_fields = alias.inherit('Alias',alias_fields_update, { class Aliases(Resource): @alias.doc('list_alias') @alias.marshal_with(alias_fields, as_list=True, skip_none=True, mask=None) - @alias.doc(security='apikey') + @alias.doc(security='Bearer') @common.api_token_authorization def get(self): """ List aliases """ @@ -35,7 +35,7 @@ class Aliases(Resource): @alias.response(200, 'Success', response_fields) @alias.response(400, 'Input validation exception', response_fields) @alias.response(409, 'Duplicate alias', response_fields) - @alias.doc(security='apikey') + @alias.doc(security='Bearer') @common.api_token_authorization def post(self): """ Create a new alias """ @@ -60,7 +60,7 @@ class Alias(Resource): @alias.doc('find_alias') @alias.response(200, 'Success', alias_fields) @alias.response(404, 'Alias not found', response_fields) - @alias.doc(security='apikey') + @alias.doc(security='Bearer') @common.api_token_authorization def get(self, alias): """ Find alias """ @@ -75,7 +75,7 @@ class Alias(Resource): @alias.response(200, 'Success', response_fields) @alias.response(404, 'Alias not found', response_fields) @alias.response(400, 'Input validation exception', response_fields) - @alias.doc(security='apikey') + @alias.doc(security='Bearer') @common.api_token_authorization def put(self, alias): """ Update alias """ @@ -97,7 +97,7 @@ class Alias(Resource): @alias.doc('delete_alias') @alias.response(200, 'Success', response_fields) @alias.response(404, 'Alias not found', response_fields) - @alias.doc(security='apikey') + @alias.doc(security='Bearer') @common.api_token_authorization def delete(self, alias): """ Delete alias """ @@ -113,7 +113,7 @@ class AliasWithDest(Resource): @alias.doc('find_alias_filter_domain') @alias.response(200, 'Success', alias_fields) @alias.response(404, 'Alias or domain not found', response_fields) - @alias.doc(security='apikey') + @alias.doc(security='Bearer') @common.api_token_authorization def get(self, domain): """ Find aliases of domain """ diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py index 35393b81..4eabe22e 100644 --- a/core/admin/mailu/api/v1/domains.py +++ b/core/admin/mailu/api/v1/domains.py @@ -78,7 +78,7 @@ alternative_fields = api.model('AlternativeDomain', { class Domains(Resource): @dom.doc('list_domain') @dom.marshal_with(domain_fields_get, as_list=True, skip_none=True, mask=None) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def get(self): """ List domains """ @@ -89,7 +89,7 @@ class Domains(Resource): @dom.response(200, 'Success', response_fields) @dom.response(400, 'Input validation exception', response_fields) @dom.response(409, 'Duplicate domain/alternative name', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def post(self): """ Create a new domain """ @@ -133,7 +133,7 @@ class Domain(Resource): @dom.doc('find_domain') @dom.response(200, 'Success', domain_fields) @dom.response(404, 'Domain not found', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def get(self, domain): """ Find domain by name """ @@ -150,7 +150,7 @@ class Domain(Resource): @dom.response(400, 'Input validation exception', response_fields) @dom.response(404, 'Domain not found', response_fields) @dom.response(409, 'Duplicate domain/alternative name', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def put(self, domain): """ Update an existing domain """ @@ -194,7 +194,7 @@ class Domain(Resource): @dom.response(200, 'Success', response_fields) @dom.response(400, 'Input validation exception', response_fields) @dom.response(404, 'Domain not found', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def delete(self, domain): """ Delete domain """ @@ -213,7 +213,7 @@ class Domain(Resource): @dom.response(200, 'Success', response_fields) @dom.response(400, 'Input validation exception', response_fields) @dom.response(404, 'Domain not found', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def post(self, domain): """ Generate new DKIM/DMARC keys for domain """ @@ -232,7 +232,7 @@ class Manager(Resource): @dom.marshal_with(manager_fields, as_list=True, skip_none=True, mask=None) @dom.response(400, 'Input validation exception', response_fields) @dom.response(404, 'domain not found', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def get(self, domain): """ List managers of domain """ @@ -249,7 +249,7 @@ class Manager(Resource): @dom.response(400, 'Input validation exception', response_fields) @dom.response(404, 'User or domain not found', response_fields) @dom.response(409, 'Duplicate domain manager', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def post(self, domain): """ Create a new domain manager """ @@ -275,7 +275,7 @@ class Domain(Resource): @dom.doc('find_manager') @dom.response(200, 'Success', manager_fields) @dom.response(404, 'Manager not found', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def get(self, domain, email): """ Find manager by email address """ @@ -301,7 +301,7 @@ class Domain(Resource): @dom.response(200, 'Success', response_fields) @dom.response(400, 'Input validation exception', response_fields) @dom.response(404, 'Manager not found', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def delete(self, domain, email): if not validators.email(email): @@ -327,7 +327,7 @@ class User(Resource): @dom.marshal_with(user.user_fields_get, as_list=True, skip_none=True, mask=None) @dom.response(400, 'Input validation exception', response_fields) @dom.response(404, 'Domain not found', response_fields) - @dom.doc(security='apikey') + @dom.doc(security='Bearer') @common.api_token_authorization def get(self, domain): """ List users from domain """ @@ -343,7 +343,7 @@ class Alternatives(Resource): @alt.doc('list_alternative') @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=None) - @alt.doc(security='apikey') + @alt.doc(security='Bearer') @common.api_token_authorization def get(self): """ List alternatives """ @@ -356,7 +356,7 @@ class Alternatives(Resource): @alt.response(400, 'Input validation exception', response_fields) @alt.response(404, 'Domain not found or missing', response_fields) @alt.response(409, 'Duplicate alternative domain name', response_fields) - @alt.doc(security='apikey') + @alt.doc(security='Bearer') @common.api_token_authorization def post(self): """ Create new alternative (for domain) """ @@ -379,7 +379,7 @@ class Alternatives(Resource): @alt.route('/') class Alternative(Resource): @alt.doc('find_alternative') - @alt.doc(security='apikey') + @alt.doc(security='Bearer') @common.api_token_authorization def get(self, alt): """ Find alternative (of domain) """ @@ -395,7 +395,7 @@ class Alternative(Resource): @alt.response(400, 'Input validation exception', response_fields) @alt.response(404, 'Alternative/Domain not found or missing', response_fields) @alt.response(409, 'Duplicate domain name', response_fields) - @alt.doc(security='apikey') + @alt.doc(security='Bearer') @common.api_token_authorization def delete(self, alt): """ Delete alternative (for domain) """ diff --git a/core/admin/mailu/api/v1/relay.py b/core/admin/mailu/api/v1/relay.py index 67089d9a..6b52c7d5 100644 --- a/core/admin/mailu/api/v1/relay.py +++ b/core/admin/mailu/api/v1/relay.py @@ -24,7 +24,7 @@ relay_fields_update = api.model('RelayUpdate', { class Relays(Resource): @relay.doc('list_relays') @relay.marshal_with(relay_fields, as_list=True, skip_none=True, mask=None) - @relay.doc(security='apikey') + @relay.doc(security='Bearer') @common.api_token_authorization def get(self): "List relays" @@ -35,7 +35,7 @@ class Relays(Resource): @relay.response(200, 'Success', response_fields) @relay.response(400, 'Input validation exception') @relay.response(409, 'Duplicate relay', response_fields) - @relay.doc(security='apikey') + @relay.doc(security='Bearer') @common.api_token_authorization def post(self): """ Create relay """ @@ -61,7 +61,7 @@ class Relay(Resource): @relay.doc('find_relay') @relay.response(400, 'Input validation exception', response_fields) @relay.response(404, 'Relay not found', response_fields) - @relay.doc(security='apikey') + @relay.doc(security='Bearer') @common.api_token_authorization def get(self, name): """ Find relay """ @@ -79,7 +79,7 @@ class Relay(Resource): @relay.response(400, 'Input validation exception', response_fields) @relay.response(404, 'Relay not found', response_fields) @relay.response(409, 'Duplicate relay', response_fields) - @relay.doc(security='apikey') + @relay.doc(security='Bearer') @common.api_token_authorization def put(self, name): """ Update relay """ @@ -105,7 +105,7 @@ class Relay(Resource): @relay.response(200, 'Success', response_fields) @relay.response(400, 'Input validation exception', response_fields) @relay.response(404, 'Relay not found', response_fields) - @relay.doc(security='apikey') + @relay.doc(security='Bearer') @common.api_token_authorization def delete(self, name): """ Delete relay """ diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py index 01dac786..cbe1fe6d 100644 --- a/core/admin/mailu/api/v1/user.py +++ b/core/admin/mailu/api/v1/user.py @@ -64,7 +64,7 @@ user_fields_put = api.model('UserUpdate', { class Users(Resource): @user.doc('list_users') @user.marshal_with(user_fields_get, as_list=True, skip_none=True, mask=None) - @user.doc(security='apikey') + @user.doc(security='Bearer') @common.api_token_authorization def get(self): "List users" @@ -75,7 +75,7 @@ class Users(Resource): @user.response(200, 'Success', response_fields) @user.response(400, 'Input validation exception') @user.response(409, 'Duplicate user', response_fields) - @user.doc(security='apikey') + @user.doc(security='Bearer') @common.api_token_authorization def post(self): """ Create user """ @@ -141,7 +141,7 @@ class User(Resource): @user.doc('find_user') @user.response(400, 'Input validation exception', response_fields) @user.response(404, 'User not found', response_fields) - @user.doc(security='apikey') + @user.doc(security='Bearer') @common.api_token_authorization def get(self, email): """ Find user """ @@ -159,7 +159,7 @@ class User(Resource): @user.response(400, 'Input validation exception', response_fields) @user.response(404, 'User not found', response_fields) @user.response(409, 'Duplicate user', response_fields) - @user.doc(security='apikey') + @user.doc(security='Bearer') @common.api_token_authorization def put(self, email): """ Update user """ @@ -222,7 +222,7 @@ class User(Resource): @user.response(200, 'Success', response_fields) @user.response(400, 'Input validation exception', response_fields) @user.response(404, 'User not found', response_fields) - @user.doc(security='apikey') + @user.doc(security='Bearer') @common.api_token_authorization def delete(self, email): """ Delete user """ diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 612bcb18..b77e2cf1 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -71,7 +71,7 @@ DEFAULT_CONFIG = { 'LOGO_BACKGROUND': None, 'API': False, # Advanced settings - 'API' : 'false', + 'API' : False, 'WEB_API' : '/api', 'API_TOKEN': None, 'LOG_LEVEL': 'WARNING', diff --git a/docs/configuration.rst b/docs/configuration.rst index 3a7d8c66..b37bec08 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -203,14 +203,13 @@ Depending on your particular deployment you most probably will want to change th Advanced settings ----------------- -The ``API`` (default: False) setting controls if the API endpoint is publicly -reachable. +The ``API`` (default: False) setting controls if the API endpoint is reachable. The ``WEB_API`` (default: /api) setting configures the endpoint that the API listens on publicly&interally. The path must always start with a leading slash. The ``API_TOKEN`` (default: None) enables the API endpoint. This token must be -passed as query parameter with requests to the API as authentication token. +passed as request header to the API as authentication token. The ``CREDENTIAL_ROUNDS`` (default: 12) setting is the number of rounds used by the password hashing scheme. The number of rounds can be reduced in case faster From 6347c18f8ab1939cf175dd257b9a889fae11fc4b Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sun, 27 Nov 2022 11:15:40 +0000 Subject: [PATCH 11/47] Process review comments (PR2464) --- core/admin/mailu/api/common.py | 17 +++++++++++++---- core/admin/mailu/api/v1/domains.py | 4 ++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py index 1b69f4d7..331fdf4e 100644 --- a/core/admin/mailu/api/common.py +++ b/core/admin/mailu/api/common.py @@ -11,7 +11,8 @@ def fqdn_in_use(name): d = models.db.session.query(label('name', models.Domain.name)) a = models.db.session.query(label('name', models.Alternative.name)) r = models.db.session.query(label('name', models.Relay.name)) - if d.union_all(a).union_all(r).filter_by(name=name).count() > 0: + u = d.union_all(a).union_all(r).filter_by(name=name) + if models.db.session.query(u.exists()).scalar(): return True return False @@ -23,11 +24,19 @@ def api_token_authorization(func): if utils.limiter.should_rate_limit_ip(client_ip): abort(429, 'Too many attempts from your IP (rate-limit)' ) if not request.headers.get('Authorization'): - abort(401, 'A valid API token is expected which is provided as request header') - if not hmac.compare_digest(request.headers.get('Authorization'), v1.api_token): + abort(401, 'A valid Bearer token is expected which is provided as request header') + #Client provides 'Authentication: Bearer ' + if (' ' in request.headers.get('Authorization') + and not hmac.compare_digest(request.headers.get('Authorization'), 'Bearer ' + v1.api_token)): utils.limiter.rate_limit_ip(client_ip) flask.current_app.logger.warn(f'Invalid API token provided by {client_ip}.') - abort(403, 'A valid API token is expected which is provided as request header') + abort(403, 'A valid Bearer token is expected which is provided as request header') + #Client provides 'Authentication: ' + elif (' ' not in request.headers.get('Authorization') + and not hmac.compare_digest(request.headers.get('Authorization'), v1.api_token)): + utils.limiter.rate_limit_ip(client_ip) + flask.current_app.logger.warn(f'Invalid API token provided by {client_ip}.') + abort(403, 'A valid Bearer token is expected which is provided as request header') flask.current_app.logger.info(f'Valid API token provided by {client_ip}.') return func(*args, **kwds) return decorated_function diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py index 4eabe22e..76554a02 100644 --- a/core/admin/mailu/api/v1/domains.py +++ b/core/admin/mailu/api/v1/domains.py @@ -401,10 +401,10 @@ class Alternative(Resource): """ Delete alternative (for domain) """ if not validators.domain(alt): return { 'code': 400, 'message': f'Alternative domain {alt} is not a valid domain'}, 400 - alternative = models.Alternative.query.filter_by(name=alt).first + alternative = models.Alternative.query.filter_by(name=alt).scalar() if not alternative: return { 'code': 404, 'message': f'Alternative domain {alt} does not exist'}, 404 - domain = alternative.domain + domain = alternative.domain_name db.session.delete(alternative) db.session.commit() return {'code': 200, 'message': f'Alternative {alt} for domain {domain} has been deleted'}, 200 From f9b26bd93449598788919db49c200e9f7146293e Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Mon, 5 Dec 2022 08:58:54 +0000 Subject: [PATCH 12/47] Update User with newly introduced allow spoofing field --- core/admin/mailu/api/v1/user.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py index cbe1fe6d..79875c04 100644 --- a/core/admin/mailu/api/v1/user.py +++ b/core/admin/mailu/api/v1/user.py @@ -23,6 +23,7 @@ user_fields_post = api.model('UserCreate', { 'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'), 'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'), 'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'), + 'allow_spoofing': fields.Boolean(description='Allow the user to spoof the sender (send email as anyone)'), 'forward_enabled': fields.Boolean(description='Enable auto forwarding'), 'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'), 'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'), @@ -45,6 +46,7 @@ user_fields_put = api.model('UserUpdate', { 'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'), 'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'), 'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'), + 'allow_spoofing': fields.Boolean(description='Allow the user to spoof the sender (send email as anyone)'), 'forward_enabled': fields.Boolean(description='Enable auto forwarding'), 'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'), 'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'), @@ -102,6 +104,8 @@ class Users(Resource): user_new.enable_imap = data['enable_imap'] if 'enable_pop' in data: user_new.enable_pop = data['enable_pop'] + if 'allow_spoofing' in data: + user.allow_spoofing = data['allow_spoofing'] if 'forward_enabled' in data: user_new.forward_enabled = data['forward_enabled'] if 'forward_destination' in data: @@ -184,6 +188,8 @@ class User(Resource): user_found.enable_imap = data['enable_imap'] if 'enable_pop' in data: user_found.enable_pop = data['enable_pop'] + if 'allow_spoofing' in data: + user.allow_spoofing = data['allow_spoofing'] if 'forward_enabled' in data: user_found.forward_enabled = data['forward_enabled'] if 'forward_destination' in data: From 39b0d44079989e73dda9a3a2b7708d5a273c845a Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 13 Dec 2022 10:35:42 +0000 Subject: [PATCH 13/47] Use first() instead of all() for better performance Actually return all data for Get user Remove non-used code --- core/admin/mailu/api/v1/alias.py | 4 ++-- core/admin/mailu/api/v1/relay.py | 1 - core/admin/mailu/api/v1/user.py | 25 ++++++++++++++++++++++--- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/core/admin/mailu/api/v1/alias.py b/core/admin/mailu/api/v1/alias.py index 7635aa19..3605c68f 100644 --- a/core/admin/mailu/api/v1/alias.py +++ b/core/admin/mailu/api/v1/alias.py @@ -41,7 +41,7 @@ class Aliases(Resource): """ Create a new alias """ data = api.payload - alias_found = models.Alias.query.filter_by(email = data['email']).all() + alias_found = models.Alias.query.filter_by(email = data['email']).first() if alias_found: return { 'code': 409, 'message': f'Duplicate alias {data["email"]}'}, 409 @@ -64,7 +64,7 @@ class Alias(Resource): @common.api_token_authorization def get(self, alias): """ Find alias """ - alias_found = models.Alias.query.filter_by(email = alias).all() + alias_found = models.Alias.query.filter_by(email = alias).first() if alias_found is None: return { 'code': 404, 'message': f'Alias {alias} cannot be found'}, 404 else: diff --git a/core/admin/mailu/api/v1/relay.py b/core/admin/mailu/api/v1/relay.py index 6b52c7d5..f4588d22 100644 --- a/core/admin/mailu/api/v1/relay.py +++ b/core/admin/mailu/api/v1/relay.py @@ -44,7 +44,6 @@ class Relays(Resource): if not validators.domain(name): return { 'code': 400, 'message': f'Relayed domain {name} is not a valid domain'}, 400 - relay_found = models.Relay.query.filter_by(name=data['name']).all() if common.fqdn_in_use(data['name']): return { 'code': 409, 'message': f'Duplicate domain {data["name"]}'}, 409 relay_model = models.Relay(name=data['name']) diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py index 79875c04..953d246b 100644 --- a/core/admin/mailu/api/v1/user.py +++ b/core/admin/mailu/api/v1/user.py @@ -12,6 +12,25 @@ user = api.namespace('user', description='User operations') user_fields_get = api.model('UserGet', { 'email': fields.String(description='The email address of the user', example='John.Doe@example.com', attribute='_email'), 'password': fields.String(description="Hash of the user's password; Example='$bcrypt-sha256$v=2,t=2b,r=12$fmsAdJbYAD1gGQIE5nfJq.$zLkQUEs2XZfTpAEpcix/1k5UTNPm0jO'"), + 'comment': fields.String(description='A description for the user. This description is shown on the Users page', example='my comment'), + 'quota_bytes': fields.Integer(description='The maximum quota for the user’s email box in bytes', example='1000000000'), + 'global_admin': fields.Boolean(description='Make the user a global administrator'), + 'enabled': fields.Boolean(description='Enable the user. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail'), + 'enable_imap': fields.Boolean(description='Allow email retrieval via IMAP'), + 'enable_pop': fields.Boolean(description='Allow email retrieval via POP3'), + 'allow_spoofing': fields.Boolean(description='Allow the user to spoof the sender (send email as anyone)'), + 'forward_enabled': fields.Boolean(description='Enable auto forwarding'), + 'forward_destination': fields.List(fields.String(description='Email address to forward emails to'), example='Other@example.com'), + 'forward_keep': fields.Boolean(description='Keep a copy of the forwarded email in the inbox'), + 'reply_enabled': fields.Boolean(description='Enable automatic replies. This is also known as out of office (ooo) or out of facility (oof) replies'), + 'reply_subject': fields.String(description='Optional subject for the automatic reply', example='Out of office'), + 'reply_body': fields.String(description='The body of the automatic reply email', example='Hello, I am out of office. I will respond when I am back.'), + 'reply_startdate': fields.Date(description='Start date for automatic replies in YYYY-MM-DD format.', example='2022-02-10'), + 'reply_enddate': fields.Date(description='End date for automatic replies in YYYY-MM-DD format.', example='2022-02-22'), + 'displayed_name': fields.String(description='The display name of the user within the Admin GUI', example='John Doe'), + 'spam_enabled': fields.Boolean(description='Enable the spam filter'), + 'spam_mark_as_read': fields.Boolean(description='Enable marking spam mails as read'), + 'spam_threshold': fields.Integer(description='The user defined spam filter tolerance', example='80'), }) user_fields_post = api.model('UserCreate', { @@ -105,7 +124,7 @@ class Users(Resource): if 'enable_pop' in data: user_new.enable_pop = data['enable_pop'] if 'allow_spoofing' in data: - user.allow_spoofing = data['allow_spoofing'] + user_new.allow_spoofing = data['allow_spoofing'] if 'forward_enabled' in data: user_new.forward_enabled = data['forward_enabled'] if 'forward_destination' in data: @@ -170,7 +189,7 @@ class User(Resource): data = api.payload if not validators.email(email): return { 'code': 400, 'message': f'Provided email address {data["email"]} is not a valid email address'}, 400 - user_found = models.User.query.filter_by(email=email).first() + user_found = models.User.query.get(email) if not user_found: return {'code': 404, 'message': f'User {email} cannot be found'}, 404 @@ -189,7 +208,7 @@ class User(Resource): if 'enable_pop' in data: user_found.enable_pop = data['enable_pop'] if 'allow_spoofing' in data: - user.allow_spoofing = data['allow_spoofing'] + user_found.allow_spoofing = data['allow_spoofing'] if 'forward_enabled' in data: user_found.forward_enabled = data['forward_enabled'] if 'forward_destination' in data: From 3cb8358090824e72e845fa829780699f161e0c60 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 23 Dec 2022 13:56:19 +0000 Subject: [PATCH 14/47] Process review comments PR#2464 - When visiting root of WEB_API, the swaggerui is shown - simplify the condition for endpoint WEB_API --- core/admin/mailu/api/__init__.py | 7 ++++--- core/admin/mailu/api/v1/__init__.py | 2 +- core/nginx/conf/nginx.conf | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py index 3fd9d495..da7325ae 100644 --- a/core/admin/mailu/api/__init__.py +++ b/core/admin/mailu/api/__init__.py @@ -1,8 +1,7 @@ -from flask import redirect, url_for +from flask import redirect, url_for, Blueprint from flask_restx import apidoc from . import v1 as APIv1 - def register(app, web_api_root): APIv1.app = app @@ -12,9 +11,11 @@ def register(app, web_api_root): app.register_blueprint(APIv1.blueprint, url_prefix=f'{web_api_root}/v{int(APIv1.VERSION)}') # add redirect to current api version - @app.route(f'{web_api_root}/') + redirect_api = Blueprint('redirect_api', __name__) + @redirect_api.route('/') def redir(): return redirect(url_for(f'{APIv1.blueprint.name}.root')) + app.register_blueprint(redirect_api, url_prefix=f'{web_api_root}') # swagger ui config app.config.SWAGGER_UI_DOC_EXPANSION = 'list' diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py index 5cb1fc82..44b6ec57 100644 --- a/core/admin/mailu/api/v1/__init__.py +++ b/core/admin/mailu/api/v1/__init__.py @@ -21,7 +21,7 @@ api = Api( validate=True, authorizations=authorization, security='Bearer', - doc='/swaggerui/' + doc='/' ) response_fields = api.model('Response', { diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 722e5e50..463cee20 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -244,7 +244,7 @@ http { {% endif %} {% endif %} - {% if API == 'true' %} + {% if API %} location ~ {{ WEB_API }} { include /etc/nginx/proxy.conf; proxy_pass http://$admin; From be407813946665ce571a5124a82f08920a32cd44 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 27 Dec 2022 14:28:25 +0100 Subject: [PATCH 15/47] Add default for WEB_API, re-add flask-restx to deps, remove whitespace --- core/admin/mailu/configuration.py | 5 ++--- core/base/requirements-dev.txt | 1 + core/base/requirements-prod.txt | 1 + core/nginx/conf/nginx.conf | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index b77e2cf1..59b692ec 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -69,10 +69,9 @@ DEFAULT_CONFIG = { 'RECAPTCHA_PRIVATE_KEY': '', 'LOGO_URL': None, 'LOGO_BACKGROUND': None, - 'API': False, # Advanced settings - 'API' : False, - 'WEB_API' : '/api', + 'API': False, + 'WEB_API': '/api', 'API_TOKEN': None, 'LOG_LEVEL': 'WARNING', 'SESSION_KEY_BITS': 128, diff --git a/core/base/requirements-dev.txt b/core/base/requirements-dev.txt index 5a9df4b1..32041642 100644 --- a/core/base/requirements-dev.txt +++ b/core/base/requirements-dev.txt @@ -15,6 +15,7 @@ Flask-DebugToolbar Flask-Login flask-marshmallow Flask-Migrate +Flask-RESTX Flask-SQLAlchemy<3 Flask-WTF gunicorn diff --git a/core/base/requirements-prod.txt b/core/base/requirements-prod.txt index 0da5219b..918d85f4 100644 --- a/core/base/requirements-prod.txt +++ b/core/base/requirements-prod.txt @@ -25,6 +25,7 @@ Flask-DebugToolbar==0.13.1 Flask-Login==0.6.2 flask-marshmallow==0.14.0 Flask-Migrate==3.1.0 +Flask-RESTX==1.0.3 Flask-SQLAlchemy==2.5.1 Flask-WTF==1.0.1 frozenlist==1.3.1 diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 463cee20..3965805c 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -245,7 +245,7 @@ http { {% endif %} {% if API %} - location ~ {{ WEB_API }} { + location ~ {{ WEB_API or '/api' }} { include /etc/nginx/proxy.conf; proxy_pass http://$admin; } From 4ae0d7d768f6a166a7f0dd14a5e8b421558973f9 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 28 Dec 2022 14:17:00 +0100 Subject: [PATCH 16/47] Enable HAPROXY protocol in between front and imap With this we avoid running into the limitations of mail_max_userip_connections (see #894 amd #1364) and the logfiles as well as ``doveadm who`` give an accurate picture. --- core/admin/mailu/internal/views/dovecot.py | 2 +- core/dovecot/conf/dovecot.conf | 3 +++ core/nginx/conf/nginx.conf | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 07fce5b2..f9a07556 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -17,7 +17,7 @@ def dovecot_passdb_dict(user_email): return flask.jsonify({ "password": None, "nopassword": "Y", - "allow_nets": ",".join(allow_nets) + "allow_real_nets": ",".join(allow_nets) }) @internal.route("/dovecot/userdb/") diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index d9b85172..60c94238 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -11,6 +11,8 @@ default_internal_user = dovecot default_login_user = mail default_internal_group = dovecot +haproxy_trusted_networks = {{ SUBNET }} {{ SUBNET6 }} + ############### # Mailboxes ############### @@ -109,6 +111,7 @@ protocol pop3 { service imap-login { inet_listener imap { port = 143 + haproxy = yes } } diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index b373fb13..7e5e7b5c 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -338,6 +338,7 @@ mail { starttls only; {% endif %} protocol imap; + proxy_protocol on; imap_auth plain; auth_http_header Auth-Port 143; } @@ -349,6 +350,7 @@ mail { starttls only; {% endif %} protocol pop3; + proxy_protocol on; pop3_auth plain; auth_http_header Auth-Port 110; } @@ -377,6 +379,7 @@ mail { listen 993 ssl; listen [::]:993 ssl; protocol imap; + proxy_protocol on; imap_auth plain; auth_http_header Auth-Port 993; } @@ -385,6 +388,7 @@ mail { listen 995 ssl; listen [::]:995 ssl; protocol pop3; + proxy_protocol on; pop3_auth plain; auth_http_header Auth-Port 995; } From 55c1e555294c4232b2d8385c67f1a9a81691dc26 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 28 Dec 2022 15:21:28 +0100 Subject: [PATCH 17/47] Same for front-smtp This should enable postfix to have visibility on TLS usage and fix the following: #1705 --- core/nginx/conf/nginx.conf | 7 +++---- core/postfix/conf/main.cf | 7 ++++--- core/postfix/conf/master.cf | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 7e5e7b5c..7dc3be90 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -292,6 +292,9 @@ mail { pop3_capabilities TOP UIDL RESP-CODES PIPELINING AUTH-RESP-CODE USER; imap_capabilities IMAP4 IMAP4rev1 UIDPLUS SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+; + # ensure we talk HAPROXY protocol to the backends + proxy_protocol on; + # Default SMTP server for the webmail (no encryption, but authentication) server { listen 10025; @@ -338,7 +341,6 @@ mail { starttls only; {% endif %} protocol imap; - proxy_protocol on; imap_auth plain; auth_http_header Auth-Port 143; } @@ -350,7 +352,6 @@ mail { starttls only; {% endif %} protocol pop3; - proxy_protocol on; pop3_auth plain; auth_http_header Auth-Port 110; } @@ -379,7 +380,6 @@ mail { listen 993 ssl; listen [::]:993 ssl; protocol imap; - proxy_protocol on; imap_auth plain; auth_http_header Auth-Port 993; } @@ -388,7 +388,6 @@ mail { listen 995 ssl; listen [::]:995 ssl; protocol pop3; - proxy_protocol on; pop3_auth plain; auth_http_header Auth-Port 995; } diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 2f0275b7..474bf42c 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -22,6 +22,8 @@ alias_maps = # Podop configuration podop = socketmap:unix:/tmp/podop.socket: +postscreen_upstream_proxy_protocol = haproxy + # Only accept virtual emails mydestination = @@ -37,9 +39,8 @@ smtp_sasl_tls_security_options = noanonymous # Recipient delimiter for extended addresses recipient_delimiter = {{ RECIPIENT_DELIMITER }} -# Only the front server is allowed to perform xclient -# In kubernetes and Docker swarm, such address cannot be determined using the hostname. Allow for the whole Mailu subnet instead. -smtpd_authorized_xclient_hosts={{ SUBNET }} +# We need to allow everything to do xclient and rely on front to filter-out "bad" requests +smtpd_authorized_xclient_hosts=0.0.0.0/0 [::0]/0 ############### # TLS diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index bec96a30..116633f1 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -2,10 +2,10 @@ # (yes) (yes) (yes) (never) (100) # Exposed SMTP service -smtp inet n - n - - smtpd +smtp inet n - n - 1 postscreen # Internal SMTP service -10025 inet n - n - - smtpd +10025 inet n - n - 1 postscreen -o smtpd_sasl_auth_enable=yes -o smtpd_discard_ehlo_keywords=pipelining -o smtpd_client_restrictions=$check_ratelimit,reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit @@ -44,6 +44,7 @@ verify unix - - n - 1 verify flush unix n - n 1000? 0 flush proxymap unix - - n - - proxymap smtp unix - - n - - smtp +smtpd pass - - n - - smtpd relay unix - - n - - smtp error unix - - n - - error retry unix - - n - - error @@ -52,4 +53,3 @@ lmtp unix - - n - - lmtp anvil unix - - n - 1 anvil scache unix - - n - 1 scache postlog unix-dgram n - n - 1 postlogd - From 163261d95169ac55931e528caeb3795e19858396 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 28 Dec 2022 15:45:47 +0100 Subject: [PATCH 18/47] Towncrier --- towncrier/newsfragments/2603.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/2603.bugfix diff --git a/towncrier/newsfragments/2603.bugfix b/towncrier/newsfragments/2603.bugfix new file mode 100644 index 00000000..7fdb9ef2 --- /dev/null +++ b/towncrier/newsfragments/2603.bugfix @@ -0,0 +1 @@ +Speak HAPROXY protocol in between front and smtp and front and imap. This ensures the backend is aware of the real client IP and whether TLS was used. From 7a2d06401af7694385a243a10f7d9ba5292df154 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 28 Dec 2022 16:05:39 +0100 Subject: [PATCH 19/47] Tweak postfix logging --- core/postfix/conf/main.cf | 1 + core/postfix/conf/rsyslog.conf | 2 ++ 2 files changed, 3 insertions(+) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 474bf42c..32996095 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -23,6 +23,7 @@ alias_maps = podop = socketmap:unix:/tmp/podop.socket: postscreen_upstream_proxy_protocol = haproxy +compatibility_level=3.6 # Only accept virtual emails mydestination = diff --git a/core/postfix/conf/rsyslog.conf b/core/postfix/conf/rsyslog.conf index 6423eb4d..b1d2f389 100644 --- a/core/postfix/conf/rsyslog.conf +++ b/core/postfix/conf/rsyslog.conf @@ -31,6 +31,8 @@ module(load="imuxsock") # Discard messages from local test requests :msg, contains, "connect from localhost[127.0.0.1]" ~ :msg, contains, "connect from localhost[::1]" ~ +:msg, contains, "haproxy read: short protocol header: QUIT" ~ +:msg, contains, "discarding EHLO keywords: PIPELINING" ~ {% if POSTFIX_LOG_FILE %} # Log mail logs to file From 83ea70849018d175c3a60d2750f791fb9b705794 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 28 Dec 2022 16:26:46 +0100 Subject: [PATCH 20/47] fix healthcheck --- core/postfix/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index dab4396c..df902dd4 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -15,7 +15,7 @@ COPY start.py / RUN echo $VERSION >/version EXPOSE 25/tcp 10025/tcp -HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" +HEALTHCHECK --start-period=350s CMD /usr/sbin/postfix status VOLUME ["/queue"] From 36b3a9f4fb1099d4477142c25f6e6ad2a8eef7c3 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 28 Dec 2022 17:05:34 +0100 Subject: [PATCH 21/47] Will fix it in another PR --- core/postfix/conf/rsyslog.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/postfix/conf/rsyslog.conf b/core/postfix/conf/rsyslog.conf index b1d2f389..6423eb4d 100644 --- a/core/postfix/conf/rsyslog.conf +++ b/core/postfix/conf/rsyslog.conf @@ -31,8 +31,6 @@ module(load="imuxsock") # Discard messages from local test requests :msg, contains, "connect from localhost[127.0.0.1]" ~ :msg, contains, "connect from localhost[::1]" ~ -:msg, contains, "haproxy read: short protocol header: QUIT" ~ -:msg, contains, "discarding EHLO keywords: PIPELINING" ~ {% if POSTFIX_LOG_FILE %} # Log mail logs to file From 6f71ea833b68964803bb3ac466920ddc236ae1f4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 29 Dec 2022 15:36:07 +0100 Subject: [PATCH 22/47] Update python dependencies as suggested by dependabot --- core/admin/run_dev.sh | 26 ++++++++++++++++++++++---- core/base/requirements-build.txt | 6 +++--- core/base/requirements-prod.txt | 12 ++++++++---- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 947ad873..0f7c6e05 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -11,6 +11,7 @@ DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}" [[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}" DEV_ADMIN="${DEV_ADMIN:-admin@example.com}" DEV_PASSWORD="${DEV_PASSWORD:-letmein}" +DEV_ARGS=( "$@" ) ### MAIN @@ -90,7 +91,8 @@ EOF # build chmod -R u+rwX,go+rX . -"${docker}" build --tag "${DEV_NAME}:latest" . +echo Running: "${docker/*\/}" build --tag "${DEV_NAME}:latest" "${DEV_ARGS[@]}" . +"${docker}" build --tag "${DEV_NAME}:latest" "${DEV_ARGS[@]}" . # gather volumes to map into container volumes=() @@ -110,6 +112,7 @@ done cat <$(realpath "${base}")/requirements-new.txt + +============================================================================= + The Mailu UI can be found here: http://${DEV_LISTEN}/sso/login EOF [[ -z "${DEV_DB}" ]] && echo "You can log in with user ${DEV_ADMIN} and password ${DEV_PASSWORD}" cat < Date: Thu, 29 Dec 2022 17:02:05 +0100 Subject: [PATCH 23/47] Use the size other implementations default to --- core/admin/mailu/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 9b308b32..f9fabb7f 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -75,7 +75,7 @@ class CommaSeparatedList(db.TypeDecorator): """ Stores a list as a comma-separated string, compatible with Postfix. """ - impl = db.Text + impl = db.String(255) cache_ok = True python_type = list @@ -96,7 +96,7 @@ class JSONEncoded(db.TypeDecorator): """ Represents an immutable structure as a json-encoded string. """ - impl = db.Text + impl = db.String(255) cache_ok = True python_type = str From 4d80c95c41a1e2046a37848d11947bf804c0bbab Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 3 Jan 2023 15:57:57 +0100 Subject: [PATCH 24/47] Fix authentication submission Don't talk haproxy to postfix; it's more headaches than it is currently worth. --- core/nginx/conf/nginx.conf | 13 ++++++++++--- core/postfix/conf/master.cf | 4 ++-- towncrier/newsfragments/2608.fix | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 towncrier/newsfragments/2608.fix diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 8bfddace..d1b4923e 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -292,9 +292,6 @@ mail { pop3_capabilities TOP UIDL RESP-CODES PIPELINING AUTH-RESP-CODE USER; imap_capabilities IMAP4 IMAP4rev1 UIDPLUS SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+; - # ensure we talk HAPROXY protocol to the backends - proxy_protocol on; - # Default SMTP server for the webmail (no encryption, but authentication) server { listen 10025; @@ -309,6 +306,8 @@ mail { protocol imap; smtp_auth plain; auth_http_header Auth-Port 10143; + # ensure we talk HAPROXY protocol to the backends + proxy_protocol on; } # SMTP is always enabled, to avoid losing emails when TLS is failing @@ -343,6 +342,8 @@ mail { protocol imap; imap_auth plain; auth_http_header Auth-Port 143; + # ensure we talk HAPROXY protocol to the backends + proxy_protocol on; } server { @@ -354,6 +355,8 @@ mail { protocol pop3; pop3_auth plain; auth_http_header Auth-Port 110; + # ensure we talk HAPROXY protocol to the backends + proxy_protocol on; } server { @@ -382,6 +385,8 @@ mail { protocol imap; imap_auth plain; auth_http_header Auth-Port 993; + # ensure we talk HAPROXY protocol to the backends + proxy_protocol on; } server { @@ -390,6 +395,8 @@ mail { protocol pop3; pop3_auth plain; auth_http_header Auth-Port 995; + # ensure we talk HAPROXY protocol to the backends + proxy_protocol on; } {% endif %} {% endif %} diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index 116633f1..86659460 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -2,10 +2,10 @@ # (yes) (yes) (yes) (never) (100) # Exposed SMTP service -smtp inet n - n - 1 postscreen +smtp inet n - n - 1 smtpd # Internal SMTP service -10025 inet n - n - 1 postscreen +10025 inet n - n - 1 smtpd -o smtpd_sasl_auth_enable=yes -o smtpd_discard_ehlo_keywords=pipelining -o smtpd_client_restrictions=$check_ratelimit,reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit diff --git a/towncrier/newsfragments/2608.fix b/towncrier/newsfragments/2608.fix new file mode 100644 index 00000000..850e647c --- /dev/null +++ b/towncrier/newsfragments/2608.fix @@ -0,0 +1 @@ +Don't talk haproxy to postfix yet. From 202ff8db1400fa17dbd4c525abaa8b1543b17bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Sitarski?= Date: Tue, 3 Jan 2023 16:53:58 +0100 Subject: [PATCH 25/47] Remove duplicated 'actionstart = ' in fail2ban conf. In fail2ban example configuration for ipset option, there was a duplicated string which makes the ipset and fail2ban fail to create the set. Fail2ban will never ban any ip due to this error. --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index bd0f4d17..df6fb664 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -665,7 +665,7 @@ Using iptables with ipset might reduce the system load in such attacks significa [Definition] - actionstart = actionstart = ipset --create f2b-bad-auth iphash + actionstart = ipset --create f2b-bad-auth iphash iptables -I DOCKER-USER -m set --match-set f2b-bad-auth src -j DROP actionstop = iptables -D DOCKER-USER -m set --match-set f2b-bad-auth src -j DROP From bf0c345bb97d6c926f131279d2d0654693514d3d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 4 Jan 2023 12:42:13 +0100 Subject: [PATCH 26/47] Fix snappymail --- webmails/snuffleupagus.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index fbb8a776..a2304fe0 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -98,7 +98,7 @@ sp.disable_function.function("is_callable").param("value").value("eval").drop(); sp.disable_function.function("is_callable").param("value").value("exec").drop(); sp.disable_function.function("is_callable").param("value").value("system").drop(); sp.disable_function.function("is_callable").param("value").value("shell_exec").drop(); -sp.disable_function.function("is_callable").filename_r("^/var/www/snappymail/snappymail/v/\d+\.\d+\.\d+/app/libraries/snappymail/pgp/gpg\.php$").param("value").value("proc_open").allow(); +sp.disable_function.function("is_callable").filename_r("^/var/www/snappymail/snappymail/v/[0-9]+\.[0-9]+\.[0-9]+/app/libraries/snappymail/pgp/gpg\.php$").param("value").value("proc_open").allow(); sp.disable_function.function("is_callable").param("value").value("proc_open").drop(); sp.disable_function.function("is_callable").param("value").value("passthru").drop(); From b263db72dfc4198cb61005cc63c4532c588ba055 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 4 Jan 2023 09:40:52 +0100 Subject: [PATCH 27/47] Restrict XHOST to where useful --- core/postfix/conf/main.cf | 3 --- core/postfix/conf/master.cf | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 32996095..2c9f71f3 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -40,9 +40,6 @@ smtp_sasl_tls_security_options = noanonymous # Recipient delimiter for extended addresses recipient_delimiter = {{ RECIPIENT_DELIMITER }} -# We need to allow everything to do xclient and rely on front to filter-out "bad" requests -smtpd_authorized_xclient_hosts=0.0.0.0/0 [::0]/0 - ############### # TLS ############### diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index 86659460..21d42b1b 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -11,6 +11,7 @@ smtp inet n - n - 1 smtpd -o smtpd_client_restrictions=$check_ratelimit,reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit -o smtpd_reject_unlisted_recipient={% if REJECT_UNLISTED_RECIPIENT %}{{ REJECT_UNLISTED_RECIPIENT }}{% else %}no{% endif %} -o cleanup_service_name=outclean + -o smtpd_authorized_xclient_hosts={{ SUBNET}},{{ SUBNET6 }} outclean unix n - n - 0 cleanup -o header_checks=pcre:/etc/postfix/outclean_header_filter.cf -o nested_header_checks= From 92c0016e32483e67dbddeb11f122dcc4aef3cb69 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 4 Jan 2023 12:42:13 +0100 Subject: [PATCH 28/47] Fix snappymail --- webmails/snuffleupagus.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index fbb8a776..a2304fe0 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -98,7 +98,7 @@ sp.disable_function.function("is_callable").param("value").value("eval").drop(); sp.disable_function.function("is_callable").param("value").value("exec").drop(); sp.disable_function.function("is_callable").param("value").value("system").drop(); sp.disable_function.function("is_callable").param("value").value("shell_exec").drop(); -sp.disable_function.function("is_callable").filename_r("^/var/www/snappymail/snappymail/v/\d+\.\d+\.\d+/app/libraries/snappymail/pgp/gpg\.php$").param("value").value("proc_open").allow(); +sp.disable_function.function("is_callable").filename_r("^/var/www/snappymail/snappymail/v/[0-9]+\.[0-9]+\.[0-9]+/app/libraries/snappymail/pgp/gpg\.php$").param("value").value("proc_open").allow(); sp.disable_function.function("is_callable").param("value").value("proc_open").drop(); sp.disable_function.function("is_callable").param("value").value("passthru").drop(); From e85a2a7e9903e7c00c2db3b23b0e3c0352d538c0 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 4 Jan 2023 11:01:50 +0100 Subject: [PATCH 29/47] Step1: expose managesieve, make the webmails use it --- core/dovecot/conf/dovecot.conf | 6 ++++++ core/nginx/Dockerfile | 5 +++-- core/nginx/conf/nginx.conf | 21 +++++++++++++++++++++ webmails/roundcube/config/config.inc.php | 2 +- webmails/snappymail/defaults/default.json | 4 ++-- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 60c94238..c60f7491 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -135,10 +135,16 @@ service lmtp { service managesieve-login { inet_listener sieve { port = 4190 + haproxy = yes } } +protocol sieve { + ssl = no +} + service managesieve { + process_limit = 1024 } plugin { diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index f271fc07..27757920 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -17,7 +17,8 @@ ARG VERSION LABEL version=$VERSION RUN set -euxo pipefail \ - ; apk add --no-cache certbot nginx nginx-mod-mail openssl + ; apk add --no-cache certbot nginx nginx-mod-http-brotli nginx-mod-stream nginx-mod-mail openssl \ + ; rm /etc/nginx/conf.d/stream.conf COPY conf/ /conf/ COPY --from=static /static/ /static/ @@ -25,7 +26,7 @@ COPY *.py / RUN echo $VERSION >/version -EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp +EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 14190/tcp 10025/tcp 10143/tcp HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://localhost/health VOLUME ["/certs", "/overrides"] diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index d1b4923e..44fca726 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -1,9 +1,11 @@ # Basic configuration user nginx; worker_processes auto; +pcre_jit on; error_log /dev/stderr notice; pid /var/run/nginx.pid; load_module "modules/ngx_mail_module.so"; +load_module "modules/ngx_stream_module.so"; events { worker_connections 1024; @@ -275,6 +277,25 @@ http { include /etc/nginx/conf.d/*.conf; } +stream { + log_format main '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$upstream_addr" ' + '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"'; + access_log /dev/stdout main; + + # managesieve + server { + listen 14190; + resolver {{ RESOLVER }} valid=30s; + + proxy_connect_timeout 1s; + proxy_timeout 1m; + proxy_protocol on; + proxy_pass {{ IMAP_ADDRESS }}:4190; + } +} + mail { server_name {{ HOSTNAMES.split(",")[0] }}; auth_http http://127.0.0.1:8000/auth/email; diff --git a/webmails/roundcube/config/config.inc.php b/webmails/roundcube/config/config.inc.php index 665ef1ad..f271eebc 100644 --- a/webmails/roundcube/config/config.inc.php +++ b/webmails/roundcube/config/config.inc.php @@ -28,7 +28,7 @@ $config['default_host'] = '{{ FRONT_ADDRESS or "front" }}'; $config['default_port'] = '10143'; // Sieve script management -$config['managesieve_host'] = '{{ IMAP_ADDRESS or "imap" }}'; +$config['managesieve_host'] = '{{ FRONT_ADDRESS or "front" }}:14190'; // We access the IMAP and SMTP servers locally with internal names, SSL // will obviously fail but this sounds better than allowing insecure login diff --git a/webmails/snappymail/defaults/default.json b/webmails/snappymail/defaults/default.json index ecbf116c..0d49bfb4 100644 --- a/webmails/snappymail/defaults/default.json +++ b/webmails/snappymail/defaults/default.json @@ -32,8 +32,8 @@ "usePhpMail": false }, "Sieve": { - "host": "{{ IMAP_ADDRESS }}", - "port": 4190, + "host": "{{ FRONT_ADDRESS }}", + "port": 14190, "secure": 0, "shortLogin": false, "ssl": { From f18776fa0faf639f2d00b68ab3b238602231f2e5 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 4 Jan 2023 15:03:47 +0100 Subject: [PATCH 30/47] Step2: put radicale and webmails on their own network --- setup/flavors/compose/docker-compose.yml | 28 ++++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 6eb4c409..87b4183b 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -36,6 +36,9 @@ services: - "{{ bind6 }}:{{ port }}:{{ port }}" {% endif %} {% endfor %} + networks: + - default + - webmail volumes: - "{{ root }}/certs:/certs" - "{{ root }}/overrides/nginx:/overrides:ro" @@ -169,12 +172,8 @@ services: env_file: {{ env }} volumes: - "{{ root }}/dav:/data" - {% if resolver_enabled %} - depends_on: - - resolver - dns: - - {{ dns }} - {% endif %} + networks: + - radicale {% endif %} {% if fetchmail_enabled %} @@ -204,13 +203,10 @@ services: volumes: - "{{ root }}/webmail:/data" - "{{ root }}/overrides/{{ webmail_type }}:/overrides:ro" + networks: + - webmail depends_on: - - imap - {% if resolver_enabled %} - - resolver - dns: - - {{ dns }} - {% endif %} + - front {% endif %} networks: @@ -226,6 +222,14 @@ networks: {% if ipv6_enabled %} - subnet: {{ subnet6 }} {% endif %} +{% if webdav_enabled %} + radicale: + driver: bridge +{% endif %} +{% if webmail_type != 'none' %} + webmail: + driver: bridge +{% endif %} {% if oletools_enabled %} noinet: driver: bridge From 8b9bb350ec7d3c2f6cd39886f23adb1f897a3af4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 4 Jan 2023 15:11:29 +0100 Subject: [PATCH 31/47] towncrier --- towncrier/newsfragments/2613.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/2613.feature diff --git a/towncrier/newsfragments/2613.feature b/towncrier/newsfragments/2613.feature new file mode 100644 index 00000000..453f59a3 --- /dev/null +++ b/towncrier/newsfragments/2613.feature @@ -0,0 +1 @@ +Isolate radicale and webmail on their own network. This ensures they don't have privileged access to any of the other containers. From 9d555b0eec2c87869599b3dce3c4e0295129b37f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 4 Jan 2023 19:19:43 +0100 Subject: [PATCH 32/47] Don't expose any port (suggestion from ghost) --- core/admin/Dockerfile | 2 +- core/nginx/Dockerfile | 3 ++- core/oletools/Dockerfile | 2 +- core/postfix/Dockerfile | 2 +- core/rspamd/Dockerfile | 2 +- optional/clamav/Dockerfile | 2 +- optional/radicale/Dockerfile | 2 +- optional/unbound/Dockerfile | 2 +- webmails/Dockerfile | 2 +- 9 files changed, 10 insertions(+), 9 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 600c3e9f..e6d70e61 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -22,7 +22,7 @@ RUN set -euxo pipefail \ RUN echo $VERSION >/version -EXPOSE 80/tcp +#EXPOSE 80/tcp HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index VOLUME ["/data","/dkim"] diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 27757920..76c3906a 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -26,7 +26,8 @@ COPY *.py / RUN echo $VERSION >/version -EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 14190/tcp 10025/tcp 10143/tcp +EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp +# EXPOSE 10025/tcp 10143/tcp 14190/tcp HEALTHCHECK --start-period=60s CMD curl -skfLo /dev/null http://localhost/health VOLUME ["/certs", "/overrides"] diff --git a/core/oletools/Dockerfile b/core/oletools/Dockerfile index 8bb98cd9..8526e506 100644 --- a/core/oletools/Dockerfile +++ b/core/oletools/Dockerfile @@ -14,7 +14,7 @@ RUN set -euxo pipefail \ RUN echo $VERSION >/version HEALTHCHECK --start-period=60s CMD echo PING|nc -q1 127.0.0.1 11343|grep "PONG" -EXPOSE 11343/tcp +#EXPOSE 11343/tcp USER nobody:nobody diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index df902dd4..8565d865 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -14,7 +14,7 @@ COPY start.py / RUN echo $VERSION >/version -EXPOSE 25/tcp 10025/tcp +#EXPOSE 25/tcp 10025/tcp HEALTHCHECK --start-period=350s CMD /usr/sbin/postfix status VOLUME ["/queue"] diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index eca8e62b..08ac0871 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -15,7 +15,7 @@ COPY start.py / RUN echo $VERSION >/version -EXPOSE 11332/tcp 11334/tcp 11335/tcp +#EXPOSE 11332/tcp 11334/tcp 11335/tcp HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost:11334/ VOLUME ["/var/lib/rspamd"] diff --git a/optional/clamav/Dockerfile b/optional/clamav/Dockerfile index 9beded99..bfe02780 100644 --- a/optional/clamav/Dockerfile +++ b/optional/clamav/Dockerfile @@ -14,7 +14,7 @@ COPY start.py / RUN echo $VERSION >/version -EXPOSE 3310/tcp +#EXPOSE 3310/tcp HEALTHCHECK --start-period=350s CMD echo PING|nc localhost 3310|grep "PONG" VOLUME ["/data"] diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index 56606494..904e47db 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -10,7 +10,7 @@ COPY radicale.conf / RUN echo $VERSION >/version -EXPOSE 5232/tcp +#EXPOSE 5232/tcp HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 VOLUME ["/data"] diff --git a/optional/unbound/Dockerfile b/optional/unbound/Dockerfile index 831476ab..95c63707 100644 --- a/optional/unbound/Dockerfile +++ b/optional/unbound/Dockerfile @@ -18,7 +18,7 @@ COPY start.py / RUN echo $VERSION >/version -EXPOSE 53/udp 53/tcp +#EXPOSE 53/udp 53/tcp HEALTHCHECK CMD dig @127.0.0.1 || exit 1 CMD /start.py diff --git a/webmails/Dockerfile b/webmails/Dockerfile index fb0d9090..9dc3514a 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -86,7 +86,7 @@ COPY php-webmail.conf /etc/php81/php-fpm.d/ COPY nginx-webmail.conf /conf/ COPY snuffleupagus.rules /etc/snuffleupagus.rules.tpl -EXPOSE 80/tcp +# EXPOSE 80/tcp VOLUME /data VOLUME /overrides From ee6975b109a3e8392b7b6e06e0afa1dec834e8d5 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 5 Jan 2023 18:14:19 +0100 Subject: [PATCH 33/47] doh --- core/postfix/conf/main.cf | 2 ++ core/postfix/conf/master.cf | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 2c9f71f3..7a2d0fe6 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -121,6 +121,8 @@ smtpd_relay_restrictions = unverified_recipient_reject_reason = Address lookup failure +smtpd_authorized_xclient_hosts={{ SUBNET}},{{ SUBNET6 }} + ############### # Milter ############### diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index 21d42b1b..86659460 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -11,7 +11,6 @@ smtp inet n - n - 1 smtpd -o smtpd_client_restrictions=$check_ratelimit,reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit -o smtpd_reject_unlisted_recipient={% if REJECT_UNLISTED_RECIPIENT %}{{ REJECT_UNLISTED_RECIPIENT }}{% else %}no{% endif %} -o cleanup_service_name=outclean - -o smtpd_authorized_xclient_hosts={{ SUBNET}},{{ SUBNET6 }} outclean unix n - n - 0 cleanup -o header_checks=pcre:/etc/postfix/outclean_header_filter.cf -o nested_header_checks= From 052f8e41ba4cd46bc49c277124a4fd8fcbbd2449 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 10 Jan 2023 12:28:38 +0100 Subject: [PATCH 34/47] Upgrade to snuffleupagus 0.9.0 --- core/base/Dockerfile | 2 +- towncrier/newsfragments/2618.misc | 1 + webmails/snuffleupagus.rules | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 towncrier/newsfragments/2618.misc diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 82c8cb9b..7919738a 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -60,7 +60,7 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" COPY requirements-${MAILU_DEPS}.txt ./ COPY libs/ libs/ -ARG SNUFFLEUPAGUS_VERSION=0.8.3 +ARG SNUFFLEUPAGUS_VERSION=0.9.0 ENV SNUFFLEUPAGUS_URL https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v$SNUFFLEUPAGUS_VERSION.tar.gz RUN set -euxo pipefail \ diff --git a/towncrier/newsfragments/2618.misc b/towncrier/newsfragments/2618.misc new file mode 100644 index 00000000..bb1d340a --- /dev/null +++ b/towncrier/newsfragments/2618.misc @@ -0,0 +1 @@ +Upgrade to snuffleupagus 0.9.0 diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index a2304fe0..cec99c29 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -130,5 +130,4 @@ sp.cookie.name("roundcube_sessid").samesite("strict"); sp.ini_protection.policy_silent_fail(); # roundcube uses unserialize() everywhere. -# This should do the job until https://github.com/jvoisin/snuffleupagus/issues/438 is implemented. -sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:[\"{]").drop(); +sp.unserialize_noclass.enable(); From e76e857ae75034ac8c6ccff57df6157379850c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vet=C3=A9si=20Zolt=C3=A1n?= Date: Wed, 11 Jan 2023 18:05:19 +0100 Subject: [PATCH 35/47] Fix smtplib.LMTP wrong argument name: ip -> host --- core/admin/mailu/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index f9fabb7f..8022709b 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -421,7 +421,7 @@ class Email(object): """ send an email to the address """ try: f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' - with smtplib.LMTP(ip=app.config['IMAP_ADDRESS'], port=2525) as lmtp: + with smtplib.LMTP(host=app.config['IMAP_ADDRESS'], port=2525) as lmtp: to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' msg = text.MIMEText(body) msg['Subject'] = subject From b0569035aecdbf0b8a312d5fd35b8fb0518562c0 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 12 Jan 2023 10:55:49 +0000 Subject: [PATCH 36/47] Change PUT method to PATCH method. This better reflects what the interface does. --- core/admin/mailu/api/v1/alias.py | 5 ++--- core/admin/mailu/api/v1/domains.py | 2 +- core/admin/mailu/api/v1/relay.py | 2 +- core/admin/mailu/api/v1/user.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/core/admin/mailu/api/v1/alias.py b/core/admin/mailu/api/v1/alias.py index 3605c68f..600ccc04 100644 --- a/core/admin/mailu/api/v1/alias.py +++ b/core/admin/mailu/api/v1/alias.py @@ -77,7 +77,7 @@ class Alias(Resource): @alias.response(400, 'Input validation exception', response_fields) @alias.doc(security='Bearer') @common.api_token_authorization - def put(self, alias): + def patch(self, alias): """ Update alias """ data = api.payload alias_found = models.Alias.query.filter_by(email = alias).first() @@ -86,8 +86,7 @@ class Alias(Resource): if 'comment' in data: alias_found.comment = data['comment'] if 'destination' in data: - destination_csl = ",".join(data['destination']) - alias_found.destination = destination_csl + alias_found.destination = data['destination'] if 'wildcard' in data: alias_found.wildcard = data['wildcard'] db.session.add(alias_found) diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py index 76554a02..7043da3d 100644 --- a/core/admin/mailu/api/v1/domains.py +++ b/core/admin/mailu/api/v1/domains.py @@ -152,7 +152,7 @@ class Domain(Resource): @dom.response(409, 'Duplicate domain/alternative name', response_fields) @dom.doc(security='Bearer') @common.api_token_authorization - def put(self, domain): + def patch(self, domain): """ Update an existing domain """ if not validators.domain(domain): return { 'code': 400, 'message': f'Domain {domain} is not a valid domain'}, 400 diff --git a/core/admin/mailu/api/v1/relay.py b/core/admin/mailu/api/v1/relay.py index f4588d22..356f8426 100644 --- a/core/admin/mailu/api/v1/relay.py +++ b/core/admin/mailu/api/v1/relay.py @@ -80,7 +80,7 @@ class Relay(Resource): @relay.response(409, 'Duplicate relay', response_fields) @relay.doc(security='Bearer') @common.api_token_authorization - def put(self, name): + def patch(self, name): """ Update relay """ data = api.payload diff --git a/core/admin/mailu/api/v1/user.py b/core/admin/mailu/api/v1/user.py index 953d246b..8e3d00a9 100644 --- a/core/admin/mailu/api/v1/user.py +++ b/core/admin/mailu/api/v1/user.py @@ -184,7 +184,7 @@ class User(Resource): @user.response(409, 'Duplicate user', response_fields) @user.doc(security='Bearer') @common.api_token_authorization - def put(self, email): + def patch(self, email): """ Update user """ data = api.payload if not validators.email(email): From 3b08b113bfbea68df632efb710e0237b1b28b2f4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 12 Jan 2023 15:15:52 +0100 Subject: [PATCH 37/47] Fix ipv6 subnet for xclient_hosts --- core/postfix/conf/main.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 7a2d0fe6..37fa70d0 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -121,7 +121,7 @@ smtpd_relay_restrictions = unverified_recipient_reject_reason = Address lookup failure -smtpd_authorized_xclient_hosts={{ SUBNET}},{{ SUBNET6 }} +smtpd_authorized_xclient_hosts={{ SUBNET}}{% if SUBNET6 %},[{{ SUBNET6 }}]{% endif %} ############### # Milter From d558be20f68afddf2de3696ad45d371877110daf Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 12 Jan 2023 15:16:53 +0100 Subject: [PATCH 38/47] Move runtime environment variables to the end --- core/base/Dockerfile | 52 +++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 7919738a..50316720 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -13,27 +13,9 @@ RUN set -euxo pipefail \ ; addgroup -Sg ${MAILU_GID} mailu \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \ - ; machine="$(uname -m)" \ - ; ! [[ "${machine}" == x86_64 ]] \ + ; ! [[ "$(uname -m)" == x86_64 ]] \ || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0 -ENV \ - LD_PRELOAD="/usr/lib/libhardened_malloc.so" \ - CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \ - CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \ - CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" \ - LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" \ - ADMIN_ADDRESS="admin" \ - FRONT_ADDRESS="front" \ - SMTP_ADDRESS="smtp" \ - IMAP_ADDRESS="imap" \ - OLETOOLS_ADDRESS="oletools" \ - REDIS_ADDRESS="redis" \ - ANTIVIRUS_ADDRESS="antivirus" \ - ANTISPAM_ADDRESS="antispam" \ - WEBMAIL_ADDRESS="webmail" \ - WEBDAV_ADDRESS="webdav" - WORKDIR /app CMD /bin/bash @@ -43,6 +25,7 @@ CMD /bin/bash FROM system as build ARG MAILU_DEPS=prod +ARG SNUFFLEUPAGUS_VERSION=0.9.0 ENV VIRTUAL_ENV=/app/venv @@ -55,13 +38,16 @@ RUN set -euxo pipefail \ ; apk del -r py3-pip \ ; rm -f /tmp/*.pem -ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" - COPY requirements-${MAILU_DEPS}.txt ./ COPY libs/ libs/ -ARG SNUFFLEUPAGUS_VERSION=0.9.0 -ENV SNUFFLEUPAGUS_URL https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v$SNUFFLEUPAGUS_VERSION.tar.gz +ENV \ + PATH="${VIRTUAL_ENV}/bin:${PATH}" \ + CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \ + CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \ + CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" \ + LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" \ + SNUFFLEUPAGUS_URL="https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v${SNUFFLEUPAGUS_VERSION}.tar.gz" RUN set -euxo pipefail \ ; machine="$(uname -m)" \ @@ -73,8 +59,8 @@ RUN set -euxo pipefail \ mkdir -p /root/.cargo/registry/index && \ git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ ; pip install -r requirements-${MAILU_DEPS}.txt \ - ; curl -sL ${SNUFFLEUPAGUS_URL} | tar xz \ - ; cd snuffleupagus-$SNUFFLEUPAGUS_VERSION \ + ; curl -sL ${SNUFFLEUPAGUS_URL} | tar xz \ + ; cd snuffleupagus-${SNUFFLEUPAGUS_VERSION} \ ; rm -rf src/tests/*php7*/ src/tests/*session*/ src/tests/broken_configuration/ src/tests/*cookie* src/tests/upload_validation/ \ ; apk add --virtual .build-deps php81-dev php81-cgi php81-simplexml php81-xml pcre-dev build-base php81-pear php81-openssl re2c \ ; pecl install vld-beta \ @@ -89,5 +75,17 @@ COPY --from=build /app/venv/ /app/venv/ COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/ RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn 'cap_net_bind_service=+ep' /usr/bin/python3.10 -ENV VIRTUAL_ENV=/app/venv -ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" +ENV \ + VIRTUAL_ENV=/app/venv \ + PATH="${VIRTUAL_ENV}/bin:${PATH}" \ + LD_PRELOAD="/usr/lib/libhardened_malloc.so" \ + ADMIN_ADDRESS="admin" \ + FRONT_ADDRESS="front" \ + SMTP_ADDRESS="smtp" \ + IMAP_ADDRESS="imap" \ + OLETOOLS_ADDRESS="oletools" \ + REDIS_ADDRESS="redis" \ + ANTIVIRUS_ADDRESS="antivirus" \ + ANTISPAM_ADDRESS="antispam" \ + WEBMAIL_ADDRESS="webmail" \ + WEBDAV_ADDRESS="webdav" From 712679b4d8a08ab35af9eecdb8a0e8d8b16a8c68 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 12 Jan 2023 18:19:35 +0100 Subject: [PATCH 39/47] Duh --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 50316720..3b394c87 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -77,7 +77,7 @@ RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn 'cap_net_bind_servi ENV \ VIRTUAL_ENV=/app/venv \ - PATH="${VIRTUAL_ENV}/bin:${PATH}" \ + PATH="/app/venv/bin:${PATH}" \ LD_PRELOAD="/usr/lib/libhardened_malloc.so" \ ADMIN_ADDRESS="admin" \ FRONT_ADDRESS="front" \ From 1697da6e23c317e2068fdaeca12139cfe1879935 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 23 Jan 2023 20:50:56 +0100 Subject: [PATCH 40/47] Disable "Fetched accounts" button in user list. --- core/admin/mailu/ui/templates/user/list.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/admin/mailu/ui/templates/user/list.html b/core/admin/mailu/ui/templates/user/list.html index 1c845062..873c37e8 100644 --- a/core/admin/mailu/ui/templates/user/list.html +++ b/core/admin/mailu/ui/templates/user/list.html @@ -36,7 +36,9 @@     + {%- if config["FETCHMAIL_ENABLED"] -%}   + {%- endif -%} {{ user }} From 10562233caf1b4a9d67db55202751281f788d4b8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 24 Jan 2023 12:15:36 +0100 Subject: [PATCH 41/47] Add SUBNET6 to places where SUBNET is used --- core/admin/mailu/configuration.py | 4 ++-- core/postfix/conf/main.cf | 4 ++-- core/rspamd/conf/worker-controller.inc | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 59b692ec..c992fbc8 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -86,7 +86,7 @@ DEFAULT_CONFIG = { 'PROXY_AUTH_HEADER': 'X-Auth-Email', 'PROXY_AUTH_CREATE': False, 'SUBNET': '192.168.203.0/24', - 'SUBNET6': None + 'SUBNET6': None, } class ConfigManager: @@ -96,7 +96,7 @@ class ConfigManager: DB_TEMPLATES = { 'sqlite': 'sqlite:////{SQLITE_DATABASE_FILE}', 'postgresql': 'postgresql://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}', - 'mysql': 'mysql+mysqlconnector://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}' + 'mysql': 'mysql+mysqlconnector://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}', } def __init__(self): diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 37fa70d0..0f6fd392 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -14,7 +14,7 @@ queue_directory = /queue message_size_limit = {{ MESSAGE_SIZE_LIMIT }} # Relayed networks -mynetworks = 127.0.0.1/32 [::1]/128 {{ SUBNET }} {% if RELAYNETS %}{{ RELAYNETS.split(",") | join(' ') }}{% endif %} +mynetworks = 127.0.0.1/32 [::1]/128 {{ SUBNET }} {% if SUBNET6 %}{{ "[{}]/{}".format(*SUBNET6.split("/")) }}{% endif %} {% if RELAYNETS %}{{ RELAYNETS.split(",") | join(" ") }}{% endif %} # Empty alias list to override the configuration variable and disable NIS alias_maps = @@ -121,7 +121,7 @@ smtpd_relay_restrictions = unverified_recipient_reject_reason = Address lookup failure -smtpd_authorized_xclient_hosts={{ SUBNET}}{% if SUBNET6 %},[{{ SUBNET6 }}]{% endif %} +smtpd_authorized_xclient_hosts={{ SUBNET }}{% if SUBNET6 %},[{{ SUBNET6 }}]{% endif %} ############### # Milter diff --git a/core/rspamd/conf/worker-controller.inc b/core/rspamd/conf/worker-controller.inc index a720c3df..d1bb251d 100644 --- a/core/rspamd/conf/worker-controller.inc +++ b/core/rspamd/conf/worker-controller.inc @@ -2,3 +2,6 @@ type = "controller"; bind_socket = "*:11334"; password = "mailu"; secure_ip = "{{ SUBNET }}"; +{%- if SUBNET6 %} +secure_ip = "{{ SUBNET6 }}"; +{%- endif %} From 5c968256e658c95d1517cb2f1d03dae40fce9997 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 28 Dec 2022 17:44:16 +0100 Subject: [PATCH 42/47] Really fix creation of deep structures using import in update mode --- core/admin/mailu/schemas.py | 78 ++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 2af3f03b..878164b3 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -5,6 +5,7 @@ from copy import deepcopy from collections import Counter from datetime import timezone +import inspect import json import logging import yaml @@ -669,20 +670,15 @@ class Storage: context = {} - def _bind(self, key, bind): - if bind is True: - return (self.__class__, key) - if isinstance(bind, str): - return (get_schema(self.recall(bind).__class__), key) - return (bind, key) - - def store(self, key, value, bind=None): + def store(self, key, value): """ store value under key """ - self.context.setdefault('_track', {})[self._bind(key, bind)]= value + key = f'{self.__class__.__name__}.{key}' + self.context.setdefault('_track', {})[key] = value - def recall(self, key, bind=None): + def recall(self, key): """ recall value from key """ - return self.context['_track'][self._bind(key, bind)] + key = f'{self.__class__.__name__}.{key}' + return self.context['_track'][key] class BaseOpts(SQLAlchemyAutoSchemaOpts): """ Option class with sqla session @@ -790,10 +786,16 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): for key, value in data.items() } - def _call_and_store(self, *args, **kwargs): - """ track current parent field for pruning """ - self.store('field', kwargs['field_name'], True) - return super()._call_and_store(*args, **kwargs) + def get_parent(self): + """ helper to determine parent of current object """ + for x in inspect.stack(): + loc = x[0].f_locals + if 'ret_d' in loc: + if isinstance(loc['self'], MailuSchema): + return self.context.get('config'), loc['attr_name'] + else: + return loc['self'].get_instance(loc['ret_d']), loc['attr_name'] + return None, None # this is only needed to work around the declared attr "email" primary key in model def get_instance(self, data): @@ -803,9 +805,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): if keys := getattr(self.Meta, 'primary_keys', None): filters = {key: data.get(key) for key in keys} if None not in filters.values(): - res= self.session.query(self.opts.model).filter_by(**filters).first() - return res - res= super().get_instance(data) + try: + res = self.session.query(self.opts.model).filter_by(**filters).first() + except sqlalchemy.exc.StatementError as exc: + raise ValidationError(f'Invalid {keys[0]}: {data.get(keys[0])!r}', data.get(keys[0])) from exc + else: + return res + res = super().get_instance(data) return res @pre_load(pass_many=True) @@ -829,6 +835,10 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): want_prune = [] def patch(count, data): + # we only process objects here + if type(data) is not dict: + raise ValidationError(f'Invalid item. {self.Meta.model.__tablename__.title()} needs to be an object.', f'{data!r}') + # don't allow __delete__ coming from input if '__delete__' in data: raise ValidationError('Unknown field.', f'{count}.__delete__') @@ -882,10 +892,10 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): ] # remember if prune was requested for _prune_items@post_load - self.store('prune', bool(want_prune), True) + self.store('prune', bool(want_prune)) # remember original items to stabilize password-changes in _add_instance@post_load - self.store('original', items, True) + self.store('original', items) return items @@ -909,23 +919,18 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): # stabilize import of auto-increment primary keys (not required), # by matching import data to existing items and setting primary key if not self._primary in data: - parent = self.recall('parent') + parent, field = self.get_parent() if parent is not None: - for item in getattr(parent, self.recall('field', 'parent')): + for item in getattr(parent, field): existing = self.dump(item, many=False) this = existing.pop(self._primary) if data == existing: - instance = item + self.instance = item data[self._primary] = this break # try to load instance instance = self.instance or self.get_instance(data) - - # remember instance as parent for pruning siblings - if not self.Meta.sibling and self.context.get('update'): - self.store('parent', instance) - if instance is None: if '__delete__' in data: @@ -1001,7 +1006,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): return items # get prune flag from _patch_many@pre_load - want_prune = self.recall('prune', True) + want_prune = self.recall('prune') # prune: determine if existing items in db need to be added or marked for deletion add_items = False @@ -1018,16 +1023,17 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): del_items = True if add_items or del_items: - parent = self.recall('parent') + parent, field = self.get_parent() if parent is not None: existing = {item[self._primary] for item in items if self._primary in item} - for item in getattr(parent, self.recall('field', 'parent')): + for item in getattr(parent, field): key = getattr(item, self._primary) if key not in existing: if add_items: items.append({self._primary: key}) else: - items.append({self._primary: key, '__delete__': '?'}) + if self.context.get('update'): + self.opts.sqla_session.delete(self.instance or self.get_instance({self._primary: key})) return items @@ -1048,7 +1054,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): # did we hash a new plaintext password? original = None pkey = getattr(item, self._primary) - for data in self.recall('original', True): + for data in self.recall('original'): if 'hash_password' in data and data.get(self._primary) == pkey: original = data['password'] break @@ -1244,12 +1250,6 @@ class MailuSchema(Schema, Storage): if field in fieldlist: fieldlist[field] = fieldlist.pop(field) - def _call_and_store(self, *args, **kwargs): - """ track current parent and field for pruning """ - self.store('field', kwargs['field_name'], True) - self.store('parent', self.context.get('config')) - return super()._call_and_store(*args, **kwargs) - @pre_load def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument """ create config object in context if missing From c4ca1cffafe29cedaec72ba918b068a7c83234dd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 25 Jan 2023 12:20:17 +0100 Subject: [PATCH 43/47] Set default for FETCHMAIL_ENABLED --- core/admin/mailu/configuration.py | 2 +- docs/configuration.rst | 5 +++-- docs/webadministration.rst | 1 + setup/flavors/compose/mailu.env | 2 +- tests/compose/core/mailu.env | 3 +++ tests/compose/fetchmail/mailu.env | 3 +++ tests/compose/filters/mailu.env | 3 +++ tests/compose/webdav/mailu.env | 3 +++ tests/compose/webmail/mailu.env | 3 +++ 9 files changed, 21 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index c992fbc8..004fe504 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -17,7 +17,7 @@ DEFAULT_CONFIG = { 'DOMAIN_REGISTRATION': False, 'TEMPLATES_AUTO_RELOAD': True, 'MEMORY_SESSIONS': False, - 'FETCHMAIL_ENABLED': False, + 'FETCHMAIL_ENABLED': True, # Database settings 'DB_FLAVOR': None, 'DB_USER': 'mailu', diff --git a/docs/configuration.rst b/docs/configuration.rst index b37bec08..882658ff 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -110,8 +110,9 @@ to reject emails containing documents with malicious macros. Under the hood, it .. _fetchmail: -When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is enabled in the admin interface. -The container itself still needs to be deployed manually. ``FETCHMAIL_ENABLED`` defaults to ``True``. +When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is enabled and +shown in the admin interface. The container itself still needs to be deployed manually. +``FETCHMAIL_ENABLED`` defaults to ``True``. The ``FETCHMAIL_DELAY`` is a delay (in seconds) for the fetchmail service to go and fetch new email if available. Do not use too short delays if you do not diff --git a/docs/webadministration.rst b/docs/webadministration.rst index 7e2a5728..6e5ca94d 100644 --- a/docs/webadministration.rst +++ b/docs/webadministration.rst @@ -162,6 +162,7 @@ You can add a fetched account by clicking on the `Add an account` button on the * Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled. Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``. +Make sure ``FETCHMAIL_ENABLED`` is set to ``true`` in ``mailu.env`` to enable fetching and showing fetchmail in the admin interface. Authentication tokens diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 980788ce..f1ecf4e2 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -82,7 +82,7 @@ RELAYNETS= # Will relay all outgoing mails if configured RELAYHOST={{ relayhost }} -# Show fetchmail functionality in admin interface +# Enable fetchmail FETCHMAIL_ENABLED={{ fetchmail_enabled or 'False' }} # Fetchmail delay diff --git a/tests/compose/core/mailu.env b/tests/compose/core/mailu.env index 7ea86f56..134b8e84 100644 --- a/tests/compose/core/mailu.env +++ b/tests/compose/core/mailu.env @@ -83,6 +83,9 @@ RELAYNETS= # Will relay all outgoing mails if configured RELAYHOST= +# Show fetchmail functionality in admin interface +FETCHMAIL_ENABLED=false + # Fetchmail delay FETCHMAIL_DELAY=600 diff --git a/tests/compose/fetchmail/mailu.env b/tests/compose/fetchmail/mailu.env index 0355f168..6f1e8b69 100644 --- a/tests/compose/fetchmail/mailu.env +++ b/tests/compose/fetchmail/mailu.env @@ -83,6 +83,9 @@ RELAYNETS= # Will relay all outgoing mails if configured RELAYHOST= +# Show fetchmail functionality in admin interface +FETCHMAIL_ENABLED=false + # Fetchmail delay FETCHMAIL_DELAY=600 diff --git a/tests/compose/filters/mailu.env b/tests/compose/filters/mailu.env index cf9f7f2d..2df09f61 100644 --- a/tests/compose/filters/mailu.env +++ b/tests/compose/filters/mailu.env @@ -83,6 +83,9 @@ RELAYNETS= # Will relay all outgoing mails if configured RELAYHOST= +# Show fetchmail functionality in admin interface +FETCHMAIL_ENABLED=false + # Fetchmail delay FETCHMAIL_DELAY=600 diff --git a/tests/compose/webdav/mailu.env b/tests/compose/webdav/mailu.env index b7a9b718..f0842dd4 100644 --- a/tests/compose/webdav/mailu.env +++ b/tests/compose/webdav/mailu.env @@ -83,6 +83,9 @@ RELAYNETS= # Will relay all outgoing mails if configured RELAYHOST= +# Show fetchmail functionality in admin interface +FETCHMAIL_ENABLED=false + # Fetchmail delay FETCHMAIL_DELAY=600 diff --git a/tests/compose/webmail/mailu.env b/tests/compose/webmail/mailu.env index ddc845fa..c1e0dff0 100644 --- a/tests/compose/webmail/mailu.env +++ b/tests/compose/webmail/mailu.env @@ -83,6 +83,9 @@ RELAYNETS= # Will relay all outgoing mails if configured RELAYHOST= +# Show fetchmail functionality in admin interface +FETCHMAIL_ENABLED=false + # Fetchmail delay FETCHMAIL_DELAY=600 From d6e7314f05f8a2702efab925c33146a7c1e1b5e8 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 25 Jan 2023 15:26:10 +0000 Subject: [PATCH 44/47] Make API configurable via the setup utility Fix some small bugs in the setup utility Improve documentation on the API. --- core/admin/mailu/__init__.py | 3 +-- core/admin/mailu/api/__init__.py | 32 ++++++++++++++--------- docs/api.rst | 20 ++++++++------- docs/configuration.rst | 14 +++++------ setup/flavors/compose/mailu.env | 10 ++++++++ setup/static/render.js | 42 +++++++++++++++++++++++++------ setup/templates/steps/config.html | 13 ++++++++++ 7 files changed, 96 insertions(+), 38 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index de3503d0..e29eff91 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -74,8 +74,7 @@ def create_app_from_config(config): app.register_blueprint(ui.ui, url_prefix=app.config['WEB_ADMIN']) app.register_blueprint(internal.internal, url_prefix='/internal') app.register_blueprint(sso.sso, url_prefix='/sso') - if app.config.get('API_TOKEN'): - api.register(app, web_api_root=app.config.get('WEB_API')) + api.register(app, web_api_root=app.config.get('WEB_API')) return app diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py index da7325ae..0465c783 100644 --- a/core/admin/mailu/api/__init__.py +++ b/core/admin/mailu/api/__init__.py @@ -8,17 +8,25 @@ def register(app, web_api_root): # register api bluprint(s) apidoc.apidoc.url_prefix = f'{web_api_root}/v{int(APIv1.VERSION)}' APIv1.api_token = app.config['API_TOKEN'] - app.register_blueprint(APIv1.blueprint, url_prefix=f'{web_api_root}/v{int(APIv1.VERSION)}') + if app.config['API_TOKEN'] != '': + app.register_blueprint(APIv1.blueprint, url_prefix=f'{web_api_root}/v{int(APIv1.VERSION)}') - # add redirect to current api version - redirect_api = Blueprint('redirect_api', __name__) - @redirect_api.route('/') - def redir(): - return redirect(url_for(f'{APIv1.blueprint.name}.root')) - app.register_blueprint(redirect_api, url_prefix=f'{web_api_root}') + # add redirect to current api version + redirect_api = Blueprint('redirect_api', __name__) + @redirect_api.route('/') + def redir(): + return redirect(url_for(f'{APIv1.blueprint.name}.root')) + app.register_blueprint(redirect_api, url_prefix=f'{web_api_root}') - # swagger ui config - app.config.SWAGGER_UI_DOC_EXPANSION = 'list' - app.config.SWAGGER_UI_OPERATION_ID = True - app.config.SWAGGER_UI_REQUEST_DURATION = True - app.config.RESTX_MASK_SWAGGER = False + # swagger ui config + app.config.SWAGGER_UI_DOC_EXPANSION = 'list' + app.config.SWAGGER_UI_OPERATION_ID = True + app.config.SWAGGER_UI_REQUEST_DURATION = True + app.config.RESTX_MASK_SWAGGER = False + else: + api = Blueprint('api', __name__) + @api.route('/', defaults={'path': ''}) + @api.route('/') + def api_token_missing(path): + return "

Error: API_TOKEN is not configured

", 500 + app.register_blueprint(api, url_prefix=f'{web_api_root}') diff --git a/docs/api.rst b/docs/api.rst index b18398ac..e5a18a03 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -5,21 +5,22 @@ Mailu offers a RESTful API for changing the Mailu configuration. Anything that can be configured via the Mailu web administration interface, can also be configured via the API. -The Mailu API is disabled by default. It can be enabled and configured via -the settings: +The Mailu API can be configured via the setup utility (setup.mailu.io). +It can also be manually configured via mailu.env: -* ``API`` -* ``WEB_API`` -* ``API_TOKEN`` +* ``API`` - Expose the API interface (value: true, false) +* ``WEB_API`` - Path to the API interface +* ``API_TOKEN`` - API token for authentication -For more information see the section :ref:`Advanced configuration ` -in the configuration reference. +For more information refer to the detailed descriptions in the +:ref:`configuration reference `. Swagger.json ------------ -The swagger.json file can be retrieved via: https://myserver/api/v1/swagger.json. +The swagger.json file can be retrieved via: https://myserver/api/v1/swagger.json +(WEB_API=/api) The swagger.json file can be consumed in programs such as Postman for generating all API calls. @@ -28,4 +29,5 @@ In-built SwaggerUI The Mailu API comes with an in-built SwaggerUI. It is a web client that allows anyone to visualize and interact with the Mailu API. -It is accessible via the URL: https://myserver/api/v1/swaggerui +Assuming ``/api`` is configured as value for ``WEB_API``, it +is accessible via the URL: https://myserver/api/ diff --git a/docs/configuration.rst b/docs/configuration.rst index b37bec08..51b2554d 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -140,13 +140,15 @@ Web settings - ``WEB_WEBMAIL`` contains the path to the Web email client. +- ``WEB_API`` contains the path to the RESTful API. + - ``WEBROOT_REDIRECT`` redirects all non-found queries to the set path. An empty ``WEBROOT_REDIRECT`` value disables redirecting and enables classic behavior of a 404 result when not found. Alternatively, ``WEBROOT_REDIRECT`` can be set to ``none`` if you are using an Nginx override for ``location /``. -All three options need a leading slash (``/``) to work. +All four options need a leading slash (``/``) to work. .. note:: ``WEBROOT_REDIRECT`` has to point to a valid path on the webserver. This means it cannot point to any services which are not enabled. @@ -203,13 +205,9 @@ Depending on your particular deployment you most probably will want to change th Advanced settings ----------------- -The ``API`` (default: False) setting controls if the API endpoint is reachable. - -The ``WEB_API`` (default: /api) setting configures the endpoint that the API -listens on publicly&interally. The path must always start with a leading slash. - -The ``API_TOKEN`` (default: None) enables the API endpoint. This token must be -passed as request header to the API as authentication token. +The ``API_TOKEN`` (default: None) configures the authentication token. +This token must be passed as request header to the API as authentication token. +This is a mandatory setting for using the RESTful API. The ``CREDENTIAL_ROUNDS`` (default: 12) setting is the number of rounds used by the password hashing scheme. The number of rounds can be reduced in case faster diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 980788ce..4fb25dc0 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -52,6 +52,9 @@ ADMIN={{ admin_enabled or 'false' }} # Choose which webmail to run if any (values: roundcube, snappymail, none) WEBMAIL={{ webmail_type }} +# Expose the API interface (value: true, false) +API={{ api_enabled or 'false' }} + # Dav server implementation (value: radicale, none) WEBDAV={{ webdav_enabled or 'none' }} @@ -131,6 +134,9 @@ WEB_WEBMAIL=/ WEB_WEBMAIL={{ webmail_path }} {% endif %} +# Path to the API interface if enabled +WEB_API={{ api_path }} + # Website name SITENAME={{ site_name }} @@ -182,6 +188,10 @@ TZ=Etc/UTC # Default spam threshold used for new users DEFAULT_SPAM_THRESHOLD=80 +# API token required for authenticating to the RESTful API. +# This is a mandatory setting for using the RESTful API. +API_TOKEN={{ api_token }} + ################################### # Database settings ################################### diff --git a/setup/static/render.js b/setup/static/render.js index f1b8e0a5..2d847a2d 100644 --- a/setup/static/render.js +++ b/setup/static/render.js @@ -1,18 +1,18 @@ $(document).ready(function() { if ($("#webmail").val() == 'none') { $("#webmail_path").hide(); - $("#webmail_path").attr("value", ""); + $("#webmail_path").val(""); } else { $("#webmail_path").show(); - $("#webmail_path").attr("value", "/webmail"); + $("#webmail_path").val("/webmail"); } $("#webmail").click(function() { if (this.value == 'none') { $("#webmail_path").hide(); - $("#webmail_path").attr("value", ""); + $("#webmail_path").val(""); } else { $("#webmail_path").show(); - $("#webmail_path").attr("value", "/webmail"); + $("#webmail_path").val("/webmail"); } }); }); @@ -20,15 +20,43 @@ $(document).ready(function() { $(document).ready(function() { if ($('#admin').prop('checked')) { $("#admin_path").show(); - $("#admin_path").attr("value", "/admin"); + $("#admin_path").val("/admin"); } $("#admin").change(function() { if ($(this).is(":checked")) { $("#admin_path").show(); - $("#admin_path").attr("value", "/admin"); + $("#admin_path").val("/admin"); } else { $("#admin_path").hide(); - $("#admin_path").attr("value", ""); + $("#admin_path").val(""); + } + }); +}); + +$(document).ready(function() { + if ($('#api').prop('checked')) { + $("#api_path").show(); + $("#api_path").val("/api") + $("#api_token").show(); + $("#api_token").prop('required',true); + $("#api_token").val(""); + $("#api_token_label").show(); + } + $("#api").change(function() { + if ($(this).is(":checked")) { + $("#api_path").show(); + $("#api_path").val("/api"); + $("#api_token").show(); + $("#api_token").prop('required',true); + $("#api_token").val("") + $("#api_token_label").show(); + } else { + $("#api_path").hide(); + $("#api_path").val("/api") + $("#api_token").hide(); + $("#api_token").prop('required',false); + $("#api_token").val(""); + $("#api_token_label").hide(); } }); }); diff --git a/setup/templates/steps/config.html b/setup/templates/steps/config.html index 5e503230..83a3a813 100644 --- a/setup/templates/steps/config.html +++ b/setup/templates/steps/config.html @@ -87,6 +87,19 @@ manage your email domains, users, etc.

+

The API interface is a RESTful API for changing the Mailu configuration. + Anything that can be configured via the Mailu web administration interface, + can also be configured via the RESTful API. For enabling the API, an API token must be configured. + It is not possible to use the API without an API token.

+ +
+ + + + + +
+ From 18b900699cfa06bd9ac5a9aebe615de5877c250a Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Wed, 25 Jan 2023 16:12:14 +0000 Subject: [PATCH 45/47] Bump version of Flask-RESTX to 1.0.5. This resolves all deprecation warnings caused by Flask-RESTX. --- core/base/requirements-prod.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/requirements-prod.txt b/core/base/requirements-prod.txt index e9f664b4..5119520e 100644 --- a/core/base/requirements-prod.txt +++ b/core/base/requirements-prod.txt @@ -27,7 +27,7 @@ Flask-DebugToolbar==0.13.1 Flask-Login==0.6.2 flask-marshmallow==0.14.0 Flask-Migrate==3.1.0 -Flask-RESTX==1.0.3 +Flask-RESTX==1.0.5 Flask-SQLAlchemy==2.5.1 Flask-WTF==1.0.1 frozenlist==1.3.1 From 02c48624278f483cc5840f9be7386e84cf2a96e4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 25 Jan 2023 20:22:38 +0100 Subject: [PATCH 46/47] Enable fetchmail for fetchmail test case --- tests/compose/fetchmail/mailu.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compose/fetchmail/mailu.env b/tests/compose/fetchmail/mailu.env index 6f1e8b69..0cc263e2 100644 --- a/tests/compose/fetchmail/mailu.env +++ b/tests/compose/fetchmail/mailu.env @@ -84,10 +84,10 @@ RELAYNETS= RELAYHOST= # Show fetchmail functionality in admin interface -FETCHMAIL_ENABLED=false +FETCHMAIL_ENABLED=true # Fetchmail delay -FETCHMAIL_DELAY=600 +FETCHMAIL_DELAY=15 # Recipient delimiter, character used to delimiter localpart from custom address part RECIPIENT_DELIMITER=+ From 8cb7265eb2f7d8f2837801a717c0b36b7bf2b2c0 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 27 Jan 2023 13:17:36 +0000 Subject: [PATCH 47/47] By default disable the API in the setup utility. Generate a sample token value for API_TOKEN. Fix small rendering issue when API was disabled in setup. --- setup/static/render.js | 21 +++++++++++++++++++-- setup/templates/steps/config.html | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/setup/static/render.js b/setup/static/render.js index 2d847a2d..b2cdc7c8 100644 --- a/setup/static/render.js +++ b/setup/static/render.js @@ -1,3 +1,13 @@ +//API_TOKEN generator +var chars = "0123456789abcdefghijklmnopqrstuvwxyz!@#$%^&*()ABCDEFGHIJKLMNOPQRSTUVWXYZ"; +var tokenLength = 12; +var token = ""; + +for (var i = 0; i <= tokenLength; i++) { + var randomNumber = Math.floor(Math.random() * chars.length); + token += chars.substring(randomNumber, randomNumber +1); + } + $(document).ready(function() { if ($("#webmail").val() == 'none') { $("#webmail_path").hide(); @@ -39,8 +49,15 @@ $(document).ready(function() { $("#api_path").val("/api") $("#api_token").show(); $("#api_token").prop('required',true); - $("#api_token").val(""); + $("#api_token").val(token); $("#api_token_label").show(); + } else { + $("#api_path").hide(); + $("#api_path").val("/api") + $("#api_token").hide(); + $("#api_token").prop('required',false); + $("#api_token").val(""); + $("#api_token_label").hide(); } $("#api").change(function() { if ($(this).is(":checked")) { @@ -48,7 +65,7 @@ $(document).ready(function() { $("#api_path").val("/api"); $("#api_token").show(); $("#api_token").prop('required',true); - $("#api_token").val("") + $("#api_token").val(token) $("#api_token_label").show(); } else { $("#api_path").hide(); diff --git a/setup/templates/steps/config.html b/setup/templates/steps/config.html index 83a3a813..19736448 100644 --- a/setup/templates/steps/config.html +++ b/setup/templates/steps/config.html @@ -93,7 +93,7 @@ manage your email domains, users, etc.

It is not possible to use the API without an API token.

- +