From 262e82a36777e296811a0447b99c78381e36bb2a Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Tue, 3 Jul 2018 20:13:00 +0200 Subject: [PATCH 01/37] Add a postfix socketmap to http proxy --- core/postfix/conf/postproxy.py | 175 +++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100755 core/postfix/conf/postproxy.py diff --git a/core/postfix/conf/postproxy.py b/core/postfix/conf/postproxy.py new file mode 100755 index 00000000..83126353 --- /dev/null +++ b/core/postfix/conf/postproxy.py @@ -0,0 +1,175 @@ +#!/usr/bin/python3 + +# Postfix socketmap proxy +# +# This script provides a proxy from Postfix socketmap to a variety of backends. +# For now, only HTTP backends are supported. + +import asyncio +import aiohttp +import logging +import urllib +import argparse + + +class NetstringProtocol(asyncio.Protocol): + """ Netstring asyncio protocol implementation. + + For protocol details, see https://cr.yp.to/proto/netstrings.txt + """ + + # Length of the smallest allocated buffer, larger buffers will be + # allocated dynamically + BASE_BUFFER = 1024 + + # Maximum length of a buffer, will crash when exceeded + MAX_BUFFER = 65535 + + def __init__(self): + super(NetstringProtocol, self).__init__() + self.init_buffer() + + def init_buffer(self): + self.len = None # None when waiting for a length to be sent) + self.separator = -1 # -1 when not yet detected (str.find) + self.index = 0 # relative to the buffer + self.buffer = bytearray(NetstringProtocol.BASE_BUFFER) + + def data_received(self, data): + # Manage the buffer + missing = len(data) - len(self.buffer) + self.index + if missing > 0: + if len(self.buffer) + missing > NetstringProtocol.MAX_BUFFER: + raise IOError("Not enough space when decoding netstring") + self.buffer.append(bytearray(missing + 1)) + new_index = self.index + len(data) + self.buffer[self.index:new_index] = data + self.index = new_index + # Try to detect a length at the beginning of the string + if self.len is None: + self.separator = self.buffer.find(0x3a) + if self.separator != -1 and self.buffer[:self.separator].isdigit(): + self.len = int(self.buffer[:self.separator], 10) + # Then get the complete string + if self.len is not None: + if self.index - self.separator == self.len + 2: + string = self.buffer[self.separator + 1:self.index - 1] + self.init_buffer() + self.string_received(string) + + def string_received(self, string): + pass + + def send_string(self, string): + """ Send a netstring + """ + self.transport.write(str(len(string)).encode('ascii')) + self.transport.write(b':') + self.transport.write(string) + self.transport.write(b',') + + +class SocketmapProtocol(NetstringProtocol): + """ TCP protocol to answer Postfix socketmap and proxify lookups to + an outside object. + + See http://www.postfix.org/socketmap_table.5.html for details on the + protocol. + + A table map must be provided as a dictionary to lookup tables. + """ + + def __init__(self, table_map): + self.table_map = table_map + super(SocketmapProtocol, self).__init__() + + def connection_made(self, transport): + logging.info('Connect {}'.format(transport.get_extra_info('peername'))) + self.transport = transport + + def string_received(self, string): + space = string.find(0x20) + if space != -1: + name = string[:space].decode('ascii') + key = string[space+1:].decode('utf8') + return asyncio.async(self.process_request(name, key)) + + def send_string(self, string): + logging.debug("Send {}".format(string)) + super(SocketmapProtocol, self).send_string(string) + + async def process_request(self, name, key): + """ Process a request by querying the provided map. + """ + logging.debug("Request {}/{}".format(name, key)) + try: + table = self.table_map.get(name) + except KeyError: + return self.send_string(b'TEMP no such map') + try: + result = await table.get(key) + return self.send_string(b'OK ' + result.encode('utf8')) + except KeyError: + return self.send_string(b'NOTFOUND ') + except Exception: + logging.exception("Error when processing request") + return self.send_string(b'TEMP unknown error') + + @classmethod + def factory(cls, table_map): + """ Provide a protocol factory for a given map instance. + """ + return lambda: cls(table_map) + + +class UrlTable(object): + """ Resolve an entry by querying a parametrized GET URL. + """ + + def __init__(self, url_pattern): + """ url_pattern must contain a format ``{}`` so the key is injected in + the url before the query, the ``§`` character will be replaced with + ``{}`` for easier setup. + """ + self.url_pattern = url_pattern.replace('§', '{}') + + async def get(self, key): + async with aiohttp.ClientSession() as session: + async with session.get(self.url_pattern.format(key)) as request: + if request.status == 200: + result = await request.text() + return result + + +def main(): + """ Run the asyncio loop. + """ + parser = argparse.ArgumentParser("Postfix Socketmap proxy") + parser.add_argument("--bind", help="address to bind to", required=True) + parser.add_argument("--port", type=int, help="port to bind to", required=True) + parser.add_argument("--name", help="name of the table", action="append") + parser.add_argument("--type", help="type of the table", action="append") + parser.add_argument("--param", help="table parameter", action="append") + args = parser.parse_args() + # Prepare the maps + table_types = dict(url=UrlTable) + table_map = {name: table_types[table_type](param) + for name, table_type, param + in zip(args.name, args.type, args.param)} + # Run the main loop + logging.basicConfig(level=logging.DEBUG) + loop = asyncio.get_event_loop() + server = loop.run_until_complete(loop.create_server( + SocketmapProtocol.factory(table_map), args.bind, args.port + )) + try: + loop.run_forever() + except KeyboardInterrupt: + pass + server.close() + loop.run_until_complete(server.wait_closed()) + loop.close() + + +if __name__ == "__main__": + main() From 70175f8c28767de6cda5a3d4f49db256731166bd Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Sun, 15 Jul 2018 15:30:16 +0200 Subject: [PATCH 02/37] Add postproxy support for Dovecot dict protocol --- core/postfix/conf/postproxy.py | 94 ++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 9 deletions(-) diff --git a/core/postfix/conf/postproxy.py b/core/postfix/conf/postproxy.py index 83126353..234d8c34 100755 --- a/core/postfix/conf/postproxy.py +++ b/core/postfix/conf/postproxy.py @@ -70,7 +70,7 @@ class NetstringProtocol(asyncio.Protocol): class SocketmapProtocol(NetstringProtocol): - """ TCP protocol to answer Postfix socketmap and proxify lookups to + """ Protocol to answer Postfix socketmap and proxify lookups to an outside object. See http://www.postfix.org/socketmap_table.5.html for details on the @@ -122,6 +122,79 @@ class SocketmapProtocol(NetstringProtocol): return lambda: cls(table_map) +class DictProtocol(asyncio.Protocol): + """ Protocol to answer Dovecot dict requests, as implemented in Dict proxy. + + There is very little documentation about the protocol, most of it was + reverse-engineered from : + + https://github.com/dovecot/core/blob/master/src/dict/dict-connection.c + https://github.com/dovecot/core/blob/master/src/dict/dict-commands.c + https://github.com/dovecot/core/blob/master/src/lib-dict/dict-client.h + """ + + DATA_TYPES = {0: str, 1: int} + + def __init__(self, table_map): + self.table_map = table_map + self.major_version = None + self.minor_version = None + self.dict = None + super(DictProtocol, self).__init__() + + def connection_made(self, transport): + logging.info('Connect {}'.format(transport.get_extra_info('peername'))) + self.transport = transport + + def data_received(self, data): + logging.debug("Received {}".format(data)) + for line in data.split(b"\n"): + if len(line) < 2: + continue + command = DictProtocol.COMMANDS.get(line[0]) + if command is None: + logging.warning('Unknown command {}'.format(line[0])) + return self.transport.abort() + args = line[1:].strip().split(b"\t") + try: + command(self, *args) + except Exception: + logging.exception("Error when processing request") + return self.transport.abort() + + def process_hello(self, major, minor, value_type, user, dict_name): + self.major, self.minor = int(major), int(minor) + logging.debug('Client version {}.{}'.format(self.major, self.minor)) + assert self.major == 2 + self.value_type = DictProtocol.DATA_TYPES[int(value_type)] + self.user = user + self.dict = self.table_map[dict_name.decode("ascii")] + logging.debug("Value type {}, user {}, dict {}".format( + self.value_type, self.user, dict_name)) + + def process_lookup(self, key): + logging.debug("Looking up {}".format(key)) + self.reply(b"O", json.dumps({})) + + def reply(self, command, *args): + logging.debug("Replying {} with {}".format(command, args)) + self.transport.write(command) + for arg in args: + self.transport.write("b\t" + arg.replace(b"\t", b"\t\t")) + self.transport.write("\n") + + @classmethod + def factory(cls, table_map): + """ Provide a protocol factory for a given map instance. + """ + return lambda: cls(table_map) + + COMMANDS = { + ord("H"): process_hello, + ord("L"): process_lookup + } + + class UrlTable(object): """ Resolve an entry by querying a parametrized GET URL. """ @@ -144,23 +217,26 @@ class UrlTable(object): def main(): """ Run the asyncio loop. """ - parser = argparse.ArgumentParser("Postfix Socketmap proxy") - parser.add_argument("--bind", help="address to bind to", required=True) - parser.add_argument("--port", type=int, help="port to bind to", required=True) + # Reference tables + server_types = dict(postfix=SocketmapProtocol, dovecot=DictProtocol) + table_types = dict(url=UrlTable) + # Argument parsing + parser = argparse.ArgumentParser("Postfix and Dovecot map proxy") + parser.add_argument("--socket", help="path to a socket", required=True) + parser.add_argument("--mode", choices=server_types.keys(), required=True) parser.add_argument("--name", help="name of the table", action="append") - parser.add_argument("--type", help="type of the table", action="append") + parser.add_argument("--type", choices=table_types.keys(), action="append") parser.add_argument("--param", help="table parameter", action="append") args = parser.parse_args() # Prepare the maps - table_types = dict(url=UrlTable) table_map = {name: table_types[table_type](param) for name, table_type, param - in zip(args.name, args.type, args.param)} + in zip(args.name, args.type, args.param)} if args.name else {} # Run the main loop logging.basicConfig(level=logging.DEBUG) loop = asyncio.get_event_loop() - server = loop.run_until_complete(loop.create_server( - SocketmapProtocol.factory(table_map), args.bind, args.port + server = loop.run_until_complete(loop.create_unix_server( + server_types[args.mode].factory(table_map), args.socket )) try: loop.run_forever() From 2b2ab864d19cc226459056f2006fb2ad110155ca Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Sun, 15 Jul 2018 15:35:35 +0200 Subject: [PATCH 03/37] Add support for querying the table in Dovecot proxy --- core/postfix/conf/postproxy.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/core/postfix/conf/postproxy.py b/core/postfix/conf/postproxy.py index 234d8c34..29b616ae 100755 --- a/core/postfix/conf/postproxy.py +++ b/core/postfix/conf/postproxy.py @@ -10,6 +10,7 @@ import aiohttp import logging import urllib import argparse +import json class NetstringProtocol(asyncio.Protocol): @@ -108,7 +109,7 @@ class SocketmapProtocol(NetstringProtocol): return self.send_string(b'TEMP no such map') try: result = await table.get(key) - return self.send_string(b'OK ' + result.encode('utf8')) + return self.send_string(b'OK ' + str(result).encode('utf8')) except KeyError: return self.send_string(b'NOTFOUND ') except Exception: @@ -157,7 +158,7 @@ class DictProtocol(asyncio.Protocol): return self.transport.abort() args = line[1:].strip().split(b"\t") try: - command(self, *args) + return command(self, *args) except Exception: logging.exception("Error when processing request") return self.transport.abort() @@ -172,9 +173,11 @@ class DictProtocol(asyncio.Protocol): logging.debug("Value type {}, user {}, dict {}".format( self.value_type, self.user, dict_name)) - def process_lookup(self, key): + async def process_lookup(self, key): logging.debug("Looking up {}".format(key)) - self.reply(b"O", json.dumps({})) + result = await self.dict.get(key) + response = result if type(result) is str else json.dumps(result) + return self.reply(b"O", response) def reply(self, command, *args): logging.debug("Replying {} with {}".format(command, args)) @@ -210,7 +213,7 @@ class UrlTable(object): async with aiohttp.ClientSession() as session: async with session.get(self.url_pattern.format(key)) as request: if request.status == 200: - result = await request.text() + result = await request.json() return result From 76617a3c972060394a7406b55fbeea28a98c134c Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:38:21 +0200 Subject: [PATCH 04/37] Store the quota status in database --- core/admin/mailu/models.py | 26 ++++++++++++++--- .../migrations/versions/25fd6c7bcb4a_.py | 28 +++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 core/admin/migrations/versions/25fd6c7bcb4a_.py diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 71c5f4d7..7e13682b 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,11 +1,11 @@ -from mailu import app, db, dkim, login_manager, quota +from mailu import app, db, dkim, login_manager from sqlalchemy.ext import declarative from passlib import context, hash from datetime import datetime, date from email.mime import text - +import sqlalchemy import re import time import os @@ -235,6 +235,7 @@ class User(Base, Email): backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) quota_bytes = db.Column(db.Integer(), nullable=False, default=10**9) + quota_bytes_used = db.Column(db.Integer(), nullable=False, default=0) global_admin = db.Column(db.Boolean(), nullable=False, default=False) enabled = db.Column(db.Boolean(), nullable=False, default=True) @@ -266,8 +267,14 @@ class User(Base, Email): return self.email @property - def quota_bytes_used(self): - return quota.get(self.email + "/quota/storage") or 0 + def destination(self): + if self.foward_enabled: + result = self.self.forward_destination + if self.forward_keep: + result += ',' + self.email + return result + else: + return self.email scheme_dict = {'SHA512-CRYPT': "sha512_crypt", 'SHA256-CRYPT': "sha256_crypt", @@ -329,6 +336,17 @@ class Alias(Base, Email): wildcard = db.Column(db.Boolean(), nullable=False, default=False) destination = db.Column(CommaSeparatedList, nullable=False, default=[]) + @classmethod + def resolve(cls, localpart, domain_name): + return cls.query.filter( + sqlalchemy._and(cls.domain_name == domain_name, + sqlalchemy._or( + cls.localpart == localpart, + cls.wildcard.like(localpart) + ) + ) + ) + class Token(Base): """ A token is an application password for a given user. diff --git a/core/admin/migrations/versions/25fd6c7bcb4a_.py b/core/admin/migrations/versions/25fd6c7bcb4a_.py new file mode 100644 index 00000000..cf9a0fc7 --- /dev/null +++ b/core/admin/migrations/versions/25fd6c7bcb4a_.py @@ -0,0 +1,28 @@ +""" Add a column for used quota + +Revision ID: 25fd6c7bcb4a +Revises: 049fed905da7 +Create Date: 2018-07-25 21:56:09.729153 + +""" + +# revision identifiers, used by Alembic. +revision = '25fd6c7bcb4a' +down_revision = '049fed905da7' + +from alembic import op +import sqlalchemy as sa + + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + with op.batch_alter_table('user') as batch: + batch.add_column(sa.Column('quota_bytes_used', sa.Integer(), nullable=False, server_default='0')) + + +def downgrade(): + with op.batch_alter_table('user') as batch: + batch.drop_column('user', 'quota_bytes_used') From 28001213d48f5fb917e1063c6130dd025f83d9bc Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:39:30 +0200 Subject: [PATCH 05/37] Remove the redis-based quota code --- core/admin/mailu/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index a3f0353c..d77420e6 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -11,7 +11,6 @@ import os import docker import socket import uuid -import redis from werkzeug.contrib import fixers @@ -89,9 +88,6 @@ manager.add_command('db', flask_migrate.MigrateCommand) babel = flask_babel.Babel(app) translations = list(map(str, babel.list_translations())) -# Quota manager -quota = redis.Redis.from_url(app.config.get("QUOTA_STORAGE_URL")) - @babel.localeselector def get_locale(): return flask.request.accept_languages.best_match(translations) From 809fe78f82250591b2b06052e6eec3872745472b Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:40:44 +0200 Subject: [PATCH 06/37] Add dovecot views to the internal API --- core/admin/mailu/internal/views.py | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/core/admin/mailu/internal/views.py b/core/admin/mailu/internal/views.py index b97d329e..b5b1da77 100644 --- a/core/admin/mailu/internal/views.py +++ b/core/admin/mailu/internal/views.py @@ -1,5 +1,6 @@ from mailu import db, models, app, limiter from mailu.internal import internal, nginx +from sqlalchemy import or_ import flask import flask_login @@ -49,3 +50,47 @@ def basic_authentication(): response = flask.Response(status=401) response.headers["WWW-Authenticate"] = 'Basic realm="Login Required"' return response + + +@internal.route("/postfix/alias/") +def postfix_alias_map(alias): + localpart, domain = alias.split('@', 1) if '@' in alias else (None, alias) + alternative = models.Alternative.query.get(domain) + if alternative: + domain = alternative.domain_name + email = '{}@{}'.format(localpart, domain) + if localpart is None: + return domain + else: + alias_obj = models.Alias.resolve(localpart, domain) + if alias_obj: + return alias_obj.destination + user_obj = models.User.query.get(email) + if user_obj: + return user_obj.destination + flask.abort(404) + + +@internal.route("/dovecot/auth/passdb/") +def dovecot_passdb_dict(user_email): + user = models.User.query.get(user_email) or flask.abort(403) + return flask.jsonify({ + "password": user.password, + }) + + +@internal.route("/dovecot/auth/userdb/") +def dovecot_userdb_dict(user_email): + user = models.User.query.get(user_email) or flask.abort(403) + return flask.jsonify({ + "quota_rule": "*:bytes={}".format(user.quota_bytes) + }) + + +@internal.route("/dovecot/quota/quota//", methods=["POST"]) +def dovecot_quota(ns, user_email): + user = models.User.query.get(user_email) or flask.abort(403) + if ns == "storage": + user.quota_bytes_used = flask.request.get_json() + db.session.commit() + return flask.jsonify(None) From b5d6b938693d30aa176b456739495b494c668f42 Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:41:11 +0200 Subject: [PATCH 07/37] Switch to using Podop in Dovecot --- core/dovecot/Dockerfile | 8 ++-- core/dovecot/conf/auth.conf | 5 +++ core/dovecot/conf/dovecot-sql.conf.ext | 18 --------- core/dovecot/conf/dovecot.conf | 51 ++++++------------------- core/dovecot/conf/pigeonhole-sieve.dict | 43 --------------------- core/dovecot/start.py | 20 +++++++--- 6 files changed, 36 insertions(+), 109 deletions(-) create mode 100644 core/dovecot/conf/auth.conf delete mode 100644 core/dovecot/conf/dovecot-sql.conf.ext delete mode 100644 core/dovecot/conf/pigeonhole-sieve.dict diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index cacfe354..484ab1d6 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -1,9 +1,9 @@ FROM alpine:edge -RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ - && apk add --no-cache \ - dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \ - rspamd-client@testing python py-jinja2 +RUN apk add --no-cache \ + dovecot dovecot-pop3d dovecot-lmtpd dovecot-pigeonhole-plugin rspamd-client \ + python3 py3-pip \ + && pip3 install jinja2 podop COPY conf /conf COPY sieve /var/lib/dovecot diff --git a/core/dovecot/conf/auth.conf b/core/dovecot/conf/auth.conf new file mode 100644 index 00000000..44a874ba --- /dev/null +++ b/core/dovecot/conf/auth.conf @@ -0,0 +1,5 @@ +uri = proxy:/tmp/podop.socket:auth +iterate_disable = yes +default_pass_scheme = plain +password_key = passdb/%u +user_key = userdb/%u diff --git a/core/dovecot/conf/dovecot-sql.conf.ext b/core/dovecot/conf/dovecot-sql.conf.ext deleted file mode 100644 index b55a5557..00000000 --- a/core/dovecot/conf/dovecot-sql.conf.ext +++ /dev/null @@ -1,18 +0,0 @@ -driver = sqlite -connect = /data/main.db - -# Return the user hashed password -password_query = \ - SELECT NULL as password, 'Y' as nopassword, '{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% if WEBMAIL_ADDRESS %},{{ WEBMAIL_ADDRESS }}{% endif %}{% endif %}' as allow_nets \ - FROM user \ - WHERE user.email = '%u' - -# Mostly get the user quota -user_query = \ - SELECT '*:bytes=' || user.quota_bytes AS quota_rule \ - FROM user \ - WHERE user.email = '%u' - -# For using doveadm -A: -iterate_query = \ - SELECT user.email AS user FROM user diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 94c43901..d6aee60c 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -7,17 +7,6 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }} hostname = {{ HOSTNAMES.split(",")[0] }} submission_host = {{ FRONT_ADDRESS }} -service dict { - unix_listener dict { - group = mail - mode = 0660 - } -} - -dict { - sieve = sqlite:/etc/dovecot/pigeonhole-sieve.dict -} - ############### # Mailboxes ############### @@ -36,28 +25,18 @@ mail_plugins = $mail_plugins quota quota_clone namespace inbox { inbox = yes - mailbox Trash { + {% for mailbox in ("Trash", "Drafts", "Sent", "Junk") %} + mailbox {{ mailbox }} { auto = subscribe - special_use = \Trash - } - mailbox Drafts { - auto = subscribe - special_use = \Drafts - } - mailbox Sent { - auto = subscribe - special_use = \Sent - } - mailbox Junk { - auto = subscribe - special_use = \Junk + special_use = \{{ mailbox }} } + {% endfor %} } plugin { quota = count:User quota quota_vsizes = yes - quota_clone_dict = redis:host={{ REDIS_ADDRESS }}:port=6379:db=1 + quota_clone_dict = proxy:/tmp/podop.socket:quota } ############### @@ -65,16 +44,15 @@ plugin { ############### auth_mechanisms = plain login disable_plaintext_auth = no -ssl_protocols = !SSLv3 passdb { - driver = sql - args = /etc/dovecot/dovecot-sql.conf.ext + driver = dict + args = /etc/dovecot/auth.conf } userdb { - driver = sql - args = /etc/dovecot/dovecot-sql.conf.ext + driver = dict + args = /etc/dovecot/auth.conf } service auth { @@ -95,7 +73,6 @@ service auth-worker { ############### # IMAP & POP ############### - protocol imap { mail_plugins = $mail_plugins imap_quota imap_sieve } @@ -113,7 +90,6 @@ service imap-login { ############### # Delivery ############### - protocol lmtp { mail_plugins = $mail_plugins sieve recipient_delimiter = {{ RECIPIENT_DELIMITER }} @@ -125,11 +101,9 @@ service lmtp { } } - ############### # Filtering ############### - service managesieve-login { inet_listener sieve { port = 4190 @@ -140,13 +114,12 @@ service managesieve { } plugin { - sieve = file:~/sieve;active=~/.dovecot.sieve - sieve_plugins = sieve_extdata sieve_imapsieve sieve_extprograms - sieve_global_extensions = +vnd.dovecot.extdata +spamtest +spamtestplus +vnd.dovecot.execute +editheader + sieve = dict:proxy:/tmp/podop.socket:sieve + sieve_plugins = sieve_imapsieve sieve_extprograms + sieve_global_extensions = +spamtest +spamtestplus +vnd.dovecot.execute +editheader sieve_before = /var/lib/dovecot/before.sieve sieve_default = /var/lib/dovecot/default.sieve sieve_after = /var/lib/dovecot/after.sieve - sieve_extdata_dict_uri = proxy::sieve # Sieve execute sieve_execute_bin_dir = /var/lib/dovecot/bin diff --git a/core/dovecot/conf/pigeonhole-sieve.dict b/core/dovecot/conf/pigeonhole-sieve.dict deleted file mode 100644 index 917fce83..00000000 --- a/core/dovecot/conf/pigeonhole-sieve.dict +++ /dev/null @@ -1,43 +0,0 @@ -connect = /data/main.db - -map { - pattern = priv/spam_enabled - table = user - username_field = email - value_field = spam_enabled -} - -map { - pattern = priv/spam_threshold - table = user - username_field = email - value_field = spam_threshold -} - -map { - pattern = priv/reply_enabled - table = user - username_field = email - value_field = reply_enabled -} - -map { - pattern = priv/reply_subject - table = user - username_field = email - value_field = reply_subject -} - -map { - pattern = priv/reply_body - table = user - username_field = email - value_field = reply_body -} - -map { - pattern = priv/reply_enddate - table = user - username_field = email - value_field = reply_enddate -} diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 83f91fab..48e9377c 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -1,21 +1,31 @@ -#!/usr/bin/python +#!/usr/bin/python3 import jinja2 import os import socket import glob +import multiprocessing + +from podop import run_server + + +def start_podop(): + os.setuid(8) + run_server(40, "dovecot", "/tmp/podop.socket", [ + ("quota", "url", "http://admin/internal/dovecot/quota/§"), + ("auth", "url", "http://admin/internal/dovecot/auth/§"), + ("sieve", "url", "http://admin/internal/dovecot/sieve/§"), + ]) convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) # Actual startup script os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) -os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis")) -if os.environ["WEBMAIL"] != "none": - os.environ["WEBMAIL_ADDRESS"] = socket.gethostbyname(os.environ.get("WEBMAIL_ADDRESS", "webmail")) for dovecot_file in glob.glob("/conf/*"): convert(dovecot_file, os.path.join("/etc/dovecot", os.path.basename(dovecot_file))) -# Run postfix +# Run Podop, then postfix +multiprocessing.Process(target=start_podop).start() os.system("chown -R mail:mail /mail /var/lib/dovecot") os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"]) From 82e738cc533f1de83af0bb4fa8250a5034536733 Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:45:33 +0200 Subject: [PATCH 08/37] Remove the old code of postproxy --- core/postfix/conf/postproxy.py | 254 --------------------------------- 1 file changed, 254 deletions(-) delete mode 100755 core/postfix/conf/postproxy.py diff --git a/core/postfix/conf/postproxy.py b/core/postfix/conf/postproxy.py deleted file mode 100755 index 29b616ae..00000000 --- a/core/postfix/conf/postproxy.py +++ /dev/null @@ -1,254 +0,0 @@ -#!/usr/bin/python3 - -# Postfix socketmap proxy -# -# This script provides a proxy from Postfix socketmap to a variety of backends. -# For now, only HTTP backends are supported. - -import asyncio -import aiohttp -import logging -import urllib -import argparse -import json - - -class NetstringProtocol(asyncio.Protocol): - """ Netstring asyncio protocol implementation. - - For protocol details, see https://cr.yp.to/proto/netstrings.txt - """ - - # Length of the smallest allocated buffer, larger buffers will be - # allocated dynamically - BASE_BUFFER = 1024 - - # Maximum length of a buffer, will crash when exceeded - MAX_BUFFER = 65535 - - def __init__(self): - super(NetstringProtocol, self).__init__() - self.init_buffer() - - def init_buffer(self): - self.len = None # None when waiting for a length to be sent) - self.separator = -1 # -1 when not yet detected (str.find) - self.index = 0 # relative to the buffer - self.buffer = bytearray(NetstringProtocol.BASE_BUFFER) - - def data_received(self, data): - # Manage the buffer - missing = len(data) - len(self.buffer) + self.index - if missing > 0: - if len(self.buffer) + missing > NetstringProtocol.MAX_BUFFER: - raise IOError("Not enough space when decoding netstring") - self.buffer.append(bytearray(missing + 1)) - new_index = self.index + len(data) - self.buffer[self.index:new_index] = data - self.index = new_index - # Try to detect a length at the beginning of the string - if self.len is None: - self.separator = self.buffer.find(0x3a) - if self.separator != -1 and self.buffer[:self.separator].isdigit(): - self.len = int(self.buffer[:self.separator], 10) - # Then get the complete string - if self.len is not None: - if self.index - self.separator == self.len + 2: - string = self.buffer[self.separator + 1:self.index - 1] - self.init_buffer() - self.string_received(string) - - def string_received(self, string): - pass - - def send_string(self, string): - """ Send a netstring - """ - self.transport.write(str(len(string)).encode('ascii')) - self.transport.write(b':') - self.transport.write(string) - self.transport.write(b',') - - -class SocketmapProtocol(NetstringProtocol): - """ Protocol to answer Postfix socketmap and proxify lookups to - an outside object. - - See http://www.postfix.org/socketmap_table.5.html for details on the - protocol. - - A table map must be provided as a dictionary to lookup tables. - """ - - def __init__(self, table_map): - self.table_map = table_map - super(SocketmapProtocol, self).__init__() - - def connection_made(self, transport): - logging.info('Connect {}'.format(transport.get_extra_info('peername'))) - self.transport = transport - - def string_received(self, string): - space = string.find(0x20) - if space != -1: - name = string[:space].decode('ascii') - key = string[space+1:].decode('utf8') - return asyncio.async(self.process_request(name, key)) - - def send_string(self, string): - logging.debug("Send {}".format(string)) - super(SocketmapProtocol, self).send_string(string) - - async def process_request(self, name, key): - """ Process a request by querying the provided map. - """ - logging.debug("Request {}/{}".format(name, key)) - try: - table = self.table_map.get(name) - except KeyError: - return self.send_string(b'TEMP no such map') - try: - result = await table.get(key) - return self.send_string(b'OK ' + str(result).encode('utf8')) - except KeyError: - return self.send_string(b'NOTFOUND ') - except Exception: - logging.exception("Error when processing request") - return self.send_string(b'TEMP unknown error') - - @classmethod - def factory(cls, table_map): - """ Provide a protocol factory for a given map instance. - """ - return lambda: cls(table_map) - - -class DictProtocol(asyncio.Protocol): - """ Protocol to answer Dovecot dict requests, as implemented in Dict proxy. - - There is very little documentation about the protocol, most of it was - reverse-engineered from : - - https://github.com/dovecot/core/blob/master/src/dict/dict-connection.c - https://github.com/dovecot/core/blob/master/src/dict/dict-commands.c - https://github.com/dovecot/core/blob/master/src/lib-dict/dict-client.h - """ - - DATA_TYPES = {0: str, 1: int} - - def __init__(self, table_map): - self.table_map = table_map - self.major_version = None - self.minor_version = None - self.dict = None - super(DictProtocol, self).__init__() - - def connection_made(self, transport): - logging.info('Connect {}'.format(transport.get_extra_info('peername'))) - self.transport = transport - - def data_received(self, data): - logging.debug("Received {}".format(data)) - for line in data.split(b"\n"): - if len(line) < 2: - continue - command = DictProtocol.COMMANDS.get(line[0]) - if command is None: - logging.warning('Unknown command {}'.format(line[0])) - return self.transport.abort() - args = line[1:].strip().split(b"\t") - try: - return command(self, *args) - except Exception: - logging.exception("Error when processing request") - return self.transport.abort() - - def process_hello(self, major, minor, value_type, user, dict_name): - self.major, self.minor = int(major), int(minor) - logging.debug('Client version {}.{}'.format(self.major, self.minor)) - assert self.major == 2 - self.value_type = DictProtocol.DATA_TYPES[int(value_type)] - self.user = user - self.dict = self.table_map[dict_name.decode("ascii")] - logging.debug("Value type {}, user {}, dict {}".format( - self.value_type, self.user, dict_name)) - - async def process_lookup(self, key): - logging.debug("Looking up {}".format(key)) - result = await self.dict.get(key) - response = result if type(result) is str else json.dumps(result) - return self.reply(b"O", response) - - def reply(self, command, *args): - logging.debug("Replying {} with {}".format(command, args)) - self.transport.write(command) - for arg in args: - self.transport.write("b\t" + arg.replace(b"\t", b"\t\t")) - self.transport.write("\n") - - @classmethod - def factory(cls, table_map): - """ Provide a protocol factory for a given map instance. - """ - return lambda: cls(table_map) - - COMMANDS = { - ord("H"): process_hello, - ord("L"): process_lookup - } - - -class UrlTable(object): - """ Resolve an entry by querying a parametrized GET URL. - """ - - def __init__(self, url_pattern): - """ url_pattern must contain a format ``{}`` so the key is injected in - the url before the query, the ``§`` character will be replaced with - ``{}`` for easier setup. - """ - self.url_pattern = url_pattern.replace('§', '{}') - - async def get(self, key): - async with aiohttp.ClientSession() as session: - async with session.get(self.url_pattern.format(key)) as request: - if request.status == 200: - result = await request.json() - return result - - -def main(): - """ Run the asyncio loop. - """ - # Reference tables - server_types = dict(postfix=SocketmapProtocol, dovecot=DictProtocol) - table_types = dict(url=UrlTable) - # Argument parsing - parser = argparse.ArgumentParser("Postfix and Dovecot map proxy") - parser.add_argument("--socket", help="path to a socket", required=True) - parser.add_argument("--mode", choices=server_types.keys(), required=True) - parser.add_argument("--name", help="name of the table", action="append") - parser.add_argument("--type", choices=table_types.keys(), action="append") - parser.add_argument("--param", help="table parameter", action="append") - args = parser.parse_args() - # Prepare the maps - table_map = {name: table_types[table_type](param) - for name, table_type, param - in zip(args.name, args.type, args.param)} if args.name else {} - # Run the main loop - logging.basicConfig(level=logging.DEBUG) - loop = asyncio.get_event_loop() - server = loop.run_until_complete(loop.create_unix_server( - server_types[args.mode].factory(table_map), args.socket - )) - try: - loop.run_forever() - except KeyboardInterrupt: - pass - server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() - - -if __name__ == "__main__": - main() From bb73933e1e7ed99970dfafb80d9d0721a04e67ea Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:57:21 +0200 Subject: [PATCH 09/37] Switch postfix to Podop --- core/postfix/Dockerfile | 4 +++- core/postfix/conf/main.cf | 16 ++++++++-------- core/postfix/start.py | 21 ++++++++++++++++++--- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index bb5831a2..81ffc95b 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,6 +1,8 @@ FROM alpine -RUN apk add --no-cache postfix postfix-sqlite postfix-pcre rsyslog python py-jinja2 +RUN apk add --no-cache postfix postfix-pcre rsyslog \ + python3 py3-pip \ + && pip3 install jinja2 podop COPY conf /conf COPY start.py /start.py diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 2f2c6990..19c2d0c2 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -19,8 +19,8 @@ mynetworks = 127.0.0.1/32 [::1]/128 {{ RELAYNETS }} # Empty alias list to override the configuration variable and disable NIS alias_maps = -# SQLite configuration -sql = sqlite:${config_directory}/ +# Podop configuration +podop = socketmap:unix:/tmp/podop.socket: # Only accept virtual emails mydestination = @@ -56,13 +56,13 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache # The alias map actually returns both aliases and local mailboxes, which is # required for reject_unlisted_sender to work properly -virtual_alias_maps = ${sql}sqlite-virtual_alias_maps.cf -virtual_mailbox_domains = ${sql}sqlite-virtual_mailbox_domains.cf -virtual_mailbox_maps = $virtual_alias_maps +virtual_alias_maps = ${podop}alias +virtual_mailbox_domains = ${podop}domains +virtual_mailbox_maps = ${podop}mailbox # Mails are transported if required, then forwarded to Dovecot for delivery -relay_domains = ${sql}sqlite-transport.cf -transport_maps = ${sql}sqlite-transport.cf +relay_domains = ${podop}transport +transport_maps = ${podop}transport virtual_transport = lmtp:inet:{{ HOST_LMTP }} # In order to prevent Postfix from running DNS query, enforce the use of the @@ -84,7 +84,7 @@ smtpd_helo_required = yes smtpd_recipient_restrictions = permit_mynetworks, - check_sender_access ${sql}sqlite-reject-spoofed.cf, + check_sender_access ${podop}spoofed reject_non_fqdn_sender, reject_unknown_sender_domain, reject_unknown_recipient_domain, diff --git a/core/postfix/start.py b/core/postfix/start.py index 4dbf2206..38905224 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -1,11 +1,25 @@ -#!/usr/bin/python +#!/usr/bin/python3 import jinja2 import os import socket import glob import shutil - +import multiprocessing + +from podop import run_server + + +def start_podop(): + os.setuid(100) + run_server(40, "postfix", "/tmp/podop.socket", [ + ("transport", "url", "http://admin/internal/postfix/transport/§"), + ("alias", "url", "http://admin/internal/postfix/alias/§"), + ("domains", "url", "http://admin/internal/postfix/domains/§"), + ("mailbox", "url", "http://admin/internal/postfix/mailbox/§"), + ("spoofed", "url", "http://admin/internal/postfix/spoofed/§"), + ]) + convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) # Actual startup script @@ -32,7 +46,8 @@ for map_file in glob.glob("/overrides/*.map"): convert("/conf/rsyslog.conf", "/etc/rsyslog.conf") -# Run postfix +# Run Podop and Postfix +multiprocessing.Process(target=start_podop).start() if os.path.exists("/var/run/rsyslogd.pid"): os.remove("/var/run/rsyslogd.pid") os.system("/usr/lib/postfix/post-install meta_directory=/etc/postfix create-missing") From c04e58498de43c7bf6549808b75c495cd31cadac Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:57:48 +0200 Subject: [PATCH 10/37] Remove unused postfix sqlite files --- core/postfix/conf/sqlite-reject-spoofed.cf | 5 ---- core/postfix/conf/sqlite-transport.cf | 3 --- .../postfix/conf/sqlite-virtual_alias_maps.cf | 23 ------------------- .../conf/sqlite-virtual_mailbox_domains.cf | 5 ---- 4 files changed, 36 deletions(-) delete mode 100644 core/postfix/conf/sqlite-reject-spoofed.cf delete mode 100644 core/postfix/conf/sqlite-transport.cf delete mode 100644 core/postfix/conf/sqlite-virtual_alias_maps.cf delete mode 100644 core/postfix/conf/sqlite-virtual_mailbox_domains.cf diff --git a/core/postfix/conf/sqlite-reject-spoofed.cf b/core/postfix/conf/sqlite-reject-spoofed.cf deleted file mode 100644 index 9cdd6c45..00000000 --- a/core/postfix/conf/sqlite-reject-spoofed.cf +++ /dev/null @@ -1,5 +0,0 @@ -dbpath = /data/main.db -query = - SELECT 'REJECT' FROM domain WHERE name='%s' - UNION - SELECT 'REJECT' FROM alternative WHERE name='%s' diff --git a/core/postfix/conf/sqlite-transport.cf b/core/postfix/conf/sqlite-transport.cf deleted file mode 100644 index 6295523b..00000000 --- a/core/postfix/conf/sqlite-transport.cf +++ /dev/null @@ -1,3 +0,0 @@ -dbpath = /data/main.db -query = - SELECT 'smtp:['||smtp||']' FROM relay WHERE name='%s' diff --git a/core/postfix/conf/sqlite-virtual_alias_maps.cf b/core/postfix/conf/sqlite-virtual_alias_maps.cf deleted file mode 100644 index dcf0a0c6..00000000 --- a/core/postfix/conf/sqlite-virtual_alias_maps.cf +++ /dev/null @@ -1,23 +0,0 @@ -dbpath = /data/main.db -query = - SELECT destination - FROM - (SELECT destination, email, wildcard, localpart, localpart||'@'||alternative.name AS alt_email FROM alias LEFT JOIN alternative ON alias.domain_name = alternative.domain_name - UNION - SELECT (CASE WHEN forward_enabled=1 THEN (CASE WHEN forward_keep=1 THEN email||',' ELSE '' END)||forward_destination ELSE email END) AS destination, email, 0 as wildcard, localpart, localpart||'@'||alternative.name as alt_email FROM user LEFT JOIN alternative ON user.domain_name = alternative.domain_name - UNION - SELECT '@'||domain_name as destination, '@'||name as email, 0 as wildcard, '' as localpart, NULL AS alt_email FROM alternative) - WHERE - ( - wildcard = 0 - AND - (email = '%s' OR alt_email = '%s') - ) OR ( - wildcard = 1 - AND - '%s' LIKE email - ) - ORDER BY - wildcard ASC, - length(localpart) DESC - LIMIT 1 diff --git a/core/postfix/conf/sqlite-virtual_mailbox_domains.cf b/core/postfix/conf/sqlite-virtual_mailbox_domains.cf deleted file mode 100644 index af453bce..00000000 --- a/core/postfix/conf/sqlite-virtual_mailbox_domains.cf +++ /dev/null @@ -1,5 +0,0 @@ -dbpath = /data/main.db -query = - SELECT name FROM domain WHERE name='%s' - UNION - SELECT name FROM alternative WHERE name='%s' From 0085b6f1e61347b22330364b0348fef6c4b24817 Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Thu, 26 Jul 2018 21:58:50 +0200 Subject: [PATCH 11/37] Remove the data mount where unused --- docs/compose/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/compose/docker-compose.yml b/docs/compose/docker-compose.yml index 740a5ffc..1a171af5 100644 --- a/docs/compose/docker-compose.yml +++ b/docs/compose/docker-compose.yml @@ -39,7 +39,6 @@ services: restart: always env_file: .env volumes: - - "$ROOT/data:/data" - "$ROOT/mail:/mail" - "$ROOT/overrides:/overrides" depends_on: @@ -50,7 +49,6 @@ services: restart: always env_file: .env volumes: - - "$ROOT/data:/data" - "$ROOT/overrides:/overrides" depends_on: - front From 73ca5fb3d3f296c43ec2bf8556bcb732a0cfdb0d Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 24 Sep 2018 23:19:06 +0200 Subject: [PATCH 12/37] Provide a more generic skeletton for postfix virtual lookups --- core/admin/mailu/internal/views.py | 20 ++++++++++++++++++++ core/postfix/conf/main.cf | 6 +++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/internal/views.py b/core/admin/mailu/internal/views.py index b5b1da77..2a71e786 100644 --- a/core/admin/mailu/internal/views.py +++ b/core/admin/mailu/internal/views.py @@ -71,6 +71,26 @@ def postfix_alias_map(alias): flask.abort(404) +@internal.route("/postfix/alias/domain/") +def postfix_alias_domain(domain): + pass + + +@internal.route("/postfix/alias/map/") +def postfix_alias_map(alias): + pass + + +@internal.route("/postfix/mailbox/domain/") +def postfix_mailbox_domain(domain): + pass + + +@internal.route("/postfix/mailbox/map/") +def postfix_mailbox_map(domain): + pass + + @internal.route("/dovecot/auth/passdb/") def dovecot_passdb_dict(user_email): user = models.User.query.get(user_email) or flask.abort(403) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 19c2d0c2..eee01f69 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -56,9 +56,9 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache # The alias map actually returns both aliases and local mailboxes, which is # required for reject_unlisted_sender to work properly -virtual_alias_maps = ${podop}alias -virtual_mailbox_domains = ${podop}domains -virtual_mailbox_maps = ${podop}mailbox +virtual_alias_maps = ${podop}alias/domain +virtual_mailbox_domains = ${podop}mailbox/domain +virtual_mailbox_maps = ${podop}mailbox/map # Mails are transported if required, then forwarded to Dovecot for delivery relay_domains = ${podop}transport From 39cd0d5034af33108223db4be4788f860d835094 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 00:14:30 +0200 Subject: [PATCH 13/37] Upgrade to alpine 3.8 for smtp and imap --- core/dovecot/Dockerfile | 4 ++-- core/postfix/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index b15b83c3..670acc9b 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -1,7 +1,7 @@ -FROM alpine:3.7 +FROM alpine:3.8 RUN apk add --no-cache \ - dovecot dovecot-pop3d dovecot-lmtpd dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ + dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ python3 py3-pip \ && pip3 install jinja2 podop diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index e7501a54..30d1103e 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.7 +FROM alpine:3.8 RUN apk add --no-cache postfix postfix-pcre rsyslog \ python3 py3-pip \ From 7143fb8c47d5e70775e868de99f6852e571ec60f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 00:14:46 +0200 Subject: [PATCH 14/37] Implement some basic views for podop --- core/admin/mailu/internal/views.py | 42 ++++++++++++++++-------------- core/admin/mailu/models.py | 8 +++--- core/postfix/start.py | 2 +- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/core/admin/mailu/internal/views.py b/core/admin/mailu/internal/views.py index 2a71e786..fff9b481 100644 --- a/core/admin/mailu/internal/views.py +++ b/core/admin/mailu/internal/views.py @@ -52,6 +52,18 @@ def basic_authentication(): return response +@internal.route("/postfix/domain/") +def postfix_mailbox_domain(domain_name): + domain = models.Domain.query.get(domain_name) or flask.abort(404) + return flask.jsonify(domain.name) + + +@internal.route("/postfix/mailbox/") +def postfix_mailbox_map(email): + user = models.User.query.get(email) or flask.abort(404) + return flask.jsonify(user.email) + + @internal.route("/postfix/alias/") def postfix_alias_map(alias): localpart, domain = alias.split('@', 1) if '@' in alias else (None, alias) @@ -60,35 +72,25 @@ def postfix_alias_map(alias): domain = alternative.domain_name email = '{}@{}'.format(localpart, domain) if localpart is None: - return domain + return flask.jsonify(domain) else: alias_obj = models.Alias.resolve(localpart, domain) if alias_obj: - return alias_obj.destination + return flask.jsonify(alias_obj.destination) user_obj = models.User.query.get(email) if user_obj: - return user_obj.destination - flask.abort(404) + return flask.jsonify(user_obj.destination) + return flask.abort(404) -@internal.route("/postfix/alias/domain/") -def postfix_alias_domain(domain): - pass +@internal.route("/postfix/spoofed/") +def postfix_spoofed(email): + return flask.abort(404) -@internal.route("/postfix/alias/map/") -def postfix_alias_map(alias): - pass - - -@internal.route("/postfix/mailbox/domain/") -def postfix_mailbox_domain(domain): - pass - - -@internal.route("/postfix/mailbox/map/") -def postfix_mailbox_map(domain): - pass +@internal.route("/postfix/transport/") +def postfix_transport(email): + return flask.abort(404) @internal.route("/dovecot/auth/passdb/") diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 7e13682b..ec94f989 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -268,7 +268,7 @@ class User(Base, Email): @property def destination(self): - if self.foward_enabled: + if self.forward_enabled: result = self.self.forward_destination if self.forward_keep: result += ',' + self.email @@ -339,13 +339,13 @@ class Alias(Base, Email): @classmethod def resolve(cls, localpart, domain_name): return cls.query.filter( - sqlalchemy._and(cls.domain_name == domain_name, - sqlalchemy._or( + sqlalchemy.and_(cls.domain_name == domain_name, + sqlalchemy.or_( cls.localpart == localpart, cls.wildcard.like(localpart) ) ) - ) + ).first() class Token(Base): diff --git a/core/postfix/start.py b/core/postfix/start.py index 38905224..a4af9328 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -15,7 +15,7 @@ def start_podop(): run_server(40, "postfix", "/tmp/podop.socket", [ ("transport", "url", "http://admin/internal/postfix/transport/§"), ("alias", "url", "http://admin/internal/postfix/alias/§"), - ("domains", "url", "http://admin/internal/postfix/domains/§"), + ("domain", "url", "http://admin/internal/postfix/domain/§"), ("mailbox", "url", "http://admin/internal/postfix/mailbox/§"), ("spoofed", "url", "http://admin/internal/postfix/spoofed/§"), ]) From 697caaab81058cfa012117d18334000eed5b9051 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 26 Sep 2018 00:15:24 +0200 Subject: [PATCH 15/37] Update podop access and mail restrictions --- core/postfix/conf/main.cf | 20 +++++++++++++------- core/postfix/conf/master.cf | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index eee01f69..1306457a 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -56,13 +56,14 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache # The alias map actually returns both aliases and local mailboxes, which is # required for reject_unlisted_sender to work properly -virtual_alias_maps = ${podop}alias/domain -virtual_mailbox_domains = ${podop}mailbox/domain -virtual_mailbox_maps = ${podop}mailbox/map +virtual_alias_domains = +virtual_alias_maps = ${podop}alias +virtual_mailbox_domains = ${podop}domain +virtual_mailbox_maps = ${podop}mailbox # Mails are transported if required, then forwarded to Dovecot for delivery -relay_domains = ${podop}transport -transport_maps = ${podop}transport +# relay_domains = ${podop}transport +# transport_maps = ${podop}transport virtual_transport = lmtp:inet:{{ HOST_LMTP }} # In order to prevent Postfix from running DNS query, enforce the use of the @@ -82,15 +83,20 @@ smtpd_sender_login_maps = $virtual_alias_maps # Restrictions for incoming SMTP, other restrictions are applied in master.cf smtpd_helo_required = yes -smtpd_recipient_restrictions = +smtpd_client_restrictions = permit_mynetworks, - check_sender_access ${podop}spoofed + reject_unauth_destination, reject_non_fqdn_sender, reject_unknown_sender_domain, reject_unknown_recipient_domain, reject_unverified_recipient, permit +smtpd_relay_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject + unverified_recipient_reject_reason = Address lookup failure ############### diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index cbcc5e56..661a02f1 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -7,7 +7,7 @@ smtp inet n - n - - smtpd # Internal SMTP service 10025 inet n - n - - smtpd -o smtpd_sasl_auth_enable=yes - -o smtpd_recipient_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit + -o smtpd_client_restrictions=reject_unlisted_sender,reject_unauth_destination,reject_authenticated_sender_login_mismatch,permit -o cleanup_service_name=outclean outclean unix n - n - 0 cleanup -o header_checks=pcre:/etc/postfix/outclean_header_filter.cf From 82140baa69ff88c329ad500224d6dfa605867536 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 27 Sep 2018 07:52:43 +0200 Subject: [PATCH 16/37] Add a very simple test script for sending mails --- tests/smtp.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/smtp.py diff --git a/tests/smtp.py b/tests/smtp.py new file mode 100644 index 00000000..cfe09583 --- /dev/null +++ b/tests/smtp.py @@ -0,0 +1,17 @@ +import smtplib +import sys +from email import mime + +from email.mime.image import MIMEImage +from email.mime.multipart import MIMEMultipart + +msg = mime.multipart.MIMEMultipart() +msg['Subject'] = 'Test email' +msg['From'] = sys.argv[1] +msg['To'] = sys.argv[2] +msg.preamble = 'Test email' + +s = smtplib.SMTP('localhost') +s.set_debuglevel(1) +s.send_message(msg) +s.quit() From d8365bfbcf35b56920428cd8bc6ed830716d2cdb Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 27 Sep 2018 07:55:54 +0200 Subject: [PATCH 17/37] Use simpler routes for Dovecot --- core/admin/mailu/internal/views.py | 7 ++++--- core/dovecot/start.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/admin/mailu/internal/views.py b/core/admin/mailu/internal/views.py index fff9b481..78d1528a 100644 --- a/core/admin/mailu/internal/views.py +++ b/core/admin/mailu/internal/views.py @@ -93,7 +93,7 @@ def postfix_transport(email): return flask.abort(404) -@internal.route("/dovecot/auth/passdb/") +@internal.route("/dovecot/passdb/") def dovecot_passdb_dict(user_email): user = models.User.query.get(user_email) or flask.abort(403) return flask.jsonify({ @@ -101,7 +101,7 @@ def dovecot_passdb_dict(user_email): }) -@internal.route("/dovecot/auth/userdb/") +@internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): user = models.User.query.get(user_email) or flask.abort(403) return flask.jsonify({ @@ -109,10 +109,11 @@ def dovecot_userdb_dict(user_email): }) -@internal.route("/dovecot/quota/quota//", methods=["POST"]) +@internal.route("/dovecot/quota//", methods=["POST"]) def dovecot_quota(ns, user_email): user = models.User.query.get(user_email) or flask.abort(403) if ns == "storage": user.quota_bytes_used = flask.request.get_json() db.session.commit() return flask.jsonify(None) + diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 48e9377c..d2163e56 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -12,9 +12,9 @@ from podop import run_server def start_podop(): os.setuid(8) run_server(40, "dovecot", "/tmp/podop.socket", [ - ("quota", "url", "http://admin/internal/dovecot/quota/§"), - ("auth", "url", "http://admin/internal/dovecot/auth/§"), - ("sieve", "url", "http://admin/internal/dovecot/sieve/§"), + ("quota", "url", "http://admin/internal/dovecot/§"), + ("auth", "url", "http://admin/internal/dovecot/§"), + ("sieve", "url", "http://admin/internal/dovecot/§"), ]) convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) From cfeaa189f96d357d82928658c86001cc3923236a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 27 Sep 2018 08:00:31 +0200 Subject: [PATCH 18/37] Use proper 404 return codes for missing objects --- core/admin/mailu/internal/views.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/internal/views.py b/core/admin/mailu/internal/views.py index 78d1528a..83cd4718 100644 --- a/core/admin/mailu/internal/views.py +++ b/core/admin/mailu/internal/views.py @@ -95,7 +95,7 @@ def postfix_transport(email): @internal.route("/dovecot/passdb/") def dovecot_passdb_dict(user_email): - user = models.User.query.get(user_email) or flask.abort(403) + user = models.User.query.get(user_email) or flask.abort(404) return flask.jsonify({ "password": user.password, }) @@ -103,7 +103,7 @@ def dovecot_passdb_dict(user_email): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - user = models.User.query.get(user_email) or flask.abort(403) + user = models.User.query.get(user_email) or flask.abort(404) return flask.jsonify({ "quota_rule": "*:bytes={}".format(user.quota_bytes) }) @@ -111,9 +111,18 @@ def dovecot_userdb_dict(user_email): @internal.route("/dovecot/quota//", methods=["POST"]) def dovecot_quota(ns, user_email): - user = models.User.query.get(user_email) or flask.abort(403) + user = models.User.query.get(user_email) or flask.abort(404) if ns == "storage": user.quota_bytes_used = flask.request.get_json() db.session.commit() return flask.jsonify(None) + +@internal.route("/dovecot/sieve/name/