From b1b0aeb69dc8d87dfce527f6ab67582683c73b1d Mon Sep 17 00:00:00 2001 From: Pierre Jaury Date: Sun, 22 Jul 2018 20:06:30 +0200 Subject: [PATCH] Initial commit --- core/base/libs/podop/CONTRIBUTING.md | 7 ++ core/base/libs/podop/LICENSE.md | 25 ++++++ core/base/libs/podop/README.md | 112 ++++++++++++++++++++++++ core/base/libs/podop/podop/__init__.py | 44 ++++++++++ core/base/libs/podop/podop/dovecot.py | 95 ++++++++++++++++++++ core/base/libs/podop/podop/postfix.py | 115 +++++++++++++++++++++++++ core/base/libs/podop/podop/table.py | 26 ++++++ core/base/libs/podop/scripts/podop | 25 ++++++ core/base/libs/podop/setup.py | 14 +++ 9 files changed, 463 insertions(+) create mode 100644 core/base/libs/podop/CONTRIBUTING.md create mode 100644 core/base/libs/podop/LICENSE.md create mode 100644 core/base/libs/podop/README.md create mode 100644 core/base/libs/podop/podop/__init__.py create mode 100644 core/base/libs/podop/podop/dovecot.py create mode 100644 core/base/libs/podop/podop/postfix.py create mode 100644 core/base/libs/podop/podop/table.py create mode 100755 core/base/libs/podop/scripts/podop create mode 100644 core/base/libs/podop/setup.py diff --git a/core/base/libs/podop/CONTRIBUTING.md b/core/base/libs/podop/CONTRIBUTING.md new file mode 100644 index 00000000..6a09c85d --- /dev/null +++ b/core/base/libs/podop/CONTRIBUTING.md @@ -0,0 +1,7 @@ +This project is open source, and your contributions are all welcome. There are mostly three different ways one can contribute to the project: + +1. use Podop, either on test or on production servers, and report meaningful bugs when you find some; +2. write and publish, or contribute to mail distributions based on Podop, like Mailu; +2. contribute code and/or configuration to the repository (see [the development guidelines](https://mailu.io/contributors/guide.html) for details); + +Either way, keep in mind that the code you write must be licensed under the same conditions as the project itself. Additionally, all contributors are considered equal co-authors of the project. diff --git a/core/base/libs/podop/LICENSE.md b/core/base/libs/podop/LICENSE.md new file mode 100644 index 00000000..8aa0da5d --- /dev/null +++ b/core/base/libs/podop/LICENSE.md @@ -0,0 +1,25 @@ +MIT License + +Copyright (c) 2018 All Podop contributors at the date + +This software consists of voluntary contributions made by multiple individuals. +For exact contribution history, see the revision history available at +https://github.com/Mailu/podop.git + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/base/libs/podop/README.md b/core/base/libs/podop/README.md new file mode 100644 index 00000000..208b3ce5 --- /dev/null +++ b/core/base/libs/podop/README.md @@ -0,0 +1,112 @@ +Podop is a piece of middleware designed to run between Postfix or Dovecot +on one side, any Python implementation of a table lookup protocol on the +other side. + +It is thus able to forward Postfix maps and Dovecot dicts to the same +(or multiple) backends in order to write a single, more flexible backend +for a mail distribution. + +Examples +======== + +- Connect Postfix to a DNS lookup so that every domain that has a proper MX + record to your Postfix is actually accepted as a local domain +- Connect both Postfix and Dovecot to an HTTP microservice to run a high + availability microservice-based mail service +- Use a single database server running any Python-compatible API for both + your Postfix and Dovecot servers + +Configure Podop tables +====================== + +Podop tables are configured through CLI arguments when running the server. +You must provide a ``--name`` for the table, a ``--type`` for the table and +a ``--param`` that parametrizes the map. + +URL table +--------- + +The URL table will initiate an HTTP GET request for read access and an HTTP +POST request for write access to a table. The table is parametrized with +a template URL containing ``§`` (or ``{}``) for inserting the table key. + +``` +--name test --type url --param http://microservice/api/v1/map/tests/§ +``` + +GET requests should return ``200`` and a JSON-encoded object +that will be passed either to Postfix or Dovecot. They should return ``4XX`` +for access issues that will result in lookup miss, and ``5XX`` for backend +issues that will result in a temporary failure. + +POST requests will contain a JSON-encoded object in the request body, that +will be saved in the table. + +Postfix usage +============= + +In order to access Podop tables from Postfix, you should setup ``socketmap`` +Postfix maps. For instance, in order to access the ``test`` table on a Podop +socket at ``/tmp/podop.socket``, use the following setup: + +``` +virtual_alias_maps = socketmap:unix:/tmp/podop.socket:test +``` + +Multiple maps or identical maps can be configured for various usages. + +``` +virtual_alias_maps = socketmap:unix:/tmp/podop.socket:alias +virtual_mailbox_domains = socketmap:unix:/tmp/podop.socket:domain +virtual_mailbox_maps = socketmap:unix:/tmp/podop.socket:alias +``` + +In order to simplify the configuration, you can setup a shortcut. + +``` +podop = socketmap:unic:/tmp/podop.socket +virtual_alias_maps = ${podop}:alias +virtual_mailbox_domains = ${podop}:domain +virtual_mailbox_maps = ${podop}:alias +``` + +Dovecot usage +============= + +In order to access Podop tables from Dovecot, you should setup a ``proxy`` +Dovecot dictionary. For instance, in order to access the ``test`` table on +a Podop socket at ``/tmp/podop.socket``, use the following setup: + +``` +mail_attribute_dict = proxy:/tmp/podop.socket:test +``` + +Multiple maps or identical maps can be configured for various usages. + +``` +mail_attribute_dict = proxy:/tmp/podop.socket:meta + +passdb { + driver = dict + args = /etc/dovecot/auth.conf +} + +userdb { + driver = dict + args = /etc/dovecot/auth.conf +} + +# then in auth.conf +uri = proxy:/tmp/podop.socket:auth +iterate_disable = yes +default_pass_scheme = plain +password_key = passdb/%u +user_key = userdb/%u +``` + +Contributing +============ + +Podop is free software, open to suggestions and contributions. All +components are free software and compatible with the MIT license. All +the code is placed under the MIT license. diff --git a/core/base/libs/podop/podop/__init__.py b/core/base/libs/podop/podop/__init__.py new file mode 100644 index 00000000..8c2c4d8d --- /dev/null +++ b/core/base/libs/podop/podop/__init__.py @@ -0,0 +1,44 @@ +""" Podop is a *Po*stfix and *Do*vecot proxy + +It is able to proxify postfix maps and dovecot dicts to any table +""" + +import asyncio +import logging + +from podop import postfix, dovecot, table + + +SERVER_TYPES = dict( + postfix=postfix.SocketmapProtocol, + dovecot=dovecot.DictProtocol +) + +TABLE_TYPES = dict( + url=table.UrlTable +) + + +def run_server(server_type, socket, tables): + """ Run the server, given its type, socket path and table list + + The table list must be a list of tuples (name, type, param) + """ + # Prepare the maps + table_map = { + name: TABLE_TYPES[table_type](param) + for name, table_type, param in tables + } + # Run the main loop + logging.basicConfig(level=logging.DEBUG) + loop = asyncio.get_event_loop() + server = loop.run_until_complete(loop.create_unix_server( + SERVER_TYPES[server_type].factory(table_map), socket + )) + try: + loop.run_forever() + except KeyboardInterrupt: + pass + server.close() + loop.run_until_complete(server.wait_closed()) + loop.close() diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py new file mode 100644 index 00000000..bbf134cc --- /dev/null +++ b/core/base/libs/podop/podop/dovecot.py @@ -0,0 +1,95 @@ +""" Dovecot dict proxy implementation +""" + +import asyncio +import logging + + +class DictProtocol(asyncio.Protocol): + """ Protocol to answer Dovecot dict requests, as implemented in Dict proxy. + + There is very little documentation about the protocol, most of it was + reverse-engineered from : + + https://github.com/dovecot/core/blob/master/src/dict/dict-connection.c + https://github.com/dovecot/core/blob/master/src/dict/dict-commands.c + https://github.com/dovecot/core/blob/master/src/lib-dict/dict-client.h + """ + + DATA_TYPES = {0: str, 1: int} + + def __init__(self, table_map): + self.table_map = table_map + self.major_version = None + self.minor_version = None + self.dict = None + super(DictProtocol, self).__init__() + + def connection_made(self, transport): + logging.info('Connect {}'.format(transport.get_extra_info('peername'))) + self.transport = transport + + def data_received(self, data): + logging.debug("Received {}".format(data)) + results = [] + for line in data.split(b"\n"): + logging.debug("Line {}".format(line)) + if len(line) < 2: + continue + command = DictProtocol.COMMANDS.get(line[0]) + if command is None: + logging.warning('Unknown command {}'.format(line[0])) + return self.transport.abort() + args = line[1:].strip().split(b"\t") + try: + future = command(self, *args) + if future: + results.append(future) + except Exception: + logging.exception("Error when processing request") + return self.transport.abort() + logging.debug("Results {}".format(results)) + return asyncio.gather(*results) + + def process_hello(self, major, minor, value_type, user, dict_name): + self.major, self.minor = int(major), int(minor) + logging.debug('Client version {}.{}'.format(self.major, self.minor)) + assert self.major == 2 + self.value_type = DictProtocol.DATA_TYPES[int(value_type)] + self.user = user + self.dict = self.table_map[dict_name.decode("ascii")] + logging.debug("Value type {}, user {}, dict {}".format( + self.value_type, self.user, dict_name)) + + async def process_lookup(self, key): + logging.debug("Looking up {}".format(key)) + result = await self.dict.get(key.decode("utf8")) + if result is not None: + if type(result) is str: + response = result.encode("utf8") + elif type(result) is bytes: + response = result + else: + response = json.dumps(result).encode("ascii") + return self.reply(b"O", response) + else: + return self.reply(b"N") + + def reply(self, command, *args): + logging.debug("Replying {} with {}".format(command, args)) + self.transport.write(command) + self.transport.write(b"\t".join( + arg.replace(b"\t", b"\t\t") for arg in args + )) + self.transport.write(b"\n") + + @classmethod + def factory(cls, table_map): + """ Provide a protocol factory for a given map instance. + """ + return lambda: cls(table_map) + + COMMANDS = { + ord("H"): process_hello, + ord("L"): process_lookup + } diff --git a/core/base/libs/podop/podop/postfix.py b/core/base/libs/podop/podop/postfix.py new file mode 100644 index 00000000..122cf962 --- /dev/null +++ b/core/base/libs/podop/podop/postfix.py @@ -0,0 +1,115 @@ +""" Postfix map proxy implementation +""" + +import asyncio +import logging + + +class NetstringProtocol(asyncio.Protocol): + """ Netstring asyncio protocol implementation. + + For protocol details, see https://cr.yp.to/proto/netstrings.txt + """ + + # Length of the smallest allocated buffer, larger buffers will be + # allocated dynamically + BASE_BUFFER = 1024 + + # Maximum length of a buffer, will crash when exceeded + MAX_BUFFER = 65535 + + def __init__(self): + super(NetstringProtocol, self).__init__() + self.init_buffer() + + def init_buffer(self): + self.len = None # None when waiting for a length to be sent) + self.separator = -1 # -1 when not yet detected (str.find) + self.index = 0 # relative to the buffer + self.buffer = bytearray(NetstringProtocol.BASE_BUFFER) + + def data_received(self, data): + # Manage the buffer + missing = len(data) - len(self.buffer) + self.index + if missing > 0: + if len(self.buffer) + missing > NetstringProtocol.MAX_BUFFER: + raise IOError("Not enough space when decoding netstring") + self.buffer.append(bytearray(missing + 1)) + new_index = self.index + len(data) + self.buffer[self.index:new_index] = data + self.index = new_index + # Try to detect a length at the beginning of the string + if self.len is None: + self.separator = self.buffer.find(0x3a) + if self.separator != -1 and self.buffer[:self.separator].isdigit(): + self.len = int(self.buffer[:self.separator], 10) + # Then get the complete string + if self.len is not None: + if self.index - self.separator == self.len + 2: + string = self.buffer[self.separator + 1:self.index - 1] + self.init_buffer() + self.string_received(string) + + def string_received(self, string): + pass + + def send_string(self, string): + """ Send a netstring + """ + self.transport.write(str(len(string)).encode('ascii')) + self.transport.write(b':') + self.transport.write(string) + self.transport.write(b',') + + +class SocketmapProtocol(NetstringProtocol): + """ Protocol to answer Postfix socketmap and proxify lookups to + an outside object. + + See http://www.postfix.org/socketmap_table.5.html for details on the + protocol. + + A table map must be provided as a dictionary to lookup tables. + """ + + def __init__(self, table_map): + self.table_map = table_map + super(SocketmapProtocol, self).__init__() + + def connection_made(self, transport): + logging.info('Connect {}'.format(transport.get_extra_info('peername'))) + self.transport = transport + + def string_received(self, string): + space = string.find(0x20) + if space != -1: + name = string[:space].decode('ascii') + key = string[space+1:].decode('utf8') + return asyncio.async(self.process_request(name, key)) + + def send_string(self, string): + logging.debug("Send {}".format(string)) + super(SocketmapProtocol, self).send_string(string) + + async def process_request(self, name, key): + """ Process a request by querying the provided map. + """ + logging.debug("Request {}/{}".format(name, key)) + try: + table = self.table_map.get(name) + except KeyError: + return self.send_string(b'TEMP no such map') + try: + result = await table.get(key) + return self.send_string(b'OK ' + str(result).encode('utf8')) + except KeyError: + return self.send_string(b'NOTFOUND ') + except Exception: + logging.exception("Error when processing request") + return self.send_string(b'TEMP unknown error') + + @classmethod + def factory(cls, table_map): + """ Provide a protocol factory for a given map instance. + """ + return lambda: cls(table_map) diff --git a/core/base/libs/podop/podop/table.py b/core/base/libs/podop/podop/table.py new file mode 100644 index 00000000..f3b8cc1e --- /dev/null +++ b/core/base/libs/podop/podop/table.py @@ -0,0 +1,26 @@ +""" Table lookup backends for podop +""" + +import aiohttp +import logging + + +class UrlTable(object): + """ Resolve an entry by querying a parametrized GET URL. + """ + + def __init__(self, url_pattern): + """ url_pattern must contain a format ``{}`` so the key is injected in + the url before the query, the ``§`` character will be replaced with + ``{}`` for easier setup. + """ + self.url_pattern = url_pattern.replace('§', '{}') + + async def get(self, key): + logging.debug("Getting {} from url table".format(key)) + async with aiohttp.ClientSession() as session: + async with session.get(self.url_pattern.format(key)) as request: + if request.status == 200: + result = await request.json() + logging.debug("Got {} from url table".format(result)) + return result diff --git a/core/base/libs/podop/scripts/podop b/core/base/libs/podop/scripts/podop new file mode 100755 index 00000000..b22c830d --- /dev/null +++ b/core/base/libs/podop/scripts/podop @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import argparse + +from podop import run_server, SERVER_TYPES, TABLE_TYPES + + +def main(): + """ Run a podop server based on CLI arguments + """ + parser = argparse.ArgumentParser("Postfix and Dovecot proxy") + parser.add_argument("--socket", help="path to the socket", required=True) + parser.add_argument("--mode", choices=SERVER_TYPES.keys(), required=True) + parser.add_argument("--name", help="name of the table", action="append") + parser.add_argument("--type", choices=TABLE_TYPES.keys(), action="append") + parser.add_argument("--param", help="table parameter", action="append") + args = parser.parse_args() + run_server( + args.mode, args.socket, + zip(args.name, args.type, args.param) if args.name else [] + ) + + +if __name__ == "__main__": + main() diff --git a/core/base/libs/podop/setup.py b/core/base/libs/podop/setup.py new file mode 100644 index 00000000..4d1e2ea3 --- /dev/null +++ b/core/base/libs/podop/setup.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +from distutils.core import setup + +setup( + name="Podop", + version="0.1", + description="Postfix and Dovecot proxy", + author="Pierre Jaury", + author_email="pierre@jaury.eu", + url="https://github.com/mailu/podop.git", + packages=["podop"], + scripts=["scripts/podop"] +)