diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 9c576404..5708327e 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,40 +1,46 @@ -from mailu import models +""" Mailu command line interface +""" -from flask import current_app as app -from flask import cli as flask_cli - -import flask +import sys import os import socket import uuid + import click +import yaml + +from flask import current_app as app +from flask.cli import FlaskGroup, with_appcontext + +from mailu import models +from mailu.schemas import MailuSchema, Logger, RenderJSON db = models.db -@click.group() -def mailu(cls=flask_cli.FlaskGroup): +@click.group(cls=FlaskGroup, context_settings={'help_option_names': ['-?', '-h', '--help']}) +def mailu(): """ Mailu command line """ @mailu.command() -@flask_cli.with_appcontext +@with_appcontext def advertise(): """ Advertise this server against statistic services. """ - if os.path.isfile(app.config["INSTANCE_ID_PATH"]): - with open(app.config["INSTANCE_ID_PATH"], "r") as handle: + if os.path.isfile(app.config['INSTANCE_ID_PATH']): + with open(app.config['INSTANCE_ID_PATH'], 'r') as handle: instance_id = handle.read() else: instance_id = str(uuid.uuid4()) - with open(app.config["INSTANCE_ID_PATH"], "w") as handle: + with open(app.config['INSTANCE_ID_PATH'], 'w') as handle: handle.write(instance_id) - if not app.config["DISABLE_STATISTICS"]: + if not app.config['DISABLE_STATISTICS']: try: - socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) - except: + socket.gethostbyname(app.config['STATS_ENDPOINT'].format(instance_id)) + except OSError: pass @@ -43,7 +49,7 @@ def advertise(): @click.argument('domain_name') @click.argument('password') @click.option('-m', '--mode') -@flask_cli.with_appcontext +@with_appcontext def admin(localpart, domain_name, password, mode='create'): """ Create an admin user 'mode' can be: @@ -58,7 +64,7 @@ def admin(localpart, domain_name, password, mode='create'): user = None if mode == 'ifmissing' or mode == 'update': - email = '{}@{}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' user = models.User.query.get(email) if user and mode == 'ifmissing': @@ -86,7 +92,7 @@ def admin(localpart, domain_name, password, mode='create'): @click.argument('localpart') @click.argument('domain_name') @click.argument('password') -@flask_cli.with_appcontext +@with_appcontext def user(localpart, domain_name, password): """ Create a user """ @@ -108,16 +114,16 @@ def user(localpart, domain_name, password): @click.argument('localpart') @click.argument('domain_name') @click.argument('password') -@flask_cli.with_appcontext +@with_appcontext def password(localpart, domain_name, password): """ Change the password of an user """ - email = '{0}@{1}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' user = models.User.query.get(email) if user: user.set_password(password) else: - print("User " + email + " not found.") + print(f'User {email} not found.') db.session.commit() @@ -126,7 +132,7 @@ def password(localpart, domain_name, password): @click.option('-u', '--max-users') @click.option('-a', '--max-aliases') @click.option('-q', '--max-quota-bytes') -@flask_cli.with_appcontext +@with_appcontext def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): """ Create a domain """ @@ -142,9 +148,9 @@ def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): @click.argument('localpart') @click.argument('domain_name') @click.argument('password_hash') -@flask_cli.with_appcontext +@with_appcontext def user_import(localpart, domain_name, password_hash): - """ Import a user along with password hash. + """ Import a user along with password hash """ domain = models.Domain.query.get(domain_name) if not domain: @@ -160,14 +166,14 @@ def user_import(localpart, domain_name, password_hash): db.session.commit() +# TODO: remove deprecated config_update function? @mailu.command() @click.option('-v', '--verbose') @click.option('-d', '--delete-objects') -@flask_cli.with_appcontext +@with_appcontext def config_update(verbose=False, delete_objects=False): - """sync configuration with data from YAML-formatted stdin""" - import yaml - import sys + """ Sync configuration with data from YAML (deprecated) + """ new_config = yaml.safe_load(sys.stdin) # print new_config domains = new_config.get('domains', []) @@ -187,13 +193,13 @@ def config_update(verbose=False, delete_objects=False): max_aliases=max_aliases, max_quota_bytes=max_quota_bytes) db.session.add(domain) - print("Added " + str(domain_config)) + print(f'Added {domain_config}') else: domain.max_users = max_users domain.max_aliases = max_aliases domain.max_quota_bytes = max_quota_bytes db.session.add(domain) - print("Updated " + str(domain_config)) + print(f'Updated {domain_config}') users = new_config.get('users', []) tracked_users = set() @@ -209,7 +215,7 @@ def config_update(verbose=False, delete_objects=False): domain_name = user_config['domain'] password_hash = user_config.get('password_hash', None) domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' optional_params = {} for k in user_optional_params: if k in user_config: @@ -239,13 +245,13 @@ def config_update(verbose=False, delete_objects=False): print(str(alias_config)) localpart = alias_config['localpart'] domain_name = alias_config['domain'] - if type(alias_config['destination']) is str: + if isinstance(alias_config['destination'], str): destination = alias_config['destination'].split(',') else: destination = alias_config['destination'] wildcard = alias_config.get('wildcard', False) domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' if not domain: domain = models.Domain(name=domain_name) db.session.add(domain) @@ -275,7 +281,7 @@ def config_update(verbose=False, delete_objects=False): domain_name = manager_config['domain'] user_name = manager_config['user'] domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) + manageruser = models.User.query.get(f'{user_name}@{domain_name}') if manageruser not in domain.managers: domain.managers.append(manageruser) db.session.add(domain) @@ -284,26 +290,117 @@ def config_update(verbose=False, delete_objects=False): if delete_objects: for user in db.session.query(models.User).all(): - if not (user.email in tracked_users): + if not user.email in tracked_users: if verbose: - print("Deleting user: " + str(user.email)) + print(f'Deleting user: {user.email}') db.session.delete(user) for alias in db.session.query(models.Alias).all(): - if not (alias.email in tracked_aliases): + if not alias.email in tracked_aliases: if verbose: - print("Deleting alias: " + str(alias.email)) + print(f'Deleting alias: {alias.email}') db.session.delete(alias) for domain in db.session.query(models.Domain).all(): - if not (domain.name in tracked_domains): + if not domain.name in tracked_domains: if verbose: - print("Deleting domain: " + str(domain.name)) + print(f'Deleting domain: {domain.name}') db.session.delete(domain) db.session.commit() +@mailu.command() +@click.option('-v', '--verbose', count=True, help='Increase verbosity.') +@click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.') +@click.option('-d', '--debug', is_flag=True, help='Enable debug output.') +@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') +@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.') +@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.') +@click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) +@with_appcontext +def config_import(verbose=0, secrets=False, debug=False, quiet=False, color=False, + update=False, dry_run=False, source=None): + """ Import configuration as YAML or JSON from stdin or file + """ + + log = Logger(want_color=color or None, can_color=sys.stdout.isatty(), secrets=secrets, debug=debug) + log.lexer = 'python' + log.strip = True + log.verbose = 0 if quiet else verbose + log.quiet = quiet + + context = { + 'import': True, + 'update': update, + 'clear': not update, + 'callback': log.track_serialize, + } + + schema = MailuSchema(only=MailuSchema.Meta.order, context=context) + + try: + # import source + with models.db.session.no_autoflush: + config = schema.loads(source) + # flush session to show/count all changes + if not quiet and (dry_run or verbose): + db.session.flush() + # check for duplicate domain names + config.check() + except Exception as exc: + if msg := log.format_exception(exc): + raise click.ClickException(msg) from exc + raise + + # don't commit when running dry + if dry_run: + log.changes('Dry run. Not committing changes.') + db.session.rollback() + else: + log.changes('Committing changes.') + db.session.commit() + + +@mailu.command() +@click.option('-f', '--full', is_flag=True, help='Include attributes with default value.') +@click.option('-s', '--secrets', is_flag=True, + help='Include secret attributes (dkim-key, passwords).') +@click.option('-d', '--dns', is_flag=True, help='Include dns records.') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') +@click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), + help='Save configuration to file.') +@click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.') +@click.argument('only', metavar='[FILTER]...', nargs=-1) +@with_appcontext +def config_export(full=False, secrets=False, color=False, dns=False, output=None, as_json=False, only=None): + """ Export configuration as YAML or JSON to stdout or file + """ + + log = Logger(want_color=color or None, can_color=output.isatty()) + + only = only or MailuSchema.Meta.order + + context = { + 'full': full, + 'secrets': secrets, + 'dns': dns, + } + + try: + schema = MailuSchema(only=only, context=context) + if as_json: + schema.opts.render_module = RenderJSON + log.lexer = 'json' + log.strip = True + print(log.colorize(schema.dumps(models.MailuConfig())), file=output) + except Exception as exc: + if msg := log.format_exception(exc): + raise click.ClickException(msg) from exc + raise + + @mailu.command() @click.argument('email') -@flask_cli.with_appcontext +@with_appcontext def user_delete(email): """delete user""" user = models.User.query.get(email) @@ -314,7 +411,7 @@ def user_delete(email): @mailu.command() @click.argument('email') -@flask_cli.with_appcontext +@with_appcontext def alias_delete(email): """delete alias""" alias = models.Alias.query.get(email) @@ -328,7 +425,7 @@ def alias_delete(email): @click.argument('domain_name') @click.argument('destination') @click.option('-w', '--wildcard', is_flag=True) -@flask_cli.with_appcontext +@with_appcontext def alias(localpart, domain_name, destination, wildcard=False): """ Create an alias """ @@ -341,7 +438,7 @@ def alias(localpart, domain_name, destination, wildcard=False): domain=domain, wildcard=wildcard, destination=destination.split(','), - email="%s@%s" % (localpart, domain_name) + email=f'{localpart}@{domain_name}' ) db.session.add(alias) db.session.commit() @@ -352,7 +449,7 @@ def alias(localpart, domain_name, destination, wildcard=False): @click.argument('max_users') @click.argument('max_aliases') @click.argument('max_quota_bytes') -@flask_cli.with_appcontext +@with_appcontext def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): """ Set domain limits """ @@ -367,16 +464,12 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): @mailu.command() @click.argument('domain_name') @click.argument('user_name') -@flask_cli.with_appcontext +@with_appcontext def setmanager(domain_name, user_name='manager'): """ Make a user manager of a domain """ domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) + manageruser = models.User.query.get(f'{user_name}@{domain_name}') domain.managers.append(manageruser) db.session.add(domain) db.session.commit() - - -if __name__ == '__main__': - cli() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 796e467a..42711cf0 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,21 +1,34 @@ -from mailu import dkim +""" Mailu config storage model +""" -from sqlalchemy.ext import declarative -from passlib import context, hash, registry -from datetime import datetime, date +import os +import smtplib +import json + +from datetime import date from email.mime import text -from flask import current_app as app +from itertools import chain import flask_sqlalchemy import sqlalchemy +import passlib.context +import passlib.hash +import passlib.registry import time import os -import glob import hmac import smtplib import idna import dns +from flask import current_app as app +from sqlalchemy.ext import declarative +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.inspection import inspect +from werkzeug.utils import cached_property + +from mailu import dkim + db = flask_sqlalchemy.SQLAlchemy() @@ -27,11 +40,14 @@ class IdnaDomain(db.TypeDecorator): impl = db.String(80) def process_bind_param(self, value, dialect): - return idna.encode(value).decode("ascii").lower() + """ encode unicode domain name to punycode """ + return idna.encode(value.lower()).decode('ascii') def process_result_value(self, value, dialect): + """ decode punycode domain name to unicode """ return idna.decode(value) + python_type = str class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) @@ -40,22 +56,19 @@ class IdnaEmail(db.TypeDecorator): impl = db.String(255) def process_bind_param(self, value, dialect): - try: - localpart, domain_name = value.split('@') - return "{0}@{1}".format( - localpart, - idna.encode(domain_name).decode('ascii'), - ).lower() - except ValueError: - pass + """ encode unicode domain part of email address to punycode """ + localpart, domain_name = value.rsplit('@', 1) + if '@' in localpart: + raise ValueError('email local part must not contain "@"') + domain_name = domain_name.lower() + return f'{localpart}@{idna.encode(domain_name).decode("ascii")}' def process_result_value(self, value, dialect): - localpart, domain_name = value.split('@') - return "{0}@{1}".format( - localpart, - idna.decode(domain_name), - ) + """ decode punycode domain part of email to unicode """ + localpart, domain_name = value.rsplit('@', 1) + return f'{localpart}@{idna.decode(domain_name)}' + python_type = str class CommaSeparatedList(db.TypeDecorator): """ Stores a list as a comma-separated string, compatible with Postfix. @@ -64,29 +77,35 @@ class CommaSeparatedList(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): - if type(value) is not list: - raise TypeError("Shoud be a list") + """ join list of items to comma separated string """ + if not isinstance(value, (list, tuple, set)): + raise TypeError('Must be a list of strings') for item in value: - if "," in item: - raise ValueError("No item should contain a comma") - return ",".join(value) + if ',' in item: + raise ValueError('list item must not contain ","') + return ','.join(sorted(set(value))) def process_result_value(self, value, dialect): - return list(filter(bool, value.split(","))) if value else [] + """ split comma separated string to list """ + return list(filter(bool, (item.strip() for item in value.split(',')))) if value else [] + python_type = list class JSONEncoded(db.TypeDecorator): - """Represents an immutable structure as a json-encoded string. + """ Represents an immutable structure as a json-encoded string. """ impl = db.String def process_bind_param(self, value, dialect): + """ encode data as json """ return json.dumps(value) if value else None def process_result_value(self, value, dialect): + """ decode json to data """ return json.loads(value) if value else None + python_type = str class Base(db.Model): """ Base class for all models @@ -96,14 +115,43 @@ class Base(db.Model): metadata = sqlalchemy.schema.MetaData( naming_convention={ - "fk": "%(table_name)s_%(column_0_name)s_fkey", - "pk": "%(table_name)s_pkey" + 'fk': '%(table_name)s_%(column_0_name)s_fkey', + 'pk': '%(table_name)s_pkey' } ) created_at = db.Column(db.Date, nullable=False, default=date.today) updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) - comment = db.Column(db.String(255), nullable=True) + comment = db.Column(db.String(255), nullable=True, default='') + + def __str__(self): + pkey = self.__table__.primary_key.columns.values()[0].name + if pkey == 'email': + # ugly hack for email declared attr. _email is not always up2date + return str(f'{self.localpart}@{self.domain_name}') + return str(getattr(self, pkey)) + + def __repr__(self): + return f'<{self.__class__.__name__} {str(self)!r}>' + + def __eq__(self, other): + if isinstance(other, self.__class__): + pkey = self.__table__.primary_key.columns.values()[0].name + this = getattr(self, pkey, None) + other = getattr(other, pkey, None) + return this is not None and other is not None and str(this) == str(other) + else: + return NotImplemented + + # we need hashable instances here for sqlalchemy to update collections + # in collections.bulk_replace, but auto-incrementing don't always have + # a valid primary key, in this case we use the object's id + __hashed = None + def __hash__(self): + if self.__hashed is None: + primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) + self.__hashed = id(self) if primary is None else hash(primary) + return self.__hashed # Many-to-many association table for domain managers @@ -121,99 +169,154 @@ class Config(Base): value = db.Column(JSONEncoded) +def _save_dkim_keys(session): + """ store DKIM keys after commit """ + for obj in session.identity_map.values(): + if isinstance(obj, Domain): + obj.save_dkim_key() + class Domain(Base): """ A DNS domain that has mail addresses associated to it. """ - __tablename__ = "domain" + + __tablename__ = 'domain' name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, backref=db.backref('manager_of'), lazy='dynamic') max_users = db.Column(db.Integer, nullable=False, default=-1) max_aliases = db.Column(db.Integer, nullable=False, default=-1) - max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) - signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + max_quota_bytes = db.Column(db.BigInteger, nullable=False, default=0) + signup_enabled = db.Column(db.Boolean, nullable=False, default=False) + + _dkim_key = None + _dkim_key_on_disk = None + + def _dkim_file(self): + """ return filename for active DKIM key """ + return app.config['DKIM_PATH'].format( + domain=self.name, + selector=app.config['DKIM_SELECTOR'] + ) + + def save_dkim_key(self): + """ save changed DKIM key to disk """ + if self._dkim_key != self._dkim_key_on_disk: + file_path = self._dkim_file() + if self._dkim_key: + with open(file_path, 'wb') as handle: + handle.write(self._dkim_key) + elif os.path.exists(file_path): + os.unlink(file_path) + self._dkim_key_on_disk = self._dkim_key + + @property + def dns_mx(self): + """ return MX record for domain """ + hostname = app.config['HOSTNAMES'].split(',', 1)[0] + return f'{self.name}. 600 IN MX 10 {hostname}.' + + @property + def dns_spf(self): + """ return SPF record for domain """ + hostname = app.config['HOSTNAMES'].split(',', 1)[0] + return f'{self.name}. 600 IN TXT "v=spf1 mx a:{hostname} ~all"' + + @property + def dns_dkim(self): + """ return DKIM record for domain """ + if self.dkim_key: + selector = app.config['DKIM_SELECTOR'] + return ( + f'{selector}._domainkey.{self.name}. 600 IN TXT' + f'"v=DKIM1; k=rsa; p={self.dkim_publickey}"' + ) + + @property + def dns_dmarc(self): + """ return DMARC record for domain """ + if self.dkim_key: + domain = app.config['DOMAIN'] + rua = app.config['DMARC_RUA'] + rua = f' rua=mailto:{rua}@{domain};' if rua else '' + ruf = app.config['DMARC_RUF'] + ruf = f' ruf=mailto:{ruf}@{domain};' if ruf else '' + return f'_dmarc.{self.name}. 600 IN TXT "v=DMARC1; p=reject;{rua}{ruf} adkim=s; aspf=s"' @property def dkim_key(self): - file_path = app.config["DKIM_PATH"].format( - domain=self.name, selector=app.config["DKIM_SELECTOR"]) - if os.path.exists(file_path): - with open(file_path, "rb") as handle: - return handle.read() + """ return private DKIM key """ + if self._dkim_key is None: + file_path = self._dkim_file() + if os.path.exists(file_path): + with open(file_path, 'rb') as handle: + self._dkim_key = self._dkim_key_on_disk = handle.read() + else: + self._dkim_key = self._dkim_key_on_disk = b'' + return self._dkim_key if self._dkim_key else None @dkim_key.setter def dkim_key(self, value): - file_path = app.config["DKIM_PATH"].format( - domain=self.name, selector=app.config["DKIM_SELECTOR"]) - with open(file_path, "wb") as handle: - handle.write(value) + """ set private DKIM key """ + old_key = self.dkim_key + self._dkim_key = value if value is not None else b'' + if self._dkim_key != old_key: + if not sqlalchemy.event.contains(db.session, 'after_commit', _save_dkim_keys): + sqlalchemy.event.listen(db.session, 'after_commit', _save_dkim_keys) @property def dkim_publickey(self): + """ return public part of DKIM key """ dkim_key = self.dkim_key if dkim_key: - return dkim.strip_key(self.dkim_key).decode("utf8") + return dkim.strip_key(dkim_key).decode('utf8') def generate_dkim_key(self): + """ generate and activate new DKIM key """ self.dkim_key = dkim.gen_key() def has_email(self, localpart): - for email in self.users + self.aliases: + """ checks if localpart is configured for domain """ + for email in chain(self.users, self.aliases): if email.localpart == localpart: return True - else: - return False + return False def check_mx(self): + """ checks if MX record for domain points to mailu host """ try: - hostnames = app.config['HOSTNAMES'].split(',') + hostnames = set(app.config['HOSTNAMES'].split(',')) return any( - str(rset).split()[-1][:-1] in hostnames + rset.exchange.to_text().rstrip('.') in hostnames for rset in dns.resolver.query(self.name, 'MX') ) - except Exception as e: - return False - - def __str__(self): - return self.name - - def __eq__(self, other): - try: - return self.name == other.name - except AttributeError: + except dns.exception.DNSException: return False class Alternative(Base): """ Alternative name for a served domain. - The name "domain alias" was avoided to prevent some confusion. + The name "domain alias" was avoided to prevent some confusion. """ - __tablename__ = "alternative" + __tablename__ = 'alternative' name = db.Column(IdnaDomain, primary_key=True, nullable=False) domain_name = db.Column(IdnaDomain, db.ForeignKey(Domain.name)) domain = db.relationship(Domain, backref=db.backref('alternatives', cascade='all, delete-orphan')) - def __str__(self): - return self.name - class Relay(Base): """ Relayed mail domain. The domain is either relayed publicly or through a specified SMTP host. """ - __tablename__ = "relay" + __tablename__ = 'relay' name = db.Column(IdnaDomain, primary_key=True, nullable=False) smtp = db.Column(db.String(80), nullable=True) - def __str__(self): - return self.name - class Email(object): """ Abstraction for an email address (localpart and domain). @@ -223,6 +326,7 @@ class Email(object): @declarative.declared_attr def domain_name(cls): + """ the domain part of the email address """ return db.Column(IdnaDomain, db.ForeignKey(Domain.name), nullable=False, default=IdnaDomain) @@ -230,36 +334,61 @@ class Email(object): # It is however very useful for quick lookups without joining tables, # especially when the mail server is reading the database. @declarative.declared_attr - def email(cls): - updater = lambda context: "{0}@{1}".format( - context.current_parameters["localpart"], - context.current_parameters["domain_name"], - ) - return db.Column(IdnaEmail, - primary_key=True, nullable=False, - default=updater) + def _email(cls): + """ the complete email address (localpart@domain) """ + + def updater(ctx): + key = f'{cls.__tablename__}_email' + if key in ctx.current_parameters: + return ctx.current_parameters[key] + return '{localpart}@{domain_name}'.format_map(ctx.current_parameters) + + return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater) + + # We need to keep email, localpart and domain_name in sync. + # But IMHO using email as primary key was not a good idea in the first place. + @hybrid_property + def email(self): + """ getter for email - gets _email """ + return self._email + + @email.setter + def email(self, value): + """ setter for email - sets _email, localpart and domain_name at once """ + self.localpart, self.domain_name = value.rsplit('@', 1) + self._email = value + + @staticmethod + def _update_localpart(target, value, *_): + if target.domain_name: + target._email = f'{value}@{target.domain_name}' + + @staticmethod + def _update_domain_name(target, value, *_): + if target.localpart: + target._email = f'{target.localpart}@{value}' + + @classmethod + def __declare_last__(cls): + # gets called after mappings are completed + sqlalchemy.event.listen(User.localpart, 'set', cls._update_localpart, propagate=True) + sqlalchemy.event.listen(User.domain_name, 'set', cls._update_domain_name, propagate=True) def sendmail(self, subject, body): - """ Send an email to the address. - """ - from_address = "{0}@{1}".format( - app.config['POSTMASTER'], - idna.encode(app.config['DOMAIN']).decode('ascii'), - ) + """ send an email to the address """ + f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: - to_address = "{0}@{1}".format( - self.localpart, - idna.encode(self.domain_name).decode('ascii'), - ) + to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' msg = text.MIMEText(body) msg['Subject'] = subject - msg['From'] = from_address + msg['From'] = f_addr msg['To'] = to_address - smtp.sendmail(from_address, [to_address], msg.as_string()) + smtp.sendmail(f_addr, [to_address], msg.as_string()) @classmethod def resolve_domain(cls, email): - localpart, domain_name = email.split('@', 1) if '@' in email else (None, email) + """ resolves domain alternative to real domain """ + localpart, domain_name = email.rsplit('@', 1) if '@' in email else (None, email) alternative = Alternative.query.get(domain_name) if alternative: domain_name = alternative.domain_name @@ -267,17 +396,19 @@ class Email(object): @classmethod def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False): + """ return destination for email address localpart@domain_name """ + localpart_stripped = None stripped_alias = None if os.environ.get('RECIPIENT_DELIMITER') in localpart: localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0] - user = User.query.get('{}@{}'.format(localpart, domain_name)) + user = User.query.get(f'{localpart}@{domain_name}') if not user and localpart_stripped: - user = User.query.get('{}@{}'.format(localpart_stripped, domain_name)) + user = User.query.get(f'{localpart_stripped}@{domain_name}') if user: - email = '{}@{}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' if user.forward_enabled: destination = user.forward_destination @@ -292,50 +423,52 @@ class Email(object): if pure_alias and not pure_alias.wildcard: return pure_alias.destination - elif stripped_alias: + + if stripped_alias: return stripped_alias.destination - elif pure_alias: + + if pure_alias: return pure_alias.destination - def __str__(self): - return self.email + return None class User(Base, Email): """ A user is an email address that has a password to access a mailbox. """ - __tablename__ = "user" + + __tablename__ = 'user' _ctx = None _credential_cache = {} domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) - quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) - global_admin = db.Column(db.Boolean(), nullable=False, default=False) - enabled = db.Column(db.Boolean(), nullable=False, default=True) + quota_bytes = db.Column(db.BigInteger, nullable=False, default=10**9) + quota_bytes_used = db.Column(db.BigInteger, nullable=False, default=0) + global_admin = db.Column(db.Boolean, nullable=False, default=False) + enabled = db.Column(db.Boolean, nullable=False, default=True) # Features - enable_imap = db.Column(db.Boolean(), nullable=False, default=True) - enable_pop = db.Column(db.Boolean(), nullable=False, default=True) + enable_imap = db.Column(db.Boolean, nullable=False, default=True) + enable_pop = db.Column(db.Boolean, nullable=False, default=True) # Filters - forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) - forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) - forward_keep = db.Column(db.Boolean(), nullable=False, default=True) - reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) + forward_enabled = db.Column(db.Boolean, nullable=False, default=False) + forward_destination = db.Column(CommaSeparatedList, nullable=True, default=list) + forward_keep = db.Column(db.Boolean, nullable=False, default=True) + reply_enabled = db.Column(db.Boolean, nullable=False, default=False) reply_subject = db.Column(db.String(255), nullable=True, default=None) - reply_body = db.Column(db.Text(), nullable=True, default=None) + reply_body = db.Column(db.Text, nullable=True, default=None) reply_startdate = db.Column(db.Date, nullable=False, default=date(1900, 1, 1)) reply_enddate = db.Column(db.Date, nullable=False, default=date(2999, 12, 31)) # Settings - displayed_name = db.Column(db.String(160), nullable=False, default="") - spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) - spam_threshold = db.Column(db.Integer(), nullable=False, default=80) + displayed_name = db.Column(db.String(160), nullable=False, default='') + spam_enabled = db.Column(db.Boolean, nullable=False, default=True) + spam_threshold = db.Column(db.Integer, nullable=False, default=80) # Flask-login attributes is_authenticated = True @@ -343,20 +476,23 @@ class User(Base, Email): is_anonymous = False def get_id(self): + """ return users email address """ return self.email @property def destination(self): + """ returns comma separated string of destinations """ if self.forward_enabled: - result = self.forward_destination + result = list(self.forward_destination) if self.forward_keep: - result += ',' + self.email - return result + result.append(self.email) + return ','.join(result) else: return self.email @property def reply_active(self): + """ returns status of autoreply function """ now = date.today() return ( self.reply_enabled and @@ -364,45 +500,48 @@ class User(Base, Email): self.reply_enddate > now ) - def get_password_context(): - if User._ctx: - return User._ctx + @classmethod + def get_password_context(cls): + """ create password context for hashing and verification + """ + if cls._ctx: + return cls._ctx - schemes = registry.list_crypt_handlers() + schemes = passlib.registry.list_crypt_handlers() # scrypt throws a warning if the native wheels aren't found schemes.remove('scrypt') # we can't leave plaintext schemes as they will be misidentified for scheme in schemes: if scheme.endswith('plaintext'): schemes.remove(scheme) - User._ctx = context.CryptContext( + cls._ctx = passlib.context.CryptContext( schemes=schemes, default='bcrypt_sha256', bcrypt_sha256__rounds=app.config['CREDENTIAL_ROUNDS'], deprecated='auto' ) - return User._ctx + return cls._ctx def check_password(self, password): + """ verifies password against stored hash + and updates hash if outdated + """ cache_result = self._credential_cache.get(self.get_id()) current_salt = self.password.split('$')[3] if len(self.password.split('$')) == 5 else None if cache_result and current_salt: cache_salt, cache_hash = cache_result if cache_salt == current_salt: - return hash.pbkdf2_sha256.verify(password, cache_hash) + return passlib.hash.pbkdf2_sha256.verify(password, cache_hash) else: # the cache is local per gunicorn; the password has changed # so the local cache can be invalidated del self._credential_cache[self.get_id()] - reference = self.password # strip {scheme} if that's something mailu has added # passlib will identify *crypt based hashes just fine # on its own - if self.password.startswith("{"): - scheme = self.password.split('}')[0][1:] - if scheme in ['PBKDF2', 'BLF-CRYPT', 'SHA512-CRYPT', 'SHA256-CRYPT', 'MD5-CRYPT', 'CRYPT']: - reference = reference[len(scheme)+2:] + if reference.startswith(('{PBKDF2}', '{BLF-CRYPT}', '{SHA512-CRYPT}', '{SHA256-CRYPT}', '{MD5-CRYPT}', '{CRYPT}')): + reference = reference.split('}', 1)[1] result, new_hash = User.get_password_context().verify_and_update(password, reference) if new_hash: @@ -419,25 +558,24 @@ we have little control over GC and string interning anyways. An attacker that can dump the process' memory is likely to find credentials in clear-text regardless of the presence of the cache. """ - self._credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) + self._credential_cache[self.get_id()] = (self.password.split('$')[3], passlib.hash.pbkdf2_sha256.using(rounds=1).hash(password)) return result - def set_password(self, password, hash_scheme=None, raw=False): - """Set password for user with specified encryption scheme - @password: plain text password to encrypt (if raw == True the hash itself) + def set_password(self, password, raw=False): + """ Set password for user + @password: plain text password to encrypt (or, if raw is True: the hash itself) """ - if raw: - self.password = password - else: - self.password = User.get_password_context().hash(password) + self.password = password if raw else User.get_password_context().hash(password) def get_managed_domains(self): + """ return list of domains this user can manage """ if self.global_admin: return Domain.query.all() else: return self.manager_of def get_managed_emails(self, include_aliases=True): + """ returns list of email addresses this user can manage """ emails = [] for domain in self.get_managed_domains(): emails.extend(domain.users) @@ -446,23 +584,25 @@ in clear-text regardless of the presence of the cache. return emails def send_welcome(self): - if app.config["WELCOME"]: - self.sendmail(app.config["WELCOME_SUBJECT"], - app.config["WELCOME_BODY"]) + """ send welcome email to user """ + if app.config['WELCOME']: + self.sendmail(app.config['WELCOME_SUBJECT'], app.config['WELCOME_BODY']) @classmethod def get(cls, email): + """ find user object for email address """ return cls.query.get(email) @classmethod def login(cls, email, password): + """ login user when enabled and password is valid """ user = cls.query.get(email) return user if (user and user.enabled and user.check_password(password)) else None @classmethod def get_temp_token(cls, email): user = cls.query.get(email) - return hmac.new(app.temp_token_key, bytearray("{}|{}".format(datetime.utcnow().strftime("%Y%m%d"), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None + return hmac.new(app.temp_token_key, bytearray("{}|{}".format(time.strftime('%Y%m%d'), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None def verify_temp_token(self, token): return hmac.compare_digest(self.get_temp_token(self.email), token) @@ -472,24 +612,27 @@ in clear-text regardless of the presence of the cache. class Alias(Base, Email): """ An alias is an email address that redirects to some destination. """ - __tablename__ = "alias" + + __tablename__ = 'alias' domain = db.relationship(Domain, backref=db.backref('aliases', cascade='all, delete-orphan')) - wildcard = db.Column(db.Boolean(), nullable=False, default=False) - destination = db.Column(CommaSeparatedList, nullable=False, default=[]) + wildcard = db.Column(db.Boolean, nullable=False, default=False) + destination = db.Column(CommaSeparatedList, nullable=False, default=list) @classmethod def resolve(cls, localpart, domain_name): + """ find aliases matching email address localpart@domain_name """ + alias_preserve_case = cls.query.filter( sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( sqlalchemy.and_( - cls.wildcard == False, + cls.wildcard is False, cls.localpart == localpart ), sqlalchemy.and_( - cls.wildcard == True, - sqlalchemy.bindparam("l", localpart).like(cls.localpart) + cls.wildcard is True, + sqlalchemy.bindparam('l', localpart).like(cls.localpart) ) ) ) @@ -500,34 +643,37 @@ class Alias(Base, Email): sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( sqlalchemy.and_( - cls.wildcard == False, + cls.wildcard is False, sqlalchemy.func.lower(cls.localpart) == localpart_lower ), sqlalchemy.and_( - cls.wildcard == True, - sqlalchemy.bindparam("l", localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) + cls.wildcard is True, + sqlalchemy.bindparam('l', localpart_lower).like( + sqlalchemy.func.lower(cls.localpart)) ) ) ) - ).order_by(cls.wildcard, sqlalchemy.func.char_length(sqlalchemy.func.lower(cls.localpart)).desc()).first() + ).order_by(cls.wildcard, sqlalchemy.func.char_length( + sqlalchemy.func.lower(cls.localpart)).desc()).first() if alias_preserve_case and alias_lower_case: - if alias_preserve_case.wildcard: - return alias_lower_case - else: - return alias_preserve_case - elif alias_preserve_case and not alias_lower_case: + return alias_lower_case if alias_preserve_case.wildcard else alias_preserve_case + + if alias_preserve_case and not alias_lower_case: return alias_preserve_case - elif alias_lower_case and not alias_preserve_case: + + if alias_lower_case and not alias_preserve_case: return alias_lower_case - else: - return None + + return None + class Token(Base): """ A token is an application password for a given user. """ - __tablename__ = "token" - id = db.Column(db.Integer(), primary_key=True) + __tablename__ = 'token' + + id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) user = db.relationship(User, @@ -536,40 +682,259 @@ class Token(Base): ip = db.Column(db.String(255)) def check_password(self, password): + """ verifies password against stored hash + and updates hash if outdated + """ if self.password.startswith("$5$"): - if hash.sha256_crypt.verify(password, self.password): + if passlib.hash.sha256_crypt.verify(password, self.password): self.set_password(password) db.session.add(self) db.session.commit() return True return False - return hash.pbkdf2_sha256.verify(password, self.password) + return passlib.hash.pbkdf2_sha256.verify(password, self.password) def set_password(self, password): + """ sets password using pbkdf2_sha256 (1 round) """ # tokens have 128bits of entropy, they are not bruteforceable - self.password = hash.pbkdf2_sha256.using(rounds=1).hash(password) + self.password = passlib.hash.pbkdf2_sha256.using(rounds=1).hash(password) - def __str__(self): - return self.comment + def __repr__(self): + return f'' class Fetch(Base): - """ A fetched account is a repote POP/IMAP account fetched into a local + """ A fetched account is a remote POP/IMAP account fetched into a local account. """ - __tablename__ = "fetch" - id = db.Column(db.Integer(), primary_key=True) + __tablename__ = 'fetch' + + id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) user = db.relationship(User, backref=db.backref('fetches', cascade='all, delete-orphan')) protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) host = db.Column(db.String(255), nullable=False) - port = db.Column(db.Integer(), nullable=False) - tls = db.Column(db.Boolean(), nullable=False) + port = db.Column(db.Integer, nullable=False) + tls = db.Column(db.Boolean, nullable=False, default=False) username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) - keep = db.Column(db.Boolean(), nullable=False) + keep = db.Column(db.Boolean, nullable=False, default=False) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) + + def __repr__(self): + return ( + f'' + ) + + +class MailuConfig: + """ Class which joins whole Mailu config for dumping + and loading + """ + + class MailuCollection: + """ Provides dict- and list-like access to instances + of a sqlalchemy model + """ + + def __init__(self, model : db.Model): + self.model = model + + def __repr__(self): + return f'<{self.model.__name__}-Collection>' + + @cached_property + def _items(self): + return { + inspect(item).identity: item + for item in self.model.query.all() + } + + def __len__(self): + return len(self._items) + + def __iter__(self): + return iter(self._items.values()) + + def __getitem__(self, key): + return self._items[key] + + def __setitem__(self, key, item): + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items[key] = item + + def __delitem__(self, key): + del self._items[key] + + def append(self, item, update=False): + """ list-like append """ + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + key = inspect(item).identity + if key in self._items: + if not update: + raise ValueError(f'item {key!r} already present in collection') + self._items[key] = item + + def extend(self, items, update=False): + """ list-like extend """ + add = {} + for item in items: + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + key = inspect(item).identity + if not update and key in self._items: + raise ValueError(f'item {key!r} already present in collection') + add[key] = item + self._items.update(add) + + def pop(self, *args): + """ list-like (no args) and dict-like (1 or 2 args) pop """ + if args: + if len(args) > 2: + raise TypeError(f'pop expected at most 2 arguments, got {len(args)}') + return self._items.pop(*args) + else: + return self._items.popitem()[1] + + def popitem(self): + """ dict-like popitem """ + return self._items.popitem() + + def remove(self, item): + """ list-like remove """ + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + key = inspect(item).identity + if not key in self._items: + raise ValueError(f'item {key!r} not found in collection') + del self._items[key] + + def clear(self): + """ dict-like clear """ + while True: + try: + self.pop() + except IndexError: + break + + def update(self, items): + """ dict-like update """ + for key, item in items: + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items.update(items) + + def setdefault(self, key, item=None): + """ dict-like setdefault """ + if key in self._items: + return self._items[key] + if item is None: + return None + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items[key] = item + return item + + def __init__(self): + + # section-name -> attr + self._sections = { + name: getattr(self, name) + for name in dir(self) + if isinstance(getattr(self, name), self.MailuCollection) + } + + # known models + self._models = tuple(section.model for section in self._sections.values()) + + # model -> attr + self._sections.update({ + section.model: section for section in self._sections.values() + }) + + def _get_model(self, section): + if section is None: + return None + model = self._sections.get(section) + if model is None: + raise ValueError(f'Invalid section: {section!r}') + if isinstance(model, self.MailuCollection): + return model.model + return model + + def _add(self, items, section, update): + + model = self._get_model(section) + if isinstance(items, self._models): + items = [items] + elif not hasattr(items, '__iter__'): + raise ValueError(f'{items!r} is not iterable') + + for item in items: + if model is not None and not isinstance(item, model): + what = item.__class__.__name__.capitalize() + raise ValueError(f'{what} can not be added to section {section!r}') + self._sections[type(item)].append(item, update=update) + + def add(self, items, section=None): + """ add item to config """ + self._add(items, section, update=False) + + def update(self, items, section=None): + """ add or replace item in config """ + self._add(items, section, update=True) + + def remove(self, items, section=None): + """ remove item from config """ + model = self._get_model(section) + if isinstance(items, self._models): + items = [items] + elif not hasattr(items, '__iter__'): + raise ValueError(f'{items!r} is not iterable') + + for item in items: + if isinstance(item, str): + if section is None: + raise ValueError(f'Cannot remove key {item!r} without section') + del self._sections[model][item] + elif model is not None and not isinstance(item, model): + what = item.__class__.__name__.capitalize() + raise ValueError(f'{what} can not be removed from section {section!r}') + self._sections[type(item)].remove(item,) + + def clear(self, models=None): + """ remove complete configuration """ + for model in self._models: + if models is None or model in models: + db.session.query(model).delete() + + def check(self): + """ check for duplicate domain names """ + dup = set() + for fqdn in chain( + db.session.query(Domain.name), + db.session.query(Alternative.name), + db.session.query(Relay.name) + ): + if fqdn in dup: + raise ValueError(f'Duplicate domain name: {fqdn}') + dup.add(fqdn) + + domain = MailuCollection(Domain) + user = MailuCollection(User) + alias = MailuCollection(Alias) + relay = MailuCollection(Relay) + config = MailuCollection(Config) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py new file mode 100644 index 00000000..2742edf1 --- /dev/null +++ b/core/admin/mailu/schemas.py @@ -0,0 +1,1269 @@ +""" Mailu marshmallow fields and schema +""" + +from copy import deepcopy +from collections import Counter +from datetime import timezone + +import json +import logging +import yaml + +import sqlalchemy + +from marshmallow import pre_load, post_load, post_dump, fields, Schema +from marshmallow.utils import ensure_text_type +from marshmallow.exceptions import ValidationError +from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts +from marshmallow_sqlalchemy.fields import RelatedList + +from flask_marshmallow import Marshmallow + +from OpenSSL import crypto + +from pygments import highlight +from pygments.token import Token +from pygments.lexers import get_lexer_by_name +from pygments.lexers.data import YamlLexer +from pygments.formatters import get_formatter_by_name + +from mailu import models, dkim + + +ma = Marshmallow() + + +### import logging and schema colorization ### + +_model2schema = {} + +def get_schema(cls=None): + """ return schema class for model """ + if cls is None: + return _model2schema.values() + return _model2schema.get(cls) + +def mapped(cls): + """ register schema in model2schema map """ + _model2schema[cls.Meta.model] = cls + return cls + +class Logger: + """ helps with counting and colorizing + imported and exported data + """ + + class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + + def __init__(self, want_color=None, can_color=False, debug=False, secrets=False): + + self.lexer = 'yaml' + self.formatter = 'terminal' + self.strip = False + self.verbose = 0 + self.quiet = False + self.secrets = secrets + self.debug = debug + self.print = print + + self.color = want_color or can_color + + self._counter = Counter() + self._schemas = {} + + # log contexts + self._diff_context = { + 'full': True, + 'secrets': secrets, + } + log_context = { + 'secrets': secrets, + } + + # register listeners + for schema in get_schema(): + model = schema.Meta.model + self._schemas[model] = schema(context=log_context) + sqlalchemy.event.listen(model, 'after_insert', self._listen_insert) + sqlalchemy.event.listen(model, 'after_update', self._listen_update) + sqlalchemy.event.listen(model, 'after_delete', self._listen_delete) + + # special listener for dkim_key changes + # TODO: _listen_dkim can be removed when dkim keys are stored in database + self._dedupe_dkim = set() + sqlalchemy.event.listen(models.db.session, 'after_flush', self._listen_dkim) + + # register debug logger for sqlalchemy + if self.debug: + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + def _log(self, action, target, message=None): + if message is None: + try: + message = self._schemas[target.__class__].dump(target) + except KeyError: + message = target + if not isinstance(message, str): + message = repr(message) + self.print(f'{action} {target.__table__}: {self.colorize(message)}') + + def _listen_insert(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + self._counter.update([('Created', target.__table__.name)]) + if self.verbose: + self._log('Created', target) + + def _listen_update(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + + changes = {} + inspection = sqlalchemy.inspect(target) + for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: + history = getattr(inspection.attrs, attr.key).history + if history.has_changes() and history.deleted: + before = history.deleted[-1] + after = getattr(target, attr.key) + # TODO: this can be removed when comment is not nullable in model + if attr.key == 'comment' and not before and not after: + pass + # only remember changed keys + elif before != after: + if self.verbose: + changes[str(attr.key)] = (before, after) + else: + break + + if self.verbose: + # use schema to log changed attributes + schema = get_schema(target.__class__) + only = set(changes.keys()) & set(schema().fields.keys()) + if only: + for key, value in schema( + only=only, + context=self._diff_context + ).dump(target).items(): + before, after = changes[key] + if value == HIDDEN: + before = HIDDEN if before else before + after = HIDDEN if after else after + else: + # also hide this + after = value + self._log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') + + if changes: + self._counter.update([('Modified', target.__table__.name)]) + + def _listen_delete(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + self._counter.update([('Deleted', target.__table__.name)]) + if self.verbose: + self._log('Deleted', target) + + # TODO: _listen_dkim can be removed when dkim keys are stored in database + def _listen_dkim(self, session, flush_context): # pylint: disable=unused-argument + """ callback method to track import """ + for target in session.identity_map.values(): + # look at Domains originally loaded from db + if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: + continue + before = target._dkim_key_on_disk + after = target._dkim_key + # "de-dupe" messages; this event is fired at every flush + if before == after or (target, before, after) in self._dedupe_dkim: + continue + self._dedupe_dkim.add((target, before, after)) + self._counter.update([('Modified', target.__table__.name)]) + if self.verbose: + if self.secrets: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = HIDDEN if before else '' + after = HIDDEN if after else '' + self._log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') + + def track_serialize(self, obj, item, backref=None): + """ callback method to track import """ + # called for backref modification? + if backref is not None: + self._log( + 'Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format_map(backref)) + return + # show input data? + if self.verbose < 2: + return + # hide secrets in data + if not self.secrets: + item = self._schemas[obj.opts.model].hide(item) + if 'hash_password' in item: + item['password'] = HIDDEN + if 'fetches' in item: + for fetch in item['fetches']: + fetch['password'] = HIDDEN + self._log('Handling', obj.opts.model, item) + + def changes(self, *messages, **kwargs): + """ show changes gathered in counter """ + if self.quiet: + return + if self._counter: + changes = [] + last = None + for (action, what), count in sorted(self._counter.items()): + if action != last: + if last: + changes.append('/') + changes.append(f'{action}:') + last = action + changes.append(f'{what}({count})') + else: + changes = ['No changes.'] + self.print(*messages, *changes, **kwargs) + + def _format_errors(self, store, path=None): + + res = [] + if path is None: + path = [] + for key in sorted(store): + location = path + [str(key)] + value = store[key] + if isinstance(value, dict): + res.extend(self._format_errors(value, location)) + else: + for message in value: + res.append((".".join(location), message)) + + if path: + return res + + maxlen = max(len(loc) for loc, msg in res) + res = [f' - {loc.ljust(maxlen)} : {msg}' for loc, msg in res] + errors = f'{len(res)} error{["s",""][len(res)==1]}' + res.insert(0, f'[ValidationError] {errors} occurred during input validation') + + return '\n'.join(res) + + def _is_validation_error(self, exc): + """ walk traceback to extract invalid field from marshmallow """ + path = [] + trace = exc.__traceback__ + while trace: + if trace.tb_frame.f_code.co_name == '_serialize': + if 'attr' in trace.tb_frame.f_locals: + path.append(trace.tb_frame.f_locals['attr']) + elif trace.tb_frame.f_code.co_name == '_init_fields': + spec = ', '.join( + '.'.join(path + [key]) + for key in trace.tb_frame.f_locals['invalid_fields']) + return f'Invalid filter: {spec}' + trace = trace.tb_next + return None + + def format_exception(self, exc): + """ format ValidationErrors and other exceptions when not debugging """ + if isinstance(exc, ValidationError): + return self._format_errors(exc.messages) + if isinstance(exc, ValueError): + if msg := self._is_validation_error(exc): + return msg + if self.debug: + return None + msg = ' '.join(str(exc).split()) + return f'[{exc.__class__.__name__}] {msg}' + + colorscheme = { + Token: ('', ''), + Token.Name.Tag: ('cyan', 'cyan'), + Token.Literal.Scalar: ('green', 'green'), + Token.Literal.String: ('green', 'green'), + Token.Name.Constant: ('green', 'green'), # multiline strings + Token.Keyword.Constant: ('magenta', 'magenta'), + Token.Literal.Number: ('magenta', 'magenta'), + Token.Error: ('red', 'red'), + Token.Name: ('red', 'red'), + Token.Operator: ('red', 'red'), + } + + def colorize(self, data, lexer=None, formatter=None, color=None, strip=None): + """ add ANSI color to data """ + + if color is False or not self.color: + return data + + lexer = lexer or self.lexer + lexer = Logger.MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer) + formatter = get_formatter_by_name(formatter or self.formatter, colorscheme=self.colorscheme) + if strip is None: + strip = self.strip + + res = highlight(data, lexer, formatter) + if strip: + return res.rstrip('\n') + return res + + +### marshmallow render modules ### + +# hidden attributes +class _Hidden: + def __bool__(self): + return False + def __copy__(self): + return self + def __deepcopy__(self, _): + return self + def __eq__(self, other): + return str(other) == '' + def __repr__(self): + return '' + __str__ = __repr__ + +yaml.add_representer( + _Hidden, + lambda dumper, data: dumper.represent_data(str(data)) +) + +HIDDEN = _Hidden() + +# multiline attributes +class _Multiline(str): + pass + +yaml.add_representer( + _Multiline, + lambda dumper, data: dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') + +) + +# yaml render module +class RenderYAML: + """ Marshmallow YAML Render Module + """ + + class SpacedDumper(yaml.Dumper): + """ YAML Dumper to add a newline between main sections + and double the indent used + """ + + def write_line_break(self, data=None): + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + @staticmethod + def _augment(kwargs, defaults): + """ add defaults to kwargs if missing + """ + for key, value in defaults.items(): + if key not in kwargs: + kwargs[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load yaml data from string + """ + cls._augment(kwargs, cls._load_defaults) + return yaml.safe_load(*args, **kwargs) + + _dump_defaults = { + 'Dumper': SpacedDumper, + 'default_flow_style': False, + 'allow_unicode': True, + 'sort_keys': False, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump data to yaml string + """ + cls._augment(kwargs, cls._dump_defaults) + return yaml.dump(*args, **kwargs) + +# json encoder +class JSONEncoder(json.JSONEncoder): + """ JSONEncoder supporting serialization of HIDDEN """ + def default(self, o): + """ serialize HIDDEN """ + if isinstance(o, _Hidden): + return str(o) + return json.JSONEncoder.default(self, o) + +# json render module +class RenderJSON: + """ Marshmallow JSON Render Module + """ + + @staticmethod + def _augment(kwargs, defaults): + """ add defaults to kwargs if missing + """ + for key, value in defaults.items(): + if key not in kwargs: + kwargs[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load json data from string + """ + cls._augment(kwargs, cls._load_defaults) + return json.loads(*args, **kwargs) + + _dump_defaults = { + 'separators': (',',':'), + 'cls': JSONEncoder, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump data to json string + """ + cls._augment(kwargs, cls._dump_defaults) + return json.dumps(*args, **kwargs) + + +### marshmallow: custom fields ### + +def _rfc3339(datetime): + """ dump datetime according to rfc3339 """ + if datetime.tzinfo is None: + datetime = datetime.astimezone(timezone.utc) + res = datetime.isoformat() + if res.endswith('+00:00'): + return f'{res[:-6]}Z' + return res + +fields.DateTime.SERIALIZATION_FUNCS['rfc3339'] = _rfc3339 +fields.DateTime.DESERIALIZATION_FUNCS['rfc3339'] = fields.DateTime.DESERIALIZATION_FUNCS['iso'] +fields.DateTime.DEFAULT_FORMAT = 'rfc3339' + +class LazyStringField(fields.String): + """ Field that serializes a "false" value to the empty string + """ + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize None to the empty string + """ + return value if value else '' + +class CommaSeparatedListField(fields.Raw): + """ Deserialize a string containing comma-separated values to + a list of strings + """ + + default_error_messages = { + "invalid": "Not a valid string or list.", + "invalid_utf8": "Not a valid utf-8 string or list.", + } + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize comma separated string to list of strings + """ + + # empty + if not value: + return [] + + # handle list + if isinstance(value, list): + try: + value = [ensure_text_type(item) for item in value] + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + + # handle text + else: + if not isinstance(value, (str, bytes)): + raise self.make_error("invalid") + try: + value = ensure_text_type(value) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + else: + value = filter(bool, (item.strip() for item in value.split(','))) + + return list(value) + + +class DkimKeyField(fields.String): + """ Serialize a dkim key to a multiline string and + deserialize a dkim key data as string or list of strings + to a valid dkim key + """ + + default_error_messages = { + "invalid": "Not a valid string or list.", + "invalid_utf8": "Not a valid utf-8 string or list.", + } + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize dkim key as multiline string + """ + + # map empty string and None to None + if not value: + return '' + + # return multiline string + return _Multiline(value.decode('utf-8')) + + def _wrap_key(self, begin, data, end): + """ generator to wrap key into RFC 7468 format """ + yield begin + pos = 0 + while pos < len(data): + yield data[pos:pos+64] + pos += 64 + yield end + yield '' + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize a string or list of strings to dkim key data + with verification + """ + + # convert list to str + if isinstance(value, list): + try: + value = ''.join(ensure_text_type(item) for item in value).strip() + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + + # only text is allowed + else: + if not isinstance(value, (str, bytes)): + raise self.make_error("invalid") + try: + value = ensure_text_type(value).strip() + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + + # generate new key? + if value.lower() == '-generate-': + return dkim.gen_key() + + # no key? + if not value: + return None + + # remember part of value for ValidationError + bad_key = value + + # strip header and footer, clean whitespace and wrap to 64 characters + try: + if value.startswith('-----BEGIN '): + end = value.index('-----', 11) + 5 + header = value[:end] + value = value[end:] + else: + header = '-----BEGIN PRIVATE KEY-----' + + if (pos := value.find('-----END ')) >= 0: + end = value.index('-----', pos+9) + 5 + footer = value[pos:end] + value = value[:pos] + else: + footer = '-----END PRIVATE KEY-----' + except ValueError: + raise ValidationError(f'invalid dkim key {bad_key!r}') from exc + + # remove whitespace from key data + value = ''.join(value.split()) + + # remember part of value for ValidationError + bad_key = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value + + # wrap key according to RFC 7468 + value = ('\n'.join(self._wrap_key(header, value, footer))).encode('ascii') + + # check key validity + try: + crypto.load_privatekey(crypto.FILETYPE_PEM, value) + except crypto.Error as exc: + raise ValidationError(f'invalid dkim key {bad_key!r}') from exc + else: + return value + +class PasswordField(fields.Str): + """ Serialize a hashed password hash by stripping the obsolete {SCHEME} + Deserialize a plain password or hashed password into a hashed password + """ + + _hashes = {'PBKDF2', 'BLF-CRYPT', 'SHA512-CRYPT', 'SHA256-CRYPT', 'MD5-CRYPT', 'CRYPT'} + + def _serialize(self, value, attr, obj, **kwargs): + """ strip obsolete {password-hash} when serializing """ + # strip scheme spec if in database - it's obsolete + if value.startswith('{') and (end := value.find('}', 1)) >= 0: + if value[1:end] in self._hashes: + return value[end+1:] + return value + + def _deserialize(self, value, attr, data, **kwargs): + """ hashes plain password or checks hashed password + also strips obsolete {password-hash} when deserializing + """ + + # when hashing is requested: use model instance to hash plain password + if data.get('hash_password'): + # hash password using model instance + inst = self.metadata['model']() + inst.set_password(value) + value = inst.password + del inst + + # strip scheme spec when specified - it's obsolete + if value.startswith('{') and (end := value.find('}', 1)) >= 0: + if value[1:end] in self._hashes: + value = value[end+1:] + + # check if algorithm is supported + inst = self.metadata['model'](password=value) + try: + # just check against empty string to see if hash is valid + inst.check_password('') + except ValueError as exc: + # ValueError: hash could not be identified + raise ValidationError(f'invalid password hash {value!r}') from exc + del inst + + return value + + +### base schema ### + +class Storage: + """ Storage class to save information in context + """ + + context = {} + + def _bind(self, key, bind): + if bind is True: + return (self.__class__, key) + if isinstance(bind, str): + return (get_schema(self.recall(bind).__class__), key) + return (bind, key) + + def store(self, key, value, bind=None): + """ store value under key """ + self.context.setdefault('_track', {})[self._bind(key, bind)]= value + + def recall(self, key, bind=None): + """ recall value from key """ + return self.context['_track'][self._bind(key, bind)] + +class BaseOpts(SQLAlchemyAutoSchemaOpts): + """ Option class with sqla session + """ + def __init__(self, meta, ordered=False): + if not hasattr(meta, 'sqla_session'): + meta.sqla_session = models.db.session + if not hasattr(meta, 'sibling'): + meta.sibling = False + super(BaseOpts, self).__init__(meta, ordered=ordered) + +class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): + """ Marshmallow base schema with custom exclude logic + and option to hide sqla defaults + """ + + OPTIONS_CLASS = BaseOpts + + class Meta: + """ Schema config """ + include_by_context = {} + exclude_by_value = {} + hide_by_context = {} + order = [] + sibling = False + + def __init__(self, *args, **kwargs): + + # prepare only to auto-include explicitly specified attributes + only = set(kwargs.get('only') or []) + + # get context + context = kwargs.get('context', {}) + flags = {key for key, value in context.items() if value is True} + + # compile excludes + exclude = set(kwargs.get('exclude', [])) + + # always exclude + exclude.update({'created_at', 'updated_at'} - only) + + # add include_by_context + if context is not None: + for need, what in getattr(self.Meta, 'include_by_context', {}).items(): + if not flags & set(need): + exclude |= what - only + + # update excludes + kwargs['exclude'] = exclude + + # init SQLAlchemyAutoSchema + super().__init__(*args, **kwargs) + + # exclude_by_value + self._exclude_by_value = { + key: values for key, values in getattr(self.Meta, 'exclude_by_value', {}).items() + if key not in only + } + + # exclude default values + if not context.get('full'): + for column in self.opts.model.__table__.columns: + if column.name not in exclude and column.name not in only: + self._exclude_by_value.setdefault(column.name, []).append( + None if column.default is None else column.default.arg + ) + + # hide by context + self._hide_by_context = set() + if context is not None: + for need, what in getattr(self.Meta, 'hide_by_context', {}).items(): + if not flags & set(need): + self._hide_by_context |= what - only + + # remember primary keys + self._primary = str(self.opts.model.__table__.primary_key.columns.values()[0].name) + + # determine attribute order + if hasattr(self.Meta, 'order'): + # use user-defined order + order = self.Meta.order + else: + # default order is: primary_key + other keys alphabetically + order = list(sorted(self.fields.keys())) + if self._primary in order: + order.remove(self._primary) + order.insert(0, self._primary) + + # order fieldlists + for fieldlist in (self.fields, self.load_fields, self.dump_fields): + for field in order: + if field in fieldlist: + fieldlist[field] = fieldlist.pop(field) + + # move post_load hook "_add_instance" to the end (after load_instance mixin) + hooks = self._hooks[('post_load', False)] + hooks.remove('_add_instance') + hooks.append('_add_instance') + + def hide(self, data): + """ helper method to hide input data for logging """ + # always returns a copy of data + return { + key: HIDDEN if key in self._hide_by_context else deepcopy(value) + for key, value in data.items() + } + + def _call_and_store(self, *args, **kwargs): + """ track current parent field for pruning """ + self.store('field', kwargs['field_name'], True) + return super()._call_and_store(*args, **kwargs) + + # this is only needed to work around the declared attr "email" primary key in model + def get_instance(self, data): + """ lookup item by defined primary key instead of key(s) from model """ + if self.transient: + return None + if keys := getattr(self.Meta, 'primary_keys', None): + filters = {key: data.get(key) for key in keys} + if None not in filters.values(): + res= self.session.query(self.opts.model).filter_by(**filters).first() + return res + res= super().get_instance(data) + return res + + @pre_load(pass_many=True) + def _patch_many(self, items, many, **kwargs): # pylint: disable=unused-argument + """ - flush sqla session before serializing a section when requested + (make sure all objects that could be referred to later are created) + - when in update mode: patch input data before deserialization + - handle "prune" and "delete" items + - replace values in keys starting with '-' with default + """ + + # flush sqla session + if not self.Meta.sibling: + self.opts.sqla_session.flush() + + # stop early when not updating + if not self.context.get('update'): + return items + + # patch "delete", "prune" and "default" + want_prune = [] + def patch(count, data): + + # don't allow __delete__ coming from input + if '__delete__' in data: + raise ValidationError('Unknown field.', f'{count}.__delete__') + + # fail when hash_password is specified without password + if 'hash_password' in data and not 'password' in data: + raise ValidationError( + 'Nothing to hash. Field "password" is missing.', + field_name = f'{count}.hash_password', + ) + + # handle "prune list" and "delete item" (-pkey: none and -pkey: id) + for key in data: + if key.startswith('-'): + if key[1:] == self._primary: + # delete or prune + if data[key] is None: + # prune + want_prune.append(True) + return None + # mark item for deletion + return {key[1:]: data[key], '__delete__': count} + + # handle "set to default value" (-key: none) + def set_default(key, value): + if not key.startswith('-'): + return (key, value) + key = key[1:] + if not key in self.opts.model.__table__.columns: + return (key, None) + if value is not None: + raise ValidationError( + 'Value must be "null" when resetting to default.', + f'{count}.{key}' + ) + value = self.opts.model.__table__.columns[key].default + if value is None: + raise ValidationError( + 'Field has no default value.', + f'{count}.{key}' + ) + return (key, value.arg) + + return dict(set_default(key, value) for key, value in data.items()) + + # convert items to "delete" and filter "prune" item + items = [ + item for item in [ + patch(count, item) for count, item in enumerate(items) + ] if item + ] + + # remember if prune was requested for _prune_items@post_load + self.store('prune', bool(want_prune), True) + + # remember original items to stabilize password-changes in _add_instance@post_load + self.store('original', items, True) + + return items + + @pre_load + def _patch_item(self, data, many, **kwargs): # pylint: disable=unused-argument + """ - call callback function to track import + - stabilize import of items with auto-increment primary key + - delete items + - delete/prune list attributes + - add missing required attributes + """ + + # callback + if callback := self.context.get('callback'): + callback(self, data) + + # stop early when not updating + if not self.opts.load_instance or not self.context.get('update'): + return data + + # stabilize import of auto-increment primary keys (not required), + # by matching import data to existing items and setting primary key + if not self._primary in data: + for item in getattr(self.recall('parent'), self.recall('field', 'parent')): + existing = self.dump(item, many=False) + this = existing.pop(self._primary) + if data == existing: + instance = item + data[self._primary] = this + break + + # try to load instance + instance = self.instance or self.get_instance(data) + if instance is None: + + if '__delete__' in data: + # deletion of non-existent item requested + raise ValidationError( + f'Item to delete not found: {data[self._primary]!r}.', + field_name = f'{data["__delete__"]}.{self._primary}', + ) + + else: + + if self.context.get('update'): + # remember instance as parent for pruning siblings + if not self.Meta.sibling: + self.store('parent', instance) + # delete instance from session when marked + if '__delete__' in data: + self.opts.sqla_session.delete(instance) + # delete item from lists or prune lists + # currently: domain.alternatives, user.forward_destination, + # user.manager_of, aliases.destination + for key, value in data.items(): + if not isinstance(self.fields.get(key), ( + RelatedList, CommaSeparatedListField, fields.Raw) + ) or not isinstance(value, list): + continue + # deduplicate new value + new_value = set(value) + # handle list pruning + if '-prune-' in value: + value.remove('-prune-') + new_value.remove('-prune-') + else: + for old in getattr(instance, key): + # using str() is okay for now (see above) + new_value.add(str(old)) + # handle item deletion + for item in value: + if item.startswith('-'): + new_value.remove(item) + try: + new_value.remove(item[1:]) + except KeyError as exc: + raise ValidationError( + f'Item to delete not found: {item[1:]!r}.', + field_name=f'?.{key}', + ) from exc + # sort list of new values + data[key] = sorted(new_value) + # log backref modification not catched by modify hook + if isinstance(self.fields[key], RelatedList): + if callback := self.context.get('callback'): + before = {str(v) for v in getattr(instance, key)} + after = set(data[key]) + if before != after: + callback(self, instance, { + 'key': key, + 'target': str(instance), + 'before': before, + 'after': after, + }) + + # add attributes required for validation from db + for attr_name, field_obj in self.load_fields.items(): + if field_obj.required and attr_name not in data: + data[attr_name] = getattr(instance, attr_name) + + return data + + @post_load(pass_many=True) + def _prune_items(self, items, many, **kwargs): # pylint: disable=unused-argument + """ handle list pruning """ + + # stop early when not updating + if not self.context.get('update'): + return items + + # get prune flag from _patch_many@pre_load + want_prune = self.recall('prune', True) + + # prune: determine if existing items in db need to be added or marked for deletion + add_items = False + del_items = False + if self.Meta.sibling: + # parent prunes automatically + if not want_prune: + # no prune requested => add old items + add_items = True + else: + # parent does not prune automatically + if want_prune: + # prune requested => mark old items for deletion + del_items = True + + if add_items or del_items: + existing = {item[self._primary] for item in items if self._primary in item} + for item in getattr(self.recall('parent'), self.recall('field', 'parent')): + key = getattr(item, self._primary) + if key not in existing: + if add_items: + items.append({self._primary: key}) + else: + items.append({self._primary: key, '__delete__': '?'}) + + return items + + @post_load + def _add_instance(self, item, many, **kwargs): # pylint: disable=unused-argument + """ - undo password change in existing instances when plain password did not change + - add new instances to sqla session + """ + + if not item in self.opts.sqla_session: + self.opts.sqla_session.add(item) + return item + + # stop early when not updating or item has no password attribute + if not self.context.get('update') or not hasattr(item, 'password'): + return item + + # did we hash a new plaintext password? + original = None + pkey = getattr(item, self._primary) + for data in self.recall('original', True): + if 'hash_password' in data and data.get(self._primary) == pkey: + original = data['password'] + break + if original is None: + # password was hashed by us + return item + + # reset hash if plain password matches hash from db + if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): + if attr.history.has_changes() and attr.history.deleted: + try: + # reset password hash + inst = type(item)(password=attr.history.deleted[-1]) + if inst.check_password(original): + item.password = inst.password + except ValueError: + # hash in db is invalid + pass + else: + del inst + + return item + + @post_dump + def _hide_values(self, data, many, **kwargs): # pylint: disable=unused-argument + """ hide secrets """ + + # stop early when not excluding/hiding + if not self._exclude_by_value and not self._hide_by_context: + return data + + # exclude or hide values + full = self.context.get('full') + return type(data)( + (key, HIDDEN if key in self._hide_by_context else value) + for key, value in data.items() + if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] + ) + + # this field is used to mark items for deletion + mark_delete = fields.Boolean(data_key='__delete__', load_only=True) + + # TODO: this can be removed when comment is not nullable in model + comment = LazyStringField() + + +### schema definitions ### + +@mapped +class DomainSchema(BaseSchema): + """ Marshmallow schema for Domain model """ + class Meta: + """ Schema config """ + model = models.Domain + load_instance = True + include_relationships = True + exclude = ['users', 'managers', 'aliases'] + + include_by_context = { + ('dns',): {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, + } + hide_by_context = { + ('secrets',): {'dkim_key'}, + } + exclude_by_value = { + 'alternatives': [[]], + 'dkim_key': [None], + 'dkim_publickey': [None], + 'dns_mx': [None], + 'dns_spf': [None], + 'dns_dkim': [None], + 'dns_dmarc': [None], + } + + dkim_key = DkimKeyField(allow_none=True) + dkim_publickey = fields.String(dump_only=True) + dns_mx = fields.String(dump_only=True) + dns_spf = fields.String(dump_only=True) + dns_dkim = fields.String(dump_only=True) + dns_dmarc = fields.String(dump_only=True) + + +@mapped +class TokenSchema(BaseSchema): + """ Marshmallow schema for Token model """ + class Meta: + """ Schema config """ + model = models.Token + load_instance = True + + sibling = True + + password = PasswordField(required=True, metadata={'model': models.User}) + hash_password = fields.Boolean(load_only=True, missing=False) + + +@mapped +class FetchSchema(BaseSchema): + """ Marshmallow schema for Fetch model """ + class Meta: + """ Schema config """ + model = models.Fetch + load_instance = True + + sibling = True + include_by_context = { + ('full', 'import'): {'last_check', 'error'}, + } + hide_by_context = { + ('secrets',): {'password'}, + } + + +@mapped +class UserSchema(BaseSchema): + """ Marshmallow schema for User model """ + class Meta: + """ Schema config """ + model = models.User + load_instance = True + include_relationships = True + exclude = ['_email', 'domain', 'localpart', 'domain_name', 'quota_bytes_used'] + + primary_keys = ['email'] + exclude_by_value = { + 'forward_destination': [[]], + 'tokens': [[]], + 'fetches': [[]], + 'manager_of': [[]], + 'reply_enddate': ['2999-12-31'], + 'reply_startdate': ['1900-01-01'], + } + + email = fields.String(required=True) + tokens = fields.Nested(TokenSchema, many=True) + fetches = fields.Nested(FetchSchema, many=True) + + password = PasswordField(required=True, metadata={'model': models.User}) + hash_password = fields.Boolean(load_only=True, missing=False) + + +@mapped +class AliasSchema(BaseSchema): + """ Marshmallow schema for Alias model """ + class Meta: + """ Schema config """ + model = models.Alias + load_instance = True + exclude = ['_email', 'domain', 'localpart', 'domain_name'] + + primary_keys = ['email'] + exclude_by_value = { + 'destination': [[]], + } + + email = fields.String(required=True) + destination = CommaSeparatedListField() + + +@mapped +class ConfigSchema(BaseSchema): + """ Marshmallow schema for Config model """ + class Meta: + """ Schema config """ + model = models.Config + load_instance = True + + +@mapped +class RelaySchema(BaseSchema): + """ Marshmallow schema for Relay model """ + class Meta: + """ Schema config """ + model = models.Relay + load_instance = True + + +@mapped +class MailuSchema(Schema, Storage): + """ Marshmallow schema for complete Mailu config """ + class Meta: + """ Schema config """ + model = models.MailuConfig + render_module = RenderYAML + + order = ['domain', 'user', 'alias', 'relay'] # 'config' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # order fieldlists + for fieldlist in (self.fields, self.load_fields, self.dump_fields): + for field in self.Meta.order: + if field in fieldlist: + fieldlist[field] = fieldlist.pop(field) + + def _call_and_store(self, *args, **kwargs): + """ track current parent and field for pruning """ + self.store('field', kwargs['field_name'], True) + self.store('parent', self.context.get('config')) + return super()._call_and_store(*args, **kwargs) + + @pre_load + def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument + """ create config object in context if missing + and clear it if requested + """ + if 'config' not in self.context: + self.context['config'] = models.MailuConfig() + if self.context.get('clear'): + self.context['config'].clear( + models = {field.nested.opts.model for field in self.fields.values()} + ) + return data + + @post_load + def _make_config(self, data, many, **kwargs): # pylint: disable=unused-argument + """ update and return config object """ + config = self.context['config'] + for section in self.Meta.order: + if section in data: + config.update(data[section], section) + + return config + + domain = fields.Nested(DomainSchema, many=True) + user = fields.Nested(UserSchema, many=True) + alias = fields.Nested(AliasSchema, many=True) + relay = fields.Nested(RelaySchema, many=True) +# config = fields.Nested(ConfigSchema, many=True) diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index 54cf9a14..5c291e24 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -16,6 +16,7 @@ Flask-DebugToolbar==0.10.1 Flask-KVSession==0.6.2 Flask-Limiter==1.0.1 Flask-Login==0.4.1 +flask-marshmallow==0.14.0 Flask-Migrate==2.4.0 Flask-Script==2.0.6 Flask-SQLAlchemy==2.4.0 @@ -30,9 +31,12 @@ limits==1.3 Mako==1.0.9 MarkupSafe==1.1.1 mysqlclient==1.4.2.post1 +marshmallow==3.10.0 +marshmallow-sqlalchemy==0.24.1 passlib==1.7.4 psycopg2==2.8.2 pycparser==2.19 +Pygments==2.8.1 pyOpenSSL==19.0.0 python-dateutil==2.8.0 python-editor==1.0.4 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index abb37234..46500295 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -17,6 +17,7 @@ gunicorn tabulate PyYAML PyOpenSSL +Pygments dnspython bcrypt tenacity @@ -24,3 +25,6 @@ mysqlclient psycopg2 idna srslib +marshmallow +flask-marshmallow +marshmallow-sqlalchemy diff --git a/docs/cli.rst b/docs/cli.rst index 8e94026b..957e47e4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -11,6 +11,8 @@ Managing users and aliases can be done from CLI using commands: * user-import * user-delete * config-update +* config-export +* config-import alias ----- @@ -62,7 +64,7 @@ primary difference with simple `user` command is that password is being imported docker-compose run --rm admin flask mailu user-import myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' 'SHA512-CRYPT' user-delete ------------- +----------- .. code-block:: bash @@ -94,7 +96,7 @@ where mail-config.yml looks like: without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. Users ------ +^^^^^ following are additional parameters that could be defined for users: @@ -113,8 +115,197 @@ following are additional parameters that could be defined for users: * spam_threshold Alias ------ +^^^^^ additional fields: * wildcard + +config-export +------------- + +The purpose of this command is to export the complete configuration in YAML or JSON format. + +.. code-block:: bash + + $ docker-compose exec admin flask mailu config-export --help + + Usage: flask mailu config-export [OPTIONS] [FILTER]... + + Export configuration as YAML or JSON to stdout or file + + Options: + -f, --full Include attributes with default value. + -s, --secrets Include secret attributes (dkim-key, passwords). + -d, --dns Include dns records. + -c, --color Force colorized output. + -o, --output-file FILENAME Save configuration to file. + -j, --json Export configuration in json format. + -?, -h, --help Show this message and exit. + +Only non-default attributes are exported. If you want to export all attributes use ``--full``. +If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option. +To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. + +By default all configuration objects are exported (domain, user, alias, relay). You can specify +filters to export only some objects or attributes (try: ``user`` or ``domain.name``). +Attributes explicitly specified in filters are automatically exported: there is no need to add ``--secrets`` or ``--full``. + +.. code-block:: bash + + $ docker-compose exec admin flask mailu config-export --output mail-config.yml + + $ docker-compose exec admin flask mailu config-export domain.dns_mx domain.dns_spf + + $ docker-compose exec admin flask mailu config-export user.spam_threshold + +config-import +------------- + +This command imports configuration data from an external YAML or JSON source. + +.. code-block:: bash + + $ docker-compose exec admin flask mailu config-import --help + + Usage: flask mailu config-import [OPTIONS] [FILENAME|-] + + Import configuration as YAML or JSON from stdin or file + + Options: + -v, --verbose Increase verbosity. + -s, --secrets Show secret attributes in messages. + -q, --quiet Quiet mode - only show errors. + -c, --color Force colorized output. + -u, --update Update mode - merge input with existing config. + -n, --dry-run Perform a trial run with no changes made. + -?, -h, --help Show this message and exit. + +The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec instead: + +.. code-block:: bash + + docker exec -i $(docker-compose ps -q admin) flask mailu config-import -nv < mail-config.yml + +mail-config.yml contains the configuration and looks like this: + +.. code-block:: yaml + + domain: + - name: example.com + alternatives: + - alternative.example.com + + user: + - email: foo@example.com + password_hash: '$2b$12$...' + hash_scheme: MD5-CRYPT + + alias: + - email: alias1@example.com + destination: + - user1@example.com + - user2@example.com + + relay: + - name: relay.example.com + comment: test + smtp: mx.example.com + +config-import shows the number of created/modified/deleted objects after import. +To suppress all messages except error messages use ``--quiet``. +By adding the ``--verbose`` switch the import gets more detailed and shows exactly what attributes changed. +In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. +If you want to test what would be done when importing without committing any changes, use ``--dry-run``. + +By default config-import replaces the whole configuration. ``--update`` allows to modify the existing configuration instead. +New elements will be added and existing elements will be modified. +It is possible to delete a single element or prune all elements from lists and associative arrays using a special notation: + ++-----------------------------+------------------+--------------------------+ +| Delete what? | notation | example | ++=============================+==================+==========================+ +| specific array object | ``- -key: id`` | ``- -name: example.com`` | ++-----------------------------+------------------+--------------------------+ +| specific list item | ``- -id`` | ``- -user1@example.com`` | ++-----------------------------+------------------+--------------------------+ +| all remaining array objects | ``- -key: null`` | ``- -email: null`` | ++-----------------------------+------------------+--------------------------+ +| all remaining list items | ``- -prune-`` | ``- -prune-`` | ++-----------------------------+------------------+--------------------------+ + +The ``-key: null`` notation can also be used to reset an attribute to its default. +To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``. + +A new dkim key can be generated when adding or modifying a domain, by using the special value +``dkim_key: -generate-``. + +This is a complete YAML template with all additional parameters that can be defined: + +.. code-block:: yaml + + domain: + - name: example.com + alternatives: + - alternative.tld + comment: '' + dkim_key: '' + max_aliases: -1 + max_quota_bytes: 0 + max_users: -1 + signup_enabled: false + + user: + - email: postmaster@example.com + comment: '' + displayed_name: 'Postmaster' + enable_imap: true + enable_pop: false + enabled: true + fetches: + - id: 1 + comment: 'test fetch' + error: null + host: other.example.com + keep: true + last_check: '2020-12-29T17:09:48.200179' + password: 'secret' + hash_password: true + port: 993 + protocol: imap + tls: true + username: fetch-user + forward_destination: + - address@remote.example.com + forward_enabled: true + forward_keep: true + global_admin: true + manager_of: + - example.com + password: '$2b$12$...' + hash_password: true + quota_bytes: 1000000000 + reply_body: '' + reply_enabled: false + reply_enddate: '2999-12-31' + reply_startdate: '1900-01-01' + reply_subject: '' + spam_enabled: true + spam_threshold: 80 + tokens: + - id: 1 + comment: email-client + ip: 192.168.1.1 + password: '$5$rounds=1$...' + + aliases: + - email: email@example.com + comment: '' + destination: + - address@example.com + wildcard: false + + relay: + - name: relay.example.com + comment: '' + smtp: mx.example.com diff --git a/towncrier/newsfragments/1604.feature b/towncrier/newsfragments/1604.feature new file mode 100644 index 00000000..2b47791a --- /dev/null +++ b/towncrier/newsfragments/1604.feature @@ -0,0 +1 @@ +Add cli commands config-import and config-export