From 5dfccdafe9e4bc62dbcbab166a8f62b256af8af2 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 11:11:23 +0200 Subject: [PATCH 01/52] fixed some minor typos, removed unused variable --- core/admin/mailu/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index bbc00f2d..6c9a2ab1 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -65,10 +65,10 @@ class CommaSeparatedList(db.TypeDecorator): def process_bind_param(self, value, dialect): if type(value) is not list: - raise TypeError("Shoud be a list") + raise TypeError("Should be a list") for item in value: if "," in item: - raise ValueError("No item should contain a comma") + raise ValueError("Item must not contain a comma") return ",".join(value) def process_result_value(self, value, dialect): @@ -76,7 +76,7 @@ class CommaSeparatedList(db.TypeDecorator): class JSONEncoded(db.TypeDecorator): - """Represents an immutable structure as a json-encoded string. + """ Represents an immutable structure as a json-encoded string. """ impl = db.String @@ -172,7 +172,7 @@ class Domain(Base): str(rset).split()[-1][:-1] in hostnames for rset in dns.resolver.query(self.name, 'MX') ) - except Exception as e: + except Exception: return False def __str__(self): @@ -503,7 +503,7 @@ class Token(Base): class Fetch(Base): - """ A fetched account is a repote POP/IMAP account fetched into a local + """ A fetched account is a remote POP/IMAP account fetched into a local account. """ __tablename__ = "fetch" From c26ddd3c68b41b35395850a9157d61b83655ac0a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 11:19:01 +0200 Subject: [PATCH 02/52] fixed user's destination property self.forward_destination is a list (and not string) --- core/admin/mailu/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6c9a2ab1..0a447758 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -346,10 +346,10 @@ class User(Base, Email): @property def destination(self): if self.forward_enabled: - result = self.forward_destination + result = list(self.forward_destination) if self.forward_keep: - result += ',' + self.email - return result + result.append(self.email) + return ','.join(result) else: return self.email From 5c0efe82cf96f73a8c15e130051f0ccbe91a4c5a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 11:27:38 +0200 Subject: [PATCH 03/52] implemented config_update and config_dump enhanced data model with to_dict and from_dict methods added config_dump function to manage command config_update now uses new data model methods --- core/admin/mailu/manage.py | 263 ++++++++++++++++---------------- core/admin/mailu/models.py | 301 ++++++++++++++++++++++++++++++++++++- 2 files changed, 431 insertions(+), 133 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 62f214d3..bf0148df 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -8,6 +8,8 @@ import os import socket import uuid import click +import yaml +import sys db = models.db @@ -169,146 +171,147 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() +yaml_sections = [ + ('domains', models.Domain), + ('relays', models.Relay), + ('users', models.User), + ('aliases', models.Alias), +# ('config', models.Config), +] + @mailu.command() -@click.option('-v', '--verbose') -@click.option('-d', '--delete-objects') +@click.option('-v', '--verbose', is_flag=True) +@click.option('-d', '--delete-objects', is_flag=True) +@click.option('-n', '--dry-run', is_flag=True) @flask_cli.with_appcontext -def config_update(verbose=False, delete_objects=False): +def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): """sync configuration with data from YAML-formatted stdin""" - import yaml - import sys - new_config = yaml.safe_load(sys.stdin) - # print new_config - domains = new_config.get('domains', []) - tracked_domains = set() - for domain_config in domains: - if verbose: - print(str(domain_config)) - domain_name = domain_config['name'] - max_users = domain_config.get('max_users', -1) - max_aliases = domain_config.get('max_aliases', -1) - max_quota_bytes = domain_config.get('max_quota_bytes', 0) - tracked_domains.add(domain_name) - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name, - max_users=max_users, - max_aliases=max_aliases, - max_quota_bytes=max_quota_bytes) - db.session.add(domain) - print("Added " + str(domain_config)) - else: - domain.max_users = max_users - domain.max_aliases = max_aliases - domain.max_quota_bytes = max_quota_bytes - db.session.add(domain) - print("Updated " + str(domain_config)) - users = new_config.get('users', []) - tracked_users = set() - user_optional_params = ('comment', 'quota_bytes', 'global_admin', - 'enable_imap', 'enable_pop', 'forward_enabled', - 'forward_destination', 'reply_enabled', - 'reply_subject', 'reply_body', 'displayed_name', - 'spam_enabled', 'email', 'spam_threshold') - for user_config in users: - if verbose: - print(str(user_config)) - localpart = user_config['localpart'] - domain_name = user_config['domain'] - password_hash = user_config.get('password_hash', None) - hash_scheme = user_config.get('hash_scheme', None) - domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) - optional_params = {} - for k in user_optional_params: - if k in user_config: - optional_params[k] = user_config[k] - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User.query.get(email) - tracked_users.add(email) - tracked_domains.add(domain_name) - if not user: - user = models.User( - localpart=localpart, - domain=domain, - **optional_params - ) - else: - for k in optional_params: - setattr(user, k, optional_params[k]) - user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) - db.session.add(user) + out = (lambda *args: print('(DRY RUN)', *args)) if dry_run else print - aliases = new_config.get('aliases', []) - tracked_aliases = set() - for alias_config in aliases: - if verbose: - print(str(alias_config)) - localpart = alias_config['localpart'] - domain_name = alias_config['domain'] - if type(alias_config['destination']) is str: - destination = alias_config['destination'].split(',') - else: - destination = alias_config['destination'] - wildcard = alias_config.get('wildcard', False) - domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - alias = models.Alias.query.get(email) - tracked_aliases.add(email) - tracked_domains.add(domain_name) - if not alias: - alias = models.Alias( - localpart=localpart, - domain=domain, - wildcard=wildcard, - destination=destination, - email=email - ) - else: - alias.destination = destination - alias.wildcard = wildcard - db.session.add(alias) + try: + new_config = yaml.safe_load(sys.stdin) + except (yaml.scanner.ScannerError, yaml.parser.ParserError) as reason: + out(f'[ERROR] Invalid yaml: {reason}') + sys.exit(1) + else: + if type(new_config) is str: + out(f'[ERROR] Invalid yaml: {new_config!r}') + sys.exit(1) + elif new_config is None or not len(new_config): + out('[ERROR] Empty yaml: Please pipe yaml into stdin') + sys.exit(1) - db.session.commit() + error = False + tracked = {} + for section, model in yaml_sections: - managers = new_config.get('managers', []) - # tracked_managers=set() - for manager_config in managers: - if verbose: - print(str(manager_config)) - domain_name = manager_config['domain'] - user_name = manager_config['user'] - domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) - if manageruser not in domain.managers: - domain.managers.append(manageruser) - db.session.add(domain) + items = new_config.get(section) + if items is None: + if delete_objects: + out(f'[ERROR] Invalid yaml: Section "{section}" is missing') + error = True + break + else: + continue - db.session.commit() + del new_config[section] + if type(items) is not list: + out(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') + error = True + break + elif not items: + continue + + # create items + for data in items: + + if verbose: + out(f'Handling {model.__table__} data: {data!r}') + + try: + changed = model.from_dict(data, delete_objects) + except Exception as reason: + out(f'[ERROR] {reason.args[0]} in data: {data}') + error = True + break + + for item, created in changed: + + if created is True: + # flush newly created item + db.session.add(item) + db.session.flush() + if verbose: + out(f'Added {item!r}: {item.to_dict()}') + else: + out(f'Added {item!r}') + + elif len(created): + # modified instance + if verbose: + for key, old, new in created: + out(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') + else: + out(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') + + # track primary key of all items + tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) + + if error: + break + + # on error: stop early + if error: + out('An error occured. Not committing changes.') + db.session.rollback() + sys.exit(1) + + # are there sections left in new_config? + if new_config: + out(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') + error = True + + # test for conflicting domains + domains = set() + for model, items in tracked.items(): + if model in (models.Domain, models.Alternative, models.Relay): + if domains & items: + for domain in domains & items: + out(f'[ERROR] Duplicate domain name used: {domain}') + error = True + domains.update(items) + + # delete items not tracked if delete_objects: - for user in db.session.query(models.User).all(): - if not (user.email in tracked_users): - if verbose: - print("Deleting user: " + str(user.email)) - db.session.delete(user) - for alias in db.session.query(models.Alias).all(): - if not (alias.email in tracked_aliases): - if verbose: - print("Deleting alias: " + str(alias.email)) - db.session.delete(alias) - for domain in db.session.query(models.Domain).all(): - if not (domain.name in tracked_domains): - if verbose: - print("Deleting domain: " + str(domain.name)) - db.session.delete(domain) - db.session.commit() + for model, items in tracked.items(): + for item in model.query.all(): + if not item._dict_pval() in items: + out(f'Deleted {item!r} {item}') + db.session.delete(item) + + # don't commit when running dry + if dry_run: + db.session.rollback() + else: + db.session.commit() + + +@mailu.command() +@click.option('-v', '--verbose', is_flag=True) +@click.option('-s', '--secrets', is_flag=True) +@flask_cli.with_appcontext +def config_dump(verbose=False, secrets=False): + """dump configuration as YAML-formatted data to stdout""" + + config = {} + for section, model in yaml_sections: + dump = [item.to_dict(verbose, secrets) for item in model.query.all()] + if len(dump): + config[section] = dump + + yaml.dump(config, sys.stdout, default_flow_style=False, allow_unicode=True) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 0a447758..fde4d6f1 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -5,6 +5,7 @@ from passlib import context, hash from datetime import datetime, date from email.mime import text from flask import current_app as app +from textwrap import wrap import flask_sqlalchemy import sqlalchemy @@ -15,6 +16,8 @@ import glob import smtplib import idna import dns +import json +import itertools db = flask_sqlalchemy.SQLAlchemy() @@ -32,6 +35,7 @@ class IdnaDomain(db.TypeDecorator): def process_result_value(self, value, dialect): return idna.decode(value) + python_type = str class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) @@ -56,6 +60,7 @@ class IdnaEmail(db.TypeDecorator): idna.decode(domain_name), ) + python_type = str class CommaSeparatedList(db.TypeDecorator): """ Stores a list as a comma-separated string, compatible with Postfix. @@ -74,6 +79,7 @@ class CommaSeparatedList(db.TypeDecorator): def process_result_value(self, value, dialect): return list(filter(bool, value.split(","))) if value else [] + python_type = list class JSONEncoded(db.TypeDecorator): """ Represents an immutable structure as a json-encoded string. @@ -87,6 +93,7 @@ class JSONEncoded(db.TypeDecorator): def process_result_value(self, value, dialect): return json.loads(value) if value else None + python_type = str class Base(db.Model): """ Base class for all models @@ -105,6 +112,219 @@ class Base(db.Model): updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True) + @classmethod + def _dict_pkey(model): + return model.__mapper__.primary_key[0].name + + def _dict_pval(self): + return getattr(self, self._dict_pkey()) + + def to_dict(self, full=False, include_secrets=False, recursed=False, hide=None): + """ Return a dictionary representation of this model. + """ + + if recursed and not getattr(self, '_dict_recurse', False): + return str(self) + + hide = set(hide or []) | {'created_at', 'updated_at'} + if hasattr(self, '_dict_hide'): + hide |= self._dict_hide + + secret = set() + if not include_secrets and hasattr(self, '_dict_secret'): + secret |= self._dict_secret + + convert = getattr(self, '_dict_output', {}) + + res = {} + + for key in itertools.chain(self.__table__.columns.keys(), getattr(self, '_dict_show', [])): + if key in hide: + continue + if key in self.__table__.columns: + default = self.__table__.columns[key].default + if isinstance(default, sqlalchemy.sql.schema.ColumnDefault): + default = default.arg + else: + default = None + value = getattr(self, key) + if full or ((default or value) and value != default): + if key in secret: + value = '' + elif value is not None and key in convert: + value = convert[key](value) + res[key] = value + + for key in self.__mapper__.relationships.keys(): + if key in hide: + continue + if self.__mapper__.relationships[key].uselist: + items = getattr(self, key) + if self.__mapper__.relationships[key].query_class is not None: + if hasattr(items, 'all'): + items = items.all() + if full or len(items): + if key in secret: + res[key] = '' + else: + res[key] = [item.to_dict(full, include_secrets, True) for item in items] + else: + value = getattr(self, key) + if full or value is not None: + if key in secret: + res[key] = '' + else: + res[key] = value.to_dict(full, include_secrets, True) + + return res + + @classmethod + def from_dict(model, data, delete=False): + + changed = [] + + pkey = model._dict_pkey() + + # handle "primary key" only + if type(data) is not dict: + data = {pkey: data} + + # modify input data + if hasattr(model, '_dict_input'): + try: + model._dict_input(data) + except Exception as reason: + raise ValueError(f'{reason}', model, None, data) + + # check for primary key (if not recursed) + if not getattr(model, '_dict_recurse', False): + if not pkey in data: + raise KeyError(f'primary key {model.__table__}.{pkey} is missing', model, pkey, data) + + # check data keys and values + for key, value in data.items(): + + # check key + if not hasattr(model, key): + raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) + + # check value type + col = model.__mapper__.columns.get(key) + if col is not None: + if not type(value) is col.type.python_type: + raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) + else: + rel = model.__mapper__.relationships.get(key) + if rel is None: + itype = getattr(model, '_dict_types', {}).get(key) + if itype is not None: + if type(value) is not itype: + raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) + else: + raise NotImplementedError(f'type not defined for {model.__table__}.{key}') + + # handle relationships + if key in model.__mapper__.relationships: + rel_model = model.__mapper__.relationships[key].argument + if not isinstance(rel_model, sqlalchemy.orm.Mapper): + add = rel_model.from_dict(value, delete) + assert len(add) == 1 + item, updated = add[0] + changed.append((item, updated)) + data[key] = item + + # create or update item? + item = model.query.get(data[pkey]) if pkey in data else None + if item is None: + # create item + + # check for mandatory keys + missing = getattr(model, '_dict_mandatory', set()) - set(data.keys()) + if missing: + raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {model.__table__} missing', model, missing, data) + + changed.append((model(**data), True)) + + else: + # update item + + updated = [] + for key, value in data.items(): + + # skip primary key + if key == pkey: + continue + + if key in model.__mapper__.relationships: + # update relationship + rel_model = model.__mapper__.relationships[key].argument + if isinstance(rel_model, sqlalchemy.orm.Mapper): + rel_model = rel_model.class_ + # add (and create) referenced items + cur = getattr(item, key) + old = sorted(cur, key=lambda i:id(i)) + new = [] + for rel_data in value: + # get or create related item + add = rel_model.from_dict(rel_data, delete) + assert len(add) == 1 + rel_item, rel_updated = add[0] + changed.append((rel_item, rel_updated)) + if rel_item not in cur: + cur.append(rel_item) + new.append(rel_item) + + # delete referenced items missing in yaml + rel_pkey = rel_model._dict_pkey() + new_data = list([i.to_dict(True, True, True, [rel_pkey]) for i in new]) + for rel_item in old: + if rel_item not in new: + # check if item with same data exists to stabilze import without primary key + rel_data = rel_item.to_dict(True, True, True, [rel_pkey]) + try: + same_idx = new_data.index(rel_data) + except ValueError: + same = None + else: + same = new[same_idx] + + if same is None: + # delete items missing in new + if delete: + cur.remove(rel_item) + else: + new.append(rel_item) + else: + # swap found item with same data with newly created item + new.append(rel_item) + new_data.append(rel_data) + new.remove(same) + del new_data[same_idx] + for i, (ch_item, ch_update) in enumerate(changed): + if ch_item is same: + changed[i] = (rel_item, []) + db.session.flush() + db.session.delete(ch_item) + break + + # remember changes + new = sorted(new, key=lambda i:id(i)) + if new != old: + updated.append((key, old, new)) + + else: + # update key + old = getattr(item, key) + if type(old) is list and not delete: + value = old + value + if value != old: + updated.append((key, old, value)) + setattr(item, key, value) + + changed.append((item, updated)) + + return changed + # Many-to-many association table for domain managers managers = db.Table('manager', Base.metadata, @@ -126,6 +346,27 @@ class Domain(Base): """ __tablename__ = "domain" + _dict_hide = {'users', 'managers', 'aliases'} + _dict_show = {'dkim_key'} + _dict_secret = {'dkim_key'} + _dict_types = {'dkim_key': bytes} + _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} + @staticmethod + def _dict_input(data): + key = data.get('dkim_key') + if key is not None: + key = data['dkim_key'] + if type(key) is list: + key = ''.join(key) + if type(key) is str: + key = ''.join(key.strip().split()) + if key.startswith('-----BEGIN PRIVATE KEY-----'): + key = key[25:] + if key.endswith('-----END PRIVATE KEY-----'): + key = key[:-23] + key = '\n'.join(wrap(key, 64)) + data['dkim_key'] = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, backref=db.backref('manager_of'), lazy='dynamic') @@ -208,6 +449,8 @@ class Relay(Base): __tablename__ = "relay" + _dict_mandatory = {'smtp'} + name = db.Column(IdnaDomain, primary_key=True, nullable=False) smtp = db.Column(db.String(80), nullable=True) @@ -221,6 +464,16 @@ class Email(object): localpart = db.Column(db.String(80), nullable=False) + @staticmethod + def _dict_input(data): + if 'email' in data: + if 'localpart' in data or 'domain' in data: + raise ValueError('ambigous key email and localpart/domain') + elif type(data['email']) is str: + data['localpart'], data['domain'] = data['email'].rsplit('@', 1) + else: + data['email'] = f"{data['localpart']}@{data['domain']}" + @declarative.declared_attr def domain_name(cls): return db.Column(IdnaDomain, db.ForeignKey(Domain.name), @@ -306,6 +559,28 @@ class User(Base, Email): """ __tablename__ = "user" + _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} + _dict_mandatory = {'localpart', 'domain', 'password'} + @classmethod + def _dict_input(cls, data): + Email._dict_input(data) + # handle password + if 'password' in data: + if 'password_hash' in data or 'hash_scheme' in data: + raise ValueError('ambigous key password and password_hash/hash_scheme') + # check (hashed) password + password = data['password'] + if password.startswith('{') and '}' in password: + scheme = password[1:password.index('}')] + if scheme not in cls.scheme_dict: + raise ValueError(f'invalid password scheme {scheme!r}') + else: + raise ValueError(f'invalid hashed password {password!r}') + elif 'password_hash' in data and 'hash_scheme' in data: + if data['hash_scheme'] not in cls.scheme_dict: + raise ValueError(f'invalid password scheme {scheme!r}') + data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) @@ -431,6 +706,14 @@ class Alias(Base, Email): """ __tablename__ = "alias" + _dict_hide = {'domain_name', 'domain', 'localpart'} + @staticmethod + def _dict_input(data): + # handle comma delimited string for backwards compability + dst = data.get('destination') + if type(dst) is str: + data['destination'] = list([adr.strip() for adr in dst.split(',')]) + domain = db.relationship(Domain, backref=db.backref('aliases', cascade='all, delete-orphan')) wildcard = db.Column(db.Boolean(), nullable=False, default=False) @@ -484,6 +767,10 @@ class Token(Base): """ __tablename__ = "token" + _dict_recurse = True + _dict_hide = {'user', 'user_email'} + _dict_mandatory = {'password'} + id = db.Column(db.Integer(), primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) @@ -499,7 +786,7 @@ class Token(Base): self.password = hash.sha256_crypt.using(rounds=1000).hash(password) def __str__(self): - return self.comment + return self.comment or self.ip class Fetch(Base): @@ -508,6 +795,11 @@ class Fetch(Base): """ __tablename__ = "fetch" + _dict_recurse = True + _dict_hide = {'user_email', 'user', 'last_check', 'error'} + _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} + _dict_secret = {'password'} + id = db.Column(db.Integer(), primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) @@ -516,9 +808,12 @@ class Fetch(Base): protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) host = db.Column(db.String(255), nullable=False) port = db.Column(db.Integer(), nullable=False) - tls = db.Column(db.Boolean(), nullable=False) + tls = db.Column(db.Boolean(), nullable=False, default=False) username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) - keep = db.Column(db.Boolean(), nullable=False) + keep = db.Column(db.Boolean(), nullable=False, default=False) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) + + def __str__(self): + return f'{self.protocol}{"s" if self.tls else ""}://{self.username}@{self.host}:{self.port}' From 190e7a709b6e3bb134ba8086e54796f14d42a05c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 23:14:27 +0200 Subject: [PATCH 04/52] renamed config-dump option --verbose to --full --- core/admin/mailu/manage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index bf0148df..f74d24b5 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -299,15 +299,15 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @mailu.command() -@click.option('-v', '--verbose', is_flag=True) +@click.option('-f', '--full', is_flag=True) @click.option('-s', '--secrets', is_flag=True) @flask_cli.with_appcontext -def config_dump(verbose=False, secrets=False): +def config_dump(full=False, secrets=False): """dump configuration as YAML-formatted data to stdout""" config = {} for section, model in yaml_sections: - dump = [item.to_dict(verbose, secrets) for item in model.query.all()] + dump = [item.to_dict(full, secrets) for item in model.query.all()] if len(dump): config[section] = dump From 69ccf791d27dabfe1b8943d837923d5b2ba78d72 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 23:16:37 +0200 Subject: [PATCH 05/52] fixed data import via from_dict - stabilized CommaSeparatedList by sorting values - CommaSeparatedList can now handle list and set input - from_dict now handles mapped keys - from_dict now handles null values - class Domain: handle dkim-key None correctly - class User: delete obsolete keys after converting - class Alias: now uses Email._dict_input --- core/admin/mailu/models.py | 205 ++++++++++++++++++++----------------- 1 file changed, 113 insertions(+), 92 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index fde4d6f1..cbafc6a4 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -69,12 +69,12 @@ class CommaSeparatedList(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): - if type(value) is not list: + if not isinstance(value, (list, set)): raise TypeError("Should be a list") for item in value: if "," in item: raise ValueError("Item must not contain a comma") - return ",".join(value) + return ",".join(sorted(value)) def process_result_value(self, value, dialect): return list(filter(bool, value.split(","))) if value else [] @@ -205,13 +205,13 @@ class Base(db.Model): for key, value in data.items(): # check key - if not hasattr(model, key): + if not hasattr(model, key) and not key in model.__mapper__.relationships: raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) # check value type col = model.__mapper__.columns.get(key) if col is not None: - if not type(value) is col.type.python_type: + if not ((value is None and col.nullable) or (type(value) is col.type.python_type)): raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) else: rel = model.__mapper__.relationships.get(key) @@ -229,99 +229,115 @@ class Base(db.Model): if not isinstance(rel_model, sqlalchemy.orm.Mapper): add = rel_model.from_dict(value, delete) assert len(add) == 1 - item, updated = add[0] - changed.append((item, updated)) - data[key] = item + rel_item, updated = add[0] + changed.append((rel_item, updated)) + data[key] = rel_item - # create or update item? + # create item if necessary + created = False item = model.query.get(data[pkey]) if pkey in data else None if item is None: - # create item # check for mandatory keys missing = getattr(model, '_dict_mandatory', set()) - set(data.keys()) if missing: raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {model.__table__} missing', model, missing, data) - changed.append((model(**data), True)) - - else: - # update item - - updated = [] - for key, value in data.items(): - - # skip primary key - if key == pkey: - continue - + # remove mapped relationships from data + mapped = {} + for key in list(data.keys()): if key in model.__mapper__.relationships: - # update relationship - rel_model = model.__mapper__.relationships[key].argument - if isinstance(rel_model, sqlalchemy.orm.Mapper): - rel_model = rel_model.class_ - # add (and create) referenced items - cur = getattr(item, key) - old = sorted(cur, key=lambda i:id(i)) - new = [] - for rel_data in value: - # get or create related item - add = rel_model.from_dict(rel_data, delete) - assert len(add) == 1 - rel_item, rel_updated = add[0] - changed.append((rel_item, rel_updated)) - if rel_item not in cur: - cur.append(rel_item) - new.append(rel_item) + if isinstance(model.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): + mapped[key] = data[key] + del data[key] - # delete referenced items missing in yaml - rel_pkey = rel_model._dict_pkey() - new_data = list([i.to_dict(True, True, True, [rel_pkey]) for i in new]) - for rel_item in old: - if rel_item not in new: - # check if item with same data exists to stabilze import without primary key - rel_data = rel_item.to_dict(True, True, True, [rel_pkey]) - try: - same_idx = new_data.index(rel_data) - except ValueError: - same = None - else: - same = new[same_idx] + # create new item + item = model(**data) + created = True - if same is None: - # delete items missing in new - if delete: - cur.remove(rel_item) - else: - new.append(rel_item) + # and update mapped relationships (below) + data = mapped + + # update item + updated = [] + for key, value in data.items(): + + # skip primary key + if key == pkey: + continue + + if key in model.__mapper__.relationships: + # update relationship + rel_model = model.__mapper__.relationships[key].argument + if isinstance(rel_model, sqlalchemy.orm.Mapper): + rel_model = rel_model.class_ + # add (and create) referenced items + cur = getattr(item, key) + old = sorted(cur, key=lambda i:id(i)) + new = [] + for rel_data in value: + # get or create related item + add = rel_model.from_dict(rel_data, delete) + assert len(add) == 1 + rel_item, rel_updated = add[0] + changed.append((rel_item, rel_updated)) + if rel_item not in cur: + cur.append(rel_item) + new.append(rel_item) + + # delete referenced items missing in yaml + rel_pkey = rel_model._dict_pkey() + new_data = list([i.to_dict(True, True, True, [rel_pkey]) for i in new]) + for rel_item in old: + if rel_item not in new: + # check if item with same data exists to stabilze import without primary key + rel_data = rel_item.to_dict(True, True, True, [rel_pkey]) + try: + same_idx = new_data.index(rel_data) + except ValueError: + same = None + else: + same = new[same_idx] + + if same is None: + # delete items missing in new + if delete: + cur.remove(rel_item) else: - # swap found item with same data with newly created item new.append(rel_item) - new_data.append(rel_data) - new.remove(same) - del new_data[same_idx] - for i, (ch_item, ch_update) in enumerate(changed): - if ch_item is same: - changed[i] = (rel_item, []) - db.session.flush() - db.session.delete(ch_item) - break + else: + # swap found item with same data with newly created item + new.append(rel_item) + new_data.append(rel_data) + new.remove(same) + del new_data[same_idx] + for i, (ch_item, ch_update) in enumerate(changed): + if ch_item is same: + changed[i] = (rel_item, []) + db.session.flush() + db.session.delete(ch_item) + break - # remember changes - new = sorted(new, key=lambda i:id(i)) - if new != old: - updated.append((key, old, new)) + # remember changes + new = sorted(new, key=lambda i:id(i)) + if new != old: + updated.append((key, old, new)) - else: - # update key - old = getattr(item, key) - if type(old) is list and not delete: - value = old + value - if value != old: - updated.append((key, old, value)) - setattr(item, key, value) + else: + # update key + old = getattr(item, key) + if type(old) is list: + # deduplicate list value + assert type(value) is list + value = set(value) + old = set(old) + if not delete: + value = old | value + if value != old: + updated.append((key, old, value)) + setattr(item, key, value) - changed.append((item, updated)) + changed.append((item, created if created else updated)) return changed @@ -353,19 +369,21 @@ class Domain(Base): _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} @staticmethod def _dict_input(data): - key = data.get('dkim_key') - if key is not None: + if 'dkim_key' in data: key = data['dkim_key'] - if type(key) is list: - key = ''.join(key) - if type(key) is str: - key = ''.join(key.strip().split()) - if key.startswith('-----BEGIN PRIVATE KEY-----'): - key = key[25:] - if key.endswith('-----END PRIVATE KEY-----'): - key = key[:-23] - key = '\n'.join(wrap(key, 64)) - data['dkim_key'] = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + if key is None: + del data['dkim_key'] + else: + if type(key) is list: + key = ''.join(key) + if type(key) is str: + key = ''.join(key.strip().split()) + if key.startswith('-----BEGIN PRIVATE KEY-----'): + key = key[25:] + if key.endswith('-----END PRIVATE KEY-----'): + key = key[:-23] + key = '\n'.join(wrap(key, 64)) + data['dkim_key'] = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, @@ -580,6 +598,8 @@ class User(Base, Email): if data['hash_scheme'] not in cls.scheme_dict: raise ValueError(f'invalid password scheme {scheme!r}') data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + del data['hash_scheme'] + del data['password_hash'] domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) @@ -709,6 +729,7 @@ class Alias(Base, Email): _dict_hide = {'domain_name', 'domain', 'localpart'} @staticmethod def _dict_input(data): + Email._dict_input(data) # handle comma delimited string for backwards compability dst = data.get('destination') if type(dst) is str: From 3a5a15a5e6326cee38b018fcad6d05ab92c0485a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 26 Aug 2020 23:23:03 +0200 Subject: [PATCH 06/52] updated documentation and changelog added some documentation for cli commands config-dump and config-update --- CHANGELOG.md | 2 + docs/cli.rst | 127 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 98 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ccde4bc..1c20d9b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ deprecated before 1.8.0, you can switch to an external database server by then. - Features: use HTTP/1.1 for proxyied connections ([#1070](https://github.com/Mailu/Mailu/issues/1070)) - Features: Update Rainloop to 1.13.0 ([#1071](https://github.com/Mailu/Mailu/issues/1071)) - Features: Use python package socrate instead of Mailustart ([#1082](https://github.com/Mailu/Mailu/issues/1082)) +- Features: Added cli command config-dump ([#1377](https://github.com/Mailu/Mailu/issues/1377)) - Bugfixes: Use ldez/traefik-certs-dumper in our certificate dumper to have a more robust solution ([#820](https://github.com/Mailu/Mailu/issues/820)) - Bugfixes: Make aliases optionally case-insensitive: After attempting to resolve an alias in its preserved case, also attempt to match it case-insensitively ([#867](https://github.com/Mailu/Mailu/issues/867)) - Bugfixes: Fix HOST_* variable usage ([#884](https://github.com/Mailu/Mailu/issues/884)) @@ -47,6 +48,7 @@ deprecated before 1.8.0, you can switch to an external database server by then. - Enhancement: Create an Authentication Token with IPv6 address restriction ([#829](https://github.com/Mailu/Mailu/issues/829)) - Enhancement: Automatically create admin user on container startup if given appropriate environment variables - Enhancement: Missing wildcard option in alias flask command ([#869](https://github.com/Mailu/Mailu/issues/869)) +- Enhancement: Cli command config-update now updates all objects and parameters ([#1377](https://github.com/Mailu/Mailu/issues/1377)) v1.6.0 - 2019-01-18 ------------------- diff --git a/docs/cli.rst b/docs/cli.rst index a9cff41c..7476e676 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -10,6 +10,7 @@ Managing users and aliases can be done from CLI using commands: * user * user-import * user-delete +* config-dump * config-update alias @@ -62,60 +63,124 @@ primary difference with simple `user` command is that password is being imported docker-compose run --rm admin flask mailu user-import myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' 'SHA512-CRYPT' user-delete ------------- +----------- .. code-block:: bash docker-compose exec admin flask mailu user-delete foo@example.net +config-dump +----------- + +The purpose of this command is to dump domain-, relay-, alias- and user-configuration to a YAML template. +If you want to export non-hashed secrets you have to add the ``--secrets`` option. +Only non-default options are dumped. If you want to dump all options use ``--full``. + +.. code-block:: bash + + docker-compose exec admin flask mailu config-dump > mail-config.yml + config-update ------------- -The sole purpose of this command is for importing users/aliases in bulk and synchronizing DB entries with external YAML template: +The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML template. .. code-block:: bash - cat mail-config.yml | docker-compose exec -T admin flask mailu config-update --delete-objects + docker exec -i $(docker-compose ps -q admin) flask mailu config-update -nvd < mail-config.yml -where mail-config.yml looks like: +*(The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec)* -.. code-block:: bash +mail-config.yml looks like: + +.. code-block:: yaml + + domains: + - name: example.com + alternatives: + - alternative.example.com users: - - localpart: foo - domain: example.com + - email: foo@example.com password_hash: klkjhumnzxcjkajahsdqweqqwr hash_scheme: MD5-CRYPT aliases: - - localpart: alias1 - domain: example.com + - email: alias1@example.com destination: "user1@example.com,user2@example.com" -without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. + relays: + - name: relay.example.com + comment: test + smtp: mx.example.com -Users ------ +You can use ``--dry-run`` to test your YAML without omitting any changes to the database. +With ``--verbose`` config-update will show exactly what it changes in the database. +Without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. -following are additional parameters that could be defined for users: +This is a complete YAML template with all additional parameters that could be defined: -* comment -* quota_bytes -* global_admin -* enable_imap -* enable_pop -* forward_enabled -* forward_destination -* reply_enabled -* reply_subject -* reply_body -* displayed_name -* spam_enabled -* spam_threshold +.. code-block:: yaml -Alias ------ + aliases: + - email: email@example.com + comment: '' + destination: + - address@example.com + wildcard: false + + domains: + - name: example.com + alternatives: + - alternative.tld + comment: '' + dkim_key: '' + max_aliases: -1 + max_quota_bytes: 0 + max_users: -1 + signup_enabled: false + + relays: + - name: relay.example.com + comment: '' + smtp: mx.example.com + + users: + - email: postmaster@example.com + comment: '' + displayed_name: 'Postmaster' + enable_imap: true + enable_pop: false + enabled: true + fetches: + - id: 1 + comment: 'test fetch' + username: fetch-user + host: other.example.com + password: 'secret' + port: 993 + protocol: imap + tls: true + keep: true + forward_destination: + - address@remote.example.com + forward_enabled: true + forward_keep: true + global_admin: true + manager_of: + - example.com + password: '{BLF-CRYPT}$2b$12$...' + quota_bytes: 1000000000 + reply_body: '' + reply_enabled: false + reply_enddate: 2999-12-31 + reply_startdate: 1900-01-01 + reply_subject: '' + spam_enabled: true + spam_threshold: 80 + tokens: + - id: 1 + comment: email-client + ip: 192.168.1.1 + password: '$5$rounds=1000$...' -additional fields: - -* wildcard From 0cf91f35a45b7272a1b9b5cb0d8760ab4cb0bdfe Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 27 Aug 2020 16:08:15 +0200 Subject: [PATCH 07/52] moved change log entry to towncrier --- CHANGELOG.md | 2 -- towncrier/newsfragments/1604.feature | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/1604.feature diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c20d9b2..7ccde4bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,7 +28,6 @@ deprecated before 1.8.0, you can switch to an external database server by then. - Features: use HTTP/1.1 for proxyied connections ([#1070](https://github.com/Mailu/Mailu/issues/1070)) - Features: Update Rainloop to 1.13.0 ([#1071](https://github.com/Mailu/Mailu/issues/1071)) - Features: Use python package socrate instead of Mailustart ([#1082](https://github.com/Mailu/Mailu/issues/1082)) -- Features: Added cli command config-dump ([#1377](https://github.com/Mailu/Mailu/issues/1377)) - Bugfixes: Use ldez/traefik-certs-dumper in our certificate dumper to have a more robust solution ([#820](https://github.com/Mailu/Mailu/issues/820)) - Bugfixes: Make aliases optionally case-insensitive: After attempting to resolve an alias in its preserved case, also attempt to match it case-insensitively ([#867](https://github.com/Mailu/Mailu/issues/867)) - Bugfixes: Fix HOST_* variable usage ([#884](https://github.com/Mailu/Mailu/issues/884)) @@ -48,7 +47,6 @@ deprecated before 1.8.0, you can switch to an external database server by then. - Enhancement: Create an Authentication Token with IPv6 address restriction ([#829](https://github.com/Mailu/Mailu/issues/829)) - Enhancement: Automatically create admin user on container startup if given appropriate environment variables - Enhancement: Missing wildcard option in alias flask command ([#869](https://github.com/Mailu/Mailu/issues/869)) -- Enhancement: Cli command config-update now updates all objects and parameters ([#1377](https://github.com/Mailu/Mailu/issues/1377)) v1.6.0 - 2019-01-18 ------------------- diff --git a/towncrier/newsfragments/1604.feature b/towncrier/newsfragments/1604.feature new file mode 100644 index 00000000..06ee0beb --- /dev/null +++ b/towncrier/newsfragments/1604.feature @@ -0,0 +1 @@ +Added cli command config-dump and enhanced config-update From 85de70212904755a5bc6ef756cfe615dec7ff0bb Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 27 Aug 2020 16:10:53 +0200 Subject: [PATCH 08/52] small typo. Change 'omitting' to 'commiting' --- docs/cli.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cli.rst b/docs/cli.rst index 7476e676..64a4b17a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -114,7 +114,7 @@ mail-config.yml looks like: comment: test smtp: mx.example.com -You can use ``--dry-run`` to test your YAML without omitting any changes to the database. +You can use ``--dry-run`` to test your YAML without comitting any changes to the database. With ``--verbose`` config-update will show exactly what it changes in the database. Without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. From ffbeabeb6f685fa3939bbc1ef1266ef8c7ae511c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 27 Aug 2020 22:02:20 +0200 Subject: [PATCH 09/52] updated test to use --verbose flag --verbose (or -v) is now a flag and not an option --- tests/compose/core/02_forward_test.sh | 4 ++-- tests/compose/core/03_alias_test.sh | 4 ++-- tests/compose/core/04_reply_test.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/compose/core/02_forward_test.sh b/tests/compose/core/02_forward_test.sh index 595820cf..651e027c 100755 --- a/tests/compose/core/02_forward_test.sh +++ b/tests/compose/core/02_forward_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -10,7 +10,7 @@ EOF python3 tests/forward_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" diff --git a/tests/compose/core/03_alias_test.sh b/tests/compose/core/03_alias_test.sh index dce1918a..2d40903a 100755 --- a/tests/compose/core/03_alias_test.sh +++ b/tests/compose/core/03_alias_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose aliases: - localpart: alltheusers domain: mailu.io @@ -7,6 +7,6 @@ EOF python3 tests/alias_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose aliases: [] EOF diff --git a/tests/compose/core/04_reply_test.sh b/tests/compose/core/04_reply_test.sh index 83c114f6..7615a0f8 100755 --- a/tests/compose/core/04_reply_test.sh +++ b/tests/compose/core/04_reply_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -11,7 +11,7 @@ EOF python3 tests/reply_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" From 9d2327b0f1b62de4f57e0c4367434572ae85c46a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Sep 2020 12:32:51 +0200 Subject: [PATCH 10/52] add space for more human readable indentation add a newline before main sections add some spaces to indent --- core/admin/mailu/manage.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index f74d24b5..c36f45c0 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -305,13 +305,23 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) def config_dump(full=False, secrets=False): """dump configuration as YAML-formatted data to stdout""" + class spacedDumper(yaml.Dumper): + + def write_line_break(self, data=None): + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + config = {} for section, model in yaml_sections: dump = [item.to_dict(full, secrets) for item in model.query.all()] if len(dump): config[section] = dump - yaml.dump(config, sys.stdout, default_flow_style=False, allow_unicode=True) + yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True) @mailu.command() From 8e14aa80ee2ea6546bd659a03813db44bd9f0024 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Sep 2020 12:57:40 +0200 Subject: [PATCH 11/52] documented options and added help text --- core/admin/mailu/manage.py | 10 +++++----- docs/cli.rst | 40 +++++++++++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index c36f45c0..b25ab568 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -180,9 +180,9 @@ yaml_sections = [ ] @mailu.command() -@click.option('-v', '--verbose', is_flag=True) -@click.option('-d', '--delete-objects', is_flag=True) -@click.option('-n', '--dry-run', is_flag=True) +@click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') +@click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') +@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') @flask_cli.with_appcontext def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): """sync configuration with data from YAML-formatted stdin""" @@ -299,8 +299,8 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @mailu.command() -@click.option('-f', '--full', is_flag=True) -@click.option('-s', '--secrets', is_flag=True) +@click.option('-f', '--full', is_flag=True, help='Include default attributes') +@click.option('-s', '--secrets', is_flag=True, help='Include secrets (plain-text / not hashed)') @flask_cli.with_appcontext def config_dump(full=False, secrets=False): """dump configuration as YAML-formatted data to stdout""" diff --git a/docs/cli.rst b/docs/cli.rst index 64a4b17a..0036e504 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -73,8 +73,22 @@ config-dump ----------- The purpose of this command is to dump domain-, relay-, alias- and user-configuration to a YAML template. -If you want to export non-hashed secrets you have to add the ``--secrets`` option. -Only non-default options are dumped. If you want to dump all options use ``--full``. + +.. code-block:: bash + + # docker-compose exec admin flask mailu config-dump --help + + Usage: flask mailu config-dump [OPTIONS] + + dump configuration as YAML-formatted data to stdout + + Options: + -f, --full Include default attributes + -s, --secrets Include secrets (plain-text / not hashed) + --help Show this message and exit. + +If you want to export secrets (plain-text / not hashed) you have to add the ``--secrets`` option. +Only non-default attributes are dumped. If you want to dump all attributes use ``--full``. .. code-block:: bash @@ -85,13 +99,29 @@ config-update The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML template. +.. code-block:: bash + + # docker-compose exec admin flask mailu config-update --help + + Usage: flask mailu config-update [OPTIONS] + + sync configuration with data from YAML-formatted stdin + + Options: + -v, --verbose Increase verbosity + -d, --delete-objects Remove objects not included in yaml + -n, --dry-run Perform a trial run with no changes made + --help Show this message and exit. + + +The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec instead: + .. code-block:: bash docker exec -i $(docker-compose ps -q admin) flask mailu config-update -nvd < mail-config.yml -*(The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec)* -mail-config.yml looks like: +mail-config.yml looks like this: .. code-block:: yaml @@ -116,7 +146,7 @@ mail-config.yml looks like: You can use ``--dry-run`` to test your YAML without comitting any changes to the database. With ``--verbose`` config-update will show exactly what it changes in the database. -Without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. +Without ``--delete-object`` option config-update will only add/update changed values but will *not* remove any entries missing in provided YAML input. This is a complete YAML template with all additional parameters that could be defined: From acc728109bc94e9d996985c582cdfc96ba8ecde9 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:13 +0200 Subject: [PATCH 12/52] validate dkim keys and allow removal --- core/admin/mailu/models.py | 41 +++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index cbafc6a4..0dc82bf4 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -218,7 +218,7 @@ class Base(db.Model): if rel is None: itype = getattr(model, '_dict_types', {}).get(key) if itype is not None: - if type(value) is not itype: + if not isinstance(value, itype): raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) else: raise NotImplementedError(f'type not defined for {model.__table__}.{key}') @@ -365,25 +365,34 @@ class Domain(Base): _dict_hide = {'users', 'managers', 'aliases'} _dict_show = {'dkim_key'} _dict_secret = {'dkim_key'} - _dict_types = {'dkim_key': bytes} + _dict_types = {'dkim_key': (bytes, type(None))} _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} @staticmethod def _dict_input(data): if 'dkim_key' in data: key = data['dkim_key'] - if key is None: - del data['dkim_key'] - else: + if key is not None: if type(key) is list: key = ''.join(key) if type(key) is str: - key = ''.join(key.strip().split()) - if key.startswith('-----BEGIN PRIVATE KEY-----'): - key = key[25:] - if key.endswith('-----END PRIVATE KEY-----'): - key = key[:-23] - key = '\n'.join(wrap(key, 64)) - data['dkim_key'] = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + key = ''.join(key.strip().split()) # removes all whitespace + if key: + m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) + if m is not None: + key = key[m.end():] + m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) + if m is not None: + key = key[:m.start()] + key = '\n'.join(wrap(key, 64)) + key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + try: + dkim.strip_key(key) + except: + raise ValueError('invalid dkim key') + else: + data['dkim_key'] = key + else: + data['dkim_key'] = None name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, @@ -405,8 +414,12 @@ class Domain(Base): def dkim_key(self, value): file_path = app.config["DKIM_PATH"].format( domain=self.name, selector=app.config["DKIM_SELECTOR"]) - with open(file_path, "wb") as handle: - handle.write(value) + if value is None: + if os.path.exists(file_path): + os.unlink(file_path) + else: + with open(file_path, "wb") as handle: + handle.write(value) @property def dkim_publickey(self): From c46f9328f700b12cd5dda4fbe0c270d54e584ae2 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:26 +0200 Subject: [PATCH 13/52] also dump dkim_publickey. allow key generation. --- core/admin/mailu/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 0dc82bf4..a9cc8d16 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -363,7 +363,7 @@ class Domain(Base): __tablename__ = "domain" _dict_hide = {'users', 'managers', 'aliases'} - _dict_show = {'dkim_key'} + _dict_show = {'dkim_key', 'dkim_publickey'} _dict_secret = {'dkim_key'} _dict_types = {'dkim_key': (bytes, type(None))} _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} @@ -376,7 +376,9 @@ class Domain(Base): key = ''.join(key) if type(key) is str: key = ''.join(key.strip().split()) # removes all whitespace - if key: + if key == 'generate': + data['dkim_key'] = dkim.gen_key() + elif key: m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) if m is not None: key = key[m.end():] @@ -390,7 +392,7 @@ class Domain(Base): except: raise ValueError('invalid dkim key') else: - data['dkim_key'] = key + data['dkim_key'] = key else: data['dkim_key'] = None From 500967b2f585bca74a53b15a49986293f16e1750 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:29 +0200 Subject: [PATCH 14/52] ignore dkim_publickey when updating config --- core/admin/mailu/models.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a9cc8d16..6c82baa0 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -202,13 +202,14 @@ class Base(db.Model): raise KeyError(f'primary key {model.__table__}.{pkey} is missing', model, pkey, data) # check data keys and values - for key, value in data.items(): + for key in list(data.keys()): # check key if not hasattr(model, key) and not key in model.__mapper__.relationships: raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) # check value type + value = data[key] col = model.__mapper__.columns.get(key) if col is not None: if not ((value is None and col.nullable) or (type(value) is col.type.python_type)): @@ -218,7 +219,10 @@ class Base(db.Model): if rel is None: itype = getattr(model, '_dict_types', {}).get(key) if itype is not None: - if not isinstance(value, itype): + if not itype: # empty tuple => ignore value + del data[key] + continue + elif not isinstance(value, itype): raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) else: raise NotImplementedError(f'type not defined for {model.__table__}.{key}') @@ -365,7 +369,7 @@ class Domain(Base): _dict_hide = {'users', 'managers', 'aliases'} _dict_show = {'dkim_key', 'dkim_publickey'} _dict_secret = {'dkim_key'} - _dict_types = {'dkim_key': (bytes, type(None))} + _dict_types = {'dkim_key': (bytes, type(None)), 'dkim_publickey': tuple()} _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} @staticmethod def _dict_input(data): From 2a5c46c890bffac107d00f13cf6acbb9047e59e8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:31 +0200 Subject: [PATCH 15/52] Allow to dump only selected sections --- core/admin/mailu/manage.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index b25ab568..44be509c 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -300,10 +300,13 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @mailu.command() @click.option('-f', '--full', is_flag=True, help='Include default attributes') -@click.option('-s', '--secrets', is_flag=True, help='Include secrets (plain-text / not hashed)') +@click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') +@click.argument('sections', nargs=-1) @flask_cli.with_appcontext -def config_dump(full=False, secrets=False): - """dump configuration as YAML-formatted data to stdout""" +def config_dump(full=False, secrets=False, sections=None): + """dump configuration as YAML-formatted data to stdout + valid SECTIONS are: domains, relays, users, aliases + """ class spacedDumper(yaml.Dumper): @@ -315,11 +318,19 @@ def config_dump(full=False, secrets=False): def increase_indent(self, flow=False, indentless=False): return super().increase_indent(flow, False) + if sections: + check = dict(yaml_sections) + for section in sections: + if section not in check: + print(f'[ERROR] Invalid section: {section}') + return 1 + config = {} for section, model in yaml_sections: - dump = [item.to_dict(full, secrets) for item in model.query.all()] - if len(dump): - config[section] = dump + if not sections or section in sections: + dump = [item.to_dict(full, secrets) for item in model.query.all()] + if len(dump): + config[section] = dump yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True) From adc9c70c3e5b0f2f5c37ae298fd92d5d903c6d94 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:31:32 +0200 Subject: [PATCH 16/52] added dump option to dump dns data of domains --- core/admin/mailu/manage.py | 14 +++++--- core/admin/mailu/models.py | 72 ++++++++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 18 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 44be509c..42d5b5f7 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -301,11 +301,13 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @mailu.command() @click.option('-f', '--full', is_flag=True, help='Include default attributes') @click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') +@click.option('-d', '--dns', is_flag=True, help='Include dns records') @click.argument('sections', nargs=-1) @flask_cli.with_appcontext -def config_dump(full=False, secrets=False, sections=None): - """dump configuration as YAML-formatted data to stdout - valid SECTIONS are: domains, relays, users, aliases +def config_dump(full=False, secrets=False, dns=False, sections=None): + """dump configuration as YAML-formatted data to stdout + + SECTIONS can be: domains, relays, users, aliases """ class spacedDumper(yaml.Dumper): @@ -325,10 +327,14 @@ def config_dump(full=False, secrets=False, sections=None): print(f'[ERROR] Invalid section: {section}') return 1 + extra = [] + if dns: + extra.append('dns') + config = {} for section, model in yaml_sections: if not sections or section in sections: - dump = [item.to_dict(full, secrets) for item in model.query.all()] + dump = [item.to_dict(full, secrets, extra) for item in model.query.all()] if len(dump): config[section] = dump diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6c82baa0..79ce49ab 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -119,7 +119,7 @@ class Base(db.Model): def _dict_pval(self): return getattr(self, self._dict_pkey()) - def to_dict(self, full=False, include_secrets=False, recursed=False, hide=None): + def to_dict(self, full=False, include_secrets=False, include_extra=None, recursed=False, hide=None): """ Return a dictionary representation of this model. """ @@ -136,9 +136,17 @@ class Base(db.Model): convert = getattr(self, '_dict_output', {}) + extra_keys = getattr(self, '_dict_extra', {}) + if include_extra is None: + include_extra = [] + res = {} - for key in itertools.chain(self.__table__.columns.keys(), getattr(self, '_dict_show', [])): + for key in itertools.chain( + self.__table__.columns.keys(), + getattr(self, '_dict_show', []), + *[extra_keys.get(extra, []) for extra in include_extra] + ): if key in hide: continue if key in self.__table__.columns: @@ -167,14 +175,14 @@ class Base(db.Model): if key in secret: res[key] = '' else: - res[key] = [item.to_dict(full, include_secrets, True) for item in items] + res[key] = [item.to_dict(full, include_secrets, include_extra, True) for item in items] else: value = getattr(self, key) if full or value is not None: if key in secret: res[key] = '' else: - res[key] = value.to_dict(full, include_secrets, True) + res[key] = value.to_dict(full, include_secrets, include_extra, True) return res @@ -219,7 +227,7 @@ class Base(db.Model): if rel is None: itype = getattr(model, '_dict_types', {}).get(key) if itype is not None: - if not itype: # empty tuple => ignore value + if itype is False: # ignore value del data[key] continue elif not isinstance(value, itype): @@ -291,11 +299,11 @@ class Base(db.Model): # delete referenced items missing in yaml rel_pkey = rel_model._dict_pkey() - new_data = list([i.to_dict(True, True, True, [rel_pkey]) for i in new]) + new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) for rel_item in old: if rel_item not in new: # check if item with same data exists to stabilze import without primary key - rel_data = rel_item.to_dict(True, True, True, [rel_pkey]) + rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) try: same_idx = new_data.index(rel_data) except ValueError: @@ -367,10 +375,18 @@ class Domain(Base): __tablename__ = "domain" _dict_hide = {'users', 'managers', 'aliases'} - _dict_show = {'dkim_key', 'dkim_publickey'} + _dict_show = {'dkim_key'} + _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} _dict_secret = {'dkim_key'} - _dict_types = {'dkim_key': (bytes, type(None)), 'dkim_publickey': tuple()} - _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]} + _dict_types = { + 'dkim_key': (bytes, type(None)), + 'dkim_publickey': False, + 'dns_mx': False, + 'dns_spf': False, + 'dns_dkim': False, + 'dns_dmarc': False, + } + _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} @staticmethod def _dict_input(data): if 'dkim_key' in data: @@ -408,18 +424,46 @@ class Domain(Base): max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + def _dkim_file(self): + return app.config["DKIM_PATH"].format( + domain=self.name, selector=app.config["DKIM_SELECTOR"]) + + @property + def dns_mx(self): + hostname = app.config['HOSTNAMES'].split(',')[0] + return f'{self.name}. 600 IN MX 10 {hostname}.' + + @property + def dns_spf(self): + hostname = app.config['HOSTNAMES'].split(',')[0] + return f'{self.name}. 600 IN TXT "v=spf1 mx a:{hostname} ~all"' + + @property + def dns_dkim(self): + if os.path.exists(self._dkim_file()): + selector = app.config['DKIM_SELECTOR'] + return f'{selector}._domainkey.{self.name}. 600 IN TXT "v=DKIM1; k=rsa; p={self.dkim_publickey}"' + + @property + def dns_dmarc(self): + if os.path.exists(self._dkim_file()): + domain = app.config['DOMAIN'] + rua = app.config['DMARC_RUA'] + rua = f' rua=mailto:{rua}@{domain};' if rua else '' + ruf = app.config['DMARC_RUF'] + ruf = f' ruf=mailto:{ruf}@{domain};' if ruf else '' + return f'_dmarc.{self.name}. 600 IN TXT "v=DMARC1; p=reject;{rua}{ruf} adkim=s; aspf=s"' + @property def dkim_key(self): - file_path = app.config["DKIM_PATH"].format( - domain=self.name, selector=app.config["DKIM_SELECTOR"]) + file_path = self._dkim_file() if os.path.exists(file_path): with open(file_path, "rb") as handle: return handle.read() @dkim_key.setter def dkim_key(self, value): - file_path = app.config["DKIM_PATH"].format( - domain=self.name, selector=app.config["DKIM_SELECTOR"]) + file_path = self._dkim_file() if value is None: if os.path.exists(file_path): os.unlink(file_path) From 0a907a744e1027f8e2b79b6fce54fc03e27b69f8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 24 Oct 2020 22:32:08 +0200 Subject: [PATCH 17/52] updated documentation for config-dump --- docs/cli.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 0036e504..1b2ed14f 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -78,22 +78,29 @@ The purpose of this command is to dump domain-, relay-, alias- and user-configur # docker-compose exec admin flask mailu config-dump --help - Usage: flask mailu config-dump [OPTIONS] + Usage: flask mailu config-dump [OPTIONS] [SECTIONS]... dump configuration as YAML-formatted data to stdout + SECTIONS can be: domains, relays, users, aliases + Options: -f, --full Include default attributes - -s, --secrets Include secrets (plain-text / not hashed) + -s, --secrets Include secrets (dkim-key, plain-text / not hashed) + -d, --dns Include dns records --help Show this message and exit. -If you want to export secrets (plain-text / not hashed) you have to add the ``--secrets`` option. +If you want to export secrets (dkim-keys, plain-text / not hashed) you have to add the ``--secrets`` option. Only non-default attributes are dumped. If you want to dump all attributes use ``--full``. +To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. +Unless you specify some sections all sections are dumped by default. .. code-block:: bash docker-compose exec admin flask mailu config-dump > mail-config.yml + docker-compose exec admin flask mailu config-dump --dns domains + config-update ------------- From 0051b93077530c9950973ce2a50fea725a2b110e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 16 Dec 2020 22:39:50 +0100 Subject: [PATCH 18/52] removed unused variable --- core/admin/mailu/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 79ce49ab..ab9a8329 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -323,7 +323,7 @@ class Base(db.Model): new_data.append(rel_data) new.remove(same) del new_data[same_idx] - for i, (ch_item, ch_update) in enumerate(changed): + for i, (ch_item, _) in enumerate(changed): if ch_item is same: changed[i] = (rel_item, []) db.session.flush() From 3064a1dcff573f6dea99a90cc5b545bc6c3d2670 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 20 Dec 2020 23:38:55 +0100 Subject: [PATCH 19/52] removed call to (undefined) cli --- core/admin/mailu/manage.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 42d5b5f7..f70c5c85 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -416,7 +416,3 @@ def setmanager(domain_name, user_name='manager'): domain.managers.append(manageruser) db.session.add(domain) db.session.commit() - - -if __name__ == '__main__': - cli() From 0a594aaa2ca9ca1964d03e914e636a7be5296e10 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 20 Dec 2020 23:45:27 +0100 Subject: [PATCH 20/52] cosmetic changes --- core/admin/mailu/models.py | 83 ++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index ab9a8329..a752000e 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -30,7 +30,7 @@ class IdnaDomain(db.TypeDecorator): impl = db.String(80) def process_bind_param(self, value, dialect): - return idna.encode(value).decode("ascii").lower() + return idna.encode(value).decode('ascii').lower() def process_result_value(self, value, dialect): return idna.decode(value) @@ -46,7 +46,7 @@ class IdnaEmail(db.TypeDecorator): def process_bind_param(self, value, dialect): try: localpart, domain_name = value.split('@') - return "{0}@{1}".format( + return '{0}@{1}'.format( localpart, idna.encode(domain_name).decode('ascii'), ).lower() @@ -55,7 +55,7 @@ class IdnaEmail(db.TypeDecorator): def process_result_value(self, value, dialect): localpart, domain_name = value.split('@') - return "{0}@{1}".format( + return '{0}@{1}'.format( localpart, idna.decode(domain_name), ) @@ -70,14 +70,14 @@ class CommaSeparatedList(db.TypeDecorator): def process_bind_param(self, value, dialect): if not isinstance(value, (list, set)): - raise TypeError("Should be a list") + raise TypeError('Should be a list') for item in value: - if "," in item: - raise ValueError("Item must not contain a comma") - return ",".join(sorted(value)) + if ',' in item: + raise ValueError('Item must not contain a comma') + return ','.join(sorted(value)) def process_result_value(self, value, dialect): - return list(filter(bool, value.split(","))) if value else [] + return list(filter(bool, value.split(','))) if value else [] python_type = list @@ -103,8 +103,8 @@ class Base(db.Model): metadata = sqlalchemy.schema.MetaData( naming_convention={ - "fk": "%(table_name)s_%(column_0_name)s_fkey", - "pk": "%(table_name)s_pkey" + 'fk': '%(table_name)s_%(column_0_name)s_fkey', + 'pk': '%(table_name)s_pkey' } ) @@ -227,7 +227,7 @@ class Base(db.Model): if rel is None: itype = getattr(model, '_dict_types', {}).get(key) if itype is not None: - if itype is False: # ignore value + if itype is False: # ignore value. TODO: emit warning? del data[key] continue elif not isinstance(value, itype): @@ -372,7 +372,8 @@ class Config(Base): class Domain(Base): """ A DNS domain that has mail addresses associated to it. """ - __tablename__ = "domain" + + __tablename__ = 'domain' _dict_hide = {'users', 'managers', 'aliases'} _dict_show = {'dkim_key'} @@ -425,8 +426,8 @@ class Domain(Base): signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) def _dkim_file(self): - return app.config["DKIM_PATH"].format( - domain=self.name, selector=app.config["DKIM_SELECTOR"]) + return app.config['DKIM_PATH'].format( + domain=self.name, selector=app.config['DKIM_SELECTOR']) @property def dns_mx(self): @@ -458,7 +459,7 @@ class Domain(Base): def dkim_key(self): file_path = self._dkim_file() if os.path.exists(file_path): - with open(file_path, "rb") as handle: + with open(file_path, 'rb') as handle: return handle.read() @dkim_key.setter @@ -468,14 +469,14 @@ class Domain(Base): if os.path.exists(file_path): os.unlink(file_path) else: - with open(file_path, "wb") as handle: + with open(file_path, 'wb') as handle: handle.write(value) @property def dkim_publickey(self): dkim_key = self.dkim_key if dkim_key: - return dkim.strip_key(self.dkim_key).decode("utf8") + return dkim.strip_key(self.dkim_key).decode('utf8') def generate_dkim_key(self): self.dkim_key = dkim.gen_key() @@ -512,7 +513,7 @@ class Alternative(Base): The name "domain alias" was avoided to prevent some confusion. """ - __tablename__ = "alternative" + __tablename__ = 'alternative' name = db.Column(IdnaDomain, primary_key=True, nullable=False) domain_name = db.Column(IdnaDomain, db.ForeignKey(Domain.name)) @@ -528,7 +529,7 @@ class Relay(Base): The domain is either relayed publicly or through a specified SMTP host. """ - __tablename__ = "relay" + __tablename__ = 'relay' _dict_mandatory = {'smtp'} @@ -553,7 +554,7 @@ class Email(object): elif type(data['email']) is str: data['localpart'], data['domain'] = data['email'].rsplit('@', 1) else: - data['email'] = f"{data['localpart']}@{data['domain']}" + data['email'] = f'{data["localpart"]}@{data["domain"]}' @declarative.declared_attr def domain_name(cls): @@ -565,9 +566,9 @@ class Email(object): # especially when the mail server is reading the database. @declarative.declared_attr def email(cls): - updater = lambda context: "{0}@{1}".format( - context.current_parameters["localpart"], - context.current_parameters["domain_name"], + updater = lambda context: '{0}@{1}'.format( + context.current_parameters['localpart'], + context.current_parameters['domain_name'], ) return db.Column(IdnaEmail, primary_key=True, nullable=False, @@ -576,12 +577,12 @@ class Email(object): def sendmail(self, subject, body): """ Send an email to the address. """ - from_address = "{0}@{1}".format( + from_address = '{0}@{1}'.format( app.config['POSTMASTER'], idna.encode(app.config['DOMAIN']).decode('ascii'), ) with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: - to_address = "{0}@{1}".format( + to_address = '{0}@{1}'.format( self.localpart, idna.encode(self.domain_name).decode('ascii'), ) @@ -638,7 +639,8 @@ class Email(object): class User(Base, Email): """ A user is an email address that has a password to access a mailbox. """ - __tablename__ = "user" + + __tablename__ = 'user' _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} _dict_mandatory = {'localpart', 'domain', 'password'} @@ -689,7 +691,7 @@ class User(Base, Email): default=date(2999, 12, 31)) # Settings - displayed_name = db.Column(db.String(160), nullable=False, default="") + displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) spam_threshold = db.Column(db.Integer(), nullable=False, default=80) @@ -720,12 +722,12 @@ class User(Base, Email): self.reply_enddate > now ) - scheme_dict = {'PBKDF2': "pbkdf2_sha512", - 'BLF-CRYPT': "bcrypt", - 'SHA512-CRYPT': "sha512_crypt", - 'SHA256-CRYPT': "sha256_crypt", - 'MD5-CRYPT': "md5_crypt", - 'CRYPT': "des_crypt"} + scheme_dict = {'PBKDF2': 'pbkdf2_sha512', + 'BLF-CRYPT': 'bcrypt', + 'SHA512-CRYPT': 'sha512_crypt', + 'SHA256-CRYPT': 'sha256_crypt', + 'MD5-CRYPT': 'md5_crypt', + 'CRYPT': 'des_crypt'} def get_password_context(self): return context.CryptContext( @@ -745,7 +747,7 @@ class User(Base, Email): def set_password(self, password, hash_scheme=None, raw=False): """Set password for user with specified encryption scheme - @password: plain text password to encrypt (if raw == True the hash itself) + @password: plain text password to encrypt (if raw == True the hash itself) """ if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] @@ -787,7 +789,8 @@ class User(Base, Email): class Alias(Base, Email): """ An alias is an email address that redirects to some destination. """ - __tablename__ = "alias" + + __tablename__ = 'alias' _dict_hide = {'domain_name', 'domain', 'localpart'} @staticmethod @@ -813,7 +816,7 @@ class Alias(Base, Email): cls.localpart == localpart ), sqlalchemy.and_( cls.wildcard == True, - sqlalchemy.bindparam("l", localpart).like(cls.localpart) + sqlalchemy.bindparam('l', localpart).like(cls.localpart) ) ) ) @@ -828,7 +831,7 @@ class Alias(Base, Email): sqlalchemy.func.lower(cls.localpart) == localpart_lower ), sqlalchemy.and_( cls.wildcard == True, - sqlalchemy.bindparam("l", localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) + sqlalchemy.bindparam('l', localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) ) ) ) @@ -849,7 +852,8 @@ class Alias(Base, Email): class Token(Base): """ A token is an application password for a given user. """ - __tablename__ = "token" + + __tablename__ = 'token' _dict_recurse = True _dict_hide = {'user', 'user_email'} @@ -877,7 +881,8 @@ class Fetch(Base): """ A fetched account is a remote POP/IMAP account fetched into a local account. """ - __tablename__ = "fetch" + + __tablename__ = 'fetch' _dict_recurse = True _dict_hide = {'user_email', 'user', 'last_check', 'error'} From 815f47667bbd940c136175f73b1b10a8157c9075 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 20 Dec 2020 23:49:42 +0100 Subject: [PATCH 21/52] update dkim-key on commit only --- core/admin/mailu/models.py | 42 ++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a752000e..d38c0ad9 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -369,6 +369,21 @@ class Config(Base): value = db.Column(JSONEncoded) +@sqlalchemy.event.listens_for(db.session, 'after_commit') +def store_dkim_key(session): + """ Store DKIM key on commit + """ + + for obj in session.identity_map.values(): + if isinstance(obj, Domain): + if obj._dkim_key_changed: + file_path = obj._dkim_file() + if obj._dkim_key: + with open(file_path, 'wb') as handle: + handle.write(obj._dkim_key) + elif os.path.exists(file_path): + os.unlink(file_path) + class Domain(Base): """ A DNS domain that has mail addresses associated to it. """ @@ -424,6 +439,9 @@ class Domain(Base): max_aliases = db.Column(db.Integer, nullable=False, default=-1) max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + + _dkim_key = None + _dkim_key_changed = False def _dkim_file(self): return app.config['DKIM_PATH'].format( @@ -457,26 +475,28 @@ class Domain(Base): @property def dkim_key(self): - file_path = self._dkim_file() - if os.path.exists(file_path): - with open(file_path, 'rb') as handle: - return handle.read() + if self._dkim_key is None: + file_path = self._dkim_file() + if os.path.exists(file_path): + with open(file_path, 'rb') as handle: + self._dkim_key = handle.read() + else: + self._dkim_key = b'' + return self._dkim_key if self._dkim_key else None @dkim_key.setter def dkim_key(self, value): - file_path = self._dkim_file() + old_key = self.dkim_key if value is None: - if os.path.exists(file_path): - os.unlink(file_path) - else: - with open(file_path, 'wb') as handle: - handle.write(value) + value = b'' + self._dkim_key_changed = value != old_key + self._dkim_key = value @property def dkim_publickey(self): dkim_key = self.dkim_key if dkim_key: - return dkim.strip_key(self.dkim_key).decode('utf8') + return dkim.strip_key(dkim_key).decode('utf8') def generate_dkim_key(self): self.dkim_key = dkim.gen_key() From 3b35180b41ee298735946d3fbc3bab34f686ccbd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 20 Dec 2020 23:50:26 +0100 Subject: [PATCH 22/52] cosmetic changes --- core/admin/mailu/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index d38c0ad9..13ebce60 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -70,7 +70,7 @@ class CommaSeparatedList(db.TypeDecorator): def process_bind_param(self, value, dialect): if not isinstance(value, (list, set)): - raise TypeError('Should be a list') + raise TypeError('Must be a list') for item in value: if ',' in item: raise ValueError('Item must not contain a comma') @@ -792,9 +792,9 @@ class User(Base, Email): return emails def send_welcome(self): - if app.config["WELCOME"]: - self.sendmail(app.config["WELCOME_SUBJECT"], - app.config["WELCOME_BODY"]) + if app.config['WELCOME']: + self.sendmail(app.config['WELCOME_SUBJECT'], + app.config['WELCOME_BODY']) @classmethod def get(cls, email): From 7229c89de1e1aec152f275d6e55a2077a810e40e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 16:31:03 +0100 Subject: [PATCH 23/52] ConfigManager should not replace app.config Updated ConfigManager to only modify app.config and not replace it. Swagger does not play well, when app.config is not a real dict and it is not necessary to keep ConfigManager around after init. Also added "API" flag to config (default: disabled). --- core/admin/mailu/configuration.py | 48 +++++++++---------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 2cf6a478..a4d3c069 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -51,6 +51,7 @@ DEFAULT_CONFIG = { 'WEBMAIL': 'none', 'RECAPTCHA_PUBLIC_KEY': '', 'RECAPTCHA_PRIVATE_KEY': '', + 'API': False, # Advanced settings 'PASSWORD_SCHEME': 'PBKDF2', 'LOG_LEVEL': 'WARNING', @@ -71,7 +72,7 @@ DEFAULT_CONFIG = { 'POD_ADDRESS_RANGE': None } -class ConfigManager(dict): +class ConfigManager: """ Naive configuration manager that uses environment only """ @@ -86,19 +87,16 @@ class ConfigManager(dict): def get_host_address(self, name): # if MYSERVICE_ADDRESS is defined, use this - if '{}_ADDRESS'.format(name) in os.environ: - return os.environ.get('{}_ADDRESS'.format(name)) + if f'{name}_ADDRESS' in os.environ: + return os.environ.get(f'{name}_ADDRESS') # otherwise use the host name and resolve it - return system.resolve_address(self.config['HOST_{}'.format(name)]) + return system.resolve_address(self.config[f'HOST_{name}']) def resolve_hosts(self): - self.config["IMAP_ADDRESS"] = self.get_host_address("IMAP") - self.config["POP3_ADDRESS"] = self.get_host_address("POP3") - self.config["AUTHSMTP_ADDRESS"] = self.get_host_address("AUTHSMTP") - self.config["SMTP_ADDRESS"] = self.get_host_address("SMTP") - self.config["REDIS_ADDRESS"] = self.get_host_address("REDIS") - if self.config["WEBMAIL"] != "none": - self.config["WEBMAIL_ADDRESS"] = self.get_host_address("WEBMAIL") + for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']: + self.config[f'{key}_ADDRESS'] = self.get_host_address(key) + if self.config['WEBMAIL'] != 'none': + self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL') def __coerce_value(self, value): if isinstance(value, str) and value.lower() in ('true','yes'): @@ -108,6 +106,7 @@ class ConfigManager(dict): return value def init_app(self, app): + # get current app config self.config.update(app.config) # get environment variables self.config.update({ @@ -121,27 +120,8 @@ class ConfigManager(dict): template = self.DB_TEMPLATES[self.config['DB_FLAVOR']] self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config) - self.config['RATELIMIT_STORAGE_URL'] = 'redis://{0}/2'.format(self.config['REDIS_ADDRESS']) - self.config['QUOTA_STORAGE_URL'] = 'redis://{0}/1'.format(self.config['REDIS_ADDRESS']) - # update the app config itself - app.config = self + self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' + self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' - def setdefault(self, key, value): - if key not in self.config: - self.config[key] = value - return self.config[key] - - def get(self, *args): - return self.config.get(*args) - - def keys(self): - return self.config.keys() - - def __getitem__(self, key): - return self.config.get(key) - - def __setitem__(self, key, value): - self.config[key] = value - - def __contains__(self, key): - return key in self.config + # update the app config + app.config.update(self.config) From 4c258f5a6b8ba082f016cf186884b42e8a3d8549 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 16:45:55 +0100 Subject: [PATCH 24/52] cosmetic changes & make linter happy renamed single letter variables (m => match) renamed classmethod arguments to cls (model) removed shadowing of variables (hash, context) shortened unneeded lambda functions (id) converted type ... is to isinstance(...) removed unneded imports (flask) --- core/admin/mailu/manage.py | 33 ++++++------ core/admin/mailu/models.py | 107 ++++++++++++++++++------------------- 2 files changed, 69 insertions(+), 71 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index f70c5c85..c07ca2b7 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,9 +1,8 @@ from mailu import models from flask import current_app as app -from flask import cli as flask_cli +from flask.cli import FlaskGroup, with_appcontext -import flask import os import socket import uuid @@ -15,14 +14,14 @@ import sys db = models.db -@click.group() -def mailu(cls=flask_cli.FlaskGroup): +@click.group(cls=FlaskGroup) +def mailu(): """ Mailu command line """ @mailu.command() -@flask_cli.with_appcontext +@with_appcontext def advertise(): """ Advertise this server against statistic services. """ @@ -45,7 +44,7 @@ def advertise(): @click.argument('domain_name') @click.argument('password') @click.option('-m', '--mode') -@flask_cli.with_appcontext +@with_appcontext def admin(localpart, domain_name, password, mode='create'): """ Create an admin user 'mode' can be: @@ -89,7 +88,7 @@ def admin(localpart, domain_name, password, mode='create'): @click.argument('domain_name') @click.argument('password') @click.argument('hash_scheme', required=False) -@flask_cli.with_appcontext +@with_appcontext def user(localpart, domain_name, password, hash_scheme=None): """ Create a user """ @@ -114,7 +113,7 @@ def user(localpart, domain_name, password, hash_scheme=None): @click.argument('domain_name') @click.argument('password') @click.argument('hash_scheme', required=False) -@flask_cli.with_appcontext +@with_appcontext def password(localpart, domain_name, password, hash_scheme=None): """ Change the password of an user """ @@ -134,7 +133,7 @@ def password(localpart, domain_name, password, hash_scheme=None): @click.option('-u', '--max-users') @click.option('-a', '--max-aliases') @click.option('-q', '--max-quota-bytes') -@flask_cli.with_appcontext +@with_appcontext def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): """ Create a domain """ @@ -151,7 +150,7 @@ def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): @click.argument('domain_name') @click.argument('password_hash') @click.argument('hash_scheme') -@flask_cli.with_appcontext +@with_appcontext def user_import(localpart, domain_name, password_hash, hash_scheme = None): """ Import a user along with password hash. """ @@ -183,7 +182,7 @@ yaml_sections = [ @click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') @click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') -@flask_cli.with_appcontext +@with_appcontext def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): """sync configuration with data from YAML-formatted stdin""" @@ -303,7 +302,7 @@ def config_update(verbose=False, delete_objects=False, dry_run=False, file=None) @click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') @click.option('-d', '--dns', is_flag=True, help='Include dns records') @click.argument('sections', nargs=-1) -@flask_cli.with_appcontext +@with_appcontext def config_dump(full=False, secrets=False, dns=False, sections=None): """dump configuration as YAML-formatted data to stdout @@ -343,7 +342,7 @@ def config_dump(full=False, secrets=False, dns=False, sections=None): @mailu.command() @click.argument('email') -@flask_cli.with_appcontext +@with_appcontext def user_delete(email): """delete user""" user = models.User.query.get(email) @@ -354,7 +353,7 @@ def user_delete(email): @mailu.command() @click.argument('email') -@flask_cli.with_appcontext +@with_appcontext def alias_delete(email): """delete alias""" alias = models.Alias.query.get(email) @@ -368,7 +367,7 @@ def alias_delete(email): @click.argument('domain_name') @click.argument('destination') @click.option('-w', '--wildcard', is_flag=True) -@flask_cli.with_appcontext +@with_appcontext def alias(localpart, domain_name, destination, wildcard=False): """ Create an alias """ @@ -392,7 +391,7 @@ def alias(localpart, domain_name, destination, wildcard=False): @click.argument('max_users') @click.argument('max_aliases') @click.argument('max_quota_bytes') -@flask_cli.with_appcontext +@with_appcontext def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): """ Set domain limits """ @@ -407,7 +406,7 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): @mailu.command() @click.argument('domain_name') @click.argument('user_name') -@flask_cli.with_appcontext +@with_appcontext def setmanager(domain_name, user_name='manager'): """ Make a user manager of a domain """ diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 13ebce60..3bf92244 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,7 +1,6 @@ from mailu import dkim from sqlalchemy.ext import declarative -from passlib import context, hash from datetime import datetime, date from email.mime import text from flask import current_app as app @@ -12,6 +11,7 @@ import sqlalchemy import re import time import os +import passlib import glob import smtplib import idna @@ -113,8 +113,8 @@ class Base(db.Model): comment = db.Column(db.String(255), nullable=True) @classmethod - def _dict_pkey(model): - return model.__mapper__.primary_key[0].name + def _dict_pkey(cls): + return cls.__mapper__.primary_key[0].name def _dict_pval(self): return getattr(self, self._dict_pkey()) @@ -187,57 +187,57 @@ class Base(db.Model): return res @classmethod - def from_dict(model, data, delete=False): + def from_dict(cls, data, delete=False): changed = [] - pkey = model._dict_pkey() + pkey = cls._dict_pkey() # handle "primary key" only - if type(data) is not dict: + if isinstance(data, dict): data = {pkey: data} # modify input data - if hasattr(model, '_dict_input'): + if hasattr(cls, '_dict_input'): try: - model._dict_input(data) + cls._dict_input(data) except Exception as reason: - raise ValueError(f'{reason}', model, None, data) + raise ValueError(f'{reason}', cls, None, data) # check for primary key (if not recursed) - if not getattr(model, '_dict_recurse', False): + if not getattr(cls, '_dict_recurse', False): if not pkey in data: - raise KeyError(f'primary key {model.__table__}.{pkey} is missing', model, pkey, data) + raise KeyError(f'primary key {cls.__table__}.{pkey} is missing', cls, pkey, data) # check data keys and values for key in list(data.keys()): # check key - if not hasattr(model, key) and not key in model.__mapper__.relationships: - raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) + if not hasattr(cls, key) and not key in cls.__mapper__.relationships: + raise KeyError(f'unknown key {cls.__table__}.{key}', cls, key, data) # check value type value = data[key] - col = model.__mapper__.columns.get(key) + col = cls.__mapper__.columns.get(key) if col is not None: - if not ((value is None and col.nullable) or (type(value) is col.type.python_type)): - raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) + if not ((value is None and col.nullable) or (isinstance(value, col.type.python_type))): + raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) else: - rel = model.__mapper__.relationships.get(key) + rel = cls.__mapper__.relationships.get(key) if rel is None: - itype = getattr(model, '_dict_types', {}).get(key) + itype = getattr(cls, '_dict_types', {}).get(key) if itype is not None: if itype is False: # ignore value. TODO: emit warning? del data[key] continue elif not isinstance(value, itype): - raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) + raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) else: - raise NotImplementedError(f'type not defined for {model.__table__}.{key}') + raise NotImplementedError(f'type not defined for {cls.__table__}.{key}') # handle relationships - if key in model.__mapper__.relationships: - rel_model = model.__mapper__.relationships[key].argument + if key in cls.__mapper__.relationships: + rel_model = cls.__mapper__.relationships[key].argument if not isinstance(rel_model, sqlalchemy.orm.Mapper): add = rel_model.from_dict(value, delete) assert len(add) == 1 @@ -247,24 +247,24 @@ class Base(db.Model): # create item if necessary created = False - item = model.query.get(data[pkey]) if pkey in data else None + item = cls.query.get(data[pkey]) if pkey in data else None if item is None: # check for mandatory keys - missing = getattr(model, '_dict_mandatory', set()) - set(data.keys()) + missing = getattr(cls, '_dict_mandatory', set()) - set(data.keys()) if missing: - raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {model.__table__} missing', model, missing, data) + raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {cls.__table__} missing', cls, missing, data) # remove mapped relationships from data mapped = {} for key in list(data.keys()): - if key in model.__mapper__.relationships: - if isinstance(model.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): + if key in cls.__mapper__.relationships: + if isinstance(cls.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): mapped[key] = data[key] del data[key] # create new item - item = model(**data) + item = cls(**data) created = True # and update mapped relationships (below) @@ -278,14 +278,14 @@ class Base(db.Model): if key == pkey: continue - if key in model.__mapper__.relationships: + if key in cls.__mapper__.relationships: # update relationship - rel_model = model.__mapper__.relationships[key].argument + rel_model = cls.__mapper__.relationships[key].argument if isinstance(rel_model, sqlalchemy.orm.Mapper): rel_model = rel_model.class_ # add (and create) referenced items cur = getattr(item, key) - old = sorted(cur, key=lambda i:id(i)) + old = sorted(cur, key=id) new = [] for rel_data in value: # get or create related item @@ -331,16 +331,16 @@ class Base(db.Model): break # remember changes - new = sorted(new, key=lambda i:id(i)) + new = sorted(new, key=id) if new != old: updated.append((key, old, new)) else: # update key old = getattr(item, key) - if type(old) is list: + if isinstance(old, list): # deduplicate list value - assert type(value) is list + assert isinstance(value, list) value = set(value) old = set(old) if not delete: @@ -408,19 +408,19 @@ class Domain(Base): if 'dkim_key' in data: key = data['dkim_key'] if key is not None: - if type(key) is list: + if isinstance(key, list): key = ''.join(key) - if type(key) is str: + if isinstance(key, str): key = ''.join(key.strip().split()) # removes all whitespace if key == 'generate': data['dkim_key'] = dkim.gen_key() elif key: - m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) - if m is not None: - key = key[m.end():] - m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) - if m is not None: - key = key[:m.start()] + match = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) + if match is not None: + key = key[match.end():] + match = re.search('-----END (RSA )?PRIVATE KEY-----$', key) + if match is not None: + key = key[:match.start()] key = '\n'.join(wrap(key, 64)) key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') try: @@ -428,7 +428,7 @@ class Domain(Base): except: raise ValueError('invalid dkim key') else: - data['dkim_key'] = key + data['dkim_key'] = key else: data['dkim_key'] = None @@ -505,8 +505,7 @@ class Domain(Base): for email in self.users + self.aliases: if email.localpart == localpart: return True - else: - return False + return False def check_mx(self): try: @@ -519,7 +518,7 @@ class Domain(Base): return False def __str__(self): - return self.name + return str(self.name) def __eq__(self, other): try: @@ -541,7 +540,7 @@ class Alternative(Base): backref=db.backref('alternatives', cascade='all, delete-orphan')) def __str__(self): - return self.name + return str(self.name) class Relay(Base): @@ -557,7 +556,7 @@ class Relay(Base): smtp = db.Column(db.String(80), nullable=True) def __str__(self): - return self.name + return str(self.name) class Email(object): @@ -571,7 +570,7 @@ class Email(object): if 'email' in data: if 'localpart' in data or 'domain' in data: raise ValueError('ambigous key email and localpart/domain') - elif type(data['email']) is str: + elif isinstance(data['email'], str): data['localpart'], data['domain'] = data['email'].rsplit('@', 1) else: data['email'] = f'{data["localpart"]}@{data["domain"]}' @@ -653,7 +652,7 @@ class Email(object): return pure_alias.destination def __str__(self): - return self.email + return str(self.email) class User(Base, Email): @@ -750,7 +749,7 @@ class User(Base, Email): 'CRYPT': 'des_crypt'} def get_password_context(self): - return context.CryptContext( + return passlib.context.CryptContext( schemes=self.scheme_dict.values(), default=self.scheme_dict[app.config['PASSWORD_SCHEME']], ) @@ -818,7 +817,7 @@ class Alias(Base, Email): Email._dict_input(data) # handle comma delimited string for backwards compability dst = data.get('destination') - if type(dst) is str: + if isinstance(dst, str): data['destination'] = list([adr.strip() for adr in dst.split(',')]) domain = db.relationship(Domain, @@ -888,10 +887,10 @@ class Token(Base): ip = db.Column(db.String(255)) def check_password(self, password): - return hash.sha256_crypt.verify(password, self.password) + return passlib.hash.sha256_crypt.verify(password, self.password) def set_password(self, password): - self.password = hash.sha256_crypt.using(rounds=1000).hash(password) + self.password = passlib.hash.sha256_crypt.using(rounds=1000).hash(password) def __str__(self): return self.comment or self.ip From 6629aa3ff824c57b1c752fdf7a7e7ea87423bc5e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 17:05:21 +0100 Subject: [PATCH 25/52] first try at api using flask-restx & marshmallow --- core/admin/mailu/__init__.py | 6 +- core/admin/mailu/api/__init__.py | 38 +++++ core/admin/mailu/api/common.py | 8 + core/admin/mailu/api/v1/__init__.py | 27 ++++ core/admin/mailu/api/v1/domains.py | 183 +++++++++++++++++++++++ core/admin/mailu/manage.py | 33 +++-- core/admin/mailu/schemas.py | 222 ++++++++++++++++++++++++++++ 7 files changed, 502 insertions(+), 15 deletions(-) create mode 100644 core/admin/mailu/api/__init__.py create mode 100644 core/admin/mailu/api/common.py create mode 100644 core/admin/mailu/api/v1/__init__.py create mode 100644 core/admin/mailu/api/v1/domains.py create mode 100644 core/admin/mailu/schemas.py diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 4de3e580..9ab90add 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -41,15 +41,17 @@ def create_app_from_config(config): ) # Import views - from mailu import ui, internal + from mailu import ui, internal, api app.register_blueprint(ui.ui, url_prefix='/ui') app.register_blueprint(internal.internal, url_prefix='/internal') + if app.config.get('API'): + api.register(app) return app def create_app(): - """ Create a new application based on the config module + """ Create a new application based on the config module """ config = configuration.ConfigManager() return create_app_from_config(config) diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py new file mode 100644 index 00000000..6e7d6386 --- /dev/null +++ b/core/admin/mailu/api/__init__.py @@ -0,0 +1,38 @@ +from flask import redirect, url_for + +# import api version(s) +from . import v1 + +# api +ROOT='/api' +ACTIVE=v1 + +# patch url for swaggerui static assets +from flask_restx.apidoc import apidoc +apidoc.static_url_path = f'{ROOT}/swaggerui' + +def register(app): + + # register api bluprint(s) + app.register_blueprint(v1.blueprint, url_prefix=f'{ROOT}/v{int(v1.VERSION)}') + + # add redirect to current api version + @app.route(f'{ROOT}/') + def redir(): + return redirect(url_for(f'{ACTIVE.blueprint.name}.root')) + + # swagger ui config + app.config.SWAGGER_UI_DOC_EXPANSION = 'list' + app.config.SWAGGER_UI_OPERATION_ID = True + app.config.SWAGGER_UI_REQUEST_DURATION = True + + # TODO: remove patch of static assets for debugging + import os + if 'STATIC_ASSETS' in os.environ: + app.blueprints['ui'].static_folder = os.environ['STATIC_ASSETS'] + +# TODO: authentication via username + password +# TODO: authentication via api token +# TODO: api access for all users (via token) +# TODO: use permissions from "manager_of" +# TODO: switch to marshmallow, as parser is deprecated. use flask_accepts? diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py new file mode 100644 index 00000000..700835ac --- /dev/null +++ b/core/admin/mailu/api/common.py @@ -0,0 +1,8 @@ +from .. import models + +def fqdn_in_use(*names): + for name in names: + for model in models.Domain, models.Alternative, models.Relay: + if model.query.get(name): + return model + return None diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py new file mode 100644 index 00000000..c6de6fa4 --- /dev/null +++ b/core/admin/mailu/api/v1/__init__.py @@ -0,0 +1,27 @@ +from flask import Blueprint +from flask_restx import Api, fields + +VERSION = 1.0 + +blueprint = Blueprint(f'api_v{int(VERSION)}', __name__) + +api = Api( + blueprint, version=f'{VERSION:.1f}', + title='Mailu API', default_label='Mailu', + validate=True +) + +response_fields = api.model('Response', { + 'code': fields.Integer, + 'message': fields.String, +}) + +error_fields = api.model('Error', { + 'errors': fields.Nested(api.model('Error_Key', { + 'key': fields.String, + 'message':fields.String + })), + 'message': fields.String, +}) + +from . import domains diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py new file mode 100644 index 00000000..7afd26b9 --- /dev/null +++ b/core/admin/mailu/api/v1/domains.py @@ -0,0 +1,183 @@ +from flask_restx import Resource, fields, abort + +from . import api, response_fields, error_fields +from .. import common +from ... import models + +db = models.db + +dom = api.namespace('domain', description='Domain operations') +alt = api.namespace('alternative', description='Alternative operations') + +domain_fields = api.model('Domain', { + 'name': fields.String(description='FQDN', example='example.com', required=True), + 'comment': fields.String(description='a comment'), + 'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1), + 'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1), + 'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0), + 'signup_enabled': fields.Boolean(description='allow signup'), +# 'dkim_key': fields.String, + 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example.com')), +}) +# TODO - name ist required on creation but immutable on change +# TODO - name and alteranatives need to be checked to be a fqdn (regex) + +domain_parser = api.parser() +domain_parser.add_argument('max_users', type=int, help='maximum number of users') +# TODO ... add more (or use marshmallow) + +alternative_fields = api.model('Domain', { + 'name': fields.String(description='alternative FQDN', example='example.com', required=True), + 'domain': fields.String(description='domain FQDN', example='example.com', required=True), + 'dkim_key': fields.String, +}) +# TODO: domain and name are not always required and can't be changed + + +@dom.route('') +class Domains(Resource): + + @dom.doc('list_domain') + @dom.marshal_with(domain_fields, as_list=True, skip_none=True, mask=['dkim_key']) + def get(self): + """ List domains """ + return models.Domain.query.all() + + @dom.doc('create_domain') + @dom.expect(domain_fields) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(409, 'Duplicate domain name', error_fields) + def post(self): + """ Create a new domain """ + data = api.payload + if common.fqdn_in_use(data['name']): + abort(409, f'Duplicate domain name {data["name"]!r}', errors={ + 'name': data['name'], + }) + for item, created in models.Domain.from_dict(data): + if not created: + abort(409, f'Duplicate domain name {item.name!r}', errors={ + 'alternatives': item.name, + }) + db.session.add(item) + db.session.commit() + +@dom.route('/') +class Domain(Resource): + + @dom.doc('get_domain') + @dom.response(200, 'Success', domain_fields) + @dom.response(404, 'Domain not found') + @dom.marshal_with(domain_fields) + def get(self, name): + """ Find domain by name """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + return domain + + @dom.doc('update_domain') + @dom.expect(domain_fields) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(404, 'Domain not found') + def put(self, name): + """ Update an existing domain """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + data = api.payload + data['name'] = name + for item, created in models.Domain.from_dict(data): + if created is True: + db.session.add(item) + db.session.commit() + + @dom.doc('modify_domain') + @dom.expect(domain_parser) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(404, 'Domain not found') + def post(self, name=None): + """ Updates domain with form data """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + data = dict(domain_parser.parse_args()) + data['name'] = name + for item, created in models.Domain.from_dict(data): + if created is True: + db.session.add(item) + # TODO: flush? + db.session.commit() + + @dom.doc('delete_domain') + @dom.response(200, 'Success', response_fields) + @dom.response(404, 'Domain not found') + def delete(self, name=None): + """ Delete domain """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + db.session.delete(domain) + db.session.commit() + + +# @dom.route('//alternative') +# @alt.route('') +# class Alternatives(Resource): + +# @alt.doc('alternatives_list') +# @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=['dkim_key']) +# def get(self, name=None): +# """ List alternatives (of domain) """ +# if name is None: +# return models.Alternative.query.all() +# else: +# return models.Alternative.query.filter_by(domain_name = name).all() + +# @alt.doc('alternative_create') +# @alt.expect(alternative_fields) +# @alt.response(200, 'Success', response_fields) +# @alt.response(400, 'Input validation exception', error_fields) +# @alt.response(404, 'Domain not found') +# @alt.response(409, 'Duplicate domain name', error_fields) +# def post(self, name=None): +# """ Create new alternative (for domain) """ +# # abort(501) +# data = api.payload +# if name is not None: +# data['name'] = name +# domain = models.Domain.query.get(name) +# if not domain: +# abort(404) +# if common.fqdn_in_use(data['name']): +# abort(409, f'Duplicate domain name {data["name"]!r}', errors={ +# 'name': data['name'], +# }) +# for item, created in models.Alternative.from_dict(data): +# # TODO: handle creation of domain +# if not created: +# abort(409, f'Duplicate domain name {item.name!r}', errors={ +# 'alternatives': item.name, +# }) +# # db.session.add(item) +# # db.session.commit() + +# @dom.route('//alternative/') +# @alt.route('/') +# class Alternative(Resource): +# def get(self, name, alt=None): +# """ Find alternative (of domain) """ +# abort(501) +# def put(self, name, alt=None): +# """ Update alternative (of domain) """ +# abort(501) +# def post(self, name, alt=None): +# """ Update alternative (of domain) with form data """ +# abort(501) +# def delete(self, name, alt=None): +# """ Delete alternative (for domain) """ +# abort(501) + diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index c07ca2b7..569c161c 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,4 +1,5 @@ from mailu import models +from .schemas import schemas from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext @@ -320,24 +321,30 @@ def config_dump(full=False, secrets=False, dns=False, sections=None): return super().increase_indent(flow, False) if sections: - check = dict(yaml_sections) for section in sections: - if section not in check: + if section not in schemas: print(f'[ERROR] Invalid section: {section}') return 1 + else: + sections = sorted(schemas.keys()) - extra = [] - if dns: - extra.append('dns') +# TODO: create World Schema and dump only this with Word.dumps ? - config = {} - for section, model in yaml_sections: - if not sections or section in sections: - dump = [item.to_dict(full, secrets, extra) for item in model.query.all()] - if len(dump): - config[section] = dump - - yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True) + for section in sections: + schema = schemas[section](many=True) + schema.context.update({ + 'full': full, + 'secrets': secrets, + 'dns': dns, + }) + yaml.dump( + {section: schema.dump(schema.Meta.model.query.all())}, + sys.stdout, + Dumper=spacedDumper, + default_flow_style=False, + allow_unicode=True + ) + sys.stdout.write('\n') @mailu.command() diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py new file mode 100644 index 00000000..20aa98c9 --- /dev/null +++ b/core/admin/mailu/schemas.py @@ -0,0 +1,222 @@ +import marshmallow +import flask_marshmallow + +from . import models + + +ma = flask_marshmallow.Marshmallow() + + +class BaseSchema(ma.SQLAlchemyAutoSchema): + + SKIP_IF = { + 'comment': {'', None} + } + + @marshmallow.post_dump + def remove_skip_values(self, data, many, **kwargs): + print(repr(data), self.context) + return { + key: value for key, value in data.items() + if key not in self.SKIP_IF or value not in self.SKIP_IF[key] + } + +class BaseMeta: + exclude = ['created_at', 'updated_at'] + + +class DomainSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Domain + + # _dict_hide = {'users', 'managers', 'aliases'} + # _dict_show = {'dkim_key'} + # _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} + # _dict_secret = {'dkim_key'} + # _dict_types = { + # 'dkim_key': (bytes, type(None)), + # 'dkim_publickey': False, + # 'dns_mx': False, + # 'dns_spf': False, + # 'dns_dkim': False, + # 'dns_dmarc': False, + # } + # _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} + # @staticmethod + # def _dict_input(data): + # if 'dkim_key' in data: + # key = data['dkim_key'] + # if key is not None: + # if type(key) is list: + # key = ''.join(key) + # if type(key) is str: + # key = ''.join(key.strip().split()) # removes all whitespace + # if key == 'generate': + # data['dkim_key'] = dkim.gen_key() + # elif key: + # m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) + # if m is not None: + # key = key[m.end():] + # m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) + # if m is not None: + # key = key[:m.start()] + # key = '\n'.join(wrap(key, 64)) + # key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + # try: + # dkim.strip_key(key) + # except: + # raise ValueError('invalid dkim key') + # else: + # data['dkim_key'] = key + # else: + # data['dkim_key'] = None + + # name = db.Column(IdnaDomain, primary_key=True, nullable=False) + # managers = db.relationship('User', secondary=managers, + # backref=db.backref('manager_of'), lazy='dynamic') + # max_users = db.Column(db.Integer, nullable=False, default=-1) + # max_aliases = db.Column(db.Integer, nullable=False, default=-1) + # max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) + # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + + +class UserSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.User + + # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} + # _dict_mandatory = {'localpart', 'domain', 'password'} + # @classmethod + # def _dict_input(cls, data): + # Email._dict_input(data) + # # handle password + # if 'password' in data: + # if 'password_hash' in data or 'hash_scheme' in data: + # raise ValueError('ambigous key password and password_hash/hash_scheme') + # # check (hashed) password + # password = data['password'] + # if password.startswith('{') and '}' in password: + # scheme = password[1:password.index('}')] + # if scheme not in cls.scheme_dict: + # raise ValueError(f'invalid password scheme {scheme!r}') + # else: + # raise ValueError(f'invalid hashed password {password!r}') + # elif 'password_hash' in data and 'hash_scheme' in data: + # if data['hash_scheme'] not in cls.scheme_dict: + # raise ValueError(f'invalid password scheme {scheme!r}') + # data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + # del data['hash_scheme'] + # del data['password_hash'] + + # domain = db.relationship(Domain, + # backref=db.backref('users', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) + # quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) + # global_admin = db.Column(db.Boolean(), nullable=False, default=False) + # enabled = db.Column(db.Boolean(), nullable=False, default=True) + + # # Features + # enable_imap = db.Column(db.Boolean(), nullable=False, default=True) + # enable_pop = db.Column(db.Boolean(), nullable=False, default=True) + + # # Filters + # forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) + # forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) + # forward_keep = db.Column(db.Boolean(), nullable=False, default=True) + # reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) + # reply_subject = db.Column(db.String(255), nullable=True, default=None) + # reply_body = db.Column(db.Text(), nullable=True, default=None) + # reply_startdate = db.Column(db.Date, nullable=False, + # default=date(1900, 1, 1)) + # reply_enddate = db.Column(db.Date, nullable=False, + # default=date(2999, 12, 31)) + + # # Settings + # displayed_name = db.Column(db.String(160), nullable=False, default='') + # spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) + # spam_threshold = db.Column(db.Integer(), nullable=False, default=80) + +class AliasSchema(BaseSchema): + class Meta(BaseMeta): + model = models.Alias + exclude = BaseMeta.exclude + ['localpart'] + # TODO look for good way to exclude secrets, unverbose and defaults + + # _dict_hide = {'domain_name', 'domain', 'localpart'} + # @staticmethod + # def _dict_input(data): + # Email._dict_input(data) + # # handle comma delimited string for backwards compability + # dst = data.get('destination') + # if type(dst) is str: + # data['destination'] = list([adr.strip() for adr in dst.split(',')]) + + +class TokenSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Token + + # _dict_recurse = True + # _dict_hide = {'user', 'user_email'} + # _dict_mandatory = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('tokens', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # ip = db.Column(db.String(255)) + + +class FetchSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Fetch + + # _dict_recurse = True + # _dict_hide = {'user_email', 'user', 'last_check', 'error'} + # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} + # _dict_secret = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('fetches', cascade='all, delete-orphan')) + # protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) + # host = db.Column(db.String(255), nullable=False) + # port = db.Column(db.Integer(), nullable=False) + # tls = db.Column(db.Boolean(), nullable=False, default=False) + # username = db.Column(db.String(255), nullable=False) + # password = db.Column(db.String(255), nullable=False) + # keep = db.Column(db.Boolean(), nullable=False, default=False) + # last_check = db.Column(db.DateTime, nullable=True) + # error = db.Column(db.String(1023), nullable=True) + + +class ConfigSchema(ma.SQLAlchemySchema): + class Meta: + model = models.Config +# created_at = ma.auto_field(dump_only=True) +# updated_at = ma.auto_field(dump_only=True) + comment = ma.auto_field() + name = ma.auto_field(required=True) + value = ma.auto_field(required=True) + +class RelaySchema(BaseSchema): + class Meta(BaseMeta): + model = models.Relay +# created_at = ma.auto_field(dump_only=True) +# updated_at = ma.auto_field(dump_only=True) +# comment = ma.auto_field() +# name = ma.auto_field(required=True) +# smtp = ma.auto_field(required=True) + +schemas = { + 'domains': DomainSchema, + 'relays': RelaySchema, + 'users': UserSchema, + 'aliases': AliasSchema, +# 'config': ConfigSchema, +} From b3f8dacdadcc562c34205fb9c8e2fe95c90394e8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 8 Jan 2021 14:17:28 +0100 Subject: [PATCH 26/52] add docstrings and make linter happy --- core/admin/mailu/api/__init__.py | 13 ++++++++++--- core/admin/mailu/api/common.py | 6 ++++++ core/admin/mailu/api/v1/__init__.py | 5 +++++ core/admin/mailu/api/v1/domains.py | 24 ++++++++++++++---------- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py index 6e7d6386..a4d8689f 100644 --- a/core/admin/mailu/api/__init__.py +++ b/core/admin/mailu/api/__init__.py @@ -1,4 +1,11 @@ +""" +Mailu API +""" + +import os + from flask import redirect, url_for +from flask_restx.apidoc import apidoc # import api version(s) from . import v1 @@ -8,10 +15,11 @@ ROOT='/api' ACTIVE=v1 # patch url for swaggerui static assets -from flask_restx.apidoc import apidoc apidoc.static_url_path = f'{ROOT}/swaggerui' def register(app): + """ Register api blueprint in flask app + """ # register api bluprint(s) app.register_blueprint(v1.blueprint, url_prefix=f'{ROOT}/v{int(v1.VERSION)}') @@ -26,8 +34,7 @@ def register(app): app.config.SWAGGER_UI_OPERATION_ID = True app.config.SWAGGER_UI_REQUEST_DURATION = True - # TODO: remove patch of static assets for debugging - import os + # TODO: remove patch of static asset location if 'STATIC_ASSETS' in os.environ: app.blueprints['ui'].static_folder = os.environ['STATIC_ASSETS'] diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py index 700835ac..000bb2d6 100644 --- a/core/admin/mailu/api/common.py +++ b/core/admin/mailu/api/common.py @@ -1,6 +1,12 @@ +""" +Common functions for all API versions +""" + from .. import models def fqdn_in_use(*names): + """ Checks if fqdn is used + """ for name in names: for model in models.Domain, models.Alternative, models.Relay: if model.query.get(name): diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py index c6de6fa4..c4379074 100644 --- a/core/admin/mailu/api/v1/__init__.py +++ b/core/admin/mailu/api/v1/__init__.py @@ -1,3 +1,7 @@ +""" +API Blueprint +""" + from flask import Blueprint from flask_restx import Api, fields @@ -24,4 +28,5 @@ error_fields = api.model('Error', { 'message': fields.String, }) +# import api namespaces (below field defs to avoid circular reference) from . import domains diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py index 7afd26b9..82ed1fde 100644 --- a/core/admin/mailu/api/v1/domains.py +++ b/core/admin/mailu/api/v1/domains.py @@ -1,3 +1,7 @@ +""" +API: domain +""" + from flask_restx import Resource, fields, abort from . import api, response_fields, error_fields @@ -9,6 +13,7 @@ db = models.db dom = api.namespace('domain', description='Domain operations') alt = api.namespace('alternative', description='Alternative operations') +# TODO: use marshmallow domain_fields = api.model('Domain', { 'name': fields.String(description='FQDN', example='example.com', required=True), 'comment': fields.String(description='a comment'), @@ -19,12 +24,12 @@ domain_fields = api.model('Domain', { # 'dkim_key': fields.String, 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example.com')), }) -# TODO - name ist required on creation but immutable on change -# TODO - name and alteranatives need to be checked to be a fqdn (regex) +# TODO: name ist required on creation but immutable on change +# TODO: name and alteranatives need to be checked to be a fqdn (regex) domain_parser = api.parser() domain_parser.add_argument('max_users', type=int, help='maximum number of users') -# TODO ... add more (or use marshmallow) +# TODO: ... add more (or use marshmallow) alternative_fields = api.model('Domain', { 'name': fields.String(description='alternative FQDN', example='example.com', required=True), @@ -65,7 +70,7 @@ class Domains(Resource): @dom.route('/') class Domain(Resource): - + @dom.doc('get_domain') @dom.response(200, 'Success', domain_fields) @dom.response(404, 'Domain not found') @@ -76,7 +81,7 @@ class Domain(Resource): if not domain: abort(404) return domain - + @dom.doc('update_domain') @dom.expect(domain_fields) @dom.response(200, 'Success', response_fields) @@ -109,7 +114,7 @@ class Domain(Resource): for item, created in models.Domain.from_dict(data): if created is True: db.session.add(item) - # TODO: flush? + # TODO: flush? db.session.commit() @dom.doc('delete_domain') @@ -123,11 +128,11 @@ class Domain(Resource): db.session.delete(domain) db.session.commit() - + # @dom.route('//alternative') # @alt.route('') # class Alternatives(Resource): - + # @alt.doc('alternatives_list') # @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=['dkim_key']) # def get(self, name=None): @@ -136,7 +141,7 @@ class Domain(Resource): # return models.Alternative.query.all() # else: # return models.Alternative.query.filter_by(domain_name = name).all() - + # @alt.doc('alternative_create') # @alt.expect(alternative_fields) # @alt.response(200, 'Success', response_fields) @@ -180,4 +185,3 @@ class Domain(Resource): # def delete(self, name, alt=None): # """ Delete alternative (for domain) """ # abort(501) - From 82cf0d843f70584e7d0779bb58088e295193c63b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 8 Jan 2021 14:22:11 +0100 Subject: [PATCH 27/52] fix sqlalchemy column definitions --- core/admin/mailu/models.py | 45 +++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 3bf92244..625d1fd6 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -369,6 +369,7 @@ class Config(Base): value = db.Column(JSONEncoded) +# TODO: use sqlalchemy.event.listen() on a store method of object? @sqlalchemy.event.listens_for(db.session, 'after_commit') def store_dkim_key(session): """ Store DKIM key on commit @@ -437,8 +438,8 @@ class Domain(Base): backref=db.backref('manager_of'), lazy='dynamic') max_users = db.Column(db.Integer, nullable=False, default=-1) max_aliases = db.Column(db.Integer, nullable=False, default=-1) - max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) - signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + max_quota_bytes = db.Column(db.BigInteger, nullable=False, default=0) + signup_enabled = db.Column(db.Boolean, nullable=False, default=False) _dkim_key = None _dkim_key_changed = False @@ -688,22 +689,22 @@ class User(Base, Email): domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) - quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) - global_admin = db.Column(db.Boolean(), nullable=False, default=False) - enabled = db.Column(db.Boolean(), nullable=False, default=True) + quota_bytes = db.Column(db.BigInteger, nullable=False, default=10**9) + quota_bytes_used = db.Column(db.BigInteger, nullable=False, default=0) + global_admin = db.Column(db.Boolean, nullable=False, default=False) + enabled = db.Column(db.Boolean, nullable=False, default=True) # Features - enable_imap = db.Column(db.Boolean(), nullable=False, default=True) - enable_pop = db.Column(db.Boolean(), nullable=False, default=True) + enable_imap = db.Column(db.Boolean, nullable=False, default=True) + enable_pop = db.Column(db.Boolean, nullable=False, default=True) # Filters - forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) - forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) - forward_keep = db.Column(db.Boolean(), nullable=False, default=True) - reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) + forward_enabled = db.Column(db.Boolean, nullable=False, default=False) + forward_destination = db.Column(CommaSeparatedList, nullable=True, default=list) + forward_keep = db.Column(db.Boolean, nullable=False, default=True) + reply_enabled = db.Column(db.Boolean, nullable=False, default=False) reply_subject = db.Column(db.String(255), nullable=True, default=None) - reply_body = db.Column(db.Text(), nullable=True, default=None) + reply_body = db.Column(db.Text, nullable=True, default=None) reply_startdate = db.Column(db.Date, nullable=False, default=date(1900, 1, 1)) reply_enddate = db.Column(db.Date, nullable=False, @@ -711,8 +712,8 @@ class User(Base, Email): # Settings displayed_name = db.Column(db.String(160), nullable=False, default='') - spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) - spam_threshold = db.Column(db.Integer(), nullable=False, default=80) + spam_enabled = db.Column(db.Boolean, nullable=False, default=True) + spam_threshold = db.Column(db.Integer, nullable=False, default=80) # Flask-login attributes is_authenticated = True @@ -822,8 +823,8 @@ class Alias(Base, Email): domain = db.relationship(Domain, backref=db.backref('aliases', cascade='all, delete-orphan')) - wildcard = db.Column(db.Boolean(), nullable=False, default=False) - destination = db.Column(CommaSeparatedList, nullable=False, default=[]) + wildcard = db.Column(db.Boolean, nullable=False, default=False) + destination = db.Column(CommaSeparatedList, nullable=False, default=list) @classmethod def resolve(cls, localpart, domain_name): @@ -878,7 +879,7 @@ class Token(Base): _dict_hide = {'user', 'user_email'} _dict_mandatory = {'password'} - id = db.Column(db.Integer(), primary_key=True) + id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) user = db.relationship(User, @@ -908,18 +909,18 @@ class Fetch(Base): _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} _dict_secret = {'password'} - id = db.Column(db.Integer(), primary_key=True) + id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) user = db.relationship(User, backref=db.backref('fetches', cascade='all, delete-orphan')) protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) host = db.Column(db.String(255), nullable=False) - port = db.Column(db.Integer(), nullable=False) - tls = db.Column(db.Boolean(), nullable=False, default=False) + port = db.Column(db.Integer, nullable=False) + tls = db.Column(db.Boolean, nullable=False, default=False) username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) - keep = db.Column(db.Boolean(), nullable=False, default=False) + keep = db.Column(db.Boolean, nullable=False, default=False) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) From dc42d375e262d9d1d4f34364dbd68c50e5fd452f Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 8 Jan 2021 14:22:59 +0100 Subject: [PATCH 28/52] added filtering of keys and default value --- core/admin/mailu/schemas.py | 88 +++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 20aa98c9..c2a3d03d 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,4 +1,5 @@ import marshmallow +import sqlalchemy import flask_marshmallow from . import models @@ -6,27 +7,51 @@ from . import models ma = flask_marshmallow.Marshmallow() - +import collections class BaseSchema(ma.SQLAlchemyAutoSchema): - SKIP_IF = { - 'comment': {'', None} - } + class Meta: + base_hide_always = {'created_at', 'updated_at'} + base_hide_secrets = set() + base_hide_by_value = { +# 'comment': {'', None} + } @marshmallow.post_dump def remove_skip_values(self, data, many, **kwargs): - print(repr(data), self.context) +# print(repr(data), self.context) + + # always hide + hide_by_key = self.Meta.base_hide_always | set(getattr(self.Meta, 'hide_always', ())) + + # hide secrets + if not self.context.get('secrets'): + hide_by_key |= self.Meta.base_hide_secrets + hide_by_key |= set(getattr(self.Meta, 'hide_secrets', ())) + + # hide by value + hide_by_value = self.Meta.base_hide_by_value | getattr(self.Meta, 'hide_by_value', {}) + + # hide defaults + if not self.context.get('full'): + for column in self.Meta.model.__table__.columns: +# print(column.name, column.default.arg if isinstance(column.default, sqlalchemy.sql.schema.ColumnDefault) else column.default) +# alias.destiantion has default [] - is this okay. how to check it? + if column.name not in hide_by_key: + hide_by_value.setdefault(column.name, set()).add(None if column.default is None else column.default.arg) + return { key: value for key, value in data.items() - if key not in self.SKIP_IF or value not in self.SKIP_IF[key] + if + not isinstance(value, collections.Hashable) + or( + key not in hide_by_key + and + (key not in hide_by_value or value not in hide_by_value[key])) } -class BaseMeta: - exclude = ['created_at', 'updated_at'] - - -class DomainSchema(ma.SQLAlchemyAutoSchema): - class Meta: +class DomainSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.Domain # _dict_hide = {'users', 'managers', 'aliases'} @@ -80,8 +105,8 @@ class DomainSchema(ma.SQLAlchemyAutoSchema): # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) -class UserSchema(ma.SQLAlchemyAutoSchema): - class Meta: +class UserSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.User # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} @@ -138,12 +163,14 @@ class UserSchema(ma.SQLAlchemyAutoSchema): # spam_threshold = db.Column(db.Integer(), nullable=False, default=80) class AliasSchema(BaseSchema): - class Meta(BaseMeta): + class Meta(BaseSchema.Meta): model = models.Alias - exclude = BaseMeta.exclude + ['localpart'] - # TODO look for good way to exclude secrets, unverbose and defaults + hide_always = {'localpart'} + hide_secrets = {'wildcard'} + hide_by_value = { + 'destination': set([]) # always hide empty lists?! + } - # _dict_hide = {'domain_name', 'domain', 'localpart'} # @staticmethod # def _dict_input(data): # Email._dict_input(data) @@ -153,8 +180,8 @@ class AliasSchema(BaseSchema): # data['destination'] = list([adr.strip() for adr in dst.split(',')]) -class TokenSchema(ma.SQLAlchemyAutoSchema): - class Meta: +class TokenSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.Token # _dict_recurse = True @@ -170,8 +197,8 @@ class TokenSchema(ma.SQLAlchemyAutoSchema): # ip = db.Column(db.String(255)) -class FetchSchema(ma.SQLAlchemyAutoSchema): - class Meta: +class FetchSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.Fetch # _dict_recurse = True @@ -195,23 +222,18 @@ class FetchSchema(ma.SQLAlchemyAutoSchema): # error = db.Column(db.String(1023), nullable=True) -class ConfigSchema(ma.SQLAlchemySchema): - class Meta: +class ConfigSchema(BaseSchema): + class Meta(BaseSchema.Meta): model = models.Config -# created_at = ma.auto_field(dump_only=True) -# updated_at = ma.auto_field(dump_only=True) - comment = ma.auto_field() +# TODO: how to mark keys as "required" while unserializing (in certain use cases/API) name = ma.auto_field(required=True) value = ma.auto_field(required=True) + class RelaySchema(BaseSchema): - class Meta(BaseMeta): + class Meta(BaseSchema.Meta): model = models.Relay -# created_at = ma.auto_field(dump_only=True) -# updated_at = ma.auto_field(dump_only=True) -# comment = ma.auto_field() -# name = ma.auto_field(required=True) -# smtp = ma.auto_field(required=True) + schemas = { 'domains': DomainSchema, From 7413f9b7b42d8797f5de6203260761f2561859f0 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 13 Jan 2021 00:05:43 +0100 Subject: [PATCH 29/52] config_dump now using marshmallow --- core/admin/mailu/manage.py | 46 +--- core/admin/mailu/models.py | 4 +- core/admin/mailu/schemas.py | 498 ++++++++++++++++++++++++------------ 3 files changed, 346 insertions(+), 202 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 569c161c..a67abdf7 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,5 +1,5 @@ from mailu import models -from .schemas import schemas +from .schemas import MailuConfig, MailuSchema from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext @@ -310,41 +310,17 @@ def config_dump(full=False, secrets=False, dns=False, sections=None): SECTIONS can be: domains, relays, users, aliases """ - class spacedDumper(yaml.Dumper): + try: + config = MailuConfig(sections) + except ValueError as reason: + print(f'[ERROR] {reason}') + return 1 - def write_line_break(self, data=None): - super().write_line_break(data) - if len(self.indents) == 1: - super().write_line_break() - - def increase_indent(self, flow=False, indentless=False): - return super().increase_indent(flow, False) - - if sections: - for section in sections: - if section not in schemas: - print(f'[ERROR] Invalid section: {section}') - return 1 - else: - sections = sorted(schemas.keys()) - -# TODO: create World Schema and dump only this with Word.dumps ? - - for section in sections: - schema = schemas[section](many=True) - schema.context.update({ - 'full': full, - 'secrets': secrets, - 'dns': dns, - }) - yaml.dump( - {section: schema.dump(schema.Meta.model.query.all())}, - sys.stdout, - Dumper=spacedDumper, - default_flow_style=False, - allow_unicode=True - ) - sys.stdout.write('\n') + MailuSchema(context={ + 'full': full, + 'secrets': secrets, + 'dns': dns, + }).dumps(config, sys.stdout) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 625d1fd6..8b77b011 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -110,7 +110,7 @@ class Base(db.Model): created_at = db.Column(db.Date, nullable=False, default=date.today) updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) - comment = db.Column(db.String(255), nullable=True) + comment = db.Column(db.String(255), nullable=True, default='') @classmethod def _dict_pkey(cls): @@ -171,7 +171,7 @@ class Base(db.Model): if self.__mapper__.relationships[key].query_class is not None: if hasattr(items, 'all'): items = items.all() - if full or len(items): + if full or items: if key in secret: res[key] = '' else: diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index c2a3d03d..bd9d228a 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,63 +1,252 @@ -import marshmallow -import sqlalchemy -import flask_marshmallow +""" +Mailu marshmallow schema +""" -from . import models +from textwrap import wrap + +import re +import yaml + +from marshmallow import post_dump, fields, Schema +from flask_marshmallow import Marshmallow +from OpenSSL import crypto + +from . import models, dkim -ma = flask_marshmallow.Marshmallow() +ma = Marshmallow() +# TODO: +# how to mark keys as "required" while unserializing (in certain use cases/API)? +# - fields withoud default => required +# - fields which are the primary key => unchangeable when updating + + +### yaml render module ### + +class RenderYAML: + """ Marshmallow YAML Render Module + """ + + class SpacedDumper(yaml.Dumper): + """ YAML Dumper to add a newline between main sections + and double the indent used + """ + + def write_line_break(self, data=None): + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + @staticmethod + def _update_dict(dict1, dict2): + """ sets missing keys in dict1 to values of dict2 + """ + for key, value in dict2.items(): + if key not in dict1: + dict1[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load yaml data from string + """ + cls._update_dict(kwargs, cls._load_defaults) + return yaml.load(*args, **kwargs) + + _dump_defaults = { + 'Dumper': SpacedDumper, + 'default_flow_style': False, + 'allow_unicode': True, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump yaml data to string + """ + cls._update_dict(kwargs, cls._dump_defaults) + return yaml.dump(*args, **kwargs) + + +### field definitions ### + +class LazyString(fields.String): + """ Field that serializes a "false" value to the empty string + """ + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize None to the empty string + """ + return value if value else '' + + +class CommaSeparatedList(fields.Raw): + """ Field that deserializes a string containing comma-separated values to + a list of strings + """ + # TODO: implement this + + +class DkimKey(fields.String): + """ Field that serializes a dkim key to a list of strings (lines) and + deserializes a string or list of strings. + """ + + _clean_re = re.compile( + r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', + flags=re.UNICODE + ) + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize dkim key to a list of strings (lines) + """ + + # map empty string and None to None + if not value: + return None + + # return list of key lines without header/footer + return value.decode('utf-8').strip().split('\n')[1:-1] + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize a string or list of strings to dkim key data + with verification + """ + + # convert list to str + if isinstance(value, list): + value = ''.join(value) + + # only strings are allowed + if not isinstance(value, str): + raise TypeError(f'invalid type: {type(value).__name__!r}') + + # clean value (remove whitespace and header/footer) + value = self._clean_re.sub('', value.strip()) + + # map empty string/list to None + if not value: + return None + + # handle special value 'generate' + elif value == 'generate': + return dkim.gen_key() + + # wrap value into valid pem layout and check validity + value = ( + '-----BEGIN PRIVATE KEY-----\n' + + '\n'.join(wrap(value, 64)) + + '\n-----END PRIVATE KEY-----\n' + ).encode('ascii') + try: + crypto.load_privatekey(crypto.FILETYPE_PEM, value) + except crypto.Error as exc: + raise ValueError('invalid dkim key') from exc + else: + return value + + +### schema definitions ### -import collections class BaseSchema(ma.SQLAlchemyAutoSchema): + """ Marshmallow base schema with custom exclude logic + and option to hide sqla defaults + """ class Meta: - base_hide_always = {'created_at', 'updated_at'} - base_hide_secrets = set() - base_hide_by_value = { -# 'comment': {'', None} - } + """ Schema config """ + model = None - @marshmallow.post_dump - def remove_skip_values(self, data, many, **kwargs): -# print(repr(data), self.context) + def __init__(self, *args, **kwargs): - # always hide - hide_by_key = self.Meta.base_hide_always | set(getattr(self.Meta, 'hide_always', ())) + # get and remove config from kwargs + context = kwargs.get('context', {}) - # hide secrets - if not self.context.get('secrets'): - hide_by_key |= self.Meta.base_hide_secrets - hide_by_key |= set(getattr(self.Meta, 'hide_secrets', ())) + # compile excludes + exclude = set(kwargs.get('exclude', [])) - # hide by value - hide_by_value = self.Meta.base_hide_by_value | getattr(self.Meta, 'hide_by_value', {}) + # always exclude + exclude.update({'created_at', 'updated_at'}) - # hide defaults - if not self.context.get('full'): + # add include_by_context + if context is not None: + for ctx, what in getattr(self.Meta, 'include_by_context', {}).items(): + if not context.get(ctx): + exclude |= set(what) + + # update excludes + kwargs['exclude'] = exclude + + # exclude_by_value + self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) + + # exclude default values + if not context.get('full'): for column in self.Meta.model.__table__.columns: -# print(column.name, column.default.arg if isinstance(column.default, sqlalchemy.sql.schema.ColumnDefault) else column.default) -# alias.destiantion has default [] - is this okay. how to check it? - if column.name not in hide_by_key: - hide_by_value.setdefault(column.name, set()).add(None if column.default is None else column.default.arg) + if column.name not in exclude: + self._exclude_by_value.setdefault(column.name, []).append( + None if column.default is None else column.default.arg + ) + # hide by context + self._hide_by_context = set() + if context is not None: + for ctx, what in getattr(self.Meta, 'hide_by_context', {}).items(): + if not context.get(ctx): + self._hide_by_context |= set(what) + + # init SQLAlchemyAutoSchema + super().__init__(*args, **kwargs) + + @post_dump + def _remove_skip_values(self, data, many, **kwargs): # pylint: disable=unused-argument + + if not self._exclude_by_value and not self._hide_by_context: + return data + + full = self.context.get('full') return { - key: value for key, value in data.items() - if - not isinstance(value, collections.Hashable) - or( - key not in hide_by_key - and - (key not in hide_by_value or value not in hide_by_value[key])) + key: '' if key in self._hide_by_context else value + for key, value in data.items() + if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] } + # TODO: remove LazyString and fix model definition (comment should not be nullable) + comment = LazyString() + class DomainSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Domain model """ + class Meta: + """ Schema config """ model = models.Domain + include_relationships = True + #include_fk = True + exclude = ['users', 'managers', 'aliases'] + + include_by_context = { + 'dns': {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, + } + hide_by_context = { + 'secrets': {'dkim_key'}, + } + exclude_by_value = { + 'alternatives': [[]], + 'dkim_key': [None], + 'dkim_publickey': [None], + 'dns_mx': [None], + 'dns_spf': [None], + 'dns_dkim': [None], + 'dns_dmarc': [None], + } + + dkim_key = DkimKey() + dkim_publickey = fields.String(dump_only=True) + dns_mx = fields.String(dump_only=True) + dns_spf = fields.String(dump_only=True) + dns_dkim = fields.String(dump_only=True) + dns_dmarc = fields.String(dump_only=True) - # _dict_hide = {'users', 'managers', 'aliases'} - # _dict_show = {'dkim_key'} - # _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} - # _dict_secret = {'dkim_key'} # _dict_types = { # 'dkim_key': (bytes, type(None)), # 'dkim_publickey': False, @@ -66,50 +255,62 @@ class DomainSchema(BaseSchema): # 'dns_dkim': False, # 'dns_dmarc': False, # } - # _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} - # @staticmethod - # def _dict_input(data): - # if 'dkim_key' in data: - # key = data['dkim_key'] - # if key is not None: - # if type(key) is list: - # key = ''.join(key) - # if type(key) is str: - # key = ''.join(key.strip().split()) # removes all whitespace - # if key == 'generate': - # data['dkim_key'] = dkim.gen_key() - # elif key: - # m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) - # if m is not None: - # key = key[m.end():] - # m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) - # if m is not None: - # key = key[:m.start()] - # key = '\n'.join(wrap(key, 64)) - # key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') - # try: - # dkim.strip_key(key) - # except: - # raise ValueError('invalid dkim key') - # else: - # data['dkim_key'] = key - # else: - # data['dkim_key'] = None - # name = db.Column(IdnaDomain, primary_key=True, nullable=False) - # managers = db.relationship('User', secondary=managers, - # backref=db.backref('manager_of'), lazy='dynamic') - # max_users = db.Column(db.Integer, nullable=False, default=-1) - # max_aliases = db.Column(db.Integer, nullable=False, default=-1) - # max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) - # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + +class TokenSchema(BaseSchema): + """ Marshmallow schema for Token model """ + class Meta: + """ Schema config """ + model = models.Token + + # _dict_recurse = True + # _dict_hide = {'user', 'user_email'} + # _dict_mandatory = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('tokens', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # ip = db.Column(db.String(255)) + + +class FetchSchema(BaseSchema): + """ Marshmallow schema for Fetch model """ + class Meta: + """ Schema config """ + model = models.Fetch + include_by_context = { + 'full': {'last_check', 'error'}, + } + hide_by_context = { + 'secrets': {'password'}, + } + +# TODO: What about mandatory keys? + # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} class UserSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for User model """ + class Meta: + """ Schema config """ model = models.User + include_relationships = True + exclude = ['localpart', 'domain', 'quota_bytes_used'] - # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} + exclude_by_value = { + 'forward_destination': [[]], + 'tokens': [[]], + 'reply_enddate': ['2999-12-31'], + 'reply_startdate': ['1900-01-01'], + } + + tokens = fields.Nested(TokenSchema, many=True) + fetches = fields.Nested(FetchSchema, many=True) + +# TODO: deserialize password/password_hash! What about mandatory keys? # _dict_mandatory = {'localpart', 'domain', 'password'} # @classmethod # def _dict_input(cls, data): @@ -133,44 +334,19 @@ class UserSchema(BaseSchema): # del data['hash_scheme'] # del data['password_hash'] - # domain = db.relationship(Domain, - # backref=db.backref('users', cascade='all, delete-orphan')) - # password = db.Column(db.String(255), nullable=False) - # quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) - # quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) - # global_admin = db.Column(db.Boolean(), nullable=False, default=False) - # enabled = db.Column(db.Boolean(), nullable=False, default=True) - - # # Features - # enable_imap = db.Column(db.Boolean(), nullable=False, default=True) - # enable_pop = db.Column(db.Boolean(), nullable=False, default=True) - - # # Filters - # forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) - # forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) - # forward_keep = db.Column(db.Boolean(), nullable=False, default=True) - # reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) - # reply_subject = db.Column(db.String(255), nullable=True, default=None) - # reply_body = db.Column(db.Text(), nullable=True, default=None) - # reply_startdate = db.Column(db.Date, nullable=False, - # default=date(1900, 1, 1)) - # reply_enddate = db.Column(db.Date, nullable=False, - # default=date(2999, 12, 31)) - - # # Settings - # displayed_name = db.Column(db.String(160), nullable=False, default='') - # spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) - # spam_threshold = db.Column(db.Integer(), nullable=False, default=80) class AliasSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Alias model """ + class Meta: + """ Schema config """ model = models.Alias - hide_always = {'localpart'} - hide_secrets = {'wildcard'} - hide_by_value = { - 'destination': set([]) # always hide empty lists?! + exclude = ['localpart'] + + exclude_by_value = { + 'destination': [[]], } +# TODO: deserialize destination! # @staticmethod # def _dict_input(data): # Email._dict_input(data) @@ -180,65 +356,57 @@ class AliasSchema(BaseSchema): # data['destination'] = list([adr.strip() for adr in dst.split(',')]) -class TokenSchema(BaseSchema): - class Meta(BaseSchema.Meta): - model = models.Token - - # _dict_recurse = True - # _dict_hide = {'user', 'user_email'} - # _dict_mandatory = {'password'} - - # id = db.Column(db.Integer(), primary_key=True) - # user_email = db.Column(db.String(255), db.ForeignKey(User.email), - # nullable=False) - # user = db.relationship(User, - # backref=db.backref('tokens', cascade='all, delete-orphan')) - # password = db.Column(db.String(255), nullable=False) - # ip = db.Column(db.String(255)) - - -class FetchSchema(BaseSchema): - class Meta(BaseSchema.Meta): - model = models.Fetch - - # _dict_recurse = True - # _dict_hide = {'user_email', 'user', 'last_check', 'error'} - # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} - # _dict_secret = {'password'} - - # id = db.Column(db.Integer(), primary_key=True) - # user_email = db.Column(db.String(255), db.ForeignKey(User.email), - # nullable=False) - # user = db.relationship(User, - # backref=db.backref('fetches', cascade='all, delete-orphan')) - # protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) - # host = db.Column(db.String(255), nullable=False) - # port = db.Column(db.Integer(), nullable=False) - # tls = db.Column(db.Boolean(), nullable=False, default=False) - # username = db.Column(db.String(255), nullable=False) - # password = db.Column(db.String(255), nullable=False) - # keep = db.Column(db.Boolean(), nullable=False, default=False) - # last_check = db.Column(db.DateTime, nullable=True) - # error = db.Column(db.String(1023), nullable=True) - - class ConfigSchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Config model """ + class Meta: + """ Schema config """ model = models.Config -# TODO: how to mark keys as "required" while unserializing (in certain use cases/API) - name = ma.auto_field(required=True) - value = ma.auto_field(required=True) class RelaySchema(BaseSchema): - class Meta(BaseSchema.Meta): + """ Marshmallow schema for Relay model """ + class Meta: + """ Schema config """ model = models.Relay -schemas = { - 'domains': DomainSchema, - 'relays': RelaySchema, - 'users': UserSchema, - 'aliases': AliasSchema, -# 'config': ConfigSchema, -} +class MailuSchema(Schema): + """ Marshmallow schema for Mailu config """ + class Meta: + """ Schema config """ + render_module = RenderYAML + domains = fields.Nested(DomainSchema, many=True) + relays = fields.Nested(RelaySchema, many=True) + users = fields.Nested(UserSchema, many=True) + aliases = fields.Nested(AliasSchema, many=True) + config = fields.Nested(ConfigSchema, many=True) + + +### config class ### + +class MailuConfig: + """ Class which joins whole Mailu config for dumping + """ + + _models = { + 'domains': models.Domain, + 'relays': models.Relay, + 'users': models.User, + 'aliases': models.Alias, +# 'config': models.Config, + } + + def __init__(self, sections): + if sections: + for section in sections: + if section not in self._models: + raise ValueError(f'Unknown section: {section!r}') + self._sections = set(sections) + else: + self._sections = set(self._models.keys()) + + def __getattr__(self, section): + if section in self._sections: + return self._models[section].query.all() + else: + raise AttributeError From c24bff1c1be3980892fddb13a139b8142bb71f96 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 14 Jan 2021 01:11:04 +0100 Subject: [PATCH 30/52] added config_import using marshmallow --- core/admin/mailu/api/__init__.py | 2 +- core/admin/mailu/manage.py | 314 ++++++++------- core/admin/mailu/models.py | 650 +++++++++++++++---------------- core/admin/mailu/schemas.py | 238 +++++------ 4 files changed, 611 insertions(+), 593 deletions(-) diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py index a4d8689f..dee43036 100644 --- a/core/admin/mailu/api/__init__.py +++ b/core/admin/mailu/api/__init__.py @@ -26,7 +26,7 @@ def register(app): # add redirect to current api version @app.route(f'{ROOT}/') - def redir(): + def _redirect_to_active_api(): return redirect(url_for(f'{ACTIVE.blueprint.name}.root')) # swagger ui config diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index a67abdf7..6103f904 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,21 +1,25 @@ -from mailu import models -from .schemas import MailuConfig, MailuSchema - -from flask import current_app as app -from flask.cli import FlaskGroup, with_appcontext +""" Mailu command line interface +""" +import sys import os import socket import uuid + import click -import yaml -import sys + +from flask import current_app as app +from flask.cli import FlaskGroup, with_appcontext +from marshmallow.exceptions import ValidationError + +from . import models +from .schemas import MailuSchema db = models.db -@click.group(cls=FlaskGroup) +@click.group(cls=FlaskGroup, context_settings={'help_option_names': ['-?', '-h', '--help']}) def mailu(): """ Mailu command line """ @@ -26,17 +30,17 @@ def mailu(): def advertise(): """ Advertise this server against statistic services. """ - if os.path.isfile(app.config["INSTANCE_ID_PATH"]): - with open(app.config["INSTANCE_ID_PATH"], "r") as handle: + if os.path.isfile(app.config['INSTANCE_ID_PATH']): + with open(app.config['INSTANCE_ID_PATH'], 'r') as handle: instance_id = handle.read() else: instance_id = str(uuid.uuid4()) - with open(app.config["INSTANCE_ID_PATH"], "w") as handle: + with open(app.config['INSTANCE_ID_PATH'], 'w') as handle: handle.write(instance_id) - if not app.config["DISABLE_STATISTICS"]: + if not app.config['DISABLE_STATISTICS']: try: - socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) - except: + socket.gethostbyname(app.config['STATS_ENDPOINT'].format(instance_id)) + except OSError: pass @@ -171,156 +175,196 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() -yaml_sections = [ - ('domains', models.Domain), - ('relays', models.Relay), - ('users', models.User), - ('aliases', models.Alias), -# ('config', models.Config), -] +# @mailu.command() +# @click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') +# @click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') +# @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') +# @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) +# @with_appcontext +# def config_update(verbose=False, delete_objects=False, dry_run=False, source=None): +# """ Update configuration with data from YAML-formatted input +# """ + + # try: + # new_config = yaml.safe_load(source) + # except (yaml.scanner.ScannerError, yaml.parser.ParserError) as exc: + # print(f'[ERROR] Invalid yaml: {exc}') + # sys.exit(1) + # else: + # if isinstance(new_config, str): + # print(f'[ERROR] Invalid yaml: {new_config!r}') + # sys.exit(1) + # elif new_config is None or not new_config: + # print('[ERROR] Empty yaml: Please pipe yaml into stdin') + # sys.exit(1) + + # error = False + # tracked = {} + # for section, model in yaml_sections: + + # items = new_config.get(section) + # if items is None: + # if delete_objects: + # print(f'[ERROR] Invalid yaml: Section "{section}" is missing') + # error = True + # break + # else: + # continue + + # del new_config[section] + + # if not isinstance(items, list): + # print(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') + # error = True + # break + # elif not items: + # continue + + # # create items + # for data in items: + + # if verbose: + # print(f'Handling {model.__table__} data: {data!r}') + + # try: + # changed = model.from_dict(data, delete_objects) + # except Exception as exc: + # print(f'[ERROR] {exc.args[0]} in data: {data}') + # error = True + # break + + # for item, created in changed: + + # if created is True: + # # flush newly created item + # db.session.add(item) + # db.session.flush() + # if verbose: + # print(f'Added {item!r}: {item.to_dict()}') + # else: + # print(f'Added {item!r}') + + # elif created: + # # modified instance + # if verbose: + # for key, old, new in created: + # print(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') + # else: + # print(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') + + # # track primary key of all items + # tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) + + # if error: + # break + + # # on error: stop early + # if error: + # print('[ERROR] An error occured. Not committing changes.') + # db.session.rollback() + # sys.exit(1) + + # # are there sections left in new_config? + # if new_config: + # print(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') + # error = True + + # # test for conflicting domains + # domains = set() + # for model, items in tracked.items(): + # if model in (models.Domain, models.Alternative, models.Relay): + # if domains & items: + # for fqdn in domains & items: + # print(f'[ERROR] Duplicate domain name used: {fqdn}') + # error = True + # domains.update(items) + + # # delete items not tracked + # if delete_objects: + # for model, items in tracked.items(): + # for item in model.query.all(): + # if not item._dict_pval() in items: + # print(f'Deleted {item!r} {item}') + # db.session.delete(item) + + # # don't commit when running dry + # if dry_run: + # print('Dry run. Not commiting changes.') + # db.session.rollback() + # else: + # db.session.commit() + + +SECTIONS = {'domains', 'relays', 'users', 'aliases'} + @mailu.command() @click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') -@click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') +@click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): - """sync configuration with data from YAML-formatted stdin""" +def config_import(verbose=False, dry_run=False, source=None): + """ Import configuration YAML + """ - out = (lambda *args: print('(DRY RUN)', *args)) if dry_run else print + context = { + 'verbose': verbose, # TODO: use callback function to be verbose? + 'import': True, + } try: - new_config = yaml.safe_load(sys.stdin) - except (yaml.scanner.ScannerError, yaml.parser.ParserError) as reason: - out(f'[ERROR] Invalid yaml: {reason}') + config = MailuSchema(context=context).loads(source) + except ValidationError as exc: + print(f'[ERROR] {exc}') + # TODO: show nice errors + from pprint import pprint + pprint(exc.messages) sys.exit(1) else: - if type(new_config) is str: - out(f'[ERROR] Invalid yaml: {new_config!r}') - sys.exit(1) - elif new_config is None or not len(new_config): - out('[ERROR] Empty yaml: Please pipe yaml into stdin') - sys.exit(1) - - error = False - tracked = {} - for section, model in yaml_sections: - - items = new_config.get(section) - if items is None: - if delete_objects: - out(f'[ERROR] Invalid yaml: Section "{section}" is missing') - error = True - break - else: - continue - - del new_config[section] - - if type(items) is not list: - out(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') - error = True - break - elif not items: - continue - - # create items - for data in items: - - if verbose: - out(f'Handling {model.__table__} data: {data!r}') - - try: - changed = model.from_dict(data, delete_objects) - except Exception as reason: - out(f'[ERROR] {reason.args[0]} in data: {data}') - error = True - break - - for item, created in changed: - - if created is True: - # flush newly created item - db.session.add(item) - db.session.flush() - if verbose: - out(f'Added {item!r}: {item.to_dict()}') - else: - out(f'Added {item!r}') - - elif len(created): - # modified instance - if verbose: - for key, old, new in created: - out(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') - else: - out(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') - - # track primary key of all items - tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) - - if error: - break - - # on error: stop early - if error: - out('An error occured. Not committing changes.') - db.session.rollback() - sys.exit(1) - - # are there sections left in new_config? - if new_config: - out(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') - error = True - - # test for conflicting domains - domains = set() - for model, items in tracked.items(): - if model in (models.Domain, models.Alternative, models.Relay): - if domains & items: - for domain in domains & items: - out(f'[ERROR] Duplicate domain name used: {domain}') - error = True - domains.update(items) - - # delete items not tracked - if delete_objects: - for model, items in tracked.items(): - for item in model.query.all(): - if not item._dict_pval() in items: - out(f'Deleted {item!r} {item}') - db.session.delete(item) + print(config) + print(MailuSchema().dumps(config)) + # TODO: does not commit yet. + # TODO: delete other entries? # don't commit when running dry - if dry_run: + if True: #dry_run: + print('Dry run. Not commiting changes.') db.session.rollback() else: db.session.commit() @mailu.command() -@click.option('-f', '--full', is_flag=True, help='Include default attributes') -@click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') +@click.option('-f', '--full', is_flag=True, help='Include attributes with default value') +@click.option('-s', '--secrets', is_flag=True, + help='Include secret attributes (dkim-key, passwords)') @click.option('-d', '--dns', is_flag=True, help='Include dns records') +@click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), + help='save yaml to file') @click.argument('sections', nargs=-1) @with_appcontext -def config_dump(full=False, secrets=False, dns=False, sections=None): - """dump configuration as YAML-formatted data to stdout +def config_dump(full=False, secrets=False, dns=False, output=None, sections=None): + """ Dump configuration as YAML to stdout or file SECTIONS can be: domains, relays, users, aliases """ - try: - config = MailuConfig(sections) - except ValueError as reason: - print(f'[ERROR] {reason}') - return 1 + if sections: + for section in sections: + if section not in SECTIONS: + print(f'[ERROR] Unknown section: {section!r}') + sys.exit(1) + sections = set(sections) + else: + sections = SECTIONS - MailuSchema(context={ + context={ 'full': full, 'secrets': secrets, 'dns': dns, - }).dumps(config, sys.stdout) + } + + MailuSchema(only=sections, context=context).dumps(models.MailuConfig(), output) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 8b77b011..73d05801 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,23 +1,26 @@ -from mailu import dkim +""" Mailu config storage model +""" -from sqlalchemy.ext import declarative -from datetime import datetime, date +import re +import os +import smtplib +import json + +from datetime import date from email.mime import text -from flask import current_app as app -from textwrap import wrap import flask_sqlalchemy import sqlalchemy -import re -import time -import os import passlib -import glob -import smtplib import idna import dns -import json -import itertools + +from flask import current_app as app +from sqlalchemy.ext import declarative +from sqlalchemy.inspection import inspect +from werkzeug.utils import cached_property + +from . import dkim db = flask_sqlalchemy.SQLAlchemy() @@ -30,9 +33,11 @@ class IdnaDomain(db.TypeDecorator): impl = db.String(80) def process_bind_param(self, value, dialect): + """ encode unicode domain name to punycode """ return idna.encode(value).decode('ascii').lower() def process_result_value(self, value, dialect): + """ decode punycode domain name to unicode """ return idna.decode(value) python_type = str @@ -44,6 +49,7 @@ class IdnaEmail(db.TypeDecorator): impl = db.String(255) def process_bind_param(self, value, dialect): + """ encode unicode domain part of email address to punycode """ try: localpart, domain_name = value.split('@') return '{0}@{1}'.format( @@ -54,6 +60,7 @@ class IdnaEmail(db.TypeDecorator): pass def process_result_value(self, value, dialect): + """ decode punycode domain part of email to unicode """ localpart, domain_name = value.split('@') return '{0}@{1}'.format( localpart, @@ -69,14 +76,16 @@ class CommaSeparatedList(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): - if not isinstance(value, (list, set)): - raise TypeError('Must be a list') + """ join list of items to comma separated string """ + if not isinstance(value, (list, tuple, set)): + raise TypeError('Must be a list of strings') for item in value: if ',' in item: raise ValueError('Item must not contain a comma') return ','.join(sorted(value)) def process_result_value(self, value, dialect): + """ split comma separated string to list """ return list(filter(bool, value.split(','))) if value else [] python_type = list @@ -88,9 +97,11 @@ class JSONEncoded(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): + """ encode data as json """ return json.dumps(value) if value else None def process_result_value(self, value, dialect): + """ decode json to data """ return json.loads(value) if value else None python_type = str @@ -112,246 +123,172 @@ class Base(db.Model): updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True, default='') - @classmethod - def _dict_pkey(cls): - return cls.__mapper__.primary_key[0].name + # @classmethod + # def from_dict(cls, data, delete=False): - def _dict_pval(self): - return getattr(self, self._dict_pkey()) + # changed = [] - def to_dict(self, full=False, include_secrets=False, include_extra=None, recursed=False, hide=None): - """ Return a dictionary representation of this model. - """ + # pkey = cls._dict_pkey() - if recursed and not getattr(self, '_dict_recurse', False): - return str(self) + # # handle "primary key" only + # if not isinstance(data, dict): + # data = {pkey: data} - hide = set(hide or []) | {'created_at', 'updated_at'} - if hasattr(self, '_dict_hide'): - hide |= self._dict_hide + # # modify input data + # if hasattr(cls, '_dict_input'): + # try: + # cls._dict_input(data) + # except Exception as exc: + # raise ValueError(f'{exc}', cls, None, data) from exc - secret = set() - if not include_secrets and hasattr(self, '_dict_secret'): - secret |= self._dict_secret + # # check for primary key (if not recursed) + # if not getattr(cls, '_dict_recurse', False): + # if not pkey in data: + # raise KeyError(f'primary key {cls.__table__}.{pkey} is missing', cls, pkey, data) - convert = getattr(self, '_dict_output', {}) + # # check data keys and values + # for key in list(data.keys()): - extra_keys = getattr(self, '_dict_extra', {}) - if include_extra is None: - include_extra = [] + # # check key + # if not hasattr(cls, key) and not key in cls.__mapper__.relationships: + # raise KeyError(f'unknown key {cls.__table__}.{key}', cls, key, data) - res = {} + # # check value type + # value = data[key] + # col = cls.__mapper__.columns.get(key) + # if col is not None: + # if not ((value is None and col.nullable) or (isinstance(value, col.type.python_type))): + # raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) + # else: + # rel = cls.__mapper__.relationships.get(key) + # if rel is None: + # itype = getattr(cls, '_dict_types', {}).get(key) + # if itype is not None: + # if itype is False: # ignore value. TODO: emit warning? + # del data[key] + # continue + # elif not isinstance(value, itype): + # raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) + # else: + # raise NotImplementedError(f'type not defined for {cls.__table__}.{key}') - for key in itertools.chain( - self.__table__.columns.keys(), - getattr(self, '_dict_show', []), - *[extra_keys.get(extra, []) for extra in include_extra] - ): - if key in hide: - continue - if key in self.__table__.columns: - default = self.__table__.columns[key].default - if isinstance(default, sqlalchemy.sql.schema.ColumnDefault): - default = default.arg - else: - default = None - value = getattr(self, key) - if full or ((default or value) and value != default): - if key in secret: - value = '' - elif value is not None and key in convert: - value = convert[key](value) - res[key] = value + # # handle relationships + # if key in cls.__mapper__.relationships: + # rel_model = cls.__mapper__.relationships[key].argument + # if not isinstance(rel_model, sqlalchemy.orm.Mapper): + # add = rel_model.from_dict(value, delete) + # assert len(add) == 1 + # rel_item, updated = add[0] + # changed.append((rel_item, updated)) + # data[key] = rel_item - for key in self.__mapper__.relationships.keys(): - if key in hide: - continue - if self.__mapper__.relationships[key].uselist: - items = getattr(self, key) - if self.__mapper__.relationships[key].query_class is not None: - if hasattr(items, 'all'): - items = items.all() - if full or items: - if key in secret: - res[key] = '' - else: - res[key] = [item.to_dict(full, include_secrets, include_extra, True) for item in items] - else: - value = getattr(self, key) - if full or value is not None: - if key in secret: - res[key] = '' - else: - res[key] = value.to_dict(full, include_secrets, include_extra, True) + # # create item if necessary + # created = False + # item = cls.query.get(data[pkey]) if pkey in data else None + # if item is None: - return res + # # check for mandatory keys + # missing = getattr(cls, '_dict_mandatory', set()) - set(data.keys()) + # if missing: + # raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {cls.__table__} missing', cls, missing, data) - @classmethod - def from_dict(cls, data, delete=False): + # # remove mapped relationships from data + # mapped = {} + # for key in list(data.keys()): + # if key in cls.__mapper__.relationships: + # if isinstance(cls.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): + # mapped[key] = data[key] + # del data[key] - changed = [] + # # create new item + # item = cls(**data) + # created = True - pkey = cls._dict_pkey() + # # and update mapped relationships (below) + # data = mapped - # handle "primary key" only - if isinstance(data, dict): - data = {pkey: data} + # # update item + # updated = [] + # for key, value in data.items(): - # modify input data - if hasattr(cls, '_dict_input'): - try: - cls._dict_input(data) - except Exception as reason: - raise ValueError(f'{reason}', cls, None, data) + # # skip primary key + # if key == pkey: + # continue - # check for primary key (if not recursed) - if not getattr(cls, '_dict_recurse', False): - if not pkey in data: - raise KeyError(f'primary key {cls.__table__}.{pkey} is missing', cls, pkey, data) + # if key in cls.__mapper__.relationships: + # # update relationship + # rel_model = cls.__mapper__.relationships[key].argument + # if isinstance(rel_model, sqlalchemy.orm.Mapper): + # rel_model = rel_model.class_ + # # add (and create) referenced items + # cur = getattr(item, key) + # old = sorted(cur, key=id) + # new = [] + # for rel_data in value: + # # get or create related item + # add = rel_model.from_dict(rel_data, delete) + # assert len(add) == 1 + # rel_item, rel_updated = add[0] + # changed.append((rel_item, rel_updated)) + # if rel_item not in cur: + # cur.append(rel_item) + # new.append(rel_item) - # check data keys and values - for key in list(data.keys()): + # # delete referenced items missing in yaml + # rel_pkey = rel_model._dict_pkey() + # new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) + # for rel_item in old: + # if rel_item not in new: + # # check if item with same data exists to stabilze import without primary key + # rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) + # try: + # same_idx = new_data.index(rel_data) + # except ValueError: + # same = None + # else: + # same = new[same_idx] - # check key - if not hasattr(cls, key) and not key in cls.__mapper__.relationships: - raise KeyError(f'unknown key {cls.__table__}.{key}', cls, key, data) + # if same is None: + # # delete items missing in new + # if delete: + # cur.remove(rel_item) + # else: + # new.append(rel_item) + # else: + # # swap found item with same data with newly created item + # new.append(rel_item) + # new_data.append(rel_data) + # new.remove(same) + # del new_data[same_idx] + # for i, (ch_item, _) in enumerate(changed): + # if ch_item is same: + # changed[i] = (rel_item, []) + # db.session.flush() + # db.session.delete(ch_item) + # break - # check value type - value = data[key] - col = cls.__mapper__.columns.get(key) - if col is not None: - if not ((value is None and col.nullable) or (isinstance(value, col.type.python_type))): - raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) - else: - rel = cls.__mapper__.relationships.get(key) - if rel is None: - itype = getattr(cls, '_dict_types', {}).get(key) - if itype is not None: - if itype is False: # ignore value. TODO: emit warning? - del data[key] - continue - elif not isinstance(value, itype): - raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) - else: - raise NotImplementedError(f'type not defined for {cls.__table__}.{key}') + # # remember changes + # new = sorted(new, key=id) + # if new != old: + # updated.append((key, old, new)) - # handle relationships - if key in cls.__mapper__.relationships: - rel_model = cls.__mapper__.relationships[key].argument - if not isinstance(rel_model, sqlalchemy.orm.Mapper): - add = rel_model.from_dict(value, delete) - assert len(add) == 1 - rel_item, updated = add[0] - changed.append((rel_item, updated)) - data[key] = rel_item + # else: + # # update key + # old = getattr(item, key) + # if isinstance(old, list): + # # deduplicate list value + # assert isinstance(value, list) + # value = set(value) + # old = set(old) + # if not delete: + # value = old | value + # if value != old: + # updated.append((key, old, value)) + # setattr(item, key, value) - # create item if necessary - created = False - item = cls.query.get(data[pkey]) if pkey in data else None - if item is None: + # changed.append((item, created if created else updated)) - # check for mandatory keys - missing = getattr(cls, '_dict_mandatory', set()) - set(data.keys()) - if missing: - raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {cls.__table__} missing', cls, missing, data) - - # remove mapped relationships from data - mapped = {} - for key in list(data.keys()): - if key in cls.__mapper__.relationships: - if isinstance(cls.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): - mapped[key] = data[key] - del data[key] - - # create new item - item = cls(**data) - created = True - - # and update mapped relationships (below) - data = mapped - - # update item - updated = [] - for key, value in data.items(): - - # skip primary key - if key == pkey: - continue - - if key in cls.__mapper__.relationships: - # update relationship - rel_model = cls.__mapper__.relationships[key].argument - if isinstance(rel_model, sqlalchemy.orm.Mapper): - rel_model = rel_model.class_ - # add (and create) referenced items - cur = getattr(item, key) - old = sorted(cur, key=id) - new = [] - for rel_data in value: - # get or create related item - add = rel_model.from_dict(rel_data, delete) - assert len(add) == 1 - rel_item, rel_updated = add[0] - changed.append((rel_item, rel_updated)) - if rel_item not in cur: - cur.append(rel_item) - new.append(rel_item) - - # delete referenced items missing in yaml - rel_pkey = rel_model._dict_pkey() - new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) - for rel_item in old: - if rel_item not in new: - # check if item with same data exists to stabilze import without primary key - rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) - try: - same_idx = new_data.index(rel_data) - except ValueError: - same = None - else: - same = new[same_idx] - - if same is None: - # delete items missing in new - if delete: - cur.remove(rel_item) - else: - new.append(rel_item) - else: - # swap found item with same data with newly created item - new.append(rel_item) - new_data.append(rel_data) - new.remove(same) - del new_data[same_idx] - for i, (ch_item, _) in enumerate(changed): - if ch_item is same: - changed[i] = (rel_item, []) - db.session.flush() - db.session.delete(ch_item) - break - - # remember changes - new = sorted(new, key=id) - if new != old: - updated.append((key, old, new)) - - else: - # update key - old = getattr(item, key) - if isinstance(old, list): - # deduplicate list value - assert isinstance(value, list) - value = set(value) - old = set(old) - if not delete: - value = old | value - if value != old: - updated.append((key, old, value)) - setattr(item, key, value) - - changed.append((item, created if created else updated)) - - return changed + # return changed # Many-to-many association table for domain managers @@ -391,48 +328,6 @@ class Domain(Base): __tablename__ = 'domain' - _dict_hide = {'users', 'managers', 'aliases'} - _dict_show = {'dkim_key'} - _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} - _dict_secret = {'dkim_key'} - _dict_types = { - 'dkim_key': (bytes, type(None)), - 'dkim_publickey': False, - 'dns_mx': False, - 'dns_spf': False, - 'dns_dkim': False, - 'dns_dmarc': False, - } - _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} - @staticmethod - def _dict_input(data): - if 'dkim_key' in data: - key = data['dkim_key'] - if key is not None: - if isinstance(key, list): - key = ''.join(key) - if isinstance(key, str): - key = ''.join(key.strip().split()) # removes all whitespace - if key == 'generate': - data['dkim_key'] = dkim.gen_key() - elif key: - match = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) - if match is not None: - key = key[match.end():] - match = re.search('-----END (RSA )?PRIVATE KEY-----$', key) - if match is not None: - key = key[:match.start()] - key = '\n'.join(wrap(key, 64)) - key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') - try: - dkim.strip_key(key) - except: - raise ValueError('invalid dkim key') - else: - data['dkim_key'] = key - else: - data['dkim_key'] = None - name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, backref=db.backref('manager_of'), lazy='dynamic') @@ -440,7 +335,7 @@ class Domain(Base): max_aliases = db.Column(db.Integer, nullable=False, default=-1) max_quota_bytes = db.Column(db.BigInteger, nullable=False, default=0) signup_enabled = db.Column(db.Boolean, nullable=False, default=False) - + _dkim_key = None _dkim_key_changed = False @@ -452,17 +347,20 @@ class Domain(Base): def dns_mx(self): hostname = app.config['HOSTNAMES'].split(',')[0] return f'{self.name}. 600 IN MX 10 {hostname}.' - + @property def dns_spf(self): hostname = app.config['HOSTNAMES'].split(',')[0] return f'{self.name}. 600 IN TXT "v=spf1 mx a:{hostname} ~all"' - + @property def dns_dkim(self): if os.path.exists(self._dkim_file()): selector = app.config['DKIM_SELECTOR'] - return f'{selector}._domainkey.{self.name}. 600 IN TXT "v=DKIM1; k=rsa; p={self.dkim_publickey}"' + return ( + f'{selector}._domainkey.{self.name}. 600 IN TXT' + f'"v=DKIM1; k=rsa; p={self.dkim_publickey}"' + ) @property def dns_dmarc(self): @@ -473,7 +371,7 @@ class Domain(Base): ruf = app.config['DMARC_RUF'] ruf = f' ruf=mailto:{ruf}@{domain};' if ruf else '' return f'_dmarc.{self.name}. 600 IN TXT "v=DMARC1; p=reject;{rua}{ruf} adkim=s; aspf=s"' - + @property def dkim_key(self): if self._dkim_key is None: @@ -525,7 +423,11 @@ class Domain(Base): try: return self.name == other.name except AttributeError: - return False + return NotImplemented + + def __hash__(self): + return hash(str(self.name)) + class Alternative(Base): @@ -551,8 +453,6 @@ class Relay(Base): __tablename__ = 'relay' - _dict_mandatory = {'smtp'} - name = db.Column(IdnaDomain, primary_key=True, nullable=False) smtp = db.Column(db.String(80), nullable=True) @@ -566,18 +466,8 @@ class Email(object): localpart = db.Column(db.String(80), nullable=False) - @staticmethod - def _dict_input(data): - if 'email' in data: - if 'localpart' in data or 'domain' in data: - raise ValueError('ambigous key email and localpart/domain') - elif isinstance(data['email'], str): - data['localpart'], data['domain'] = data['email'].rsplit('@', 1) - else: - data['email'] = f'{data["localpart"]}@{data["domain"]}' - @declarative.declared_attr - def domain_name(cls): + def domain_name(self): return db.Column(IdnaDomain, db.ForeignKey(Domain.name), nullable=False, default=IdnaDomain) @@ -585,7 +475,7 @@ class Email(object): # It is however very useful for quick lookups without joining tables, # especially when the mail server is reading the database. @declarative.declared_attr - def email(cls): + def email(self): updater = lambda context: '{0}@{1}'.format( context.current_parameters['localpart'], context.current_parameters['domain_name'], @@ -662,30 +552,6 @@ class User(Base, Email): __tablename__ = 'user' - _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} - _dict_mandatory = {'localpart', 'domain', 'password'} - @classmethod - def _dict_input(cls, data): - Email._dict_input(data) - # handle password - if 'password' in data: - if 'password_hash' in data or 'hash_scheme' in data: - raise ValueError('ambigous key password and password_hash/hash_scheme') - # check (hashed) password - password = data['password'] - if password.startswith('{') and '}' in password: - scheme = password[1:password.index('}')] - if scheme not in cls.scheme_dict: - raise ValueError(f'invalid password scheme {scheme!r}') - else: - raise ValueError(f'invalid hashed password {password!r}') - elif 'password_hash' in data and 'hash_scheme' in data: - if data['hash_scheme'] not in cls.scheme_dict: - raise ValueError(f'invalid password scheme {scheme!r}') - data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] - del data['hash_scheme'] - del data['password_hash'] - domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) @@ -775,7 +641,8 @@ class User(Base, Email): if raw: self.password = '{'+hash_scheme+'}' + password else: - self.password = '{'+hash_scheme+'}' + self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) + self.password = '{'+hash_scheme+'}' + \ + self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) def get_managed_domains(self): if self.global_admin: @@ -812,15 +679,6 @@ class Alias(Base, Email): __tablename__ = 'alias' - _dict_hide = {'domain_name', 'domain', 'localpart'} - @staticmethod - def _dict_input(data): - Email._dict_input(data) - # handle comma delimited string for backwards compability - dst = data.get('destination') - if isinstance(dst, str): - data['destination'] = list([adr.strip() for adr in dst.split(',')]) - domain = db.relationship(Domain, backref=db.backref('aliases', cascade='all, delete-orphan')) wildcard = db.Column(db.Boolean, nullable=False, default=False) @@ -832,10 +690,10 @@ class Alias(Base, Email): sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( sqlalchemy.and_( - cls.wildcard == False, + cls.wildcard is False, cls.localpart == localpart ), sqlalchemy.and_( - cls.wildcard == True, + cls.wildcard is True, sqlalchemy.bindparam('l', localpart).like(cls.localpart) ) ) @@ -847,10 +705,10 @@ class Alias(Base, Email): sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( sqlalchemy.and_( - cls.wildcard == False, + cls.wildcard is False, sqlalchemy.func.lower(cls.localpart) == localpart_lower ), sqlalchemy.and_( - cls.wildcard == True, + cls.wildcard is True, sqlalchemy.bindparam('l', localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) ) ) @@ -875,10 +733,6 @@ class Token(Base): __tablename__ = 'token' - _dict_recurse = True - _dict_hide = {'user', 'user_email'} - _dict_mandatory = {'password'} - id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) @@ -904,11 +758,6 @@ class Fetch(Base): __tablename__ = 'fetch' - _dict_recurse = True - _dict_hide = {'user_email', 'user', 'last_check', 'error'} - _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} - _dict_secret = {'password'} - id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) @@ -926,3 +775,124 @@ class Fetch(Base): def __str__(self): return f'{self.protocol}{"s" if self.tls else ""}://{self.username}@{self.host}:{self.port}' + + +class MailuConfig: + """ Class which joins whole Mailu config for dumping + and loading + """ + + # TODO: add sqlalchemy session updating (.add & .del) + class MailuCollection: + """ Provides dict- and list-like access to all instances + of a sqlalchemy model + """ + + def __init__(self, model : db.Model): + self._model = model + + @cached_property + def _items(self): + return { + inspect(item).identity: item + for item in self._model.query.all() + } + + def __len__(self): + return len(self._items) + + def __iter__(self): + return iter(self._items.values()) + + def __getitem__(self, key): + return self._items[key] + + def __setitem__(self, key, item): + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items[key] = item + + def __delitem__(self, key): + del self._items[key] + + def append(self, item): + """ list-like append """ + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + key = inspect(item).identity + if key in self._items: + raise ValueError(f'item {key!r} already present in collection') + self._items[key] = item + + def extend(self, items): + """ list-like extend """ + add = {} + for item in items: + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + key = inspect(item).identity + if key in self._items: + raise ValueError(f'item {key!r} already present in collection') + add[key] = item + self._items.update(add) + + def pop(self, *args): + """ list-like (no args) and dict-like (1 or 2 args) pop """ + if args: + if len(args) > 2: + raise TypeError(f'pop expected at most 2 arguments, got {len(args)}') + return self._items.pop(*args) + else: + return self._items.popitem()[1] + + def popitem(self): + """ dict-like popitem """ + return self._items.popitem() + + def remove(self, item): + """ list-like remove """ + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + key = inspect(item).identity + if not key in self._items: + raise ValueError(f'item {key!r} not found in collection') + del self._items[key] + + def clear(self): + """ dict-like clear """ + while True: + try: + self.pop() + except IndexError: + break + + def update(self, items): + """ dict-like update """ + for key, item in items: + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + if key in self._items: + raise ValueError(f'item {key!r} already present in collection') + + def setdefault(self, key, item=None): + """ dict-like setdefault """ + if key in self._items: + return self._items[key] + if item is None: + return None + if not isinstance(item, self._model): + raise TypeError(f'expected {self._model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items[key] = item + return item + + domains = MailuCollection(Domain) + relays = MailuCollection(Relay) + users = MailuCollection(User) + aliases = MailuCollection(Alias) + config = MailuCollection(Config) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index bd9d228a..fc08b67c 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,13 +1,15 @@ +""" Mailu marshmallow fields and schema """ -Mailu marshmallow schema -""" + +import re from textwrap import wrap -import re import yaml -from marshmallow import post_dump, fields, Schema +from marshmallow import pre_load, post_dump, fields, Schema +from marshmallow.exceptions import ValidationError +from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts from flask_marshmallow import Marshmallow from OpenSSL import crypto @@ -15,9 +17,9 @@ from . import models, dkim ma = Marshmallow() -# TODO: -# how to mark keys as "required" while unserializing (in certain use cases/API)? -# - fields withoud default => required + +# TODO: how and where to mark keys as "required" while unserializing (on commandline, in api)? +# - fields without default => required # - fields which are the primary key => unchangeable when updating @@ -41,7 +43,7 @@ class RenderYAML: return super().increase_indent(flow, False) @staticmethod - def _update_dict(dict1, dict2): + def _update_items(dict1, dict2): """ sets missing keys in dict1 to values of dict2 """ for key, value in dict2.items(): @@ -53,8 +55,8 @@ class RenderYAML: def loads(cls, *args, **kwargs): """ load yaml data from string """ - cls._update_dict(kwargs, cls._load_defaults) - return yaml.load(*args, **kwargs) + cls._update_items(kwargs, cls._load_defaults) + return yaml.safe_load(*args, **kwargs) _dump_defaults = { 'Dumper': SpacedDumper, @@ -65,13 +67,33 @@ class RenderYAML: def dumps(cls, *args, **kwargs): """ dump yaml data to string """ - cls._update_dict(kwargs, cls._dump_defaults) + cls._update_items(kwargs, cls._dump_defaults) return yaml.dump(*args, **kwargs) +### functions ### + +def handle_email(data): + """ merge separate localpart and domain to email + """ + + localpart = 'localpart' in data + domain = 'domain' in data + + if 'email' in data: + if localpart or domain: + raise ValidationError('duplicate email and localpart/domain') + elif localpart and domain: + data['email'] = f'{data["localpart"]}@{data["domain"]}' + elif localpart or domain: + raise ValidationError('incomplete localpart/domain') + + return data + + ### field definitions ### -class LazyString(fields.String): +class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string """ @@ -81,14 +103,27 @@ class LazyString(fields.String): return value if value else '' -class CommaSeparatedList(fields.Raw): +class CommaSeparatedListField(fields.Raw): """ Field that deserializes a string containing comma-separated values to a list of strings """ - # TODO: implement this + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize comma separated string to list of strings + """ + + # empty + if not value: + return [] + + # split string + if isinstance(value, str): + return list([item.strip() for item in value.split(',') if item.strip()]) + else: + return value -class DkimKey(fields.String): +class DkimKeyField(fields.String): """ Field that serializes a dkim key to a list of strings (lines) and deserializes a string or list of strings. """ @@ -120,7 +155,7 @@ class DkimKey(fields.String): # only strings are allowed if not isinstance(value, str): - raise TypeError(f'invalid type: {type(value).__name__!r}') + raise ValidationError(f'invalid type {type(value).__name__!r}') # clean value (remove whitespace and header/footer) value = self._clean_re.sub('', value.strip()) @@ -133,6 +168,11 @@ class DkimKey(fields.String): elif value == 'generate': return dkim.gen_key() + # remember some keydata for error message + keydata = value + if len(keydata) > 40: + keydata = keydata[:25] + '...' + keydata[-10:] + # wrap value into valid pem layout and check validity value = ( '-----BEGIN PRIVATE KEY-----\n' + @@ -142,26 +182,37 @@ class DkimKey(fields.String): try: crypto.load_privatekey(crypto.FILETYPE_PEM, value) except crypto.Error as exc: - raise ValueError('invalid dkim key') from exc + raise ValidationError(f'invalid dkim key {keydata!r}') from exc else: return value -### schema definitions ### +### base definitions ### + +class BaseOpts(SQLAlchemyAutoSchemaOpts): + """ Option class with sqla session + """ + def __init__(self, meta, ordered=False): + if not hasattr(meta, 'sqla_session'): + meta.sqla_session = models.db.session + super(BaseOpts, self).__init__(meta, ordered=ordered) class BaseSchema(ma.SQLAlchemyAutoSchema): """ Marshmallow base schema with custom exclude logic and option to hide sqla defaults """ + OPTIONS_CLASS = BaseOpts + class Meta: """ Schema config """ model = None def __init__(self, *args, **kwargs): - # get and remove config from kwargs + # context? context = kwargs.get('context', {}) + flags = set([key for key, value in context.items() if value is True]) # compile excludes exclude = set(kwargs.get('exclude', [])) @@ -171,8 +222,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # add include_by_context if context is not None: - for ctx, what in getattr(self.Meta, 'include_by_context', {}).items(): - if not context.get(ctx): + for need, what in getattr(self.Meta, 'include_by_context', {}).items(): + if not flags & set(need): exclude |= set(what) # update excludes @@ -192,8 +243,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # hide by context self._hide_by_context = set() if context is not None: - for ctx, what in getattr(self.Meta, 'hide_by_context', {}).items(): - if not context.get(ctx): + for need, what in getattr(self.Meta, 'hide_by_context', {}).items(): + if not flags & set(need): self._hide_by_context |= set(what) # init SQLAlchemyAutoSchema @@ -212,23 +263,26 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] } - # TODO: remove LazyString and fix model definition (comment should not be nullable) - comment = LazyString() + # TODO: remove LazyString and change model (IMHO comment should not be nullable) + comment = LazyStringField() + + +### schema definitions ### class DomainSchema(BaseSchema): """ Marshmallow schema for Domain model """ class Meta: """ Schema config """ model = models.Domain + load_instance = True include_relationships = True - #include_fk = True exclude = ['users', 'managers', 'aliases'] include_by_context = { - 'dns': {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, + ('dns',): {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, } hide_by_context = { - 'secrets': {'dkim_key'}, + ('secrets',): {'dkim_key'}, } exclude_by_value = { 'alternatives': [[]], @@ -240,40 +294,20 @@ class DomainSchema(BaseSchema): 'dns_dmarc': [None], } - dkim_key = DkimKey() + dkim_key = DkimKeyField(allow_none=True) dkim_publickey = fields.String(dump_only=True) dns_mx = fields.String(dump_only=True) dns_spf = fields.String(dump_only=True) dns_dkim = fields.String(dump_only=True) dns_dmarc = fields.String(dump_only=True) - # _dict_types = { - # 'dkim_key': (bytes, type(None)), - # 'dkim_publickey': False, - # 'dns_mx': False, - # 'dns_spf': False, - # 'dns_dkim': False, - # 'dns_dmarc': False, - # } - class TokenSchema(BaseSchema): """ Marshmallow schema for Token model """ class Meta: """ Schema config """ model = models.Token - - # _dict_recurse = True - # _dict_hide = {'user', 'user_email'} - # _dict_mandatory = {'password'} - - # id = db.Column(db.Integer(), primary_key=True) - # user_email = db.Column(db.String(255), db.ForeignKey(User.email), - # nullable=False) - # user = db.relationship(User, - # backref=db.backref('tokens', cascade='all, delete-orphan')) - # password = db.Column(db.String(255), nullable=False) - # ip = db.Column(db.String(255)) + load_instance = True class FetchSchema(BaseSchema): @@ -281,58 +315,57 @@ class FetchSchema(BaseSchema): class Meta: """ Schema config """ model = models.Fetch + load_instance = True include_by_context = { - 'full': {'last_check', 'error'}, + ('full', 'import'): {'last_check', 'error'}, } hide_by_context = { - 'secrets': {'password'}, + ('secrets',): {'password'}, } -# TODO: What about mandatory keys? - # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} - class UserSchema(BaseSchema): """ Marshmallow schema for User model """ class Meta: """ Schema config """ model = models.User + load_instance = True include_relationships = True exclude = ['localpart', 'domain', 'quota_bytes_used'] exclude_by_value = { 'forward_destination': [[]], 'tokens': [[]], + 'manager_of': [[]], 'reply_enddate': ['2999-12-31'], 'reply_startdate': ['1900-01-01'], } + @pre_load + def _handle_password(self, data, many, **kwargs): # pylint: disable=unused-argument + data = handle_email(data) + if 'password' in data: + if 'password_hash' in data or 'hash_scheme' in data: + raise ValidationError('ambigous key password and password_hash/hash_scheme') + # check (hashed) password + password = data['password'] + if password.startswith('{') and '}' in password: + scheme = password[1:password.index('}')] + if scheme not in self.Meta.model.scheme_dict: + raise ValidationError(f'invalid password scheme {scheme!r}') + else: + raise ValidationError(f'invalid hashed password {password!r}') + elif 'password_hash' in data and 'hash_scheme' in data: + if data['hash_scheme'] not in self.Meta.model.scheme_dict: + raise ValidationError(f'invalid password scheme {scheme!r}') + data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + del data['hash_scheme'] + del data['password_hash'] + return data + tokens = fields.Nested(TokenSchema, many=True) fetches = fields.Nested(FetchSchema, many=True) -# TODO: deserialize password/password_hash! What about mandatory keys? - # _dict_mandatory = {'localpart', 'domain', 'password'} - # @classmethod - # def _dict_input(cls, data): - # Email._dict_input(data) - # # handle password - # if 'password' in data: - # if 'password_hash' in data or 'hash_scheme' in data: - # raise ValueError('ambigous key password and password_hash/hash_scheme') - # # check (hashed) password - # password = data['password'] - # if password.startswith('{') and '}' in password: - # scheme = password[1:password.index('}')] - # if scheme not in cls.scheme_dict: - # raise ValueError(f'invalid password scheme {scheme!r}') - # else: - # raise ValueError(f'invalid hashed password {password!r}') - # elif 'password_hash' in data and 'hash_scheme' in data: - # if data['hash_scheme'] not in cls.scheme_dict: - # raise ValueError(f'invalid password scheme {scheme!r}') - # data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] - # del data['hash_scheme'] - # del data['password_hash'] class AliasSchema(BaseSchema): @@ -340,20 +373,18 @@ class AliasSchema(BaseSchema): class Meta: """ Schema config """ model = models.Alias + load_instance = True exclude = ['localpart'] exclude_by_value = { 'destination': [[]], } -# TODO: deserialize destination! - # @staticmethod - # def _dict_input(data): - # Email._dict_input(data) - # # handle comma delimited string for backwards compability - # dst = data.get('destination') - # if type(dst) is str: - # data['destination'] = list([adr.strip() for adr in dst.split(',')]) + @pre_load + def _handle_password(self, data, many, **kwargs): # pylint: disable=unused-argument + return handle_email(data) + + destination = CommaSeparatedListField() class ConfigSchema(BaseSchema): @@ -361,6 +392,7 @@ class ConfigSchema(BaseSchema): class Meta: """ Schema config """ model = models.Config + load_instance = True class RelaySchema(BaseSchema): @@ -368,45 +400,17 @@ class RelaySchema(BaseSchema): class Meta: """ Schema config """ model = models.Relay + load_instance = True class MailuSchema(Schema): - """ Marshmallow schema for Mailu config """ + """ Marshmallow schema for complete Mailu config """ class Meta: """ Schema config """ render_module = RenderYAML + domains = fields.Nested(DomainSchema, many=True) relays = fields.Nested(RelaySchema, many=True) users = fields.Nested(UserSchema, many=True) aliases = fields.Nested(AliasSchema, many=True) config = fields.Nested(ConfigSchema, many=True) - - -### config class ### - -class MailuConfig: - """ Class which joins whole Mailu config for dumping - """ - - _models = { - 'domains': models.Domain, - 'relays': models.Relay, - 'users': models.User, - 'aliases': models.Alias, -# 'config': models.Config, - } - - def __init__(self, sections): - if sections: - for section in sections: - if section not in self._models: - raise ValueError(f'Unknown section: {section!r}') - self._sections = set(sections) - else: - self._sections = set(self._models.keys()) - - def __getattr__(self, section): - if section in self._sections: - return self._models[section].query.all() - else: - raise AttributeError From 31a903f95959e4234f21b9c5e059a73d37d8ec46 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 15 Jan 2021 13:45:36 +0100 Subject: [PATCH 31/52] revived & renamed config-fns. cosmetics. - revived original config-update function for backwards compability - renamed config-dump to config-export to be in line with config-import - converted '*'.format(*) to f-strings - converted string-concatenation to f-strings --- core/admin/mailu/manage.py | 310 +++++++++++++++++++++---------------- 1 file changed, 176 insertions(+), 134 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 6103f904..80a73230 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -7,6 +7,7 @@ import socket import uuid import click +import yaml from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext @@ -64,7 +65,7 @@ def admin(localpart, domain_name, password, mode='create'): user = None if mode == 'ifmissing' or mode == 'update': - email = '{}@{}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' user = models.User.query.get(email) if user and mode == 'ifmissing': @@ -122,14 +123,14 @@ def user(localpart, domain_name, password, hash_scheme=None): def password(localpart, domain_name, password, hash_scheme=None): """ Change the password of an user """ - email = '{0}@{1}'.format(localpart, domain_name) - user = models.User.query.get(email) + email = f'{localpart}@{domain_name}' + user = models.User.query.get(email) if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] if user: user.set_password(password, hash_scheme=hash_scheme) else: - print("User " + email + " not found.") + print(f'User {email} not found.') db.session.commit() @@ -157,7 +158,7 @@ def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): @click.argument('hash_scheme') @with_appcontext def user_import(localpart, domain_name, password_hash, hash_scheme = None): - """ Import a user along with password hash. + """ Import a user along with password hash """ if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] @@ -175,124 +176,146 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() -# @mailu.command() -# @click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') -# @click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') -# @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') -# @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) -# @with_appcontext -# def config_update(verbose=False, delete_objects=False, dry_run=False, source=None): -# """ Update configuration with data from YAML-formatted input -# """ +# TODO: remove this deprecated function +@mailu.command() +@click.option('-v', '--verbose') +@click.option('-d', '--delete-objects') +@with_appcontext +def config_update(verbose=False, delete_objects=False): + """ Sync configuration with data from YAML (deprecated) + """ + new_config = yaml.safe_load(sys.stdin) + # print new_config + domains = new_config.get('domains', []) + tracked_domains = set() + for domain_config in domains: + if verbose: + print(str(domain_config)) + domain_name = domain_config['name'] + max_users = domain_config.get('max_users', -1) + max_aliases = domain_config.get('max_aliases', -1) + max_quota_bytes = domain_config.get('max_quota_bytes', 0) + tracked_domains.add(domain_name) + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name, + max_users=max_users, + max_aliases=max_aliases, + max_quota_bytes=max_quota_bytes) + db.session.add(domain) + print(f'Added {domain_config}') + else: + domain.max_users = max_users + domain.max_aliases = max_aliases + domain.max_quota_bytes = max_quota_bytes + db.session.add(domain) + print(f'Updated {domain_config}') - # try: - # new_config = yaml.safe_load(source) - # except (yaml.scanner.ScannerError, yaml.parser.ParserError) as exc: - # print(f'[ERROR] Invalid yaml: {exc}') - # sys.exit(1) - # else: - # if isinstance(new_config, str): - # print(f'[ERROR] Invalid yaml: {new_config!r}') - # sys.exit(1) - # elif new_config is None or not new_config: - # print('[ERROR] Empty yaml: Please pipe yaml into stdin') - # sys.exit(1) + users = new_config.get('users', []) + tracked_users = set() + user_optional_params = ('comment', 'quota_bytes', 'global_admin', + 'enable_imap', 'enable_pop', 'forward_enabled', + 'forward_destination', 'reply_enabled', + 'reply_subject', 'reply_body', 'displayed_name', + 'spam_enabled', 'email', 'spam_threshold') + for user_config in users: + if verbose: + print(str(user_config)) + localpart = user_config['localpart'] + domain_name = user_config['domain'] + password_hash = user_config.get('password_hash', None) + hash_scheme = user_config.get('hash_scheme', None) + domain = models.Domain.query.get(domain_name) + email = f'{localpart}@{domain_name}' + optional_params = {} + for k in user_optional_params: + if k in user_config: + optional_params[k] = user_config[k] + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User.query.get(email) + tracked_users.add(email) + tracked_domains.add(domain_name) + if not user: + user = models.User( + localpart=localpart, + domain=domain, + **optional_params + ) + else: + for k in optional_params: + setattr(user, k, optional_params[k]) + user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) + db.session.add(user) - # error = False - # tracked = {} - # for section, model in yaml_sections: + aliases = new_config.get('aliases', []) + tracked_aliases = set() + for alias_config in aliases: + if verbose: + print(str(alias_config)) + localpart = alias_config['localpart'] + domain_name = alias_config['domain'] + if isinstance(alias_config['destination'], str): + destination = alias_config['destination'].split(',') + else: + destination = alias_config['destination'] + wildcard = alias_config.get('wildcard', False) + domain = models.Domain.query.get(domain_name) + email = f'{localpart}@{domain_name}' + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + alias = models.Alias.query.get(email) + tracked_aliases.add(email) + tracked_domains.add(domain_name) + if not alias: + alias = models.Alias( + localpart=localpart, + domain=domain, + wildcard=wildcard, + destination=destination, + email=email + ) + else: + alias.destination = destination + alias.wildcard = wildcard + db.session.add(alias) - # items = new_config.get(section) - # if items is None: - # if delete_objects: - # print(f'[ERROR] Invalid yaml: Section "{section}" is missing') - # error = True - # break - # else: - # continue + db.session.commit() - # del new_config[section] + managers = new_config.get('managers', []) + # tracked_managers=set() + for manager_config in managers: + if verbose: + print(str(manager_config)) + domain_name = manager_config['domain'] + user_name = manager_config['user'] + domain = models.Domain.query.get(domain_name) + manageruser = models.User.query.get(f'{user_name}@{domain_name}') + if manageruser not in domain.managers: + domain.managers.append(manageruser) + db.session.add(domain) - # if not isinstance(items, list): - # print(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') - # error = True - # break - # elif not items: - # continue + db.session.commit() - # # create items - # for data in items: - - # if verbose: - # print(f'Handling {model.__table__} data: {data!r}') - - # try: - # changed = model.from_dict(data, delete_objects) - # except Exception as exc: - # print(f'[ERROR] {exc.args[0]} in data: {data}') - # error = True - # break - - # for item, created in changed: - - # if created is True: - # # flush newly created item - # db.session.add(item) - # db.session.flush() - # if verbose: - # print(f'Added {item!r}: {item.to_dict()}') - # else: - # print(f'Added {item!r}') - - # elif created: - # # modified instance - # if verbose: - # for key, old, new in created: - # print(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') - # else: - # print(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') - - # # track primary key of all items - # tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) - - # if error: - # break - - # # on error: stop early - # if error: - # print('[ERROR] An error occured. Not committing changes.') - # db.session.rollback() - # sys.exit(1) - - # # are there sections left in new_config? - # if new_config: - # print(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') - # error = True - - # # test for conflicting domains - # domains = set() - # for model, items in tracked.items(): - # if model in (models.Domain, models.Alternative, models.Relay): - # if domains & items: - # for fqdn in domains & items: - # print(f'[ERROR] Duplicate domain name used: {fqdn}') - # error = True - # domains.update(items) - - # # delete items not tracked - # if delete_objects: - # for model, items in tracked.items(): - # for item in model.query.all(): - # if not item._dict_pval() in items: - # print(f'Deleted {item!r} {item}') - # db.session.delete(item) - - # # don't commit when running dry - # if dry_run: - # print('Dry run. Not commiting changes.') - # db.session.rollback() - # else: - # db.session.commit() + if delete_objects: + for user in db.session.query(models.User).all(): + if not user.email in tracked_users: + if verbose: + print(f'Deleting user: {user.email}') + db.session.delete(user) + for alias in db.session.query(models.Alias).all(): + if not alias.email in tracked_aliases: + if verbose: + print(f'Deleting alias: {alias.email}') + db.session.delete(alias) + for domain in db.session.query(models.Domain).all(): + if not domain.name in tracked_domains: + if verbose: + print(f'Deleting domain: {domain.name}') + db.session.delete(domain) + db.session.commit() SECTIONS = {'domains', 'relays', 'users', 'aliases'} @@ -304,30 +327,51 @@ SECTIONS = {'domains', 'relays', 'users', 'aliases'} @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext def config_import(verbose=False, dry_run=False, source=None): - """ Import configuration YAML + """ Import configuration from YAML """ + def log(**data): + caller = sys._getframe(1).f_code.co_name # pylint: disable=protected-access + if caller == '_track_import': + print(f'Handling {data["self"].opts.model.__table__} data: {data["data"]!r}') + + def format_errors(store, path=None): + if path is None: + path = [] + for key in sorted(store): + location = path + [str(key)] + value = store[key] + if isinstance(value, dict): + format_errors(value, location) + else: + for message in value: + print(f'[ERROR] {".".join(location)}: {message}') + context = { - 'verbose': verbose, # TODO: use callback function to be verbose? + 'callback': log if verbose else None, 'import': True, } + error = False try: config = MailuSchema(context=context).loads(source) except ValidationError as exc: - print(f'[ERROR] {exc}') - # TODO: show nice errors - from pprint import pprint - pprint(exc.messages) - sys.exit(1) + error = True + format_errors(exc.messages) else: print(config) print(MailuSchema().dumps(config)) - # TODO: does not commit yet. - # TODO: delete other entries? + # TODO: need to delete other entries - # don't commit when running dry - if True: #dry_run: + # TODO: enable commit + error = True + + # don't commit when running dry or validation errors occured + if error: + print('An error occured. Not committing changes.') + db.session.rollback() + sys.exit(2) + elif dry_run: print('Dry run. Not commiting changes.') db.session.rollback() else: @@ -343,10 +387,8 @@ def config_import(verbose=False, dry_run=False, source=None): help='save yaml to file') @click.argument('sections', nargs=-1) @with_appcontext -def config_dump(full=False, secrets=False, dns=False, output=None, sections=None): - """ Dump configuration as YAML to stdout or file - - SECTIONS can be: domains, relays, users, aliases +def config_export(full=False, secrets=False, dns=False, output=None, sections=None): + """ Export configuration as YAML to stdout or file """ if sections: @@ -407,7 +449,7 @@ def alias(localpart, domain_name, destination, wildcard=False): domain=domain, wildcard=wildcard, destination=destination.split(','), - email="%s@%s" % (localpart, domain_name) + email=f'{localpart}@{domain_name}' ) db.session.add(alias) db.session.commit() @@ -438,7 +480,7 @@ def setmanager(domain_name, user_name='manager'): """ Make a user manager of a domain """ domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) + manageruser = models.User.query.get(f'{user_name}@{domain_name}') domain.managers.append(manageruser) db.session.add(domain) db.session.commit() From 8213d044b25860046381c6766220c0855616b88c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 15 Jan 2021 13:53:47 +0100 Subject: [PATCH 32/52] added docstrings, use f-strings, cleanup - idna.encode does not encode upper-case letters, so .lower() has to be called on value not on result - split email-address on '@' only once - converted '*'.format(*) to f-strings - added docstrings - removed from_dict method - code cleanup/style (list concat, exceptions, return&else, line-length) - added TODO comments on possible future changes --- core/admin/mailu/models.py | 376 ++++++++++++------------------------- 1 file changed, 122 insertions(+), 254 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 73d05801..3187f597 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -8,6 +8,7 @@ import json from datetime import date from email.mime import text +from itertools import chain import flask_sqlalchemy import sqlalchemy @@ -30,11 +31,12 @@ class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ + # TODO: String(80) is too small? impl = db.String(80) def process_bind_param(self, value, dialect): """ encode unicode domain name to punycode """ - return idna.encode(value).decode('ascii').lower() + return idna.encode(value.lower()).decode('ascii') def process_result_value(self, value, dialect): """ decode punycode domain name to unicode """ @@ -46,26 +48,21 @@ class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ + # TODO: String(255) is too small? impl = db.String(255) def process_bind_param(self, value, dialect): """ encode unicode domain part of email address to punycode """ - try: - localpart, domain_name = value.split('@') - return '{0}@{1}'.format( - localpart, - idna.encode(domain_name).decode('ascii'), - ).lower() - except ValueError: - pass + localpart, domain_name = value.rsplit('@', 1) + if '@' in localpart: + raise ValueError('email local part must not contain "@"') + domain_name = domain_name.lower() + return f'{localpart}@{idna.encode(domain_name).decode("ascii")}' def process_result_value(self, value, dialect): """ decode punycode domain part of email to unicode """ - localpart, domain_name = value.split('@') - return '{0}@{1}'.format( - localpart, - idna.decode(domain_name), - ) + localpart, domain_name = value.rsplit('@', 1) + return f'{localpart}@{idna.decode(domain_name)}' python_type = str @@ -81,7 +78,7 @@ class CommaSeparatedList(db.TypeDecorator): raise TypeError('Must be a list of strings') for item in value: if ',' in item: - raise ValueError('Item must not contain a comma') + raise ValueError('list item must not contain ","') return ','.join(sorted(value)) def process_result_value(self, value, dialect): @@ -123,173 +120,6 @@ class Base(db.Model): updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True, default='') - # @classmethod - # def from_dict(cls, data, delete=False): - - # changed = [] - - # pkey = cls._dict_pkey() - - # # handle "primary key" only - # if not isinstance(data, dict): - # data = {pkey: data} - - # # modify input data - # if hasattr(cls, '_dict_input'): - # try: - # cls._dict_input(data) - # except Exception as exc: - # raise ValueError(f'{exc}', cls, None, data) from exc - - # # check for primary key (if not recursed) - # if not getattr(cls, '_dict_recurse', False): - # if not pkey in data: - # raise KeyError(f'primary key {cls.__table__}.{pkey} is missing', cls, pkey, data) - - # # check data keys and values - # for key in list(data.keys()): - - # # check key - # if not hasattr(cls, key) and not key in cls.__mapper__.relationships: - # raise KeyError(f'unknown key {cls.__table__}.{key}', cls, key, data) - - # # check value type - # value = data[key] - # col = cls.__mapper__.columns.get(key) - # if col is not None: - # if not ((value is None and col.nullable) or (isinstance(value, col.type.python_type))): - # raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) - # else: - # rel = cls.__mapper__.relationships.get(key) - # if rel is None: - # itype = getattr(cls, '_dict_types', {}).get(key) - # if itype is not None: - # if itype is False: # ignore value. TODO: emit warning? - # del data[key] - # continue - # elif not isinstance(value, itype): - # raise TypeError(f'{cls.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', cls, key, data) - # else: - # raise NotImplementedError(f'type not defined for {cls.__table__}.{key}') - - # # handle relationships - # if key in cls.__mapper__.relationships: - # rel_model = cls.__mapper__.relationships[key].argument - # if not isinstance(rel_model, sqlalchemy.orm.Mapper): - # add = rel_model.from_dict(value, delete) - # assert len(add) == 1 - # rel_item, updated = add[0] - # changed.append((rel_item, updated)) - # data[key] = rel_item - - # # create item if necessary - # created = False - # item = cls.query.get(data[pkey]) if pkey in data else None - # if item is None: - - # # check for mandatory keys - # missing = getattr(cls, '_dict_mandatory', set()) - set(data.keys()) - # if missing: - # raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {cls.__table__} missing', cls, missing, data) - - # # remove mapped relationships from data - # mapped = {} - # for key in list(data.keys()): - # if key in cls.__mapper__.relationships: - # if isinstance(cls.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): - # mapped[key] = data[key] - # del data[key] - - # # create new item - # item = cls(**data) - # created = True - - # # and update mapped relationships (below) - # data = mapped - - # # update item - # updated = [] - # for key, value in data.items(): - - # # skip primary key - # if key == pkey: - # continue - - # if key in cls.__mapper__.relationships: - # # update relationship - # rel_model = cls.__mapper__.relationships[key].argument - # if isinstance(rel_model, sqlalchemy.orm.Mapper): - # rel_model = rel_model.class_ - # # add (and create) referenced items - # cur = getattr(item, key) - # old = sorted(cur, key=id) - # new = [] - # for rel_data in value: - # # get or create related item - # add = rel_model.from_dict(rel_data, delete) - # assert len(add) == 1 - # rel_item, rel_updated = add[0] - # changed.append((rel_item, rel_updated)) - # if rel_item not in cur: - # cur.append(rel_item) - # new.append(rel_item) - - # # delete referenced items missing in yaml - # rel_pkey = rel_model._dict_pkey() - # new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) - # for rel_item in old: - # if rel_item not in new: - # # check if item with same data exists to stabilze import without primary key - # rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) - # try: - # same_idx = new_data.index(rel_data) - # except ValueError: - # same = None - # else: - # same = new[same_idx] - - # if same is None: - # # delete items missing in new - # if delete: - # cur.remove(rel_item) - # else: - # new.append(rel_item) - # else: - # # swap found item with same data with newly created item - # new.append(rel_item) - # new_data.append(rel_data) - # new.remove(same) - # del new_data[same_idx] - # for i, (ch_item, _) in enumerate(changed): - # if ch_item is same: - # changed[i] = (rel_item, []) - # db.session.flush() - # db.session.delete(ch_item) - # break - - # # remember changes - # new = sorted(new, key=id) - # if new != old: - # updated.append((key, old, new)) - - # else: - # # update key - # old = getattr(item, key) - # if isinstance(old, list): - # # deduplicate list value - # assert isinstance(value, list) - # value = set(value) - # old = set(old) - # if not delete: - # value = old | value - # if value != old: - # updated.append((key, old, value)) - # setattr(item, key, value) - - # changed.append((item, created if created else updated)) - - # return changed - # Many-to-many association table for domain managers managers = db.Table('manager', Base.metadata, @@ -309,9 +139,7 @@ class Config(Base): # TODO: use sqlalchemy.event.listen() on a store method of object? @sqlalchemy.event.listens_for(db.session, 'after_commit') def store_dkim_key(session): - """ Store DKIM key on commit - """ - + """ Store DKIM key on commit """ for obj in session.identity_map.values(): if isinstance(obj, Domain): if obj._dkim_key_changed: @@ -340,21 +168,27 @@ class Domain(Base): _dkim_key_changed = False def _dkim_file(self): + """ return filename for active DKIM key """ return app.config['DKIM_PATH'].format( - domain=self.name, selector=app.config['DKIM_SELECTOR']) + domain=self.name, + selector=app.config['DKIM_SELECTOR'] + ) @property def dns_mx(self): - hostname = app.config['HOSTNAMES'].split(',')[0] + """ return MX record for domain """ + hostname = app.config['HOSTNAMES'].split(',', 1)[0] return f'{self.name}. 600 IN MX 10 {hostname}.' @property def dns_spf(self): - hostname = app.config['HOSTNAMES'].split(',')[0] + """ return SPF record for domain """ + hostname = app.config['HOSTNAMES'].split(',', 1)[0] return f'{self.name}. 600 IN TXT "v=spf1 mx a:{hostname} ~all"' @property def dns_dkim(self): + """ return DKIM record for domain """ if os.path.exists(self._dkim_file()): selector = app.config['DKIM_SELECTOR'] return ( @@ -364,6 +198,7 @@ class Domain(Base): @property def dns_dmarc(self): + """ return DMARC record for domain """ if os.path.exists(self._dkim_file()): domain = app.config['DOMAIN'] rua = app.config['DMARC_RUA'] @@ -374,6 +209,7 @@ class Domain(Base): @property def dkim_key(self): + """ return private DKIM key """ if self._dkim_key is None: file_path = self._dkim_file() if os.path.exists(file_path): @@ -385,6 +221,7 @@ class Domain(Base): @dkim_key.setter def dkim_key(self, value): + """ set private DKIM key """ old_key = self.dkim_key if value is None: value = b'' @@ -393,36 +230,40 @@ class Domain(Base): @property def dkim_publickey(self): + """ return public part of DKIM key """ dkim_key = self.dkim_key if dkim_key: return dkim.strip_key(dkim_key).decode('utf8') def generate_dkim_key(self): + """ generate and activate new DKIM key """ self.dkim_key = dkim.gen_key() def has_email(self, localpart): - for email in self.users + self.aliases: + """ checks if localpart is configured for domain """ + for email in chain(self.users, self.aliases): if email.localpart == localpart: return True return False def check_mx(self): + """ checks if MX record for domain points to mailu host """ try: - hostnames = app.config['HOSTNAMES'].split(',') + hostnames = set(app.config['HOSTNAMES'].split(',')) return any( - str(rset).split()[-1][:-1] in hostnames + rset.exchange.to_text().rstrip('.') in hostnames for rset in dns.resolver.query(self.name, 'MX') ) - except Exception: + except dns.exception.DNSException: return False def __str__(self): return str(self.name) def __eq__(self, other): - try: - return self.name == other.name - except AttributeError: + if isinstance(other, self.__class__): + return str(self.name) == str(other.name) + else: return NotImplemented def __hash__(self): @@ -432,7 +273,7 @@ class Domain(Base): class Alternative(Base): """ Alternative name for a served domain. - The name "domain alias" was avoided to prevent some confusion. + The name "domain alias" was avoided to prevent some confusion. """ __tablename__ = 'alternative' @@ -454,6 +295,7 @@ class Relay(Base): __tablename__ = 'relay' name = db.Column(IdnaDomain, primary_key=True, nullable=False) + # TODO: String(80) is too small? smtp = db.Column(db.String(80), nullable=True) def __str__(self): @@ -464,10 +306,14 @@ class Email(object): """ Abstraction for an email address (localpart and domain). """ + # TODO: validate max. total length of address (<=254) + + # TODO: String(80) is too large (>64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr def domain_name(self): + """ the domain part of the email address """ return db.Column(IdnaDomain, db.ForeignKey(Domain.name), nullable=False, default=IdnaDomain) @@ -476,26 +322,18 @@ class Email(object): # especially when the mail server is reading the database. @declarative.declared_attr def email(self): - updater = lambda context: '{0}@{1}'.format( - context.current_parameters['localpart'], - context.current_parameters['domain_name'], - ) + """ the complete email address (localpart@domain) """ + updater = lambda ctx: '{localpart}@{domain_name}'.format(**ctx.current_parameters) return db.Column(IdnaEmail, primary_key=True, nullable=False, - default=updater) + default=updater + ) def sendmail(self, subject, body): - """ Send an email to the address. - """ - from_address = '{0}@{1}'.format( - app.config['POSTMASTER'], - idna.encode(app.config['DOMAIN']).decode('ascii'), - ) + """ send an email to the address """ + from_address = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: - to_address = '{0}@{1}'.format( - self.localpart, - idna.encode(self.domain_name).decode('ascii'), - ) + to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' msg = text.MIMEText(body) msg['Subject'] = subject msg['From'] = from_address @@ -504,7 +342,8 @@ class Email(object): @classmethod def resolve_domain(cls, email): - localpart, domain_name = email.split('@', 1) if '@' in email else (None, email) + """ resolves domain alternative to real domain """ + localpart, domain_name = email.rsplit('@', 1) if '@' in email else (None, email) alternative = Alternative.query.get(domain_name) if alternative: domain_name = alternative.domain_name @@ -512,17 +351,19 @@ class Email(object): @classmethod def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False): + """ return destination for email address localpart@domain_name """ + localpart_stripped = None stripped_alias = None if os.environ.get('RECIPIENT_DELIMITER') in localpart: localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0] - user = User.query.get('{}@{}'.format(localpart, domain_name)) + user = User.query.get(f'{localpart}@{domain_name}') if not user and localpart_stripped: - user = User.query.get('{}@{}'.format(localpart_stripped, domain_name)) + user = User.query.get(f'{localpart_stripped}@{domain_name}') if user: - email = '{}@{}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' if user.forward_enabled: destination = user.forward_destination @@ -537,11 +378,15 @@ class Email(object): if pure_alias and not pure_alias.wildcard: return pure_alias.destination - elif stripped_alias: + + if stripped_alias: return stripped_alias.destination - elif pure_alias: + + if pure_alias: return pure_alias.destination + return None + def __str__(self): return str(self.email) @@ -586,11 +431,15 @@ class User(Base, Email): is_active = True is_anonymous = False + # TODO: remove unused user.get_id() def get_id(self): + """ return users email address """ return self.email + # TODO: remove unused user.destination @property def destination(self): + """ returns comma separated string of destinations """ if self.forward_enabled: result = list(self.forward_destination) if self.forward_keep: @@ -601,6 +450,7 @@ class User(Base, Email): @property def reply_active(self): + """ returns status of autoreply function """ now = date.today() return ( self.reply_enabled and @@ -608,49 +458,56 @@ class User(Base, Email): self.reply_enddate > now ) - scheme_dict = {'PBKDF2': 'pbkdf2_sha512', - 'BLF-CRYPT': 'bcrypt', - 'SHA512-CRYPT': 'sha512_crypt', - 'SHA256-CRYPT': 'sha256_crypt', - 'MD5-CRYPT': 'md5_crypt', - 'CRYPT': 'des_crypt'} + scheme_dict = { + 'PBKDF2': 'pbkdf2_sha512', + 'BLF-CRYPT': 'bcrypt', + 'SHA512-CRYPT': 'sha512_crypt', + 'SHA256-CRYPT': 'sha256_crypt', + 'MD5-CRYPT': 'md5_crypt', + 'CRYPT': 'des_crypt', + } - def get_password_context(self): + def _get_password_context(self): return passlib.context.CryptContext( schemes=self.scheme_dict.values(), default=self.scheme_dict[app.config['PASSWORD_SCHEME']], ) - def check_password(self, password): - context = self.get_password_context() - reference = re.match('({[^}]+})?(.*)', self.password).group(2) - result = context.verify(password, reference) - if result and context.identify(reference) != context.default_scheme(): - self.set_password(password) + def check_password(self, plain): + """ Check password against stored hash + Update hash when default scheme has changed + """ + context = self._get_password_context() + hashed = re.match('^({[^}]+})?(.*)$', self.password).group(2) + result = context.verify(plain, hashed) + if result and context.identify(hashed) != context.default_scheme(): + self.set_password(plain) db.session.add(self) db.session.commit() return result - def set_password(self, password, hash_scheme=None, raw=False): - """Set password for user with specified encryption scheme - @password: plain text password to encrypt (if raw == True the hash itself) + # TODO: remove kwarg hash_scheme - there is no point in setting a scheme, + # when the next check updates the password to the default scheme. + def set_password(self, new, hash_scheme=None, raw=False): + """ Set password for user with specified encryption scheme + @new: plain text password to encrypt (or, if raw is True: the hash itself) """ + # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] - # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes - if raw: - self.password = '{'+hash_scheme+'}' + password - else: - self.password = '{'+hash_scheme+'}' + \ - self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) + if not raw: + new = self._get_password_context().encrypt(new, self.scheme_dict[hash_scheme]) + self.password = f'{{{hash_scheme}}}{new}' def get_managed_domains(self): + """ return list of domains this user can manage """ if self.global_admin: return Domain.query.all() else: return self.manager_of def get_managed_emails(self, include_aliases=True): + """ returns list of email addresses this user can manage """ emails = [] for domain in self.get_managed_domains(): emails.extend(domain.users) @@ -659,16 +516,18 @@ class User(Base, Email): return emails def send_welcome(self): + """ send welcome email to user """ if app.config['WELCOME']: - self.sendmail(app.config['WELCOME_SUBJECT'], - app.config['WELCOME_BODY']) + self.sendmail(app.config['WELCOME_SUBJECT'], app.config['WELCOME_BODY']) @classmethod def get(cls, email): + """ find user object for email address """ return cls.query.get(email) @classmethod def login(cls, email, password): + """ login user when enabled and password is valid """ user = cls.query.get(email) return user if (user and user.enabled and user.check_password(password)) else None @@ -686,6 +545,8 @@ class Alias(Base, Email): @classmethod def resolve(cls, localpart, domain_name): + """ find aliases matching email address localpart@domain_name """ + alias_preserve_case = cls.query.filter( sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( @@ -709,24 +570,27 @@ class Alias(Base, Email): sqlalchemy.func.lower(cls.localpart) == localpart_lower ), sqlalchemy.and_( cls.wildcard is True, - sqlalchemy.bindparam('l', localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) + sqlalchemy.bindparam('l', localpart_lower).like( + sqlalchemy.func.lower(cls.localpart)) ) ) ) - ).order_by(cls.wildcard, sqlalchemy.func.char_length(sqlalchemy.func.lower(cls.localpart)).desc()).first() + ).order_by(cls.wildcard, sqlalchemy.func.char_length( + sqlalchemy.func.lower(cls.localpart)).desc()).first() if alias_preserve_case and alias_lower_case: - if alias_preserve_case.wildcard: - return alias_lower_case - else: - return alias_preserve_case - elif alias_preserve_case and not alias_lower_case: - return alias_preserve_case - elif alias_lower_case and not alias_preserve_case: - return alias_lower_case - else: - return None + return alias_lower_case if alias_preserve_case.wildcard else alias_preserve_case + if alias_preserve_case and not alias_lower_case: + return alias_preserve_case + + if alias_lower_case and not alias_preserve_case: + return alias_lower_case + + return None + +# TODO: where are Tokens used / validated? +# TODO: what about API tokens? class Token(Base): """ A token is an application password for a given user. """ @@ -739,16 +603,20 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) + # TODO: String(80) is too large? ip = db.Column(db.String(255)) def check_password(self, password): + """ verifies password against stored hash """ return passlib.hash.sha256_crypt.verify(password, self.password) + # TODO: use crypt context and default scheme from config? def set_password(self, password): + """ sets password using sha256_crypt(rounds=1000) """ self.password = passlib.hash.sha256_crypt.using(rounds=1000).hash(password) def __str__(self): - return self.comment or self.ip + return str(self.comment or self.ip) class Fetch(Base): From 65b1ad46d97c355ac93879b5b59dc75b77c579b9 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 15 Jan 2021 13:57:20 +0100 Subject: [PATCH 33/52] order yaml data and allow callback on import - in yaml the primary key is now always first - calling a function on import allows import to be more verbose - skip "fetches" when empty --- core/admin/mailu/schemas.py | 88 ++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 16 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index fc08b67c..5dc10e17 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -3,11 +3,12 @@ import re +from collections import OrderedDict from textwrap import wrap import yaml -from marshmallow import pre_load, post_dump, fields, Schema +from marshmallow import pre_load, post_load, post_dump, fields, Schema from marshmallow.exceptions import ValidationError from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts from flask_marshmallow import Marshmallow @@ -25,6 +26,12 @@ ma = Marshmallow() ### yaml render module ### +# allow yaml module to dump OrderedDict +yaml.add_representer( + OrderedDict, + lambda cls, data: cls.represent_mapping('tag:yaml.org,2002:map', data.items()) +) + class RenderYAML: """ Marshmallow YAML Render Module """ @@ -62,6 +69,7 @@ class RenderYAML: 'Dumper': SpacedDumper, 'default_flow_style': False, 'allow_unicode': True, + 'sort_keys': False, } @classmethod def dumps(cls, *args, **kwargs): @@ -195,6 +203,8 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): def __init__(self, meta, ordered=False): if not hasattr(meta, 'sqla_session'): meta.sqla_session = models.db.session + if not hasattr(meta, 'ordered'): + meta.ordered = True super(BaseOpts, self).__init__(meta, ordered=ordered) class BaseSchema(ma.SQLAlchemyAutoSchema): @@ -206,13 +216,12 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): class Meta: """ Schema config """ - model = None def __init__(self, *args, **kwargs): # context? context = kwargs.get('context', {}) - flags = set([key for key, value in context.items() if value is True]) + flags = {key for key, value in context.items() if value is True} # compile excludes exclude = set(kwargs.get('exclude', [])) @@ -234,7 +243,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # exclude default values if not context.get('full'): - for column in self.Meta.model.__table__.columns: + for column in getattr(self.Meta, 'model').__table__.columns: if column.name not in exclude: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg @@ -250,20 +259,48 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # init SQLAlchemyAutoSchema super().__init__(*args, **kwargs) - @post_dump - def _remove_skip_values(self, data, many, **kwargs): # pylint: disable=unused-argument + # init order + if hasattr(self.Meta, 'order'): + # use user-defined order + self._order = list(reversed(getattr(self.Meta, 'order'))) + else: + # default order is: primary_key + other keys alphabetically + self._order = list(sorted(self.fields.keys())) + primary = self.opts.model.__table__.primary_key.columns.values()[0].name + self._order.remove(primary) + self._order.reverse() + self._order.append(primary) + @pre_load + def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument + call = self.context.get('callback') + if call is not None: + call(self=self, data=data, many=many, **kwargs) + return data + + @post_dump + def _hide_and_order(self, data, many, **kwargs): # pylint: disable=unused-argument + + # order output + for key in self._order: + try: + data.move_to_end(key, False) + except KeyError: + pass + + # stop early when not excluding/hiding if not self._exclude_by_value and not self._hide_by_context: return data + # exclude items or hide values full = self.context.get('full') - return { - key: '' if key in self._hide_by_context else value + return type(data)([ + (key, '' if key in self._hide_by_context else value) for key, value in data.items() if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] - } + ]) - # TODO: remove LazyString and change model (IMHO comment should not be nullable) + # TODO: remove LazyStringField and change model (IMHO comment should not be nullable) comment = LazyStringField() @@ -336,13 +373,14 @@ class UserSchema(BaseSchema): exclude_by_value = { 'forward_destination': [[]], 'tokens': [[]], + 'fetches': [[]], 'manager_of': [[]], 'reply_enddate': ['2999-12-31'], 'reply_startdate': ['1900-01-01'], } @pre_load - def _handle_password(self, data, many, **kwargs): # pylint: disable=unused-argument + def _handle_email_and_password(self, data, many, **kwargs): # pylint: disable=unused-argument data = handle_email(data) if 'password' in data: if 'password_hash' in data or 'hash_scheme' in data: @@ -358,16 +396,23 @@ class UserSchema(BaseSchema): elif 'password_hash' in data and 'hash_scheme' in data: if data['hash_scheme'] not in self.Meta.model.scheme_dict: raise ValidationError(f'invalid password scheme {scheme!r}') - data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + data['password'] = f'{{{data["hash_scheme"]}}}{data["password_hash"]}' del data['hash_scheme'] del data['password_hash'] return data + # TODO: verify password (should this be done in model?) + # scheme, hashed = re.match('^(?:{([^}]+)})?(.*)$', self.password).groups() + # if not scheme... + # ctx = passlib.context.CryptContext(schemes=[scheme], default=scheme) + # try: + # ctx.verify('', hashed) + # =>? ValueError: hash could not be identified + tokens = fields.Nested(TokenSchema, many=True) fetches = fields.Nested(FetchSchema, many=True) - class AliasSchema(BaseSchema): """ Marshmallow schema for Alias model """ class Meta: @@ -381,7 +426,7 @@ class AliasSchema(BaseSchema): } @pre_load - def _handle_password(self, data, many, **kwargs): # pylint: disable=unused-argument + def _handle_email(self, data, many, **kwargs): # pylint: disable=unused-argument return handle_email(data) destination = CommaSeparatedListField() @@ -408,9 +453,20 @@ class MailuSchema(Schema): class Meta: """ Schema config """ render_module = RenderYAML + ordered = True + order = ['config', 'domains', 'users', 'aliases', 'relays'] + @post_dump(pass_many=True) + def _order(self, data : OrderedDict, many : bool, **kwargs): # pylint: disable=unused-argument + for key in reversed(self.Meta.order): + try: + data.move_to_end(key, False) + except KeyError: + pass + return data + + config = fields.Nested(ConfigSchema, many=True) domains = fields.Nested(DomainSchema, many=True) - relays = fields.Nested(RelaySchema, many=True) users = fields.Nested(UserSchema, many=True) aliases = fields.Nested(AliasSchema, many=True) - config = fields.Nested(ConfigSchema, many=True) + relays = fields.Nested(RelaySchema, many=True) From 902b398127dc9abb6f3433ee7fccd51ddc0b3873 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 24 Jan 2021 19:07:48 +0100 Subject: [PATCH 34/52] next step for import/export yaml & json --- core/admin/mailu/manage.py | 238 ++++++++++++++++++++++++++++++------ core/admin/mailu/models.py | 176 ++++++++++++++++++-------- core/admin/mailu/schemas.py | 157 +++++++++++++++++------- 3 files changed, 441 insertions(+), 130 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 80a73230..e02d9ad4 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -4,9 +4,15 @@ import sys import os import socket +import json +import logging import uuid +from collections import Counter +from itertools import chain + import click +import sqlalchemy import yaml from flask import current_app as app @@ -14,7 +20,7 @@ from flask.cli import FlaskGroup, with_appcontext from marshmallow.exceptions import ValidationError from . import models -from .schemas import MailuSchema +from .schemas import MailuSchema, get_schema db = models.db @@ -322,60 +328,211 @@ SECTIONS = {'domains', 'relays', 'users', 'aliases'} @mailu.command() -@click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') +@click.option('-v', '--verbose', count=True, help='Increase verbosity') +@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors') +@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_import(verbose=False, dry_run=False, source=None): - """ Import configuration from YAML +def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=None): + """ Import configuration as YAML or JSON from stdin or file """ - def log(**data): - caller = sys._getframe(1).f_code.co_name # pylint: disable=protected-access - if caller == '_track_import': - print(f'Handling {data["self"].opts.model.__table__} data: {data["data"]!r}') + # verbose + # 0 : show number of changes + # 1 : also show changes + # 2 : also show secrets + # 3 : also show input data + # 4 : also show sql queries + + if quiet: + verbose = -1 + + counter = Counter() + dumper = {} def format_errors(store, path=None): + + res = [] if path is None: path = [] for key in sorted(store): location = path + [str(key)] value = store[key] if isinstance(value, dict): - format_errors(value, location) + res.extend(format_errors(value, location)) else: for message in value: - print(f'[ERROR] {".".join(location)}: {message}') + res.append((".".join(location), message)) - context = { - 'callback': log if verbose else None, + if path: + return res + + fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}' + res = [fmt.format(loc, msg) for loc, msg in res] + num = f'error{["s",""][len(res)==1]}' + res.insert(0, f'[ValidationError] {len(res)} {num} occured during input validation') + + return '\n'.join(res) + + def format_changes(*message): + if counter: + changes = [] + last = None + for (action, what), count in sorted(counter.items()): + if action != last: + if last: + changes.append('/') + changes.append(f'{action}:') + last = action + changes.append(f'{what}({count})') + else: + changes = 'no changes.' + return chain(message, changes) + + def log(action, target, message=None): + if message is None: + message = json.dumps(dumper[target.__class__].dump(target), ensure_ascii=False) + print(f'{action} {target.__table__}: {message}') + + def listen_insert(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + counter.update([('Added', target.__table__.name)]) + if verbose >= 1: + log('Added', target) + + def listen_update(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + + changed = {} + inspection = sqlalchemy.inspect(target) + for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: + if getattr(inspection.attrs, attr.key).history.has_changes(): + if sqlalchemy.orm.attributes.get_history(target, attr.key)[2]: + before = sqlalchemy.orm.attributes.get_history(target, attr.key)[2].pop() + after = getattr(target, attr.key) + # only remember changed keys + if before != after and (before or after): + if verbose >= 1: + changed[str(attr.key)] = (before, after) + else: + break + + if verbose >= 1: + # use schema with dump_context to hide secrets and sort keys + primary = json.dumps(str(target), ensure_ascii=False) + dumped = get_schema(target)(only=changed.keys(), context=dump_context).dump(target) + for key, value in dumped.items(): + before, after = changed[key] + if value == '': + before = '' if before else before + after = '' if after else after + else: + # TODO: use schema to "convert" before value? + after = value + before = json.dumps(before, ensure_ascii=False) + after = json.dumps(after, ensure_ascii=False) + log('Modified', target, f'{primary} {key}: {before} -> {after}') + + if changed: + counter.update([('Modified', target.__table__.name)]) + + def listen_delete(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + counter.update([('Deleted', target.__table__.name)]) + if verbose >= 1: + log('Deleted', target) + + # this listener should not be necessary, when: + # dkim keys should be stored in database and it should be possible to store multiple + # keys per domain. the active key would be also stored on disk on commit. + def listen_dkim(session, flush_context): # pylint: disable=unused-argument + """ callback function to track import """ + for target in session.identity_map.values(): + if not isinstance(target, models.Domain): + continue + primary = json.dumps(str(target), ensure_ascii=False) + before = target._dkim_key_on_disk + after = target._dkim_key + if before != after and (before or after): + if verbose >= 2: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = '' if before else '' + after = '' if after else '' + before = json.dumps(before, ensure_ascii=False) + after = json.dumps(after, ensure_ascii=False) + log('Modified', target, f'{primary} dkim_key: {before} -> {after}') + counter.update([('Modified', target.__table__.name)]) + + def track_serialize(self, item): + """ callback function to track import """ + log('Handling', self.opts.model, item) + + # configure contexts + dump_context = { + 'secrets': verbose >= 2, + } + load_context = { + 'callback': track_serialize if verbose >= 3 else None, + 'clear': not update, 'import': True, } - error = False + # register listeners + for schema in get_schema(): + model = schema.Meta.model + dumper[model] = schema(context=dump_context) + sqlalchemy.event.listen(model, 'after_insert', listen_insert) + sqlalchemy.event.listen(model, 'after_update', listen_update) + sqlalchemy.event.listen(model, 'after_delete', listen_delete) + + # special listener for dkim_key changes + sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim) + + if verbose >= 4: + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + try: - config = MailuSchema(context=context).loads(source) + with models.db.session.no_autoflush: + config = MailuSchema(only=SECTIONS, context=load_context).loads(source) except ValidationError as exc: - error = True - format_errors(exc.messages) - else: - print(config) + raise click.ClickException(format_errors(exc.messages)) from exc + except Exception as exc: + # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) + raise click.ClickException(f'[{exc.__class__.__name__}] {" ".join(str(exc).split())}') from exc + + # flush session to show/count all changes + if dry_run or verbose >= 1: + db.session.flush() + + # check for duplicate domain names + dup = set() + for fqdn in chain(db.session.query(models.Domain.name), + db.session.query(models.Alternative.name), + db.session.query(models.Relay.name)): + if fqdn in dup: + raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}') + dup.add(fqdn) + + # TODO: implement special update "items" + # -pkey: which - remove item "which" + # -key: null or [] or {} - set key to default + # -pkey: null or [] or {} - remove all existing items in this list + + # don't commit when running dry + if dry_run: + db.session.rollback() + if not quiet: + print(*format_changes('Dry run. Not commiting changes.')) + # TODO: remove debug print(MailuSchema().dumps(config)) - # TODO: need to delete other entries - - # TODO: enable commit - error = True - - # don't commit when running dry or validation errors occured - if error: - print('An error occured. Not committing changes.') - db.session.rollback() - sys.exit(2) - elif dry_run: - print('Dry run. Not commiting changes.') - db.session.rollback() else: db.session.commit() + if not quiet: + print(*format_changes('Commited changes.')) @mailu.command() @@ -385,28 +542,35 @@ def config_import(verbose=False, dry_run=False, source=None): @click.option('-d', '--dns', is_flag=True, help='Include dns records') @click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), help='save yaml to file') +@click.option('-j', '--json', 'as_json', is_flag=True, help='Dump in josn format') @click.argument('sections', nargs=-1) @with_appcontext -def config_export(full=False, secrets=False, dns=False, output=None, sections=None): - """ Export configuration as YAML to stdout or file +def config_export(full=False, secrets=False, dns=False, output=None, as_json=False, sections=None): + """ Export configuration as YAML or JSON to stdout or file """ if sections: for section in sections: if section not in SECTIONS: - print(f'[ERROR] Unknown section: {section!r}') - sys.exit(1) + print(f'[ERROR] Unknown section: {section}') + raise click.exceptions.Exit(1) sections = set(sections) else: sections = SECTIONS - context={ + context = { 'full': full, 'secrets': secrets, 'dns': dns, } - MailuSchema(only=sections, context=context).dumps(models.MailuConfig(), output) + if as_json: + schema = MailuSchema(only=sections, context=context) + schema.opts.render_module = json + print(schema.dumps(models.MailuConfig(), separators=(',',':')), file=output) + + else: + MailuSchema(only=sections, context=context).dumps(models.MailuConfig(), output) @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 3187f597..dac1dc70 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -12,7 +12,8 @@ from itertools import chain import flask_sqlalchemy import sqlalchemy -import passlib +import passlib.context +import passlib.hash import idna import dns @@ -79,11 +80,11 @@ class CommaSeparatedList(db.TypeDecorator): for item in value: if ',' in item: raise ValueError('list item must not contain ","') - return ','.join(sorted(value)) + return ','.join(sorted(set(value))) def process_result_value(self, value, dialect): """ split comma separated string to list """ - return list(filter(bool, value.split(','))) if value else [] + return list(filter(bool, [item.strip() for item in value.split(',')])) if value else [] python_type = list @@ -136,19 +137,11 @@ class Config(Base): value = db.Column(JSONEncoded) -# TODO: use sqlalchemy.event.listen() on a store method of object? -@sqlalchemy.event.listens_for(db.session, 'after_commit') -def store_dkim_key(session): - """ Store DKIM key on commit """ +def _save_dkim_keys(session): + """ store DKIM keys after commit """ for obj in session.identity_map.values(): if isinstance(obj, Domain): - if obj._dkim_key_changed: - file_path = obj._dkim_file() - if obj._dkim_key: - with open(file_path, 'wb') as handle: - handle.write(obj._dkim_key) - elif os.path.exists(file_path): - os.unlink(file_path) + obj.save_dkim_key() class Domain(Base): """ A DNS domain that has mail addresses associated to it. @@ -165,7 +158,7 @@ class Domain(Base): signup_enabled = db.Column(db.Boolean, nullable=False, default=False) _dkim_key = None - _dkim_key_changed = False + _dkim_key_on_disk = None def _dkim_file(self): """ return filename for active DKIM key """ @@ -174,6 +167,17 @@ class Domain(Base): selector=app.config['DKIM_SELECTOR'] ) + def save_dkim_key(self): + """ save changed DKIM key to disk """ + if self._dkim_key != self._dkim_key_on_disk: + file_path = self._dkim_file() + if self._dkim_key: + with open(file_path, 'wb') as handle: + handle.write(self._dkim_key) + elif os.path.exists(file_path): + os.unlink(file_path) + self._dkim_key_on_disk = self._dkim_key + @property def dns_mx(self): """ return MX record for domain """ @@ -189,7 +193,7 @@ class Domain(Base): @property def dns_dkim(self): """ return DKIM record for domain """ - if os.path.exists(self._dkim_file()): + if self.dkim_key: selector = app.config['DKIM_SELECTOR'] return ( f'{selector}._domainkey.{self.name}. 600 IN TXT' @@ -199,7 +203,7 @@ class Domain(Base): @property def dns_dmarc(self): """ return DMARC record for domain """ - if os.path.exists(self._dkim_file()): + if self.dkim_key: domain = app.config['DOMAIN'] rua = app.config['DMARC_RUA'] rua = f' rua=mailto:{rua}@{domain};' if rua else '' @@ -214,19 +218,19 @@ class Domain(Base): file_path = self._dkim_file() if os.path.exists(file_path): with open(file_path, 'rb') as handle: - self._dkim_key = handle.read() + self._dkim_key = self._dkim_key_on_disk = handle.read() else: - self._dkim_key = b'' + self._dkim_key = self._dkim_key_on_disk = b'' return self._dkim_key if self._dkim_key else None @dkim_key.setter def dkim_key(self, value): """ set private DKIM key """ old_key = self.dkim_key - if value is None: - value = b'' - self._dkim_key_changed = value != old_key - self._dkim_key = value + self._dkim_key = value if value is not None else b'' + if self._dkim_key != old_key: + if not sqlalchemy.event.contains(db.session, 'after_commit', _save_dkim_keys): + sqlalchemy.event.listen(db.session, 'after_commit', _save_dkim_keys) @property def dkim_publickey(self): @@ -331,14 +335,14 @@ class Email(object): def sendmail(self, subject, body): """ send an email to the address """ - from_address = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' + f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' msg = text.MIMEText(body) msg['Subject'] = subject - msg['From'] = from_address + msg['From'] = f_addr msg['To'] = to_address - smtp.sendmail(from_address, [to_address], msg.as_string()) + smtp.sendmail(f_addr, [to_address], msg.as_string()) @classmethod def resolve_domain(cls, email): @@ -589,7 +593,6 @@ class Alias(Base, Email): return None -# TODO: where are Tokens used / validated? # TODO: what about API tokens? class Token(Base): """ A token is an application password for a given user. @@ -650,20 +653,22 @@ class MailuConfig: and loading """ - # TODO: add sqlalchemy session updating (.add & .del) class MailuCollection: - """ Provides dict- and list-like access to all instances + """ Provides dict- and list-like access to instances of a sqlalchemy model """ def __init__(self, model : db.Model): - self._model = model + self.model = model + + def __str__(self): + return f'<{self.model.__name__}-Collection>' @cached_property def _items(self): return { inspect(item).identity: item - for item in self._model.query.all() + for item in self.model.query.all() } def __len__(self): @@ -676,8 +681,8 @@ class MailuConfig: return self._items[key] def __setitem__(self, key, item): - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') if key != inspect(item).identity: raise ValueError(f'item identity != key {key!r}') self._items[key] = item @@ -685,23 +690,24 @@ class MailuConfig: def __delitem__(self, key): del self._items[key] - def append(self, item): + def append(self, item, update=False): """ list-like append """ - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') key = inspect(item).identity if key in self._items: - raise ValueError(f'item {key!r} already present in collection') + if not update: + raise ValueError(f'item {key!r} already present in collection') self._items[key] = item - def extend(self, items): + def extend(self, items, update=False): """ list-like extend """ add = {} for item in items: - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') key = inspect(item).identity - if key in self._items: + if not update and key in self._items: raise ValueError(f'item {key!r} already present in collection') add[key] = item self._items.update(add) @@ -721,8 +727,8 @@ class MailuConfig: def remove(self, item): """ list-like remove """ - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') key = inspect(item).identity if not key in self._items: raise ValueError(f'item {key!r} not found in collection') @@ -739,12 +745,11 @@ class MailuConfig: def update(self, items): """ dict-like update """ for key, item in items: - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') if key != inspect(item).identity: raise ValueError(f'item identity != key {key!r}') - if key in self._items: - raise ValueError(f'item {key!r} already present in collection') + self._items.update(items) def setdefault(self, key, item=None): """ dict-like setdefault """ @@ -752,13 +757,86 @@ class MailuConfig: return self._items[key] if item is None: return None - if not isinstance(item, self._model): - raise TypeError(f'expected {self._model.name}') + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') if key != inspect(item).identity: raise ValueError(f'item identity != key {key!r}') self._items[key] = item return item + def __init__(self): + + # section-name -> attr + self._sections = { + name: getattr(self, name) + for name in dir(self) + if isinstance(getattr(self, name), self.MailuCollection) + } + + # known models + self._models = tuple(section.model for section in self._sections.values()) + + # model -> attr + self._sections.update({ + section.model: section for section in self._sections.values() + }) + + def _get_model(self, section): + if section is None: + return None + model = self._sections.get(section) + if model is None: + raise ValueError(f'Invalid section: {section!r}') + if isinstance(model, self.MailuCollection): + return model.model + return model + + def _add(self, items, section, update): + + model = self._get_model(section) + if isinstance(items, self._models): + items = [items] + elif not hasattr(items, '__iter__'): + raise ValueError(f'{items!r} is not iterable') + + for item in items: + if model is not None and not isinstance(item, model): + what = item.__class__.__name__.capitalize() + raise ValueError(f'{what} can not be added to section {section!r}') + self._sections[type(item)].append(item, update=update) + + def add(self, items, section=None): + """ add item to config """ + self._add(items, section, update=False) + + def update(self, items, section=None): + """ add or replace item in config """ + self._add(items, section, update=True) + + def remove(self, items, section=None): + """ remove item from config """ + model = self._get_model(section) + if isinstance(items, self._models): + items = [items] + elif not hasattr(items, '__iter__'): + raise ValueError(f'{items!r} is not iterable') + + for item in items: + if isinstance(item, str): + if section is None: + raise ValueError(f'Cannot remove key {item!r} without section') + del self._sections[model][item] + elif model is not None and not isinstance(item, model): + what = item.__class__.__name__.capitalize() + raise ValueError(f'{what} can not be removed from section {section!r}') + self._sections[type(item)].remove(item,) + + def clear(self, models=None): + """ remove complete configuration """ + for model in self._models: + if models is None or model in models: + db.session.query(model).delete() + domains = MailuCollection(Domain) relays = MailuCollection(Relay) users = MailuCollection(User) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 5dc10e17..04512f6d 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -24,6 +24,23 @@ ma = Marshmallow() # - fields which are the primary key => unchangeable when updating +### map model to schema ### + +_model2schema = {} + +def get_schema(model=None): + """ return schema class for model or instance of model """ + if model is None: + return _model2schema.values() + else: + return _model2schema.get(model) or _model2schema.get(model.__class__) + +def mapped(cls): + """ register schema in model2schema map """ + _model2schema[cls.Meta.model] = cls + return cls + + ### yaml render module ### # allow yaml module to dump OrderedDict @@ -79,26 +96,6 @@ class RenderYAML: return yaml.dump(*args, **kwargs) -### functions ### - -def handle_email(data): - """ merge separate localpart and domain to email - """ - - localpart = 'localpart' in data - domain = 'domain' in data - - if 'email' in data: - if localpart or domain: - raise ValidationError('duplicate email and localpart/domain') - elif localpart and domain: - data['email'] = f'{data["localpart"]}@{data["domain"]}' - elif localpart or domain: - raise ValidationError('incomplete localpart/domain') - - return data - - ### field definitions ### class LazyStringField(fields.String): @@ -177,9 +174,7 @@ class DkimKeyField(fields.String): return dkim.gen_key() # remember some keydata for error message - keydata = value - if len(keydata) > 40: - keydata = keydata[:25] + '...' + keydata[-10:] + keydata = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value # wrap value into valid pem layout and check validity value = ( @@ -197,6 +192,26 @@ class DkimKeyField(fields.String): ### base definitions ### +def handle_email(data): + """ merge separate localpart and domain to email + """ + + localpart = 'localpart' in data + domain = 'domain' in data + + if 'email' in data: + if localpart or domain: + raise ValidationError('duplicate email and localpart/domain') + data['localpart'], data['domain_name'] = data['email'].rsplit('@', 1) + elif localpart and domain: + data['domain_name'] = data['domain'] + del data['domain'] + data['email'] = f'{data["localpart"]}@{data["domain_name"]}' + elif localpart or domain: + raise ValidationError('incomplete localpart/domain') + + return data + class BaseOpts(SQLAlchemyAutoSchemaOpts): """ Option class with sqla session """ @@ -238,12 +253,15 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # update excludes kwargs['exclude'] = exclude + # init SQLAlchemyAutoSchema + super().__init__(*args, **kwargs) + # exclude_by_value self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) # exclude default values if not context.get('full'): - for column in getattr(self.Meta, 'model').__table__.columns: + for column in getattr(self.opts, 'model').__table__.columns: if column.name not in exclude: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg @@ -256,10 +274,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if not flags & set(need): self._hide_by_context |= set(what) - # init SQLAlchemyAutoSchema - super().__init__(*args, **kwargs) - - # init order + # initialize attribute order if hasattr(self.Meta, 'order'): # use user-defined order self._order = list(reversed(getattr(self.Meta, 'order'))) @@ -267,17 +282,35 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # default order is: primary_key + other keys alphabetically self._order = list(sorted(self.fields.keys())) primary = self.opts.model.__table__.primary_key.columns.values()[0].name - self._order.remove(primary) - self._order.reverse() - self._order.append(primary) + if primary in self._order: + self._order.remove(primary) + self._order.reverse() + self._order.append(primary) + + # move pre_load hook "_track_import" to the front + hooks = self._hooks[('pre_load', False)] + if '_track_import' in hooks: + hooks.remove('_track_import') + hooks.insert(0, '_track_import') + # and post_load hook "_fooo" to the end + hooks = self._hooks[('post_load', False)] + if '_add_instance' in hooks: + hooks.remove('_add_instance') + hooks.append('_add_instance') @pre_load def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument - call = self.context.get('callback') - if call is not None: - call(self=self, data=data, many=many, **kwargs) +# TODO: also handle reset, prune and delete in pre_load / post_load hooks! +# print('!!!', repr(data)) + if callback := self.context.get('callback'): + callback(self, data) return data + @post_load + def _add_instance(self, item, many, **kwargs): # pylint: disable=unused-argument + self.opts.sqla_session.add(item) + return item + @post_dump def _hide_and_order(self, data, many, **kwargs): # pylint: disable=unused-argument @@ -306,6 +339,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): ### schema definitions ### +@mapped class DomainSchema(BaseSchema): """ Marshmallow schema for Domain model """ class Meta: @@ -339,6 +373,7 @@ class DomainSchema(BaseSchema): dns_dmarc = fields.String(dump_only=True) +@mapped class TokenSchema(BaseSchema): """ Marshmallow schema for Token model """ class Meta: @@ -347,6 +382,7 @@ class TokenSchema(BaseSchema): load_instance = True +@mapped class FetchSchema(BaseSchema): """ Marshmallow schema for Fetch model """ class Meta: @@ -361,6 +397,7 @@ class FetchSchema(BaseSchema): } +@mapped class UserSchema(BaseSchema): """ Marshmallow schema for User model """ class Meta: @@ -368,7 +405,7 @@ class UserSchema(BaseSchema): model = models.User load_instance = True include_relationships = True - exclude = ['localpart', 'domain', 'quota_bytes_used'] + exclude = ['domain', 'quota_bytes_used'] exclude_by_value = { 'forward_destination': [[]], @@ -395,7 +432,7 @@ class UserSchema(BaseSchema): raise ValidationError(f'invalid hashed password {password!r}') elif 'password_hash' in data and 'hash_scheme' in data: if data['hash_scheme'] not in self.Meta.model.scheme_dict: - raise ValidationError(f'invalid password scheme {scheme!r}') + raise ValidationError(f'invalid password scheme {data["hash_scheme"]!r}') data['password'] = f'{{{data["hash_scheme"]}}}{data["password_hash"]}' del data['hash_scheme'] del data['password_hash'] @@ -409,17 +446,20 @@ class UserSchema(BaseSchema): # ctx.verify('', hashed) # =>? ValueError: hash could not be identified + localpart = fields.Str(load_only=True) + domain_name = fields.Str(load_only=True) tokens = fields.Nested(TokenSchema, many=True) fetches = fields.Nested(FetchSchema, many=True) +@mapped class AliasSchema(BaseSchema): """ Marshmallow schema for Alias model """ class Meta: """ Schema config """ model = models.Alias load_instance = True - exclude = ['localpart'] + exclude = ['domain'] exclude_by_value = { 'destination': [[]], @@ -429,9 +469,12 @@ class AliasSchema(BaseSchema): def _handle_email(self, data, many, **kwargs): # pylint: disable=unused-argument return handle_email(data) + localpart = fields.Str(load_only=True) + domain_name = fields.Str(load_only=True) destination = CommaSeparatedListField() +@mapped class ConfigSchema(BaseSchema): """ Marshmallow schema for Config model """ class Meta: @@ -440,6 +483,7 @@ class ConfigSchema(BaseSchema): load_instance = True +@mapped class RelaySchema(BaseSchema): """ Marshmallow schema for Relay model """ class Meta: @@ -453,18 +497,43 @@ class MailuSchema(Schema): class Meta: """ Schema config """ render_module = RenderYAML + ordered = True order = ['config', 'domains', 'users', 'aliases', 'relays'] - @post_dump(pass_many=True) - def _order(self, data : OrderedDict, many : bool, **kwargs): # pylint: disable=unused-argument - for key in reversed(self.Meta.order): - try: - data.move_to_end(key, False) - except KeyError: - pass + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # order fields + for field_list in self.load_fields, self.dump_fields, self.fields: + for section in reversed(self.Meta.order): + try: + field_list.move_to_end(section, False) + except KeyError: + pass + + @pre_load + def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument + """ create config object in context if missing + and clear it if requested + """ + if 'config' not in self.context: + self.context['config'] = models.MailuConfig() + if self.context.get('clear'): + self.context['config'].clear( + models = {field.nested.opts.model for field in self.fields.values()} + ) return data + @post_load + def _make_config(self, data, many, **kwargs): # pylint: disable=unused-argument + """ update and return config object """ + config = self.context['config'] + for section in self.Meta.order: + if section in data: + config.update(data[section], section) + + return config + config = fields.Nested(ConfigSchema, many=True) domains = fields.Nested(DomainSchema, many=True) users = fields.Nested(UserSchema, many=True) From 1c9abf6e481fee92d87be7b8907d8b9fde06479b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 24 Jan 2021 19:27:22 +0100 Subject: [PATCH 35/52] updated requirements for import/export api reqs (flask-restx, ...) are still missing --- core/admin/requirements-prod.txt | 3 +++ core/admin/requirements.txt | 3 +++ 2 files changed, 6 insertions(+) diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index a3c32855..8ad412cf 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -15,6 +15,7 @@ Flask-Bootstrap==3.3.7.1 Flask-DebugToolbar==0.10.1 Flask-Limiter==1.0.1 Flask-Login==0.4.1 +flask-marshmallow==0.14.0 Flask-Migrate==2.4.0 Flask-Script==2.0.6 Flask-SQLAlchemy==2.4.0 @@ -29,6 +30,8 @@ limits==1.3 Mako==1.0.9 MarkupSafe==1.1.1 mysqlclient==1.4.2.post1 +marshmallow==3.10.0 +marshmallow-sqlalchemy==0.24.1 passlib==1.7.1 psycopg2==2.8.2 pycparser==2.19 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index 9739ed3f..59383a07 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -23,3 +23,6 @@ mysqlclient psycopg2 idna srslib +marshmallow +flask-marshmallow +marshmallow-sqlalchemy From 68caf501549d08c048a7c232ac72bb8a072dd9e6 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 15 Feb 2021 00:46:59 +0100 Subject: [PATCH 36/52] new import/export using marshmallow --- core/admin/mailu/manage.py | 218 +++++---- core/admin/mailu/models.py | 132 +++--- core/admin/mailu/schemas.py | 608 +++++++++++++++++++++----- docs/cli.rst | 221 ++++++---- tests/compose/core/02_forward_test.sh | 4 +- tests/compose/core/03_alias_test.sh | 4 +- tests/compose/core/04_reply_test.sh | 4 +- towncrier/newsfragments/1604.feature | 2 +- 8 files changed, 851 insertions(+), 342 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index e02d9ad4..a8d1d3cb 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -4,7 +4,6 @@ import sys import os import socket -import json import logging import uuid @@ -20,7 +19,7 @@ from flask.cli import FlaskGroup, with_appcontext from marshmallow.exceptions import ValidationError from . import models -from .schemas import MailuSchema, get_schema +from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, RenderJSON, HIDDEN db = models.db @@ -182,7 +181,7 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() -# TODO: remove this deprecated function +# TODO: remove deprecated config_update function? @mailu.command() @click.option('-v', '--verbose') @click.option('-d', '--delete-objects') @@ -324,17 +323,16 @@ def config_update(verbose=False, delete_objects=False): db.session.commit() -SECTIONS = {'domains', 'relays', 'users', 'aliases'} - - @mailu.command() -@click.option('-v', '--verbose', count=True, help='Increase verbosity') -@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors') -@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config') -@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') +@click.option('-v', '--verbose', count=True, help='Increase verbosity.') +@click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.') +@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') +@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.') +@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.') @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=None): +def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None): """ Import configuration as YAML or JSON from stdin or file """ @@ -344,12 +342,19 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No # 2 : also show secrets # 3 : also show input data # 4 : also show sql queries + # 5 : also show tracebacks if quiet: verbose = -1 + color_cfg = { + 'color': color or sys.stdout.isatty(), + 'lexer': 'python', + 'strip': True, + } + counter = Counter() - dumper = {} + logger = {} def format_errors(store, path=None): @@ -387,19 +392,26 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No last = action changes.append(f'{what}({count})') else: - changes = 'no changes.' + changes = ['No changes.'] return chain(message, changes) def log(action, target, message=None): if message is None: - message = json.dumps(dumper[target.__class__].dump(target), ensure_ascii=False) - print(f'{action} {target.__table__}: {message}') + # TODO: convert nested OrderedDict to dict + # see: flask mailu config-import -nvv yaml/dump4.yaml + try: + message = dict(logger[target.__class__].dump(target)) + except KeyError: + message = target + if not isinstance(message, str): + message = repr(message) + print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}') def listen_insert(mapper, connection, target): # pylint: disable=unused-argument """ callback function to track import """ - counter.update([('Added', target.__table__.name)]) + counter.update([('Created', target.__table__.name)]) if verbose >= 1: - log('Added', target) + log('Created', target) def listen_update(mapper, connection, target): # pylint: disable=unused-argument """ callback function to track import """ @@ -407,32 +419,32 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No changed = {} inspection = sqlalchemy.inspect(target) for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: - if getattr(inspection.attrs, attr.key).history.has_changes(): - if sqlalchemy.orm.attributes.get_history(target, attr.key)[2]: - before = sqlalchemy.orm.attributes.get_history(target, attr.key)[2].pop() - after = getattr(target, attr.key) - # only remember changed keys - if before != after and (before or after): - if verbose >= 1: - changed[str(attr.key)] = (before, after) - else: - break + history = getattr(inspection.attrs, attr.key).history + if history.has_changes() and history.deleted: + before = history.deleted[-1] + after = getattr(target, attr.key) + # TODO: remove special handling of "comment" after modifying model + if attr.key == 'comment' and not before and not after: + pass + # only remember changed keys + elif before != after: + if verbose >= 1: + changed[str(attr.key)] = (before, after) + else: + break if verbose >= 1: # use schema with dump_context to hide secrets and sort keys - primary = json.dumps(str(target), ensure_ascii=False) - dumped = get_schema(target)(only=changed.keys(), context=dump_context).dump(target) + dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target) for key, value in dumped.items(): before, after = changed[key] - if value == '': - before = '' if before else before - after = '' if after else after + if value == HIDDEN: + before = HIDDEN if before else before + after = HIDDEN if after else after else: - # TODO: use schema to "convert" before value? + # TODO: need to use schema to "convert" before value? after = value - before = json.dumps(before, ensure_ascii=False) - after = json.dumps(after, ensure_ascii=False) - log('Modified', target, f'{primary} {key}: {before} -> {after}') + log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') if changed: counter.update([('Modified', target.__table__.name)]) @@ -443,47 +455,60 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No if verbose >= 1: log('Deleted', target) - # this listener should not be necessary, when: - # dkim keys should be stored in database and it should be possible to store multiple - # keys per domain. the active key would be also stored on disk on commit. + # TODO: this listener will not be necessary, if dkim keys would be stored in database + _dedupe_dkim = set() def listen_dkim(session, flush_context): # pylint: disable=unused-argument """ callback function to track import """ for target in session.identity_map.values(): - if not isinstance(target, models.Domain): + # look at Domains originally loaded from db + if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: continue - primary = json.dumps(str(target), ensure_ascii=False) before = target._dkim_key_on_disk after = target._dkim_key - if before != after and (before or after): - if verbose >= 2: + if before != after: + if secrets: before = before.decode('ascii', 'ignore') after = after.decode('ascii', 'ignore') else: - before = '' if before else '' - after = '' if after else '' - before = json.dumps(before, ensure_ascii=False) - after = json.dumps(after, ensure_ascii=False) - log('Modified', target, f'{primary} dkim_key: {before} -> {after}') - counter.update([('Modified', target.__table__.name)]) + before = HIDDEN if before else '' + after = HIDDEN if after else '' + # "de-dupe" messages; this event is fired at every flush + if not (target, before, after) in _dedupe_dkim: + _dedupe_dkim.add((target, before, after)) + counter.update([('Modified', target.__table__.name)]) + if verbose >= 1: + log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') - def track_serialize(self, item): + def track_serialize(obj, item): """ callback function to track import """ - log('Handling', self.opts.model, item) + # hide secrets + data = logger[obj.opts.model].hide(item) + if 'hash_password' in data: + data['password'] = HIDDEN + if 'fetches' in data: + for fetch in data['fetches']: + fetch['password'] = HIDDEN + log('Handling', obj.opts.model, data) # configure contexts - dump_context = { - 'secrets': verbose >= 2, + diff_context = { + 'full': True, + 'secrets': secrets, + } + log_context = { + 'secrets': secrets, } load_context = { - 'callback': track_serialize if verbose >= 3 else None, - 'clear': not update, 'import': True, + 'update': update, + 'clear': not update, + 'callback': track_serialize if verbose >= 2 else None, } # register listeners for schema in get_schema(): model = schema.Meta.model - dumper[model] = schema(context=dump_context) + logger[model] = schema(context=log_context) sqlalchemy.event.listen(model, 'after_insert', listen_insert) sqlalchemy.event.listen(model, 'after_update', listen_update) sqlalchemy.event.listen(model, 'after_delete', listen_delete) @@ -491,18 +516,24 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No # special listener for dkim_key changes sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim) - if verbose >= 4: + if verbose >= 3: logging.basicConfig() logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) try: with models.db.session.no_autoflush: - config = MailuSchema(only=SECTIONS, context=load_context).loads(source) + config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source) except ValidationError as exc: raise click.ClickException(format_errors(exc.messages)) from exc except Exception as exc: - # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) - raise click.ClickException(f'[{exc.__class__.__name__}] {" ".join(str(exc).split())}') from exc + if verbose >= 5: + raise + else: + # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) + raise click.ClickException( + f'[{exc.__class__.__name__}] ' + f'{" ".join(str(exc).split())}' + ) from exc # flush session to show/count all changes if dry_run or verbose >= 1: @@ -510,53 +541,47 @@ def config_import(verbose=0, quiet=False, update=False, dry_run=False, source=No # check for duplicate domain names dup = set() - for fqdn in chain(db.session.query(models.Domain.name), - db.session.query(models.Alternative.name), - db.session.query(models.Relay.name)): + for fqdn in chain( + db.session.query(models.Domain.name), + db.session.query(models.Alternative.name), + db.session.query(models.Relay.name) + ): if fqdn in dup: raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}') dup.add(fqdn) - # TODO: implement special update "items" - # -pkey: which - remove item "which" - # -key: null or [] or {} - set key to default - # -pkey: null or [] or {} - remove all existing items in this list - # don't commit when running dry if dry_run: - db.session.rollback() if not quiet: print(*format_changes('Dry run. Not commiting changes.')) - # TODO: remove debug - print(MailuSchema().dumps(config)) + db.session.rollback() else: - db.session.commit() if not quiet: - print(*format_changes('Commited changes.')) + print(*format_changes('Committing changes.')) + db.session.commit() @mailu.command() -@click.option('-f', '--full', is_flag=True, help='Include attributes with default value') +@click.option('-f', '--full', is_flag=True, help='Include attributes with default value.') @click.option('-s', '--secrets', is_flag=True, - help='Include secret attributes (dkim-key, passwords)') -@click.option('-d', '--dns', is_flag=True, help='Include dns records') + help='Include secret attributes (dkim-key, passwords).') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') +@click.option('-d', '--dns', is_flag=True, help='Include dns records.') @click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), - help='save yaml to file') -@click.option('-j', '--json', 'as_json', is_flag=True, help='Dump in josn format') -@click.argument('sections', nargs=-1) + help='Save configuration to file.') +@click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.') +@click.argument('only', metavar='[FILTER]...', nargs=-1) @with_appcontext -def config_export(full=False, secrets=False, dns=False, output=None, as_json=False, sections=None): +def config_export(full=False, secrets=False, color=False, dns=False, output=None, as_json=False, only=None): """ Export configuration as YAML or JSON to stdout or file """ - if sections: - for section in sections: - if section not in SECTIONS: - print(f'[ERROR] Unknown section: {section}') - raise click.exceptions.Exit(1) - sections = set(sections) + if only: + for spec in only: + if spec.split('.', 1)[0] not in MailuSchema.Meta.order: + raise click.ClickException(f'[ERROR] Unknown section: {spec}') else: - sections = SECTIONS + only = MailuSchema.Meta.order context = { 'full': full, @@ -564,13 +589,20 @@ def config_export(full=False, secrets=False, dns=False, output=None, as_json=Fal 'dns': dns, } - if as_json: - schema = MailuSchema(only=sections, context=context) - schema.opts.render_module = json - print(schema.dumps(models.MailuConfig(), separators=(',',':')), file=output) + schema = MailuSchema(only=only, context=context) + color_cfg = {'color': color or output.isatty()} - else: - MailuSchema(only=sections, context=context).dumps(models.MailuConfig(), output) + if as_json: + schema.opts.render_module = RenderJSON + color_cfg['lexer'] = 'json' + color_cfg['strip'] = True + + try: + print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) + except ValueError as exc: + if spec := get_fieldspec(exc): + raise click.ClickException(f'[ERROR] Invalid filter: {spec}') from exc + raise @mailu.command() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index dac1dc70..5799e282 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -19,6 +19,7 @@ import dns from flask import current_app as app from sqlalchemy.ext import declarative +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.inspection import inspect from werkzeug.utils import cached_property @@ -121,6 +122,36 @@ class Base(db.Model): updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True, default='') + def __str__(self): + pkey = self.__table__.primary_key.columns.values()[0].name + if pkey == 'email': + # ugly hack for email declared attr. _email is not always up2date + return str(f'{self.localpart}@{self.domain_name}') + elif pkey in {'name', 'email'}: + return str(getattr(self, pkey, None)) + else: + return self.__repr__() + return str(getattr(self, self.__table__.primary_key.columns.values()[0].name)) + + def __repr__(self): + return f'<{self.__class__.__name__} {str(self)!r}>' + + def __eq__(self, other): + if isinstance(other, self.__class__): + pkey = self.__table__.primary_key.columns.values()[0].name + this = getattr(self, pkey, None) + other = getattr(other, pkey, None) + return this is not None and other is not None and str(this) == str(other) + else: + return NotImplemented + + def __hash__(self): + primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) + if primary is None: + return NotImplemented + else: + return hash(primary) + # Many-to-many association table for domain managers managers = db.Table('manager', Base.metadata, @@ -261,19 +292,6 @@ class Domain(Base): except dns.exception.DNSException: return False - def __str__(self): - return str(self.name) - - def __eq__(self, other): - if isinstance(other, self.__class__): - return str(self.name) == str(other.name) - else: - return NotImplemented - - def __hash__(self): - return hash(str(self.name)) - - class Alternative(Base): """ Alternative name for a served domain. @@ -287,9 +305,6 @@ class Alternative(Base): domain = db.relationship(Domain, backref=db.backref('alternatives', cascade='all, delete-orphan')) - def __str__(self): - return str(self.name) - class Relay(Base): """ Relayed mail domain. @@ -302,9 +317,6 @@ class Relay(Base): # TODO: String(80) is too small? smtp = db.Column(db.String(80), nullable=True) - def __str__(self): - return str(self.name) - class Email(object): """ Abstraction for an email address (localpart and domain). @@ -312,11 +324,11 @@ class Email(object): # TODO: validate max. total length of address (<=254) - # TODO: String(80) is too large (>64)? + # TODO: String(80) is too large (64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr - def domain_name(self): + def domain_name(cls): """ the domain part of the email address """ return db.Column(IdnaDomain, db.ForeignKey(Domain.name), nullable=False, default=IdnaDomain) @@ -325,13 +337,33 @@ class Email(object): # It is however very useful for quick lookups without joining tables, # especially when the mail server is reading the database. @declarative.declared_attr - def email(self): + def _email(cls): """ the complete email address (localpart@domain) """ - updater = lambda ctx: '{localpart}@{domain_name}'.format(**ctx.current_parameters) - return db.Column(IdnaEmail, - primary_key=True, nullable=False, - default=updater - ) + + def updater(ctx): + key = f'{cls.__tablename__}_email' + if key in ctx.current_parameters: + return ctx.current_parameters[key] + return '{localpart}@{domain_name}'.format(**ctx.current_parameters) + + return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater) + + # We need to keep email, localpart and domain_name in sync. + # But IMHO using email as primary key was not a good idea in the first place. + @hybrid_property + def email(self): + """ getter for email - gets _email """ + return self._email + + @email.setter + def email(self, value): + """ setter for email - sets _email, localpart and domain_name at once """ + self.localpart, self.domain_name = value.rsplit('@', 1) + self._email = value + + # hack for email declared attr - when _email is not updated yet + def __str__(self): + return str(f'{self.localpart}@{self.domain_name}') def sendmail(self, subject, body): """ send an email to the address """ @@ -391,9 +423,6 @@ class Email(object): return None - def __str__(self): - return str(self.email) - class User(Base, Email): """ A user is an email address that has a password to access a mailbox. @@ -435,12 +464,10 @@ class User(Base, Email): is_active = True is_anonymous = False - # TODO: remove unused user.get_id() def get_id(self): """ return users email address """ return self.email - # TODO: remove unused user.destination @property def destination(self): """ returns comma separated string of destinations """ @@ -471,17 +498,20 @@ class User(Base, Email): 'CRYPT': 'des_crypt', } - def _get_password_context(self): + @classmethod + def get_password_context(cls): + """ Create password context for hashing and verification + """ return passlib.context.CryptContext( - schemes=self.scheme_dict.values(), - default=self.scheme_dict[app.config['PASSWORD_SCHEME']], + schemes=cls.scheme_dict.values(), + default=cls.scheme_dict[app.config['PASSWORD_SCHEME']], ) def check_password(self, plain): """ Check password against stored hash Update hash when default scheme has changed """ - context = self._get_password_context() + context = self.get_password_context() hashed = re.match('^({[^}]+})?(.*)$', self.password).group(2) result = context.verify(plain, hashed) if result and context.identify(hashed) != context.default_scheme(): @@ -490,8 +520,6 @@ class User(Base, Email): db.session.commit() return result - # TODO: remove kwarg hash_scheme - there is no point in setting a scheme, - # when the next check updates the password to the default scheme. def set_password(self, new, hash_scheme=None, raw=False): """ Set password for user with specified encryption scheme @new: plain text password to encrypt (or, if raw is True: the hash itself) @@ -500,7 +528,7 @@ class User(Base, Email): if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] if not raw: - new = self._get_password_context().encrypt(new, self.scheme_dict[hash_scheme]) + new = self.get_password_context().encrypt(new, self.scheme_dict[hash_scheme]) self.password = f'{{{hash_scheme}}}{new}' def get_managed_domains(self): @@ -593,7 +621,7 @@ class Alias(Base, Email): return None -# TODO: what about API tokens? + class Token(Base): """ A token is an application password for a given user. """ @@ -606,20 +634,19 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - # TODO: String(80) is too large? + # TODO: String(255) is too large? (43 should be sufficient) ip = db.Column(db.String(255)) def check_password(self, password): """ verifies password against stored hash """ return passlib.hash.sha256_crypt.verify(password, self.password) - # TODO: use crypt context and default scheme from config? def set_password(self, password): """ sets password using sha256_crypt(rounds=1000) """ self.password = passlib.hash.sha256_crypt.using(rounds=1000).hash(password) - def __str__(self): - return str(self.comment or self.ip) + def __repr__(self): + return f'' class Fetch(Base): @@ -644,8 +671,11 @@ class Fetch(Base): last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) - def __str__(self): - return f'{self.protocol}{"s" if self.tls else ""}://{self.username}@{self.host}:{self.port}' + def __repr__(self): + return ( + f'' + ) class MailuConfig: @@ -661,7 +691,7 @@ class MailuConfig: def __init__(self, model : db.Model): self.model = model - def __str__(self): + def __repr__(self): return f'<{self.model.__name__}-Collection>' @cached_property @@ -837,8 +867,8 @@ class MailuConfig: if models is None or model in models: db.session.query(model).delete() - domains = MailuCollection(Domain) - relays = MailuCollection(Relay) - users = MailuCollection(User) - aliases = MailuCollection(Alias) + domain = MailuCollection(Domain) + user = MailuCollection(User) + alias = MailuCollection(Alias) + relay = MailuCollection(Relay) config = MailuCollection(Config) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 04512f6d..54a2e928 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1,27 +1,66 @@ """ Mailu marshmallow fields and schema """ -import re - +from copy import deepcopy from collections import OrderedDict from textwrap import wrap +import re +import json import yaml +import sqlalchemy + from marshmallow import pre_load, post_load, post_dump, fields, Schema +from marshmallow.utils import ensure_text_type from marshmallow.exceptions import ValidationError from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts + from flask_marshmallow import Marshmallow + from OpenSSL import crypto +try: + from pygments import highlight + from pygments.token import Token + from pygments.lexers import get_lexer_by_name + from pygments.lexers.data import YamlLexer + from pygments.formatters import get_formatter_by_name +except ModuleNotFoundError: + COLOR_SUPPORTED = False +else: + COLOR_SUPPORTED = True + from . import models, dkim ma = Marshmallow() -# TODO: how and where to mark keys as "required" while unserializing (on commandline, in api)? -# - fields without default => required -# - fields which are the primary key => unchangeable when updating +# TODO: how and where to mark keys as "required" while unserializing in api? +# - when modifying, nothing is required (only the primary key, but this key is in the uri) +# - the primary key from post data must not differ from the key in the uri +# - when creating all fields without default or auto-increment are required +# TODO: what about deleting list items and prung lists? +# - domain.alternatives, user.forward_destination, user.manager_of, aliases.destination +# TODO: validate everything! + + +### class for hidden values ### + +class _Hidden: + def __bool__(self): + return False + def __copy__(self): + return self + def __deepcopy__(self, _): + return self + def __eq__(self, other): + return str(other) == '' + def __repr__(self): + return '' + __str__ = __repr__ + +HIDDEN = _Hidden() ### map model to schema ### @@ -41,13 +80,90 @@ def mapped(cls): return cls -### yaml render module ### +### helper functions ### + +def get_fieldspec(exc): + """ walk traceback to extract spec of invalid field from marshmallow """ + path = [] + tbck = exc.__traceback__ + while tbck: + if tbck.tb_frame.f_code.co_name == '_serialize': + if 'attr' in tbck.tb_frame.f_locals: + path.append(tbck.tb_frame.f_locals['attr']) + elif tbck.tb_frame.f_code.co_name == '_init_fields': + path = '.'.join(path) + spec = ', '.join([f'{path}.{key}' for key in tbck.tb_frame.f_locals['invalid_fields']]) + return spec + tbck = tbck.tb_next + return None + +def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): + """ add ANSI color to data """ + if color is None: + # autodetect colorize + color = COLOR_SUPPORTED + if not color: + # no color wanted + return data + if not COLOR_SUPPORTED: + # want color, but not supported + raise ValueError('Please install pygments to colorize output') + + scheme = { + Token: ('', ''), + Token.Name.Tag: ('cyan', 'brightcyan'), + Token.Literal.Scalar: ('green', 'green'), + Token.Literal.String: ('green', 'green'), + Token.Keyword.Constant: ('magenta', 'brightmagenta'), + Token.Literal.Number: ('magenta', 'brightmagenta'), + Token.Error: ('red', 'brightred'), + Token.Name: ('red', 'brightred'), + Token.Operator: ('red', 'brightred'), + } + + class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + + res = highlight( + data, + MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer), + get_formatter_by_name(formatter, colorscheme=scheme) + ) + + return res.rstrip('\n') if strip else res + + +### render modules ### # allow yaml module to dump OrderedDict yaml.add_representer( OrderedDict, lambda cls, data: cls.represent_mapping('tag:yaml.org,2002:map', data.items()) ) +yaml.add_representer( + _Hidden, + lambda cls, data: cls.represent_data(str(data)) +) class RenderYAML: """ Marshmallow YAML Render Module @@ -67,19 +183,19 @@ class RenderYAML: return super().increase_indent(flow, False) @staticmethod - def _update_items(dict1, dict2): - """ sets missing keys in dict1 to values of dict2 + def _augment(kwargs, defaults): + """ add default kv's to kwargs if missing """ - for key, value in dict2.items(): - if key not in dict1: - dict1[key] = value + for key, value in defaults.items(): + if key not in kwargs: + kwargs[key] = value _load_defaults = {} @classmethod def loads(cls, *args, **kwargs): """ load yaml data from string """ - cls._update_items(kwargs, cls._load_defaults) + cls._augment(kwargs, cls._load_defaults) return yaml.safe_load(*args, **kwargs) _dump_defaults = { @@ -90,13 +206,52 @@ class RenderYAML: } @classmethod def dumps(cls, *args, **kwargs): - """ dump yaml data to string + """ dump data to yaml string """ - cls._update_items(kwargs, cls._dump_defaults) + cls._augment(kwargs, cls._dump_defaults) return yaml.dump(*args, **kwargs) +class JSONEncoder(json.JSONEncoder): + """ JSONEncoder supporting serialization of HIDDEN """ + def default(self, o): + """ serialize HIDDEN """ + if isinstance(o, _Hidden): + return str(o) + return json.JSONEncoder.default(self, o) -### field definitions ### +class RenderJSON: + """ Marshmallow JSON Render Module + """ + + @staticmethod + def _augment(kwargs, defaults): + """ add default kv's to kwargs if missing + """ + for key, value in defaults.items(): + if key not in kwargs: + kwargs[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load json data from string + """ + cls._augment(kwargs, cls._load_defaults) + return json.loads(*args, **kwargs) + + _dump_defaults = { + 'separators': (',',':'), + 'cls': JSONEncoder, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump data to json string + """ + cls._augment(kwargs, cls._dump_defaults) + return json.dumps(*args, **kwargs) + + +### custom fields ### class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string @@ -107,9 +262,8 @@ class LazyStringField(fields.String): """ return value if value else '' - class CommaSeparatedListField(fields.Raw): - """ Field that deserializes a string containing comma-separated values to + """ Deserialize a string containing comma-separated values to a list of strings """ @@ -129,10 +283,15 @@ class CommaSeparatedListField(fields.Raw): class DkimKeyField(fields.String): - """ Field that serializes a dkim key to a list of strings (lines) and - deserializes a string or list of strings. + """ Serialize a dkim key to a list of strings (lines) and + Deserialize a string or list of strings to a valid dkim key """ + default_error_messages = { + "invalid": "Not a valid string or list.", + "invalid_utf8": "Not a valid utf-8 string or list.", + } + _clean_re = re.compile( r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', flags=re.UNICODE @@ -156,11 +315,19 @@ class DkimKeyField(fields.String): # convert list to str if isinstance(value, list): - value = ''.join(value) + try: + value = ''.join([ensure_text_type(item) for item in value]) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc - # only strings are allowed - if not isinstance(value, str): - raise ValidationError(f'invalid type {type(value).__name__!r}') + # only text is allowed + else: + if not isinstance(value, (str, bytes)): + raise self.make_error("invalid") + try: + value = ensure_text_type(value) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc # clean value (remove whitespace and header/footer) value = self._clean_re.sub('', value.strip()) @@ -189,28 +356,53 @@ class DkimKeyField(fields.String): else: return value - -### base definitions ### - -def handle_email(data): - """ merge separate localpart and domain to email +class PasswordField(fields.Str): + """ Serialize a hashed password hash by stripping the obsolete {SCHEME} + Deserialize a plain password or hashed password into a hashed password """ - localpart = 'localpart' in data - domain = 'domain' in data + _hashes = {'PBKDF2', 'BLF-CRYPT', 'SHA512-CRYPT', 'SHA256-CRYPT', 'MD5-CRYPT', 'CRYPT'} - if 'email' in data: - if localpart or domain: - raise ValidationError('duplicate email and localpart/domain') - data['localpart'], data['domain_name'] = data['email'].rsplit('@', 1) - elif localpart and domain: - data['domain_name'] = data['domain'] - del data['domain'] - data['email'] = f'{data["localpart"]}@{data["domain_name"]}' - elif localpart or domain: - raise ValidationError('incomplete localpart/domain') + def _serialize(self, value, attr, obj, **kwargs): + """ strip obsolete {password-hash} when serializing """ + # strip scheme spec if in database - it's obsolete + if value.startswith('{') and (end := value.find('}', 1)) >= 0: + if value[1:end] in self._hashes: + return value[end+1:] + return value - return data + def _deserialize(self, value, attr, data, **kwargs): + """ hashes plain password or checks hashed password + also strips obsolete {password-hash} when deserializing + """ + + # when hashing is requested: use model instance to hash plain password + if data.get('hash_password'): + # hash password using model instance + inst = self.metadata['model']() + inst.set_password(value) + value = inst.password + del inst + + # strip scheme spec when specified - it's obsolete + if value.startswith('{') and (end := value.find('}', 1)) >= 0: + if value[1:end] in self._hashes: + value = value[end+1:] + + # check if algorithm is supported + inst = self.metadata['model'](password=value) + try: + # just check against empty string to see if hash is valid + inst.check_password('') + except ValueError as exc: + # ValueError: hash could not be identified + raise ValidationError(f'invalid password hash {value!r}') from exc + del inst + + return value + + +### base schema ### class BaseOpts(SQLAlchemyAutoSchemaOpts): """ Option class with sqla session @@ -220,6 +412,8 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): meta.sqla_session = models.db.session if not hasattr(meta, 'ordered'): meta.ordered = True + if not hasattr(meta, 'sibling'): + meta.sibling = False super(BaseOpts, self).__init__(meta, ordered=ordered) class BaseSchema(ma.SQLAlchemyAutoSchema): @@ -231,10 +425,15 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): class Meta: """ Schema config """ + include_by_context = {} + exclude_by_value = {} + hide_by_context = {} + order = [] + sibling = False def __init__(self, *args, **kwargs): - # context? + # get context context = kwargs.get('context', {}) flags = {key for key, value in context.items() if value is True} @@ -261,7 +460,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # exclude default values if not context.get('full'): - for column in getattr(self.opts, 'model').__table__.columns: + for column in self.opts.model.__table__.columns: if column.name not in exclude: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg @@ -274,45 +473,239 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if not flags & set(need): self._hide_by_context |= set(what) + # remember primary keys + self._primary = self.opts.model.__table__.primary_key.columns.values()[0].name + # initialize attribute order if hasattr(self.Meta, 'order'): # use user-defined order - self._order = list(reversed(getattr(self.Meta, 'order'))) + self._order = list(reversed(self.Meta.order)) else: # default order is: primary_key + other keys alphabetically self._order = list(sorted(self.fields.keys())) - primary = self.opts.model.__table__.primary_key.columns.values()[0].name - if primary in self._order: - self._order.remove(primary) + if self._primary in self._order: + self._order.remove(self._primary) self._order.reverse() - self._order.append(primary) + self._order.append(self._primary) # move pre_load hook "_track_import" to the front hooks = self._hooks[('pre_load', False)] - if '_track_import' in hooks: - hooks.remove('_track_import') - hooks.insert(0, '_track_import') - # and post_load hook "_fooo" to the end + hooks.remove('_track_import') + hooks.insert(0, '_track_import') + # move pre_load hook "_add_instance" to the end + hooks.remove('_add_required') + hooks.append('_add_required') + + # move post_load hook "_add_instance" to the end hooks = self._hooks[('post_load', False)] - if '_add_instance' in hooks: - hooks.remove('_add_instance') - hooks.append('_add_instance') + hooks.remove('_add_instance') + hooks.append('_add_instance') + + def hide(self, data): + """ helper method to hide input data for logging """ + # always returns a copy of data + return { + key: HIDDEN if key in self._hide_by_context else deepcopy(value) + for key, value in data.items() + } + + def _call_and_store(self, *args, **kwargs): + """ track curent parent field for pruning """ + self.context['parent_field'] = kwargs['field_name'] + return super()._call_and_store(*args, **kwargs) + + # this is only needed to work around the declared attr "email" primary key in model + def get_instance(self, data): + """ lookup item by defined primary key instead of key(s) from model """ + if self.transient: + return None + if keys := getattr(self.Meta, 'primary_keys', None): + filters = {key: data.get(key) for key in keys} + if None not in filters.values(): + return self.session.query(self.opts.model).filter_by(**filters).first() + return super().get_instance(data) + + @pre_load(pass_many=True) + def _patch_input(self, items, many, **kwargs): # pylint: disable=unused-argument + """ - flush sqla session before serializing a section when requested + (make sure all objects that could be referred to later are created) + - when in update mode: patch input data before deserialization + - handle "prune" and "delete" items + - replace values in keys starting with '-' with default + """ + + # flush sqla session + if not self.Meta.sibling: + self.opts.sqla_session.flush() + + # stop early when not updating + if not self.context.get('update'): + return items + + # patch "delete", "prune" and "default" + want_prune = [] + def patch(count, data, prune): + + # don't allow __delete__ coming from input + if '__delete__' in data: + raise ValidationError('Unknown field.', f'{count}.__delete__') + + # handle "prune list" and "delete item" (-pkey: none and -pkey: id) + for key in data: + if key.startswith('-'): + if key[1:] == self._primary: + # delete or prune + if data[key] is None: + # prune + prune.append(True) + return None + # mark item for deletion + return {key[1:]: data[key], '__delete__': True} + + # handle "set to default value" (-key: none) + def set_default(key, value): + if not key.startswith('-'): + return (key, value) + key = key[1:] + if not key in self.opts.model.__table__.columns: + return (key, None) + if value is not None: + raise ValidationError( + 'When resetting to default value must be null.', + f'{count}.{key}' + ) + value = self.opts.model.__table__.columns[key].default + if value is None: + raise ValidationError( + 'Field has no default value.', + f'{count}.{key}' + ) + return (key, value.arg) + + return dict([set_default(key, value) for key, value in data.items()]) + + # convert items to "delete" and filter "prune" item + items = [ + item for item in [ + patch(count, item, want_prune) for count, item in enumerate(items) + ] if item + ] + + # prune: determine if existing items in db need to be added or marked for deletion + add_items = False + del_items = False + if self.Meta.sibling: + # parent prunes automatically + if not want_prune: + # no prune requested => add old items + add_items = True + else: + # parent does not prune automatically + if want_prune: + # prune requested => mark old items for deletion + del_items = True + + if add_items or del_items: + existing = {item[self._primary] for item in items if self._primary in item} + for item in getattr(self.context['parent'], self.context['parent_field']): + key = getattr(item, self._primary) + if key not in existing: + if add_items: + items.append({self._primary: key}) + else: + items.append({self._primary: key, '__delete__': True}) + + return items @pre_load def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument -# TODO: also handle reset, prune and delete in pre_load / post_load hooks! -# print('!!!', repr(data)) + """ call callback function to track import + """ + # callback if callback := self.context.get('callback'): callback(self, data) + return data - @post_load - def _add_instance(self, item, many, **kwargs): # pylint: disable=unused-argument - self.opts.sqla_session.add(item) + @pre_load + def _add_required(self, data, many, **kwargs): # pylint: disable=unused-argument + """ when updating: + allow modification of existing items having required attributes + by loading existing value from db + """ + + if not self.opts.load_instance or not self.context.get('update'): + return data + + # stabilize import of auto-increment primary keys (not required), + # by matching import data to existing items and setting primary key + if not self._primary in data: + for item in getattr(self.context['parent'], self.context['parent_field']): + existing = self.dump(item, many=False) + this = existing.pop(self._primary) + if data == existing: + instance = item + data[self._primary] = this + break + + # try to load instance + instance = self.instance or self.get_instance(data) + if instance is None: + + if '__delete__' in data: + # deletion of non-existent item requested + raise ValidationError( + f'item not found: {data[self._primary]!r}', + field_name=f'?.{self._primary}', + ) + + else: + + if self.context.get('update'): + # remember instance as parent for pruning siblings + if not self.Meta.sibling: + self.context['parent'] = instance + # delete instance when marked + if '__delete__' in data: + self.opts.sqla_session.delete(instance) + + # add attributes required for validation from db + # TODO: this will cause validation errors if value from database does not validate + for attr_name, field_obj in self.load_fields.items(): + if field_obj.required and attr_name not in data: + data[attr_name] = getattr(instance, attr_name) + + return data + + @post_load(pass_original=True) + def _add_instance(self, item, original, many, **kwargs): # pylint: disable=unused-argument + """ add new instances to sqla session """ + + if item in self.opts.sqla_session: + # item was modified + if 'hash_password' in original: + # stabilize import of passwords to be hashed, + # by not re-hashing an unchanged password + if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): + if attr.history.has_changes() and attr.history.deleted: + try: + # reset password hash, if password was not changed + inst = type(item)(password=attr.history.deleted[-1]) + if inst.check_password(original['password']): + item.password = inst.password + except ValueError: + # hash in db is invalid + pass + else: + del inst + else: + # new item + self.opts.sqla_session.add(item) return item @post_dump def _hide_and_order(self, data, many, **kwargs): # pylint: disable=unused-argument + """ hide secrets and order output """ # order output for key in self._order: @@ -325,15 +718,18 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if not self._exclude_by_value and not self._hide_by_context: return data - # exclude items or hide values + # exclude or hide values full = self.context.get('full') return type(data)([ - (key, '' if key in self._hide_by_context else value) + (key, HIDDEN if key in self._hide_by_context else value) for key, value in data.items() if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] ]) - # TODO: remove LazyStringField and change model (IMHO comment should not be nullable) + # this field is used to mark items for deletion + mark_delete = fields.Boolean(data_key='__delete__', load_only=True) + + # TODO: remove LazyStringField (when model was changed - IMHO comment should not be nullable) comment = LazyStringField() @@ -381,6 +777,11 @@ class TokenSchema(BaseSchema): model = models.Token load_instance = True + sibling = True + + password = PasswordField(required=True, metadata={'model': models.User}) + hash_password = fields.Boolean(load_only=True, missing=False) + @mapped class FetchSchema(BaseSchema): @@ -389,6 +790,8 @@ class FetchSchema(BaseSchema): """ Schema config """ model = models.Fetch load_instance = True + + sibling = True include_by_context = { ('full', 'import'): {'last_check', 'error'}, } @@ -405,52 +808,25 @@ class UserSchema(BaseSchema): model = models.User load_instance = True include_relationships = True - exclude = ['domain', 'quota_bytes_used'] + exclude = ['_email', 'domain', 'localpart', 'domain_name', 'quota_bytes_used'] + primary_keys = ['email'] exclude_by_value = { 'forward_destination': [[]], - 'tokens': [[]], - 'fetches': [[]], - 'manager_of': [[]], - 'reply_enddate': ['2999-12-31'], - 'reply_startdate': ['1900-01-01'], + 'tokens': [[]], + 'fetches': [[]], + 'manager_of': [[]], + 'reply_enddate': ['2999-12-31'], + 'reply_startdate': ['1900-01-01'], } - @pre_load - def _handle_email_and_password(self, data, many, **kwargs): # pylint: disable=unused-argument - data = handle_email(data) - if 'password' in data: - if 'password_hash' in data or 'hash_scheme' in data: - raise ValidationError('ambigous key password and password_hash/hash_scheme') - # check (hashed) password - password = data['password'] - if password.startswith('{') and '}' in password: - scheme = password[1:password.index('}')] - if scheme not in self.Meta.model.scheme_dict: - raise ValidationError(f'invalid password scheme {scheme!r}') - else: - raise ValidationError(f'invalid hashed password {password!r}') - elif 'password_hash' in data and 'hash_scheme' in data: - if data['hash_scheme'] not in self.Meta.model.scheme_dict: - raise ValidationError(f'invalid password scheme {data["hash_scheme"]!r}') - data['password'] = f'{{{data["hash_scheme"]}}}{data["password_hash"]}' - del data['hash_scheme'] - del data['password_hash'] - return data - - # TODO: verify password (should this be done in model?) - # scheme, hashed = re.match('^(?:{([^}]+)})?(.*)$', self.password).groups() - # if not scheme... - # ctx = passlib.context.CryptContext(schemes=[scheme], default=scheme) - # try: - # ctx.verify('', hashed) - # =>? ValueError: hash could not be identified - - localpart = fields.Str(load_only=True) - domain_name = fields.Str(load_only=True) + email = fields.String(required=True) tokens = fields.Nested(TokenSchema, many=True) fetches = fields.Nested(FetchSchema, many=True) + password = PasswordField(required=True, metadata={'model': models.User}) + hash_password = fields.Boolean(load_only=True, missing=False) + @mapped class AliasSchema(BaseSchema): @@ -459,18 +835,14 @@ class AliasSchema(BaseSchema): """ Schema config """ model = models.Alias load_instance = True - exclude = ['domain'] + exclude = ['_email', 'domain', 'localpart', 'domain_name'] + primary_keys = ['email'] exclude_by_value = { 'destination': [[]], } - @pre_load - def _handle_email(self, data, many, **kwargs): # pylint: disable=unused-argument - return handle_email(data) - - localpart = fields.Str(load_only=True) - domain_name = fields.Str(load_only=True) + email = fields.String(required=True) destination = CommaSeparatedListField() @@ -499,7 +871,7 @@ class MailuSchema(Schema): render_module = RenderYAML ordered = True - order = ['config', 'domains', 'users', 'aliases', 'relays'] + order = ['domain', 'user', 'alias', 'relay'] # 'config' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -511,6 +883,14 @@ class MailuSchema(Schema): except KeyError: pass + def _call_and_store(self, *args, **kwargs): + """ track current parent and field for pruning """ + self.context.update({ + 'parent': self.context.get('config'), + 'parent_field': kwargs['field_name'], + }) + return super()._call_and_store(*args, **kwargs) + @pre_load def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument """ create config object in context if missing @@ -534,8 +914,8 @@ class MailuSchema(Schema): return config - config = fields.Nested(ConfigSchema, many=True) - domains = fields.Nested(DomainSchema, many=True) - users = fields.Nested(UserSchema, many=True) - aliases = fields.Nested(AliasSchema, many=True) - relays = fields.Nested(RelaySchema, many=True) + domain = fields.Nested(DomainSchema, many=True) + user = fields.Nested(UserSchema, many=True) + alias = fields.Nested(AliasSchema, many=True) + relay = fields.Nested(RelaySchema, many=True) +# config = fields.Nested(ConfigSchema, many=True) diff --git a/docs/cli.rst b/docs/cli.rst index 1b2ed14f..497cdfc5 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -10,8 +10,9 @@ Managing users and aliases can be done from CLI using commands: * user * user-import * user-delete -* config-dump * config-update +* config-export +* config-import alias ----- @@ -69,104 +70,160 @@ user-delete docker-compose exec admin flask mailu user-delete foo@example.net -config-dump ------------ - -The purpose of this command is to dump domain-, relay-, alias- and user-configuration to a YAML template. - -.. code-block:: bash - - # docker-compose exec admin flask mailu config-dump --help - - Usage: flask mailu config-dump [OPTIONS] [SECTIONS]... - - dump configuration as YAML-formatted data to stdout - - SECTIONS can be: domains, relays, users, aliases - - Options: - -f, --full Include default attributes - -s, --secrets Include secrets (dkim-key, plain-text / not hashed) - -d, --dns Include dns records - --help Show this message and exit. - -If you want to export secrets (dkim-keys, plain-text / not hashed) you have to add the ``--secrets`` option. -Only non-default attributes are dumped. If you want to dump all attributes use ``--full``. -To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. -Unless you specify some sections all sections are dumped by default. - -.. code-block:: bash - - docker-compose exec admin flask mailu config-dump > mail-config.yml - - docker-compose exec admin flask mailu config-dump --dns domains - config-update ------------- -The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML template. +The sole purpose of this command is for importing users/aliases in bulk and synchronizing DB entries with external YAML template: .. code-block:: bash - # docker-compose exec admin flask mailu config-update --help + cat mail-config.yml | docker-compose exec -T admin flask mailu config-update --delete-objects - Usage: flask mailu config-update [OPTIONS] +where mail-config.yml looks like: - sync configuration with data from YAML-formatted stdin +.. code-block:: bash - Options: - -v, --verbose Increase verbosity - -d, --delete-objects Remove objects not included in yaml - -n, --dry-run Perform a trial run with no changes made - --help Show this message and exit. + users: + - localpart: foo + domain: example.com + password_hash: klkjhumnzxcjkajahsdqweqqwr + hash_scheme: MD5-CRYPT + aliases: + - localpart: alias1 + domain: example.com + destination: "user1@example.com,user2@example.com" + +without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. + +Users +----- + +following are additional parameters that could be defined for users: + +* comment +* quota_bytes +* global_admin +* enable_imap +* enable_pop +* forward_enabled +* forward_destination +* reply_enabled +* reply_subject +* reply_body +* displayed_name +* spam_enabled +* spam_threshold + +Alias +----- + +additional fields: + +* wildcard + +config-export +------------- + +The purpose of this command is to export domain-, relay-, alias- and user-configuration in YAML or JSON format. + +.. code-block:: bash + + # docker-compose exec admin flask mailu config-export --help + + Usage: flask mailu config-export [OPTIONS] [FILTER]... + + Export configuration as YAML or JSON to stdout or file + + Options: + -f, --full Include attributes with default value. + -s, --secrets Include secret attributes (dkim-key, passwords). + -c, --color Force colorized output. + -d, --dns Include dns records. + -o, --output-file FILENAME Save configuration to file. + -j, --json Export configuration in json format. + -?, -h, --help Show this message and exit. + +Only non-default attributes are exported. If you want to export all attributes use ``--full``. +If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option. +To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. +By default all configuration objects are exported (domain, user, alias, relay). You can specify +filters to export only some objects or attributes (try: ``user`` or ``domain.name``). + +.. code-block:: bash + + docker-compose exec admin flask mailu config-export -o mail-config.yml + + docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf + +config-import +------------- + +The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML/JOSN source. + +.. code-block:: bash + + # docker-compose exec admin flask mailu config-import --help + + Usage: flask mailu config-import [OPTIONS] [FILENAME|-] + + Import configuration as YAML or JSON from stdin or file + + Options: + -v, --verbose Increase verbosity. + -s, --secrets Show secret attributes in messages. + -q, --quiet Quiet mode - only show errors. + -c, --color Force colorized output. + -u, --update Update mode - merge input with existing config. + -n, --dry-run Perform a trial run with no changes made. + -?, -h, --help Show this message and exit. The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec instead: .. code-block:: bash - docker exec -i $(docker-compose ps -q admin) flask mailu config-update -nvd < mail-config.yml + docker exec -i $(docker-compose ps -q admin) flask mailu config-import -nv < mail-config.yml - -mail-config.yml looks like this: +mail-config.yml contains the configuration and looks like this: .. code-block:: yaml - - domains: + + domain: - name: example.com alternatives: - alternative.example.com - users: + user: - email: foo@example.com - password_hash: klkjhumnzxcjkajahsdqweqqwr + password_hash: '$2b$12$...' hash_scheme: MD5-CRYPT - aliases: + alias: - email: alias1@example.com - destination: "user1@example.com,user2@example.com" + destination: + - user1@example.com + - user2@example.com - relays: + relay: - name: relay.example.com comment: test smtp: mx.example.com -You can use ``--dry-run`` to test your YAML without comitting any changes to the database. -With ``--verbose`` config-update will show exactly what it changes in the database. -Without ``--delete-object`` option config-update will only add/update changed values but will *not* remove any entries missing in provided YAML input. +config-update shows the number of created/modified/deleted objects after import. +To suppress all messages except error messages use ``--quiet``. +By adding the ``--verbose`` switch (one or more times) the import gets more detailed and shows exactyl what attributes changed. +In all messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to show secrets. +If you want to test what would be done when importing use ``--dry-run``. +By default config-update replaces the whole configuration. You can use ``--update`` to change the existing configuration instead. +When updating you can add new and change existing objects. +To delete an object use ``-key: value`` (To delete the domain example.com ``-name: example.com`` for example). +To reset an attribute to default use ``-key: null`` (To reset enable_imap ``-enable_imap: null`` for example). -This is a complete YAML template with all additional parameters that could be defined: +This is a complete YAML template with all additional parameters that can be defined: .. code-block:: yaml - aliases: - - email: email@example.com - comment: '' - destination: - - address@example.com - wildcard: false - - domains: + domain: - name: example.com alternatives: - alternative.tld @@ -176,13 +233,8 @@ This is a complete YAML template with all additional parameters that could be de max_quota_bytes: 0 max_users: -1 signup_enabled: false - - relays: - - name: relay.example.com - comment: '' - smtp: mx.example.com - - users: + + user: - email: postmaster@example.com comment: '' displayed_name: 'Postmaster' @@ -192,13 +244,16 @@ This is a complete YAML template with all additional parameters that could be de fetches: - id: 1 comment: 'test fetch' - username: fetch-user + error: null host: other.example.com + keep: true + last_check: '2020-12-29T17:09:48.200179' password: 'secret' + hash_password: true port: 993 protocol: imap tls: true - keep: true + username: fetch-user forward_destination: - address@remote.example.com forward_enabled: true @@ -206,12 +261,13 @@ This is a complete YAML template with all additional parameters that could be de global_admin: true manager_of: - example.com - password: '{BLF-CRYPT}$2b$12$...' + password: '$2b$12$...' + hash_password: true quota_bytes: 1000000000 reply_body: '' reply_enabled: false - reply_enddate: 2999-12-31 - reply_startdate: 1900-01-01 + reply_enddate: '2999-12-31' + reply_startdate: '1900-01-01' reply_subject: '' spam_enabled: true spam_threshold: 80 @@ -219,5 +275,16 @@ This is a complete YAML template with all additional parameters that could be de - id: 1 comment: email-client ip: 192.168.1.1 - password: '$5$rounds=1000$...' + password: '$5$rounds=1$...' + aliases: + - email: email@example.com + comment: '' + destination: + - address@example.com + wildcard: false + + relay: + - name: relay.example.com + comment: '' + smtp: mx.example.com diff --git a/tests/compose/core/02_forward_test.sh b/tests/compose/core/02_forward_test.sh index 651e027c..595820cf 100755 --- a/tests/compose/core/02_forward_test.sh +++ b/tests/compose/core/02_forward_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -10,7 +10,7 @@ EOF python3 tests/forward_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" diff --git a/tests/compose/core/03_alias_test.sh b/tests/compose/core/03_alias_test.sh index 2d40903a..dce1918a 100755 --- a/tests/compose/core/03_alias_test.sh +++ b/tests/compose/core/03_alias_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 aliases: - localpart: alltheusers domain: mailu.io @@ -7,6 +7,6 @@ EOF python3 tests/alias_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 aliases: [] EOF diff --git a/tests/compose/core/04_reply_test.sh b/tests/compose/core/04_reply_test.sh index 7615a0f8..83c114f6 100755 --- a/tests/compose/core/04_reply_test.sh +++ b/tests/compose/core/04_reply_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -11,7 +11,7 @@ EOF python3 tests/reply_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" diff --git a/towncrier/newsfragments/1604.feature b/towncrier/newsfragments/1604.feature index 06ee0beb..2b47791a 100644 --- a/towncrier/newsfragments/1604.feature +++ b/towncrier/newsfragments/1604.feature @@ -1 +1 @@ -Added cli command config-dump and enhanced config-update +Add cli commands config-import and config-export From 3937986e76f7079eb4488f12164f83f1966cb0ec Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 15 Feb 2021 10:01:35 +0100 Subject: [PATCH 37/52] Convert OrderedDict to dict for output --- core/admin/mailu/manage.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index a8d1d3cb..37d91f33 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -7,7 +7,7 @@ import socket import logging import uuid -from collections import Counter +from collections import Counter, OrderedDict from itertools import chain import click @@ -396,11 +396,19 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal return chain(message, changes) def log(action, target, message=None): + + def od2d(val): + """ converts OrderedDicts to Dict for logging purposes """ + if isinstance(val, OrderedDict): + return {k: od2d(v) for k, v in val.items()} + elif isinstance(val, list): + return [od2d(v) for v in val] + else: + return val + if message is None: - # TODO: convert nested OrderedDict to dict - # see: flask mailu config-import -nvv yaml/dump4.yaml try: - message = dict(logger[target.__class__].dump(target)) + message = od2d(logger[target.__class__].dump(target)) except KeyError: message = target if not isinstance(message, str): From 8929912dea9c061632a3bde53726e0949ad455b9 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 15 Feb 2021 21:56:58 +0100 Subject: [PATCH 38/52] remove OrderedDict - not necessary in python>=3.7 --- core/admin/mailu/manage.py | 24 ++++++------------ core/admin/mailu/schemas.py | 50 +++++++++++++------------------------ 2 files changed, 25 insertions(+), 49 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 37d91f33..a20c7d6d 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -7,7 +7,7 @@ import socket import logging import uuid -from collections import Counter, OrderedDict +from collections import Counter from itertools import chain import click @@ -397,18 +397,9 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal def log(action, target, message=None): - def od2d(val): - """ converts OrderedDicts to Dict for logging purposes """ - if isinstance(val, OrderedDict): - return {k: od2d(v) for k, v in val.items()} - elif isinstance(val, list): - return [od2d(v) for v in val] - else: - return val - if message is None: try: - message = od2d(logger[target.__class__].dump(target)) + message = logger[target.__class__].dump(target) except KeyError: message = target if not isinstance(message, str): @@ -536,12 +527,11 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal except Exception as exc: if verbose >= 5: raise - else: - # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) - raise click.ClickException( - f'[{exc.__class__.__name__}] ' - f'{" ".join(str(exc).split())}' - ) from exc + # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) + raise click.ClickException( + f'[{exc.__class__.__name__}] ' + f'{" ".join(str(exc).split())}' + ) from exc # flush session to show/count all changes if dry_run or verbose >= 1: diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 54a2e928..8e91b4aa 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -2,7 +2,6 @@ """ from copy import deepcopy -from collections import OrderedDict from textwrap import wrap import re @@ -155,11 +154,7 @@ def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): ### render modules ### -# allow yaml module to dump OrderedDict -yaml.add_representer( - OrderedDict, - lambda cls, data: cls.represent_mapping('tag:yaml.org,2002:map', data.items()) -) +# allow yaml to represent hidden attributes yaml.add_representer( _Hidden, lambda cls, data: cls.represent_data(str(data)) @@ -410,8 +405,6 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): def __init__(self, meta, ordered=False): if not hasattr(meta, 'sqla_session'): meta.sqla_session = models.db.session - if not hasattr(meta, 'ordered'): - meta.ordered = True if not hasattr(meta, 'sibling'): meta.sibling = False super(BaseOpts, self).__init__(meta, ordered=ordered) @@ -474,19 +467,23 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): self._hide_by_context |= set(what) # remember primary keys - self._primary = self.opts.model.__table__.primary_key.columns.values()[0].name + self._primary = str(self.opts.model.__table__.primary_key.columns.values()[0].name) - # initialize attribute order + # determine attribute order if hasattr(self.Meta, 'order'): # use user-defined order - self._order = list(reversed(self.Meta.order)) + order = self.Meta.order else: # default order is: primary_key + other keys alphabetically - self._order = list(sorted(self.fields.keys())) - if self._primary in self._order: - self._order.remove(self._primary) - self._order.reverse() - self._order.append(self._primary) + order = list(sorted(self.fields.keys())) + if self._primary in order: + order.remove(self._primary) + order.insert(0, self._primary) + + # order dump_fields + for field in order: + if field in self.dump_fields: + self.dump_fields[field] = self.dump_fields.pop(field) # move pre_load hook "_track_import" to the front hooks = self._hooks[('pre_load', False)] @@ -704,16 +701,9 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): return item @post_dump - def _hide_and_order(self, data, many, **kwargs): # pylint: disable=unused-argument + def _hide_values(self, data, many, **kwargs): # pylint: disable=unused-argument """ hide secrets and order output """ - # order output - for key in self._order: - try: - data.move_to_end(key, False) - except KeyError: - pass - # stop early when not excluding/hiding if not self._exclude_by_value and not self._hide_by_context: return data @@ -870,18 +860,14 @@ class MailuSchema(Schema): """ Schema config """ render_module = RenderYAML - ordered = True order = ['domain', 'user', 'alias', 'relay'] # 'config' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # order fields - for field_list in self.load_fields, self.dump_fields, self.fields: - for section in reversed(self.Meta.order): - try: - field_list.move_to_end(section, False) - except KeyError: - pass + # order dump_fields + for field in self.Meta.order: + if field in self.dump_fields: + self.dump_fields[field] = self.dump_fields.pop(field) def _call_and_store(self, *args, **kwargs): """ track current parent and field for pruning """ From 70a1c79f81d4ceecba42e541947b14a2e9980bff Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 15 Feb 2021 22:57:37 +0100 Subject: [PATCH 39/52] handle prune and delete for lists and backrefs --- core/admin/mailu/manage.py | 13 ++++++++--- core/admin/mailu/schemas.py | 43 ++++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index a20c7d6d..05eae010 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -478,9 +478,16 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal if verbose >= 1: log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') - def track_serialize(obj, item): + def track_serialize(obj, item, backref=None): """ callback function to track import """ - # hide secrets + # called for backref modification? + if backref is not None: + log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref)) + return + # verbose? + if not verbose >= 2: + return + # hide secrets in data data = logger[obj.opts.model].hide(item) if 'hash_password' in data: data['password'] = HIDDEN @@ -501,7 +508,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal 'import': True, 'update': update, 'clear': not update, - 'callback': track_serialize if verbose >= 2 else None, + 'callback': track_serialize, } # register listeners diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 8e91b4aa..3e15ee26 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -14,6 +14,7 @@ from marshmallow import pre_load, post_load, post_dump, fields, Schema from marshmallow.utils import ensure_text_type from marshmallow.exceptions import ValidationError from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts +from marshmallow_sqlalchemy.fields import RelatedList from flask_marshmallow import Marshmallow @@ -39,8 +40,6 @@ ma = Marshmallow() # - when modifying, nothing is required (only the primary key, but this key is in the uri) # - the primary key from post data must not differ from the key in the uri # - when creating all fields without default or auto-increment are required -# TODO: what about deleting list items and prung lists? -# - domain.alternatives, user.forward_destination, user.manager_of, aliases.destination # TODO: validate everything! @@ -652,7 +651,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if '__delete__' in data: # deletion of non-existent item requested raise ValidationError( - f'item not found: {data[self._primary]!r}', + f'item to delete not found: {data[self._primary]!r}', field_name=f'?.{self._primary}', ) @@ -665,6 +664,44 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # delete instance when marked if '__delete__' in data: self.opts.sqla_session.delete(instance) + # delete item from lists or prune lists + # currently: domain.alternatives, user.forward_destination, + # user.manager_of, aliases.destination + for key, value in data.items(): + if isinstance(value, list): + new_value = set(value) + # handle list pruning + if '-prune-' in value: + value.remove('-prune-') + new_value.remove('-prune-') + else: + for old in getattr(instance, key): + # using str() is okay for now (see above) + new_value.add(str(old)) + # handle item deletion + for item in value: + if item.startswith('-'): + new_value.remove(item) + try: + new_value.remove(item[1:]) + except KeyError as exc: + raise ValidationError( + f'item to delete not found: {item[1:]!r}', + field_name=f'?.{key}', + ) from exc + # deduplicate and sort list + data[key] = sorted(new_value) + # log backref modification not catched by hook + if isinstance(self.fields[key], RelatedList): + if callback := self.context.get('callback'): + callback(self, instance, { + 'key': key, + 'target': str(instance), + 'before': [str(v) for v in getattr(instance, key)], + 'after': data[key], + }) + + # add attributes required for validation from db # TODO: this will cause validation errors if value from database does not validate From 1e2b5f26ab9fed0db8994ca54db7f57ce0792ce8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 16 Feb 2021 13:34:02 +0100 Subject: [PATCH 40/52] don't handle nested lists --- core/admin/mailu/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 3e15ee26..b9b8e393 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -668,7 +668,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # currently: domain.alternatives, user.forward_destination, # user.manager_of, aliases.destination for key, value in data.items(): - if isinstance(value, list): + if not isinstance(self.fields[key], fields.Nested) and isinstance(value, list): new_value = set(value) # handle list pruning if '-prune-' in value: From 10435114ec0206e3558734634d906cffbd49e783 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 16 Feb 2021 15:36:01 +0100 Subject: [PATCH 41/52] updated remarks and docs --- core/admin/mailu/manage.py | 25 ++++++++++++--------- core/admin/mailu/models.py | 12 +++++----- core/admin/mailu/schemas.py | 3 ++- docs/cli.rst | 45 ++++++++++++++++++++++++------------- 4 files changed, 51 insertions(+), 34 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 05eae010..756400ad 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -337,16 +337,19 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal """ # verbose - # 0 : show number of changes - # 1 : also show changes - # 2 : also show secrets - # 3 : also show input data - # 4 : also show sql queries - # 5 : also show tracebacks + # 0 : only show number of changes + # 1 : also show detailed changes + # 2 : also show input data + # 3 : also show sql queries (also needs -s, as sql may contain secrets) + # 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets) if quiet: verbose = -1 + if verbose > 2 and not secrets: + print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.') + verbose = 2 + color_cfg = { 'color': color or sys.stdout.isatty(), 'lexer': 'python', @@ -376,7 +379,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}' res = [fmt.format(loc, msg) for loc, msg in res] num = f'error{["s",""][len(res)==1]}' - res.insert(0, f'[ValidationError] {len(res)} {num} occured during input validation') + res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation') return '\n'.join(res) @@ -484,7 +487,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal if backref is not None: log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref)) return - # verbose? + # show input data? if not verbose >= 2: return # hide secrets in data @@ -532,7 +535,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal except ValidationError as exc: raise click.ClickException(format_errors(exc.messages)) from exc except Exception as exc: - if verbose >= 5: + if verbose >= 3: raise # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) raise click.ClickException( @@ -584,7 +587,7 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None if only: for spec in only: if spec.split('.', 1)[0] not in MailuSchema.Meta.order: - raise click.ClickException(f'[ERROR] Unknown section: {spec}') + raise click.ClickException(f'[ValidationError] Unknown section: {spec}') else: only = MailuSchema.Meta.order @@ -606,7 +609,7 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) except ValueError as exc: if spec := get_fieldspec(exc): - raise click.ClickException(f'[ERROR] Invalid filter: {spec}') from exc + raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc raise diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 5799e282..4c119984 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -33,7 +33,7 @@ class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: String(80) is too small? + # TODO: use db.String(255)? impl = db.String(80) def process_bind_param(self, value, dialect): @@ -50,7 +50,7 @@ class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: String(255) is too small? + # TODO: use db.String(254)? impl = db.String(255) def process_bind_param(self, value, dialect): @@ -314,7 +314,7 @@ class Relay(Base): __tablename__ = 'relay' name = db.Column(IdnaDomain, primary_key=True, nullable=False) - # TODO: String(80) is too small? + # TODO: use db.String(266)? transport(8):(1)[nexthop(255)](2) smtp = db.Column(db.String(80), nullable=True) @@ -322,9 +322,7 @@ class Email(object): """ Abstraction for an email address (localpart and domain). """ - # TODO: validate max. total length of address (<=254) - - # TODO: String(80) is too large (64)? + # TODO: use db.String(64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr @@ -634,7 +632,7 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - # TODO: String(255) is too large? (43 should be sufficient) + # TODO: use db.String(32)? ip = db.Column(db.String(255)) def check_password(self, password): diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index b9b8e393..7d0393f0 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -36,7 +36,7 @@ from . import models, dkim ma = Marshmallow() -# TODO: how and where to mark keys as "required" while unserializing in api? +# TODO: how and where to mark keys as "required" while deserializing in api? # - when modifying, nothing is required (only the primary key, but this key is in the uri) # - the primary key from post data must not differ from the key in the uri # - when creating all fields without default or auto-increment are required @@ -705,6 +705,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # add attributes required for validation from db # TODO: this will cause validation errors if value from database does not validate + # but there should not be an invalid value in the database for attr_name, field_obj in self.load_fields.items(): if field_obj.required and attr_name not in data: data[attr_name] = getattr(instance, attr_name) diff --git a/docs/cli.rst b/docs/cli.rst index 497cdfc5..6d48c576 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -97,7 +97,7 @@ where mail-config.yml looks like: without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. Users ------ +^^^^^ following are additional parameters that could be defined for users: @@ -116,7 +116,7 @@ following are additional parameters that could be defined for users: * spam_threshold Alias ------ +^^^^^ additional fields: @@ -125,11 +125,11 @@ additional fields: config-export ------------- -The purpose of this command is to export domain-, relay-, alias- and user-configuration in YAML or JSON format. +The purpose of this command is to export the complete configuration in YAML or JSON format. .. code-block:: bash - # docker-compose exec admin flask mailu config-export --help + $ docker-compose exec admin flask mailu config-export --help Usage: flask mailu config-export [OPTIONS] [FILTER]... @@ -152,18 +152,18 @@ filters to export only some objects or attributes (try: ``user`` or ``domain.nam .. code-block:: bash - docker-compose exec admin flask mailu config-export -o mail-config.yml + $ docker-compose exec admin flask mailu config-export -o mail-config.yml - docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf + $ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf config-import ------------- -The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML/JOSN source. +This command imports configuration data from an external YAML or JSON source. .. code-block:: bash - # docker-compose exec admin flask mailu config-import --help + $ docker-compose exec admin flask mailu config-import --help Usage: flask mailu config-import [OPTIONS] [FILENAME|-] @@ -211,13 +211,28 @@ mail-config.yml contains the configuration and looks like this: config-update shows the number of created/modified/deleted objects after import. To suppress all messages except error messages use ``--quiet``. -By adding the ``--verbose`` switch (one or more times) the import gets more detailed and shows exactyl what attributes changed. -In all messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to show secrets. -If you want to test what would be done when importing use ``--dry-run``. -By default config-update replaces the whole configuration. You can use ``--update`` to change the existing configuration instead. -When updating you can add new and change existing objects. -To delete an object use ``-key: value`` (To delete the domain example.com ``-name: example.com`` for example). -To reset an attribute to default use ``-key: null`` (To reset enable_imap ``-enable_imap: null`` for example). +By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed. +In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. +If you want to test what would be done when importing without committing any changes, use ``--dry-run``. + +By default config-update replaces the whole configuration. ``--update`` allows to modify the existing configuration instead. +New elements will be added and existing elements will be modified. +It is possible to delete a single element or prune all elements from lists and associative arrays using a special notation: + ++-----------------------------+------------------+--------------------------+ +| Delete what? | notation | example | ++=============================+==================+==========================+ +| specific array object | ``- -key: id`` | ``- -name: example.com`` | ++-----------------------------+------------------+--------------------------+ +| specific list item | ``- -id`` | ``- -user1@example.com`` | ++-----------------------------+------------------+--------------------------+ +| all remaining array objects | ``- -key: null`` | ``- -email: null`` | ++-----------------------------+------------------+--------------------------+ +| all remaining list items | ``- -prune-`` | ``- -prune-`` | ++-----------------------------+------------------+--------------------------+ + +The ``-key: null`` notation can also be used to reset an attribute to its default. +To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``. This is a complete YAML template with all additional parameters that can be defined: From e4c83e162dfa37189b8eac506b07ec6f78a5b5d0 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 16 Feb 2021 17:59:43 +0100 Subject: [PATCH 42/52] fixed colorize auto detection --- core/admin/mailu/manage.py | 6 +++--- core/admin/mailu/schemas.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 756400ad..bef49faa 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -19,7 +19,7 @@ from flask.cli import FlaskGroup, with_appcontext from marshmallow.exceptions import ValidationError from . import models -from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, RenderJSON, HIDDEN +from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, canColorize, RenderJSON, HIDDEN db = models.db @@ -351,7 +351,7 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal verbose = 2 color_cfg = { - 'color': color or sys.stdout.isatty(), + 'color': color or (canColorize and sys.stdout.isatty()), 'lexer': 'python', 'strip': True, } @@ -598,7 +598,7 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None } schema = MailuSchema(only=only, context=context) - color_cfg = {'color': color or output.isatty()} + color_cfg = {'color': color or (canColorize and output.isatty())} if as_json: schema.opts.render_module = RenderJSON diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 7d0393f0..6a8303c5 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -27,9 +27,9 @@ try: from pygments.lexers.data import YamlLexer from pygments.formatters import get_formatter_by_name except ModuleNotFoundError: - COLOR_SUPPORTED = False + canColorize = False else: - COLOR_SUPPORTED = True + canColorize = True from . import models, dkim @@ -99,11 +99,11 @@ def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): """ add ANSI color to data """ if color is None: # autodetect colorize - color = COLOR_SUPPORTED + color = canColorize if not color: # no color wanted return data - if not COLOR_SUPPORTED: + if not canColorize: # want color, but not supported raise ValueError('Please install pygments to colorize output') From bde7a2b6c4a8a2351b461cee9be413d7683e95dd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 19 Feb 2021 18:01:02 +0100 Subject: [PATCH 43/52] moved import logging to schema - yaml-import is now logged via schema.Logger - iremoved relative imports - not used in other mailu modules - removed develepment comments - added Mailconfig.check method to check for duplicate domain names - converted .format() to .format_map() where possible - switched to yaml multiline dump for dkim_key - converted dkim_key import from regex to string functions - automatically unhide/unexclude explicitly specified attributes on dump - use field order when loading to stabilize import - fail when using 'hash_password' without 'password' - fixed logging of dkim_key - fixed pruning and deleting of lists - modified error messages - added debug flag and two verbosity levels --- core/admin/mailu/manage.py | 280 ++--------- core/admin/mailu/models.py | 40 +- core/admin/mailu/schemas.py | 922 ++++++++++++++++++++++++------------ docs/cli.rst | 15 +- 4 files changed, 688 insertions(+), 569 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index bef49faa..f9add0f4 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -4,22 +4,16 @@ import sys import os import socket -import logging import uuid -from collections import Counter -from itertools import chain - import click -import sqlalchemy import yaml from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext -from marshmallow.exceptions import ValidationError -from . import models -from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, canColorize, RenderJSON, HIDDEN +from mailu import models +from mailu.schemas import MailuSchema, Logger, RenderJSON db = models.db @@ -326,246 +320,53 @@ def config_update(verbose=False, delete_objects=False): @mailu.command() @click.option('-v', '--verbose', count=True, help='Increase verbosity.') @click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.') +@click.option('-d', '--debug', is_flag=True, help='Enable debug output.') @click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.') @click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.') @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None): +def config_import(verbose=0, secrets=False, debug=False, quiet=False, color=False, + update=False, dry_run=False, source=None): """ Import configuration as YAML or JSON from stdin or file """ - # verbose - # 0 : only show number of changes - # 1 : also show detailed changes - # 2 : also show input data - # 3 : also show sql queries (also needs -s, as sql may contain secrets) - # 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets) + log = Logger(want_color=color or None, can_color=sys.stdout.isatty(), secrets=secrets, debug=debug) + log.lexer = 'python' + log.strip = True + log.verbose = 0 if quiet else verbose + log.quiet = quiet - if quiet: - verbose = -1 - - if verbose > 2 and not secrets: - print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.') - verbose = 2 - - color_cfg = { - 'color': color or (canColorize and sys.stdout.isatty()), - 'lexer': 'python', - 'strip': True, - } - - counter = Counter() - logger = {} - - def format_errors(store, path=None): - - res = [] - if path is None: - path = [] - for key in sorted(store): - location = path + [str(key)] - value = store[key] - if isinstance(value, dict): - res.extend(format_errors(value, location)) - else: - for message in value: - res.append((".".join(location), message)) - - if path: - return res - - fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}' - res = [fmt.format(loc, msg) for loc, msg in res] - num = f'error{["s",""][len(res)==1]}' - res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation') - - return '\n'.join(res) - - def format_changes(*message): - if counter: - changes = [] - last = None - for (action, what), count in sorted(counter.items()): - if action != last: - if last: - changes.append('/') - changes.append(f'{action}:') - last = action - changes.append(f'{what}({count})') - else: - changes = ['No changes.'] - return chain(message, changes) - - def log(action, target, message=None): - - if message is None: - try: - message = logger[target.__class__].dump(target) - except KeyError: - message = target - if not isinstance(message, str): - message = repr(message) - print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}') - - def listen_insert(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - counter.update([('Created', target.__table__.name)]) - if verbose >= 1: - log('Created', target) - - def listen_update(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - - changed = {} - inspection = sqlalchemy.inspect(target) - for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: - history = getattr(inspection.attrs, attr.key).history - if history.has_changes() and history.deleted: - before = history.deleted[-1] - after = getattr(target, attr.key) - # TODO: remove special handling of "comment" after modifying model - if attr.key == 'comment' and not before and not after: - pass - # only remember changed keys - elif before != after: - if verbose >= 1: - changed[str(attr.key)] = (before, after) - else: - break - - if verbose >= 1: - # use schema with dump_context to hide secrets and sort keys - dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target) - for key, value in dumped.items(): - before, after = changed[key] - if value == HIDDEN: - before = HIDDEN if before else before - after = HIDDEN if after else after - else: - # TODO: need to use schema to "convert" before value? - after = value - log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') - - if changed: - counter.update([('Modified', target.__table__.name)]) - - def listen_delete(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - counter.update([('Deleted', target.__table__.name)]) - if verbose >= 1: - log('Deleted', target) - - # TODO: this listener will not be necessary, if dkim keys would be stored in database - _dedupe_dkim = set() - def listen_dkim(session, flush_context): # pylint: disable=unused-argument - """ callback function to track import """ - for target in session.identity_map.values(): - # look at Domains originally loaded from db - if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: - continue - before = target._dkim_key_on_disk - after = target._dkim_key - if before != after: - if secrets: - before = before.decode('ascii', 'ignore') - after = after.decode('ascii', 'ignore') - else: - before = HIDDEN if before else '' - after = HIDDEN if after else '' - # "de-dupe" messages; this event is fired at every flush - if not (target, before, after) in _dedupe_dkim: - _dedupe_dkim.add((target, before, after)) - counter.update([('Modified', target.__table__.name)]) - if verbose >= 1: - log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') - - def track_serialize(obj, item, backref=None): - """ callback function to track import """ - # called for backref modification? - if backref is not None: - log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref)) - return - # show input data? - if not verbose >= 2: - return - # hide secrets in data - data = logger[obj.opts.model].hide(item) - if 'hash_password' in data: - data['password'] = HIDDEN - if 'fetches' in data: - for fetch in data['fetches']: - fetch['password'] = HIDDEN - log('Handling', obj.opts.model, data) - - # configure contexts - diff_context = { - 'full': True, - 'secrets': secrets, - } - log_context = { - 'secrets': secrets, - } - load_context = { + context = { 'import': True, 'update': update, 'clear': not update, - 'callback': track_serialize, + 'callback': log.track_serialize, } - # register listeners - for schema in get_schema(): - model = schema.Meta.model - logger[model] = schema(context=log_context) - sqlalchemy.event.listen(model, 'after_insert', listen_insert) - sqlalchemy.event.listen(model, 'after_update', listen_update) - sqlalchemy.event.listen(model, 'after_delete', listen_delete) - - # special listener for dkim_key changes - sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim) - - if verbose >= 3: - logging.basicConfig() - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + schema = MailuSchema(only=MailuSchema.Meta.order, context=context) try: + # import source with models.db.session.no_autoflush: - config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source) - except ValidationError as exc: - raise click.ClickException(format_errors(exc.messages)) from exc + config = schema.loads(source) + # flush session to show/count all changes + if not quiet and (dry_run or verbose): + db.session.flush() + # check for duplicate domain names + config.check() except Exception as exc: - if verbose >= 3: - raise - # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) - raise click.ClickException( - f'[{exc.__class__.__name__}] ' - f'{" ".join(str(exc).split())}' - ) from exc - - # flush session to show/count all changes - if dry_run or verbose >= 1: - db.session.flush() - - # check for duplicate domain names - dup = set() - for fqdn in chain( - db.session.query(models.Domain.name), - db.session.query(models.Alternative.name), - db.session.query(models.Relay.name) - ): - if fqdn in dup: - raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}') - dup.add(fqdn) + if msg := log.format_exception(exc): + raise click.ClickException(msg) from exc + raise # don't commit when running dry if dry_run: - if not quiet: - print(*format_changes('Dry run. Not commiting changes.')) + log.changes('Dry run. Not committing changes.') db.session.rollback() else: - if not quiet: - print(*format_changes('Committing changes.')) + log.changes('Committing changes.') db.session.commit() @@ -573,8 +374,8 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal @click.option('-f', '--full', is_flag=True, help='Include attributes with default value.') @click.option('-s', '--secrets', is_flag=True, help='Include secret attributes (dkim-key, passwords).') -@click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-d', '--dns', is_flag=True, help='Include dns records.') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), help='Save configuration to file.') @click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.') @@ -584,32 +385,25 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None """ Export configuration as YAML or JSON to stdout or file """ - if only: - for spec in only: - if spec.split('.', 1)[0] not in MailuSchema.Meta.order: - raise click.ClickException(f'[ValidationError] Unknown section: {spec}') - else: - only = MailuSchema.Meta.order + only = only or MailuSchema.Meta.order context = { 'full': full, 'secrets': secrets, 'dns': dns, } - - schema = MailuSchema(only=only, context=context) - color_cfg = {'color': color or (canColorize and output.isatty())} - - if as_json: - schema.opts.render_module = RenderJSON - color_cfg['lexer'] = 'json' - color_cfg['strip'] = True + log = Logger(want_color=color or None, can_color=output.isatty()) try: - print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) - except ValueError as exc: - if spec := get_fieldspec(exc): - raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc + schema = MailuSchema(only=only, context=context) + if as_json: + schema.opts.render_module = RenderJSON + log.lexer = 'json' + log.strip = True + print(log.colorize(schema.dumps(models.MailuConfig())), file=output) + except Exception as exc: + if msg := log.format_exception(exc): + raise click.ClickException(msg) from exc raise diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 4c119984..1b3c787a 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -23,7 +23,7 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.inspection import inspect from werkzeug.utils import cached_property -from . import dkim +from mailu import dkim db = flask_sqlalchemy.SQLAlchemy() @@ -33,7 +33,6 @@ class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: use db.String(255)? impl = db.String(80) def process_bind_param(self, value, dialect): @@ -50,7 +49,6 @@ class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: use db.String(254)? impl = db.String(255) def process_bind_param(self, value, dialect): @@ -127,11 +125,7 @@ class Base(db.Model): if pkey == 'email': # ugly hack for email declared attr. _email is not always up2date return str(f'{self.localpart}@{self.domain_name}') - elif pkey in {'name', 'email'}: - return str(getattr(self, pkey, None)) - else: - return self.__repr__() - return str(getattr(self, self.__table__.primary_key.columns.values()[0].name)) + return str(getattr(self, pkey)) def __repr__(self): return f'<{self.__class__.__name__} {str(self)!r}>' @@ -145,12 +139,15 @@ class Base(db.Model): else: return NotImplemented + # we need hashable instances here for sqlalchemy to update collections + # in collections.bulk_replace, but auto-incrementing don't always have + # a valid primary key, in this case we use the object's id + __hashed = None def __hash__(self): - primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) - if primary is None: - return NotImplemented - else: - return hash(primary) + if self.__hashed is None: + primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) + self.__hashed = id(self) if primary is None else hash(primary) + return self.__hashed # Many-to-many association table for domain managers @@ -314,7 +311,6 @@ class Relay(Base): __tablename__ = 'relay' name = db.Column(IdnaDomain, primary_key=True, nullable=False) - # TODO: use db.String(266)? transport(8):(1)[nexthop(255)](2) smtp = db.Column(db.String(80), nullable=True) @@ -322,7 +318,6 @@ class Email(object): """ Abstraction for an email address (localpart and domain). """ - # TODO: use db.String(64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr @@ -342,7 +337,7 @@ class Email(object): key = f'{cls.__tablename__}_email' if key in ctx.current_parameters: return ctx.current_parameters[key] - return '{localpart}@{domain_name}'.format(**ctx.current_parameters) + return '{localpart}@{domain_name}'.format_map(ctx.current_parameters) return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater) @@ -632,7 +627,6 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - # TODO: use db.String(32)? ip = db.Column(db.String(255)) def check_password(self, password): @@ -865,6 +859,18 @@ class MailuConfig: if models is None or model in models: db.session.query(model).delete() + def check(self): + """ check for duplicate domain names """ + dup = set() + for fqdn in chain( + db.session.query(Domain.name), + db.session.query(Alternative.name), + db.session.query(Relay.name) + ): + if fqdn in dup: + raise ValueError(f'Duplicate domain name: {fqdn}') + dup.add(fqdn) + domain = MailuCollection(Domain) user = MailuCollection(User) alias = MailuCollection(Alias) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 6a8303c5..4c5042ea 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -2,10 +2,10 @@ """ from copy import deepcopy -from textwrap import wrap +from collections import Counter -import re import json +import logging import yaml import sqlalchemy @@ -27,24 +27,309 @@ try: from pygments.lexers.data import YamlLexer from pygments.formatters import get_formatter_by_name except ModuleNotFoundError: - canColorize = False + COLOR_SUPPORTED = False else: - canColorize = True + COLOR_SUPPORTED = True -from . import models, dkim +from mailu import models, dkim ma = Marshmallow() -# TODO: how and where to mark keys as "required" while deserializing in api? -# - when modifying, nothing is required (only the primary key, but this key is in the uri) -# - the primary key from post data must not differ from the key in the uri -# - when creating all fields without default or auto-increment are required -# TODO: validate everything! + +### import logging and schema colorization ### + +_model2schema = {} + +def get_schema(cls=None): + """ return schema class for model """ + if cls is None: + return _model2schema.values() + return _model2schema.get(cls) + +def mapped(cls): + """ register schema in model2schema map """ + _model2schema[cls.Meta.model] = cls + return cls + +class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + +class Logger: + + def __init__(self, want_color=None, can_color=False, debug=False, secrets=False): + + self.lexer = 'yaml' + self.formatter = 'terminal' + self.strip = False + self.verbose = 0 + self.quiet = False + self.secrets = secrets + self.debug = debug + self.print = print + + if want_color and not COLOR_SUPPORTED: + raise ValueError('Please install pygments to colorize output') + + self.color = want_color or (can_color and COLOR_SUPPORTED) + + self._counter = Counter() + self._schemas = {} + + # log contexts + self._diff_context = { + 'full': True, + 'secrets': secrets, + } + log_context = { + 'secrets': secrets, + } + + # register listeners + for schema in get_schema(): + model = schema.Meta.model + self._schemas[model] = schema(context=log_context) + sqlalchemy.event.listen(model, 'after_insert', self._listen_insert) + sqlalchemy.event.listen(model, 'after_update', self._listen_update) + sqlalchemy.event.listen(model, 'after_delete', self._listen_delete) + + # special listener for dkim_key changes + # TODO: _listen_dkim can be removed when dkim keys are stored in database + self._dedupe_dkim = set() + sqlalchemy.event.listen(models.db.session, 'after_flush', self._listen_dkim) + + # register debug logger for sqlalchemy + if self.debug: + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + def _log(self, action, target, message=None): + if message is None: + try: + message = self._schemas[target.__class__].dump(target) + except KeyError: + message = target + if not isinstance(message, str): + message = repr(message) + self.print(f'{action} {target.__table__}: {self.colorize(message)}') + + def _listen_insert(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + self._counter.update([('Created', target.__table__.name)]) + if self.verbose: + self._log('Created', target) + + def _listen_update(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + + changes = {} + inspection = sqlalchemy.inspect(target) + for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: + history = getattr(inspection.attrs, attr.key).history + if history.has_changes() and history.deleted: + before = history.deleted[-1] + after = getattr(target, attr.key) + # TODO: this can be removed when comment is not nullable in model + if attr.key == 'comment' and not before and not after: + pass + # only remember changed keys + elif before != after: + if self.verbose: + changes[str(attr.key)] = (before, after) + else: + break + + if self.verbose: + # use schema to log changed attributes + schema = get_schema(target.__class__) + only = set(changes.keys()) & set(schema().fields.keys()) + if only: + for key, value in schema( + only=only, + context=self._diff_context + ).dump(target).items(): + before, after = changes[key] + if value == HIDDEN: + before = HIDDEN if before else before + after = HIDDEN if after else after + else: + # also hide this + after = value + self._log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') + + if changes: + self._counter.update([('Modified', target.__table__.name)]) + + def _listen_delete(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + self._counter.update([('Deleted', target.__table__.name)]) + if self.verbose: + self._log('Deleted', target) + + # TODO: _listen_dkim can be removed when dkim keys are stored in database + def _listen_dkim(self, session, flush_context): # pylint: disable=unused-argument + """ callback method to track import """ + for target in session.identity_map.values(): + # look at Domains originally loaded from db + if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: + continue + before = target._dkim_key_on_disk + after = target._dkim_key + if before != after: + # "de-dupe" messages; this event is fired at every flush + if not (target, before, after) in self._dedupe_dkim: + self._dedupe_dkim.add((target, before, after)) + self._counter.update([('Modified', target.__table__.name)]) + if self.verbose: + if self.secrets: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = HIDDEN if before else '' + after = HIDDEN if after else '' + self._log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') + + def track_serialize(self, obj, item, backref=None): + """ callback method to track import """ + # called for backref modification? + if backref is not None: + self._log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format_map(backref)) + return + # show input data? + if self.verbose < 2: + return + # hide secrets in data + if not self.secrets: + item = self._schemas[obj.opts.model].hide(item) + if 'hash_password' in item: + item['password'] = HIDDEN + if 'fetches' in item: + for fetch in item['fetches']: + fetch['password'] = HIDDEN + self._log('Handling', obj.opts.model, item) + + def changes(self, *messages, **kwargs): + """ show changes gathered in counter """ + if self.quiet: + return + if self._counter: + changes = [] + last = None + for (action, what), count in sorted(self._counter.items()): + if action != last: + if last: + changes.append('/') + changes.append(f'{action}:') + last = action + changes.append(f'{what}({count})') + else: + changes = ['No changes.'] + self.print(*messages, *changes, **kwargs) + + def _format_errors(self, store, path=None): + + res = [] + if path is None: + path = [] + for key in sorted(store): + location = path + [str(key)] + value = store[key] + if isinstance(value, dict): + res.extend(self._format_errors(value, location)) + else: + for message in value: + res.append((".".join(location), message)) + + if path: + return res + + maxlen = max([len(loc) for loc, msg in res]) + res = [f' - {loc.ljust(maxlen)} : {msg}' for loc, msg in res] + errors = f'{len(res)} error{["s",""][len(res)==1]}' + res.insert(0, f'[ValidationError] {errors} occurred during input validation') + + return '\n'.join(res) + + def _is_validation_error(self, exc): + """ walk traceback to extract invalid field from marshmallow """ + path = [] + trace = exc.__traceback__ + while trace: + if trace.tb_frame.f_code.co_name == '_serialize': + if 'attr' in trace.tb_frame.f_locals: + path.append(trace.tb_frame.f_locals['attr']) + elif trace.tb_frame.f_code.co_name == '_init_fields': + spec = ', '.join(['.'.join(path + [key]) for key in trace.tb_frame.f_locals['invalid_fields']]) + return f'Invalid filter: {spec}' + trace = trace.tb_next + return None + + def format_exception(self, exc): + """ format ValidationErrors and other exceptions when not debugging """ + if isinstance(exc, ValidationError): + return self._format_errors(exc.messages) + if isinstance(exc, ValueError): + if msg := self._is_validation_error(exc): + return msg + if self.debug: + return None + msg = ' '.join(str(exc).split()) + return f'[{exc.__class__.__name__}] {msg}' + + colorscheme = { + Token: ('', ''), + Token.Name.Tag: ('cyan', 'cyan'), + Token.Literal.Scalar: ('green', 'green'), + Token.Literal.String: ('green', 'green'), + Token.Name.Constant: ('green', 'green'), # multiline strings + Token.Keyword.Constant: ('magenta', 'magenta'), + Token.Literal.Number: ('magenta', 'magenta'), + Token.Error: ('red', 'red'), + Token.Name: ('red', 'red'), + Token.Operator: ('red', 'red'), + } + + def colorize(self, data, lexer=None, formatter=None, color=None, strip=None): + """ add ANSI color to data """ + + if color is False or not self.color: + return data + + lexer = lexer or self.lexer + lexer = MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer) + formatter = get_formatter_by_name(formatter or self.formatter, colorscheme=self.colorscheme) + if strip is None: + strip = self.strip + + res = highlight(data, lexer, formatter) + if strip: + return res.rstrip('\n') + return res -### class for hidden values ### +### marshmallow render modules ### +# hidden attributes class _Hidden: def __bool__(self): return False @@ -58,107 +343,24 @@ class _Hidden: return '' __str__ = __repr__ -HIDDEN = _Hidden() - - -### map model to schema ### - -_model2schema = {} - -def get_schema(model=None): - """ return schema class for model or instance of model """ - if model is None: - return _model2schema.values() - else: - return _model2schema.get(model) or _model2schema.get(model.__class__) - -def mapped(cls): - """ register schema in model2schema map """ - _model2schema[cls.Meta.model] = cls - return cls - - -### helper functions ### - -def get_fieldspec(exc): - """ walk traceback to extract spec of invalid field from marshmallow """ - path = [] - tbck = exc.__traceback__ - while tbck: - if tbck.tb_frame.f_code.co_name == '_serialize': - if 'attr' in tbck.tb_frame.f_locals: - path.append(tbck.tb_frame.f_locals['attr']) - elif tbck.tb_frame.f_code.co_name == '_init_fields': - path = '.'.join(path) - spec = ', '.join([f'{path}.{key}' for key in tbck.tb_frame.f_locals['invalid_fields']]) - return spec - tbck = tbck.tb_next - return None - -def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): - """ add ANSI color to data """ - if color is None: - # autodetect colorize - color = canColorize - if not color: - # no color wanted - return data - if not canColorize: - # want color, but not supported - raise ValueError('Please install pygments to colorize output') - - scheme = { - Token: ('', ''), - Token.Name.Tag: ('cyan', 'brightcyan'), - Token.Literal.Scalar: ('green', 'green'), - Token.Literal.String: ('green', 'green'), - Token.Keyword.Constant: ('magenta', 'brightmagenta'), - Token.Literal.Number: ('magenta', 'brightmagenta'), - Token.Error: ('red', 'brightred'), - Token.Name: ('red', 'brightred'), - Token.Operator: ('red', 'brightred'), - } - - class MyYamlLexer(YamlLexer): - """ colorize yaml constants and integers """ - def get_tokens(self, text, unfiltered=False): - for typ, value in super().get_tokens(text, unfiltered): - if typ is Token.Literal.Scalar.Plain: - if value in {'true', 'false', 'null'}: - typ = Token.Keyword.Constant - elif value == HIDDEN: - typ = Token.Error - else: - try: - int(value, 10) - except ValueError: - try: - float(value) - except ValueError: - pass - else: - typ = Token.Literal.Number.Float - else: - typ = Token.Literal.Number.Integer - yield typ, value - - res = highlight( - data, - MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer), - get_formatter_by_name(formatter, colorscheme=scheme) - ) - - return res.rstrip('\n') if strip else res - - -### render modules ### - -# allow yaml to represent hidden attributes yaml.add_representer( _Hidden, - lambda cls, data: cls.represent_data(str(data)) + lambda dumper, data: dumper.represent_data(str(data)) ) +HIDDEN = _Hidden() + +# multiline attributes +class _Multiline(str): + pass + +yaml.add_representer( + _Multiline, + lambda dumper, data: dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') + +) + +# yaml render module class RenderYAML: """ Marshmallow YAML Render Module """ @@ -178,7 +380,7 @@ class RenderYAML: @staticmethod def _augment(kwargs, defaults): - """ add default kv's to kwargs if missing + """ add defaults to kwargs if missing """ for key, value in defaults.items(): if key not in kwargs: @@ -205,6 +407,7 @@ class RenderYAML: cls._augment(kwargs, cls._dump_defaults) return yaml.dump(*args, **kwargs) +# json encoder class JSONEncoder(json.JSONEncoder): """ JSONEncoder supporting serialization of HIDDEN """ def default(self, o): @@ -213,13 +416,14 @@ class JSONEncoder(json.JSONEncoder): return str(o) return json.JSONEncoder.default(self, o) +# json render module class RenderJSON: """ Marshmallow JSON Render Module """ @staticmethod def _augment(kwargs, defaults): - """ add default kv's to kwargs if missing + """ add defaults to kwargs if missing """ for key, value in defaults.items(): if key not in kwargs: @@ -245,7 +449,7 @@ class RenderJSON: return json.dumps(*args, **kwargs) -### custom fields ### +### schema: custom fields ### class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string @@ -261,6 +465,11 @@ class CommaSeparatedListField(fields.Raw): a list of strings """ + default_error_messages = { + "invalid": "Not a valid string or list.", + "invalid_utf8": "Not a valid utf-8 string or list.", + } + def _deserialize(self, value, attr, data, **kwargs): """ deserialize comma separated string to list of strings """ @@ -269,16 +478,31 @@ class CommaSeparatedListField(fields.Raw): if not value: return [] - # split string - if isinstance(value, str): - return list([item.strip() for item in value.split(',') if item.strip()]) + # handle list + if isinstance(value, list): + try: + value = [ensure_text_type(item) for item in value] + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + + # handle text else: - return value + if not isinstance(value, (str, bytes)): + raise self.make_error("invalid") + try: + value = ensure_text_type(value) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + else: + value = filter(None, [item.strip() for item in value.split(',')]) + + return list(value) class DkimKeyField(fields.String): - """ Serialize a dkim key to a list of strings (lines) and - Deserialize a string or list of strings to a valid dkim key + """ Serialize a dkim key to a multiline string and + deserialize a dkim key data as string or list of strings + to a valid dkim key """ default_error_messages = { @@ -286,21 +510,26 @@ class DkimKeyField(fields.String): "invalid_utf8": "Not a valid utf-8 string or list.", } - _clean_re = re.compile( - r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', - flags=re.UNICODE - ) - def _serialize(self, value, attr, obj, **kwargs): - """ serialize dkim key to a list of strings (lines) + """ serialize dkim key as multiline string """ # map empty string and None to None if not value: - return None + return '' - # return list of key lines without header/footer - return value.decode('utf-8').strip().split('\n')[1:-1] + # return multiline string + return _Multiline(value.decode('utf-8')) + + def _wrap_key(self, begin, data, end): + """ generator to wrap key into RFC 7468 format """ + yield begin + pos = 0 + while pos < len(data): + yield data[pos:pos+64] + pos += 64 + yield end + yield '' def _deserialize(self, value, attr, data, **kwargs): """ deserialize a string or list of strings to dkim key data @@ -310,7 +539,7 @@ class DkimKeyField(fields.String): # convert list to str if isinstance(value, list): try: - value = ''.join([ensure_text_type(item) for item in value]) + value = ''.join([ensure_text_type(item) for item in value]).strip() except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc @@ -319,34 +548,53 @@ class DkimKeyField(fields.String): if not isinstance(value, (str, bytes)): raise self.make_error("invalid") try: - value = ensure_text_type(value) + value = ensure_text_type(value).strip() except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc - # clean value (remove whitespace and header/footer) - value = self._clean_re.sub('', value.strip()) + # generate new key? + if value.lower() == '-generate-': + return dkim.gen_key() - # map empty string/list to None + # no key? if not value: return None - # handle special value 'generate' - elif value == 'generate': - return dkim.gen_key() + # remember part of value for ValidationError + bad_key = value - # remember some keydata for error message - keydata = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value + # strip header and footer, clean whitespace and wrap to 64 characters + try: + if value.startswith('-----BEGIN '): + end = value.index('-----', 11) + 5 + header = value[:end] + value = value[end:] + else: + header = '-----BEGIN PRIVATE KEY-----' - # wrap value into valid pem layout and check validity - value = ( - '-----BEGIN PRIVATE KEY-----\n' + - '\n'.join(wrap(value, 64)) + - '\n-----END PRIVATE KEY-----\n' - ).encode('ascii') + if (pos := value.find('-----END ')) >= 0: + end = value.index('-----', pos+9) + 5 + footer = value[pos:end] + value = value[:pos] + else: + footer = '-----END PRIVATE KEY-----' + except ValueError: + raise ValidationError(f'invalid dkim key {bad_key!r}') from exc + + # remove whitespace from key data + value = ''.join(value.split()) + + # remember part of value for ValidationError + bad_key = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value + + # wrap key according to RFC 7468 + value = ('\n'.join(self._wrap_key(header, value, footer))).encode('ascii') + + # check key validity try: crypto.load_privatekey(crypto.FILETYPE_PEM, value) except crypto.Error as exc: - raise ValidationError(f'invalid dkim key {keydata!r}') from exc + raise ValidationError(f'invalid dkim key {bad_key!r}') from exc else: return value @@ -398,6 +646,27 @@ class PasswordField(fields.Str): ### base schema ### +class Storage: + """ Storage class to save information in context + """ + + context = {} + + def _bind(self, key, bind): + if bind is True: + return (self.__class__, key) + if isinstance(bind, str): + return (get_schema(self.recall(bind).__class__), key) + return (bind, key) + + def store(self, key, value, bind=None): + """ store value under key """ + self.context.setdefault('_track', {})[self._bind(key, bind)]= value + + def recall(self, key, bind=None): + """ recall value from key """ + return self.context['_track'][self._bind(key, bind)] + class BaseOpts(SQLAlchemyAutoSchemaOpts): """ Option class with sqla session """ @@ -408,7 +677,7 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): meta.sibling = False super(BaseOpts, self).__init__(meta, ordered=ordered) -class BaseSchema(ma.SQLAlchemyAutoSchema): +class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): """ Marshmallow base schema with custom exclude logic and option to hide sqla defaults """ @@ -425,6 +694,9 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): def __init__(self, *args, **kwargs): + # prepare only to auto-include explicitly specified attributes + only = set(kwargs.get('only') or []) + # get context context = kwargs.get('context', {}) flags = {key for key, value in context.items() if value is True} @@ -433,13 +705,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): exclude = set(kwargs.get('exclude', [])) # always exclude - exclude.update({'created_at', 'updated_at'}) + exclude.update({'created_at', 'updated_at'} - only) # add include_by_context if context is not None: for need, what in getattr(self.Meta, 'include_by_context', {}).items(): if not flags & set(need): - exclude |= set(what) + exclude |= what - only # update excludes kwargs['exclude'] = exclude @@ -448,12 +720,15 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): super().__init__(*args, **kwargs) # exclude_by_value - self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) + self._exclude_by_value = { + key: values for key, values in getattr(self.Meta, 'exclude_by_value', {}).items() + if key not in only + } # exclude default values if not context.get('full'): for column in self.opts.model.__table__.columns: - if column.name not in exclude: + if column.name not in exclude and column.name not in only: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg ) @@ -463,7 +738,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if context is not None: for need, what in getattr(self.Meta, 'hide_by_context', {}).items(): if not flags & set(need): - self._hide_by_context |= set(what) + self._hide_by_context |= what - only # remember primary keys self._primary = str(self.opts.model.__table__.primary_key.columns.values()[0].name) @@ -479,20 +754,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): order.remove(self._primary) order.insert(0, self._primary) - # order dump_fields - for field in order: - if field in self.dump_fields: - self.dump_fields[field] = self.dump_fields.pop(field) + # order fieldlists + for fieldlist in (self.fields, self.load_fields, self.dump_fields): + for field in order: + if field in fieldlist: + fieldlist[field] = fieldlist.pop(field) - # move pre_load hook "_track_import" to the front - hooks = self._hooks[('pre_load', False)] - hooks.remove('_track_import') - hooks.insert(0, '_track_import') - # move pre_load hook "_add_instance" to the end - hooks.remove('_add_required') - hooks.append('_add_required') - - # move post_load hook "_add_instance" to the end + # move post_load hook "_add_instance" to the end (after load_instance mixin) hooks = self._hooks[('post_load', False)] hooks.remove('_add_instance') hooks.append('_add_instance') @@ -506,8 +774,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): } def _call_and_store(self, *args, **kwargs): - """ track curent parent field for pruning """ - self.context['parent_field'] = kwargs['field_name'] + """ track current parent field for pruning """ + self.store('field', kwargs['field_name'], True) return super()._call_and_store(*args, **kwargs) # this is only needed to work around the declared attr "email" primary key in model @@ -518,11 +786,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if keys := getattr(self.Meta, 'primary_keys', None): filters = {key: data.get(key) for key in keys} if None not in filters.values(): - return self.session.query(self.opts.model).filter_by(**filters).first() - return super().get_instance(data) + res= self.session.query(self.opts.model).filter_by(**filters).first() + return res + res= super().get_instance(data) + return res @pre_load(pass_many=True) - def _patch_input(self, items, many, **kwargs): # pylint: disable=unused-argument + def _patch_many(self, items, many, **kwargs): # pylint: disable=unused-argument """ - flush sqla session before serializing a section when requested (make sure all objects that could be referred to later are created) - when in update mode: patch input data before deserialization @@ -540,12 +810,19 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # patch "delete", "prune" and "default" want_prune = [] - def patch(count, data, prune): + def patch(count, data): # don't allow __delete__ coming from input if '__delete__' in data: raise ValidationError('Unknown field.', f'{count}.__delete__') + # fail when hash_password is specified without password + if 'hash_password' in data and not 'password' in data: + raise ValidationError( + 'Nothing to hash. Field "password" is missing.', + field_name = f'{count}.hash_password', + ) + # handle "prune list" and "delete item" (-pkey: none and -pkey: id) for key in data: if key.startswith('-'): @@ -553,10 +830,10 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # delete or prune if data[key] is None: # prune - prune.append(True) + want_prune.append(True) return None # mark item for deletion - return {key[1:]: data[key], '__delete__': True} + return {key[1:]: data[key], '__delete__': count} # handle "set to default value" (-key: none) def set_default(key, value): @@ -567,7 +844,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): return (key, None) if value is not None: raise ValidationError( - 'When resetting to default value must be null.', + 'Value must be "null" when resetting to default.', f'{count}.{key}' ) value = self.opts.model.__table__.columns[key].default @@ -583,10 +860,128 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # convert items to "delete" and filter "prune" item items = [ item for item in [ - patch(count, item, want_prune) for count, item in enumerate(items) + patch(count, item) for count, item in enumerate(items) ] if item ] + # remember if prune was requested for _prune_items@post_load + self.store('prune', bool(want_prune), True) + + # remember original items to stabilize password-changes in _add_instance@post_load + self.store('original', items, True) + + return items + + @pre_load + def _patch_item(self, data, many, **kwargs): # pylint: disable=unused-argument + """ - call callback function to track import + - stabilize import of items with auto-increment primary key + - delete items + - delete/prune list attributes + - add missing required attributes + """ + + # callback + if callback := self.context.get('callback'): + callback(self, data) + + # stop early when not updating + if not self.opts.load_instance or not self.context.get('update'): + return data + + # stabilize import of auto-increment primary keys (not required), + # by matching import data to existing items and setting primary key + if not self._primary in data: + for item in getattr(self.recall('parent'), self.recall('field', 'parent')): + existing = self.dump(item, many=False) + this = existing.pop(self._primary) + if data == existing: + instance = item + data[self._primary] = this + break + + # try to load instance + instance = self.instance or self.get_instance(data) + if instance is None: + + if '__delete__' in data: + # deletion of non-existent item requested + raise ValidationError( + f'Item to delete not found: {data[self._primary]!r}.', + field_name = f'{data["__delete__"]}.{self._primary}', + ) + + else: + + if self.context.get('update'): + # remember instance as parent for pruning siblings + if not self.Meta.sibling: + self.store('parent', instance) + # delete instance from session when marked + if '__delete__' in data: + self.opts.sqla_session.delete(instance) + # delete item from lists or prune lists + # currently: domain.alternatives, user.forward_destination, + # user.manager_of, aliases.destination + for key, value in data.items(): + if not isinstance(self.fields.get(key), ( + RelatedList, CommaSeparatedListField, fields.Raw) + ) or not isinstance(value, list): + continue + # deduplicate new value + new_value = set(value) + # handle list pruning + if '-prune-' in value: + value.remove('-prune-') + new_value.remove('-prune-') + else: + for old in getattr(instance, key): + # using str() is okay for now (see above) + new_value.add(str(old)) + # handle item deletion + for item in value: + if item.startswith('-'): + new_value.remove(item) + try: + new_value.remove(item[1:]) + except KeyError as exc: + raise ValidationError( + f'Item to delete not found: {item[1:]!r}.', + field_name=f'?.{key}', + ) from exc + # sort list of new values + data[key] = sorted(new_value) + # log backref modification not catched by modify hook + if isinstance(self.fields[key], RelatedList): + if callback := self.context.get('callback'): + before = {str(v) for v in getattr(instance, key)} + after = set(data[key]) + if before != after: + callback(self, instance, { + 'key': key, + 'target': str(instance), + 'before': before, + 'after': after, + }) + + # add attributes required for validation from db + for attr_name, field_obj in self.load_fields.items(): + if field_obj.required and attr_name not in data: + data[attr_name] = getattr(instance, attr_name) + + return data + + @post_load(pass_many=True) + def _prune_items(self, items, many, **kwargs): # pylint: disable=unused-argument + """ handle list pruning """ + + # stop early when not updating + if not self.context.get('update'): + return items + + # get prune flag from _patch_many@pre_load + want_prune = self.recall('prune', True) + # prune: determine if existing items in db need to be added or marked for deletion add_items = False del_items = False @@ -603,144 +998,60 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if add_items or del_items: existing = {item[self._primary] for item in items if self._primary in item} - for item in getattr(self.context['parent'], self.context['parent_field']): + for item in getattr(self.recall('parent'), self.recall('field', 'parent')): key = getattr(item, self._primary) if key not in existing: if add_items: items.append({self._primary: key}) else: - items.append({self._primary: key, '__delete__': True}) + items.append({self._primary: key, '__delete__': '?'}) return items - @pre_load - def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument - """ call callback function to track import - """ - # callback - if callback := self.context.get('callback'): - callback(self, data) - - return data - - @pre_load - def _add_required(self, data, many, **kwargs): # pylint: disable=unused-argument - """ when updating: - allow modification of existing items having required attributes - by loading existing value from db + @post_load + def _add_instance(self, item, many, **kwargs): # pylint: disable=unused-argument + """ - undo password change in existing instances when plain password did not change + - add new instances to sqla session """ - if not self.opts.load_instance or not self.context.get('update'): - return data - - # stabilize import of auto-increment primary keys (not required), - # by matching import data to existing items and setting primary key - if not self._primary in data: - for item in getattr(self.context['parent'], self.context['parent_field']): - existing = self.dump(item, many=False) - this = existing.pop(self._primary) - if data == existing: - instance = item - data[self._primary] = this - break - - # try to load instance - instance = self.instance or self.get_instance(data) - if instance is None: - - if '__delete__' in data: - # deletion of non-existent item requested - raise ValidationError( - f'item to delete not found: {data[self._primary]!r}', - field_name=f'?.{self._primary}', - ) - - else: - - if self.context.get('update'): - # remember instance as parent for pruning siblings - if not self.Meta.sibling: - self.context['parent'] = instance - # delete instance when marked - if '__delete__' in data: - self.opts.sqla_session.delete(instance) - # delete item from lists or prune lists - # currently: domain.alternatives, user.forward_destination, - # user.manager_of, aliases.destination - for key, value in data.items(): - if not isinstance(self.fields[key], fields.Nested) and isinstance(value, list): - new_value = set(value) - # handle list pruning - if '-prune-' in value: - value.remove('-prune-') - new_value.remove('-prune-') - else: - for old in getattr(instance, key): - # using str() is okay for now (see above) - new_value.add(str(old)) - # handle item deletion - for item in value: - if item.startswith('-'): - new_value.remove(item) - try: - new_value.remove(item[1:]) - except KeyError as exc: - raise ValidationError( - f'item to delete not found: {item[1:]!r}', - field_name=f'?.{key}', - ) from exc - # deduplicate and sort list - data[key] = sorted(new_value) - # log backref modification not catched by hook - if isinstance(self.fields[key], RelatedList): - if callback := self.context.get('callback'): - callback(self, instance, { - 'key': key, - 'target': str(instance), - 'before': [str(v) for v in getattr(instance, key)], - 'after': data[key], - }) - - - - # add attributes required for validation from db - # TODO: this will cause validation errors if value from database does not validate - # but there should not be an invalid value in the database - for attr_name, field_obj in self.load_fields.items(): - if field_obj.required and attr_name not in data: - data[attr_name] = getattr(instance, attr_name) - - return data - - @post_load(pass_original=True) - def _add_instance(self, item, original, many, **kwargs): # pylint: disable=unused-argument - """ add new instances to sqla session """ - - if item in self.opts.sqla_session: - # item was modified - if 'hash_password' in original: - # stabilize import of passwords to be hashed, - # by not re-hashing an unchanged password - if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): - if attr.history.has_changes() and attr.history.deleted: - try: - # reset password hash, if password was not changed - inst = type(item)(password=attr.history.deleted[-1]) - if inst.check_password(original['password']): - item.password = inst.password - except ValueError: - # hash in db is invalid - pass - else: - del inst - else: - # new item + if not item in self.opts.sqla_session: self.opts.sqla_session.add(item) + return item + + # stop early if item has no password attribute + if not hasattr(item, 'password'): + return item + + # did we hash a new plaintext password? + original = None + pkey = getattr(item, self._primary) + for data in self.recall('original', True): + if 'hash_password' in data and data.get(self._primary) == pkey: + original = data['password'] + break + if original is None: + # password was hashed by us + return item + + # reset hash if plain password matches hash from db + if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): + if attr.history.has_changes() and attr.history.deleted: + try: + # reset password hash + inst = type(item)(password=attr.history.deleted[-1]) + if inst.check_password(original): + item.password = inst.password + except ValueError: + # hash in db is invalid + pass + else: + del inst + return item @post_dump def _hide_values(self, data, many, **kwargs): # pylint: disable=unused-argument - """ hide secrets and order output """ + """ hide secrets """ # stop early when not excluding/hiding if not self._exclude_by_value and not self._hide_by_context: @@ -757,7 +1068,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # this field is used to mark items for deletion mark_delete = fields.Boolean(data_key='__delete__', load_only=True) - # TODO: remove LazyStringField (when model was changed - IMHO comment should not be nullable) + # TODO: this can be removed when comment is not nullable in model comment = LazyStringField() @@ -892,27 +1203,28 @@ class RelaySchema(BaseSchema): load_instance = True -class MailuSchema(Schema): +@mapped +class MailuSchema(Schema, Storage): """ Marshmallow schema for complete Mailu config """ class Meta: """ Schema config """ + model = models.MailuConfig render_module = RenderYAML order = ['domain', 'user', 'alias', 'relay'] # 'config' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # order dump_fields - for field in self.Meta.order: - if field in self.dump_fields: - self.dump_fields[field] = self.dump_fields.pop(field) + # order fieldlists + for fieldlist in (self.fields, self.load_fields, self.dump_fields): + for field in self.Meta.order: + if field in fieldlist: + fieldlist[field] = fieldlist.pop(field) def _call_and_store(self, *args, **kwargs): """ track current parent and field for pruning """ - self.context.update({ - 'parent': self.context.get('config'), - 'parent_field': kwargs['field_name'], - }) + self.store('field', kwargs['field_name'], True) + self.store('parent', self.context.get('config')) return super()._call_and_store(*args, **kwargs) @pre_load diff --git a/docs/cli.rst b/docs/cli.rst index 6d48c576..891db152 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -138,8 +138,8 @@ The purpose of this command is to export the complete configuration in YAML or J Options: -f, --full Include attributes with default value. -s, --secrets Include secret attributes (dkim-key, passwords). - -c, --color Force colorized output. -d, --dns Include dns records. + -c, --color Force colorized output. -o, --output-file FILENAME Save configuration to file. -j, --json Export configuration in json format. -?, -h, --help Show this message and exit. @@ -147,14 +147,18 @@ The purpose of this command is to export the complete configuration in YAML or J Only non-default attributes are exported. If you want to export all attributes use ``--full``. If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option. To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. + By default all configuration objects are exported (domain, user, alias, relay). You can specify filters to export only some objects or attributes (try: ``user`` or ``domain.name``). +Attributes explicitly specified in filters are automatically exported: there is no need to add ``--secrets`` or ``--full``. .. code-block:: bash - $ docker-compose exec admin flask mailu config-export -o mail-config.yml + $ docker-compose exec admin flask mailu config-export --output mail-config.yml - $ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf + $ docker-compose exec admin flask mailu config-export domain.dns_mx domain.dns_spf + + $ docker-compose exec admin flask mailu config-export user.spam_threshold config-import ------------- @@ -211,7 +215,7 @@ mail-config.yml contains the configuration and looks like this: config-update shows the number of created/modified/deleted objects after import. To suppress all messages except error messages use ``--quiet``. -By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed. +By adding the ``--verbose`` switch the import gets more detailed and shows exactly what attributes changed. In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. If you want to test what would be done when importing without committing any changes, use ``--dry-run``. @@ -234,6 +238,9 @@ It is possible to delete a single element or prune all elements from lists and a The ``-key: null`` notation can also be used to reset an attribute to its default. To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``. +A new dkim key can be generated when adding or modifying a domain, by using the special value +``dkim_key: -generate-``. + This is a complete YAML template with all additional parameters that can be defined: .. code-block:: yaml From 0a9f732faa4d1addcc848cda2678976fb244314c Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 22 Feb 2021 20:35:23 +0100 Subject: [PATCH 44/52] added docstring to Logger. use generators. --- core/admin/mailu/models.py | 2 +- core/admin/mailu/schemas.py | 48 +++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 1b3c787a..d134086f 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -83,7 +83,7 @@ class CommaSeparatedList(db.TypeDecorator): def process_result_value(self, value, dialect): """ split comma separated string to list """ - return list(filter(bool, [item.strip() for item in value.split(',')])) if value else [] + return list(filter(bool, (item.strip() for item in value.split(',')))) if value else [] python_type = list diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 4c5042ea..68814170 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -76,6 +76,9 @@ class MyYamlLexer(YamlLexer): yield typ, value class Logger: + """ helps with counting and colorizing + imported and exported data + """ def __init__(self, want_color=None, can_color=False, debug=False, secrets=False): @@ -195,25 +198,26 @@ class Logger: continue before = target._dkim_key_on_disk after = target._dkim_key - if before != after: - # "de-dupe" messages; this event is fired at every flush - if not (target, before, after) in self._dedupe_dkim: - self._dedupe_dkim.add((target, before, after)) - self._counter.update([('Modified', target.__table__.name)]) - if self.verbose: - if self.secrets: - before = before.decode('ascii', 'ignore') - after = after.decode('ascii', 'ignore') - else: - before = HIDDEN if before else '' - after = HIDDEN if after else '' - self._log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') + # "de-dupe" messages; this event is fired at every flush + if before == after or (target, before, after) in self._dedupe_dkim: + continue + self._dedupe_dkim.add((target, before, after)) + self._counter.update([('Modified', target.__table__.name)]) + if self.verbose: + if self.secrets: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = HIDDEN if before else '' + after = HIDDEN if after else '' + self._log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') def track_serialize(self, obj, item, backref=None): """ callback method to track import """ # called for backref modification? if backref is not None: - self._log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format_map(backref)) + self._log( + 'Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format_map(backref)) return # show input data? if self.verbose < 2: @@ -263,7 +267,7 @@ class Logger: if path: return res - maxlen = max([len(loc) for loc, msg in res]) + maxlen = max(len(loc) for loc, msg in res) res = [f' - {loc.ljust(maxlen)} : {msg}' for loc, msg in res] errors = f'{len(res)} error{["s",""][len(res)==1]}' res.insert(0, f'[ValidationError] {errors} occurred during input validation') @@ -279,7 +283,9 @@ class Logger: if 'attr' in trace.tb_frame.f_locals: path.append(trace.tb_frame.f_locals['attr']) elif trace.tb_frame.f_code.co_name == '_init_fields': - spec = ', '.join(['.'.join(path + [key]) for key in trace.tb_frame.f_locals['invalid_fields']]) + spec = ', '.join( + '.'.join(path + [key]) + for key in trace.tb_frame.f_locals['invalid_fields']) return f'Invalid filter: {spec}' trace = trace.tb_next return None @@ -494,7 +500,7 @@ class CommaSeparatedListField(fields.Raw): except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc else: - value = filter(None, [item.strip() for item in value.split(',')]) + value = filter(bool, (item.strip() for item in value.split(','))) return list(value) @@ -539,7 +545,7 @@ class DkimKeyField(fields.String): # convert list to str if isinstance(value, list): try: - value = ''.join([ensure_text_type(item) for item in value]).strip() + value = ''.join(ensure_text_type(item) for item in value).strip() except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc @@ -855,7 +861,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): ) return (key, value.arg) - return dict([set_default(key, value) for key, value in data.items()]) + return dict(set_default(key, value) for key, value in data.items()) # convert items to "delete" and filter "prune" item items = [ @@ -1059,11 +1065,11 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): # exclude or hide values full = self.context.get('full') - return type(data)([ + return type(data)( (key, HIDDEN if key in self._hide_by_context else value) for key, value in data.items() if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] - ]) + ) # this field is used to mark items for deletion mark_delete = fields.Boolean(data_key='__delete__', load_only=True) From e90d5548a651fe933b5977e15c52cd0be954e7dd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 10 Mar 2021 18:30:28 +0100 Subject: [PATCH 45/52] use RFC3339 for last_check fixed to UTC for now --- core/admin/mailu/schemas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 68814170..284c1551 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1135,6 +1135,7 @@ class FetchSchema(BaseSchema): """ Schema config """ model = models.Fetch load_instance = True + datetimeformat = '%Y-%m-%dT%H:%M:%S.%fZ' # RFC3339, but fixed to UTC sibling = True include_by_context = { From c17bfae24050e48ae83cc21196346cfe7c6d90bc Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 10 Mar 2021 18:50:25 +0100 Subject: [PATCH 46/52] correct rfc3339 datetime serialization now using correct timezone --- core/admin/mailu/schemas.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 284c1551..277748f7 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -3,6 +3,7 @@ from copy import deepcopy from collections import Counter +from datetime import timezone import json import logging @@ -455,7 +456,20 @@ class RenderJSON: return json.dumps(*args, **kwargs) -### schema: custom fields ### +### marshmallow: custom fields ### + +def _rfc3339(datetime): + """ dump datetime according to rfc3339 """ + if datetime.tzinfo is None: + datetime = datetime.astimezone(timezone.utc) + res = datetime.isoformat() + if res.endswith('+00:00'): + return f'{res[:-6]}Z' + return res + +fields.DateTime.SERIALIZATION_FUNCS['rfc3339'] = _rfc3339 +fields.DateTime.DESERIALIZATION_FUNCS['rfc3339'] = fields.DateTime.DESERIALIZATION_FUNCS['iso'] +fields.DateTime.DEFAULT_FORMAT = 'rfc3339' class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string @@ -1135,7 +1149,6 @@ class FetchSchema(BaseSchema): """ Schema config """ model = models.Fetch load_instance = True - datetimeformat = '%Y-%m-%dT%H:%M:%S.%fZ' # RFC3339, but fixed to UTC sibling = True include_by_context = { From ce9a9ec572d8220e673b2817f1350ffb53b3215a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 10 Mar 2021 18:50:52 +0100 Subject: [PATCH 47/52] always init Logger first --- core/admin/mailu/manage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index f9c58d26..5708327e 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -375,6 +375,8 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None """ Export configuration as YAML or JSON to stdout or file """ + log = Logger(want_color=color or None, can_color=output.isatty()) + only = only or MailuSchema.Meta.order context = { @@ -382,7 +384,6 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None 'secrets': secrets, 'dns': dns, } - log = Logger(want_color=color or None, can_color=output.isatty()) try: schema = MailuSchema(only=only, context=context) From 9cb6962335d00fbb0505e7d571271232d4f54b97 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 11 Mar 2021 18:12:50 +0100 Subject: [PATCH 48/52] Moved MyYamlLexer into logger now cmdline runs without pygments --- core/admin/mailu/schemas.py | 48 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 277748f7..fdd766b3 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -53,34 +53,34 @@ def mapped(cls): _model2schema[cls.Meta.model] = cls return cls -class MyYamlLexer(YamlLexer): - """ colorize yaml constants and integers """ - def get_tokens(self, text, unfiltered=False): - for typ, value in super().get_tokens(text, unfiltered): - if typ is Token.Literal.Scalar.Plain: - if value in {'true', 'false', 'null'}: - typ = Token.Keyword.Constant - elif value == HIDDEN: - typ = Token.Error - else: - try: - int(value, 10) - except ValueError: - try: - float(value) - except ValueError: - pass - else: - typ = Token.Literal.Number.Float - else: - typ = Token.Literal.Number.Integer - yield typ, value - class Logger: """ helps with counting and colorizing imported and exported data """ + class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + def __init__(self, want_color=None, can_color=False, debug=False, secrets=False): self.lexer = 'yaml' @@ -323,7 +323,7 @@ class Logger: return data lexer = lexer or self.lexer - lexer = MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer) + lexer = Logger.MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer) formatter = get_formatter_by_name(formatter or self.formatter, colorscheme=self.colorscheme) if strip is None: strip = self.strip From 0c38128c4e459aa6fb783c86bea51ae2e19ac9b7 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 11 Mar 2021 18:38:00 +0100 Subject: [PATCH 49/52] Add pygments to requirements --- core/admin/mailu/schemas.py | 20 ++++++-------------- core/admin/requirements-prod.txt | 1 + core/admin/requirements.txt | 1 + 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index fdd766b3..28a5d6f4 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -21,16 +21,11 @@ from flask_marshmallow import Marshmallow from OpenSSL import crypto -try: - from pygments import highlight - from pygments.token import Token - from pygments.lexers import get_lexer_by_name - from pygments.lexers.data import YamlLexer - from pygments.formatters import get_formatter_by_name -except ModuleNotFoundError: - COLOR_SUPPORTED = False -else: - COLOR_SUPPORTED = True +from pygments import highlight +from pygments.token import Token +from pygments.lexers import get_lexer_by_name +from pygments.lexers.data import YamlLexer +from pygments.formatters import get_formatter_by_name from mailu import models, dkim @@ -92,10 +87,7 @@ class Logger: self.debug = debug self.print = print - if want_color and not COLOR_SUPPORTED: - raise ValueError('Please install pygments to colorize output') - - self.color = want_color or (can_color and COLOR_SUPPORTED) + self.color = want_color or can_color self._counter = Counter() self._schemas = {} diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index 7620fd95..5c291e24 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -36,6 +36,7 @@ marshmallow-sqlalchemy==0.24.1 passlib==1.7.4 psycopg2==2.8.2 pycparser==2.19 +Pygments==2.8.1 pyOpenSSL==19.0.0 python-dateutil==2.8.0 python-editor==1.0.4 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index f9d175b3..46500295 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -17,6 +17,7 @@ gunicorn tabulate PyYAML PyOpenSSL +Pygments dnspython bcrypt tenacity From 8bc44455721471a7277196190bb31db9d48d4bce Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 12 Mar 2021 17:56:17 +0100 Subject: [PATCH 50/52] Sync update of localpart, domain_name and email --- core/admin/mailu/models.py | 18 +++++++++++++++--- core/admin/mailu/schemas.py | 4 ++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index b08c5bd2..93efd016 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -355,9 +355,21 @@ class Email(object): self.localpart, self.domain_name = value.rsplit('@', 1) self._email = value - # hack for email declared attr - when _email is not updated yet - def __str__(self): - return str(f'{self.localpart}@{self.domain_name}') + @staticmethod + def _update_localpart(target, value, *_): + if target.domain_name: + target._email = f'{value}@{target.domain_name}' + + @staticmethod + def _update_domain_name(target, value, *_): + if target.localpart: + target._email = f'{target.localpart}@{value}' + + @classmethod + def __declare_last__(cls): + # gets called after mappings are completed + sqlalchemy.event.listen(User.localpart, 'set', cls._update_localpart, propagate=True) + sqlalchemy.event.listen(User.domain_name, 'set', cls._update_domain_name, propagate=True) def sendmail(self, subject, body): """ send an email to the address """ diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 28a5d6f4..2742edf1 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -1030,8 +1030,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): self.opts.sqla_session.add(item) return item - # stop early if item has no password attribute - if not hasattr(item, 'password'): + # stop early when not updating or item has no password attribute + if not self.context.get('update') or not hasattr(item, 'password'): return item # did we hash a new plaintext password? From 21a362fdaea022404e4f8e55fb1def73f1c24d35 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 8 Jun 2021 07:09:07 +0000 Subject: [PATCH 51/52] Changed config-update to config-import in config-import description. --- docs/cli.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 10669108..957e47e4 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -212,13 +212,13 @@ mail-config.yml contains the configuration and looks like this: comment: test smtp: mx.example.com -config-update shows the number of created/modified/deleted objects after import. +config-import shows the number of created/modified/deleted objects after import. To suppress all messages except error messages use ``--quiet``. By adding the ``--verbose`` switch the import gets more detailed and shows exactly what attributes changed. In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. If you want to test what would be done when importing without committing any changes, use ``--dry-run``. -By default config-update replaces the whole configuration. ``--update`` allows to modify the existing configuration instead. +By default config-import replaces the whole configuration. ``--update`` allows to modify the existing configuration instead. New elements will be added and existing elements will be modified. It is possible to delete a single element or prune all elements from lists and associative arrays using a special notation: From fbd945390d32a3804ef8abd3b5c98e479890ddef Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 29 Jun 2021 16:13:04 +0200 Subject: [PATCH 52/52] cleaned imports and fixed datetime and passlib use --- core/admin/mailu/models.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 58e38de9..42711cf0 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,7 +1,6 @@ """ Mailu config storage model """ -import re import os import smtplib import json @@ -17,7 +16,6 @@ import passlib.hash import passlib.registry import time import os -import glob import hmac import smtplib import idna @@ -533,7 +531,7 @@ class User(Base, Email): if cache_result and current_salt: cache_salt, cache_hash = cache_result if cache_salt == current_salt: - return hash.pbkdf2_sha256.verify(password, cache_hash) + return passlib.hash.pbkdf2_sha256.verify(password, cache_hash) else: # the cache is local per gunicorn; the password has changed # so the local cache can be invalidated @@ -560,7 +558,7 @@ we have little control over GC and string interning anyways. An attacker that can dump the process' memory is likely to find credentials in clear-text regardless of the presence of the cache. """ - self._credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) + self._credential_cache[self.get_id()] = (self.password.split('$')[3], passlib.hash.pbkdf2_sha256.using(rounds=1).hash(password)) return result def set_password(self, password, raw=False): @@ -604,7 +602,7 @@ in clear-text regardless of the presence of the cache. @classmethod def get_temp_token(cls, email): user = cls.query.get(email) - return hmac.new(app.temp_token_key, bytearray("{}|{}".format(datetime.utcnow().strftime("%Y%m%d"), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None + return hmac.new(app.temp_token_key, bytearray("{}|{}".format(time.strftime('%Y%m%d'), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None def verify_temp_token(self, token): return hmac.compare_digest(self.get_temp_token(self.email), token)