diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index f70c5c85..756400ad 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,42 +1,52 @@ -from mailu import models +""" Mailu command line interface +""" -from flask import current_app as app -from flask import cli as flask_cli - -import flask +import sys import os import socket +import logging import uuid + +from collections import Counter +from itertools import chain + import click +import sqlalchemy import yaml -import sys + +from flask import current_app as app +from flask.cli import FlaskGroup, with_appcontext +from marshmallow.exceptions import ValidationError + +from . import models +from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, RenderJSON, HIDDEN db = models.db -@click.group() -def mailu(cls=flask_cli.FlaskGroup): +@click.group(cls=FlaskGroup, context_settings={'help_option_names': ['-?', '-h', '--help']}) +def mailu(): """ Mailu command line """ @mailu.command() -@flask_cli.with_appcontext +@with_appcontext def advertise(): """ Advertise this server against statistic services. """ - if os.path.isfile(app.config["INSTANCE_ID_PATH"]): - with open(app.config["INSTANCE_ID_PATH"], "r") as handle: + if os.path.isfile(app.config['INSTANCE_ID_PATH']): + with open(app.config['INSTANCE_ID_PATH'], 'r') as handle: instance_id = handle.read() else: instance_id = str(uuid.uuid4()) - with open(app.config["INSTANCE_ID_PATH"], "w") as handle: + with open(app.config['INSTANCE_ID_PATH'], 'w') as handle: handle.write(instance_id) - if not app.config["DISABLE_STATISTICS"]: + if not app.config['DISABLE_STATISTICS']: try: - socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) - except: + socket.gethostbyname(app.config['STATS_ENDPOINT'].format(instance_id)) + except OSError: pass @@ -45,7 +55,7 @@ def advertise(): @click.argument('domain_name') @click.argument('password') @click.option('-m', '--mode') -@flask_cli.with_appcontext +@with_appcontext def admin(localpart, domain_name, password, mode='create'): """ Create an admin user 'mode' can be: @@ -60,7 +70,7 @@ def admin(localpart, domain_name, password, mode='create'): user = None if mode == 'ifmissing' or mode == 'update': - email = '{}@{}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' user = models.User.query.get(email) if user and mode == 'ifmissing': @@ -89,7 +99,7 @@ def admin(localpart, domain_name, password, mode='create'): @click.argument('domain_name') @click.argument('password') @click.argument('hash_scheme', required=False) -@flask_cli.with_appcontext +@with_appcontext def user(localpart, domain_name, password, hash_scheme=None): """ Create a user """ @@ -114,18 +124,18 @@ def user(localpart, domain_name, password, hash_scheme=None): @click.argument('domain_name') @click.argument('password') @click.argument('hash_scheme', required=False) -@flask_cli.with_appcontext +@with_appcontext def password(localpart, domain_name, password, hash_scheme=None): """ Change the password of an user """ - email = '{0}@{1}'.format(localpart, domain_name) - user = models.User.query.get(email) + email = f'{localpart}@{domain_name}' + user = models.User.query.get(email) if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] if user: user.set_password(password, hash_scheme=hash_scheme) else: - print("User " + email + " not found.") + print(f'User {email} not found.') db.session.commit() @@ -134,7 +144,7 @@ def password(localpart, domain_name, password, hash_scheme=None): @click.option('-u', '--max-users') @click.option('-a', '--max-aliases') @click.option('-q', '--max-quota-bytes') -@flask_cli.with_appcontext +@with_appcontext def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): """ Create a domain """ @@ -151,9 +161,9 @@ def domain(domain_name, max_users=-1, max_aliases=-1, max_quota_bytes=0): @click.argument('domain_name') @click.argument('password_hash') @click.argument('hash_scheme') -@flask_cli.with_appcontext +@with_appcontext def user_import(localpart, domain_name, password_hash, hash_scheme = None): - """ Import a user along with password hash. + """ Import a user along with password hash """ if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] @@ -171,179 +181,441 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() -yaml_sections = [ - ('domains', models.Domain), - ('relays', models.Relay), - ('users', models.User), - ('aliases', models.Alias), -# ('config', models.Config), -] +# TODO: remove deprecated config_update function? +@mailu.command() +@click.option('-v', '--verbose') +@click.option('-d', '--delete-objects') +@with_appcontext +def config_update(verbose=False, delete_objects=False): + """ Sync configuration with data from YAML (deprecated) + """ + new_config = yaml.safe_load(sys.stdin) + # print new_config + domains = new_config.get('domains', []) + tracked_domains = set() + for domain_config in domains: + if verbose: + print(str(domain_config)) + domain_name = domain_config['name'] + max_users = domain_config.get('max_users', -1) + max_aliases = domain_config.get('max_aliases', -1) + max_quota_bytes = domain_config.get('max_quota_bytes', 0) + tracked_domains.add(domain_name) + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name, + max_users=max_users, + max_aliases=max_aliases, + max_quota_bytes=max_quota_bytes) + db.session.add(domain) + print(f'Added {domain_config}') + else: + domain.max_users = max_users + domain.max_aliases = max_aliases + domain.max_quota_bytes = max_quota_bytes + db.session.add(domain) + print(f'Updated {domain_config}') + + users = new_config.get('users', []) + tracked_users = set() + user_optional_params = ('comment', 'quota_bytes', 'global_admin', + 'enable_imap', 'enable_pop', 'forward_enabled', + 'forward_destination', 'reply_enabled', + 'reply_subject', 'reply_body', 'displayed_name', + 'spam_enabled', 'email', 'spam_threshold') + for user_config in users: + if verbose: + print(str(user_config)) + localpart = user_config['localpart'] + domain_name = user_config['domain'] + password_hash = user_config.get('password_hash', None) + hash_scheme = user_config.get('hash_scheme', None) + domain = models.Domain.query.get(domain_name) + email = f'{localpart}@{domain_name}' + optional_params = {} + for k in user_optional_params: + if k in user_config: + optional_params[k] = user_config[k] + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User.query.get(email) + tracked_users.add(email) + tracked_domains.add(domain_name) + if not user: + user = models.User( + localpart=localpart, + domain=domain, + **optional_params + ) + else: + for k in optional_params: + setattr(user, k, optional_params[k]) + user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) + db.session.add(user) + + aliases = new_config.get('aliases', []) + tracked_aliases = set() + for alias_config in aliases: + if verbose: + print(str(alias_config)) + localpart = alias_config['localpart'] + domain_name = alias_config['domain'] + if isinstance(alias_config['destination'], str): + destination = alias_config['destination'].split(',') + else: + destination = alias_config['destination'] + wildcard = alias_config.get('wildcard', False) + domain = models.Domain.query.get(domain_name) + email = f'{localpart}@{domain_name}' + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + alias = models.Alias.query.get(email) + tracked_aliases.add(email) + tracked_domains.add(domain_name) + if not alias: + alias = models.Alias( + localpart=localpart, + domain=domain, + wildcard=wildcard, + destination=destination, + email=email + ) + else: + alias.destination = destination + alias.wildcard = wildcard + db.session.add(alias) + + db.session.commit() + + managers = new_config.get('managers', []) + # tracked_managers=set() + for manager_config in managers: + if verbose: + print(str(manager_config)) + domain_name = manager_config['domain'] + user_name = manager_config['user'] + domain = models.Domain.query.get(domain_name) + manageruser = models.User.query.get(f'{user_name}@{domain_name}') + if manageruser not in domain.managers: + domain.managers.append(manageruser) + db.session.add(domain) + + db.session.commit() + + if delete_objects: + for user in db.session.query(models.User).all(): + if not user.email in tracked_users: + if verbose: + print(f'Deleting user: {user.email}') + db.session.delete(user) + for alias in db.session.query(models.Alias).all(): + if not alias.email in tracked_aliases: + if verbose: + print(f'Deleting alias: {alias.email}') + db.session.delete(alias) + for domain in db.session.query(models.Domain).all(): + if not domain.name in tracked_domains: + if verbose: + print(f'Deleting domain: {domain.name}') + db.session.delete(domain) + db.session.commit() + @mailu.command() -@click.option('-v', '--verbose', is_flag=True, help='Increase verbosity') -@click.option('-d', '--delete-objects', is_flag=True, help='Remove objects not included in yaml') -@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made') -@flask_cli.with_appcontext -def config_update(verbose=False, delete_objects=False, dry_run=False, file=None): - """sync configuration with data from YAML-formatted stdin""" +@click.option('-v', '--verbose', count=True, help='Increase verbosity.') +@click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.') +@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') +@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.') +@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.') +@click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) +@with_appcontext +def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None): + """ Import configuration as YAML or JSON from stdin or file + """ - out = (lambda *args: print('(DRY RUN)', *args)) if dry_run else print + # verbose + # 0 : only show number of changes + # 1 : also show detailed changes + # 2 : also show input data + # 3 : also show sql queries (also needs -s, as sql may contain secrets) + # 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets) + + if quiet: + verbose = -1 + + if verbose > 2 and not secrets: + print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.') + verbose = 2 + + color_cfg = { + 'color': color or sys.stdout.isatty(), + 'lexer': 'python', + 'strip': True, + } + + counter = Counter() + logger = {} + + def format_errors(store, path=None): + + res = [] + if path is None: + path = [] + for key in sorted(store): + location = path + [str(key)] + value = store[key] + if isinstance(value, dict): + res.extend(format_errors(value, location)) + else: + for message in value: + res.append((".".join(location), message)) + + if path: + return res + + fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}' + res = [fmt.format(loc, msg) for loc, msg in res] + num = f'error{["s",""][len(res)==1]}' + res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation') + + return '\n'.join(res) + + def format_changes(*message): + if counter: + changes = [] + last = None + for (action, what), count in sorted(counter.items()): + if action != last: + if last: + changes.append('/') + changes.append(f'{action}:') + last = action + changes.append(f'{what}({count})') + else: + changes = ['No changes.'] + return chain(message, changes) + + def log(action, target, message=None): + + if message is None: + try: + message = logger[target.__class__].dump(target) + except KeyError: + message = target + if not isinstance(message, str): + message = repr(message) + print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}') + + def listen_insert(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + counter.update([('Created', target.__table__.name)]) + if verbose >= 1: + log('Created', target) + + def listen_update(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + + changed = {} + inspection = sqlalchemy.inspect(target) + for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: + history = getattr(inspection.attrs, attr.key).history + if history.has_changes() and history.deleted: + before = history.deleted[-1] + after = getattr(target, attr.key) + # TODO: remove special handling of "comment" after modifying model + if attr.key == 'comment' and not before and not after: + pass + # only remember changed keys + elif before != after: + if verbose >= 1: + changed[str(attr.key)] = (before, after) + else: + break + + if verbose >= 1: + # use schema with dump_context to hide secrets and sort keys + dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target) + for key, value in dumped.items(): + before, after = changed[key] + if value == HIDDEN: + before = HIDDEN if before else before + after = HIDDEN if after else after + else: + # TODO: need to use schema to "convert" before value? + after = value + log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') + + if changed: + counter.update([('Modified', target.__table__.name)]) + + def listen_delete(mapper, connection, target): # pylint: disable=unused-argument + """ callback function to track import """ + counter.update([('Deleted', target.__table__.name)]) + if verbose >= 1: + log('Deleted', target) + + # TODO: this listener will not be necessary, if dkim keys would be stored in database + _dedupe_dkim = set() + def listen_dkim(session, flush_context): # pylint: disable=unused-argument + """ callback function to track import """ + for target in session.identity_map.values(): + # look at Domains originally loaded from db + if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: + continue + before = target._dkim_key_on_disk + after = target._dkim_key + if before != after: + if secrets: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = HIDDEN if before else '' + after = HIDDEN if after else '' + # "de-dupe" messages; this event is fired at every flush + if not (target, before, after) in _dedupe_dkim: + _dedupe_dkim.add((target, before, after)) + counter.update([('Modified', target.__table__.name)]) + if verbose >= 1: + log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') + + def track_serialize(obj, item, backref=None): + """ callback function to track import """ + # called for backref modification? + if backref is not None: + log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref)) + return + # show input data? + if not verbose >= 2: + return + # hide secrets in data + data = logger[obj.opts.model].hide(item) + if 'hash_password' in data: + data['password'] = HIDDEN + if 'fetches' in data: + for fetch in data['fetches']: + fetch['password'] = HIDDEN + log('Handling', obj.opts.model, data) + + # configure contexts + diff_context = { + 'full': True, + 'secrets': secrets, + } + log_context = { + 'secrets': secrets, + } + load_context = { + 'import': True, + 'update': update, + 'clear': not update, + 'callback': track_serialize, + } + + # register listeners + for schema in get_schema(): + model = schema.Meta.model + logger[model] = schema(context=log_context) + sqlalchemy.event.listen(model, 'after_insert', listen_insert) + sqlalchemy.event.listen(model, 'after_update', listen_update) + sqlalchemy.event.listen(model, 'after_delete', listen_delete) + + # special listener for dkim_key changes + sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim) + + if verbose >= 3: + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) try: - new_config = yaml.safe_load(sys.stdin) - except (yaml.scanner.ScannerError, yaml.parser.ParserError) as reason: - out(f'[ERROR] Invalid yaml: {reason}') - sys.exit(1) - else: - if type(new_config) is str: - out(f'[ERROR] Invalid yaml: {new_config!r}') - sys.exit(1) - elif new_config is None or not len(new_config): - out('[ERROR] Empty yaml: Please pipe yaml into stdin') - sys.exit(1) + with models.db.session.no_autoflush: + config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source) + except ValidationError as exc: + raise click.ClickException(format_errors(exc.messages)) from exc + except Exception as exc: + if verbose >= 3: + raise + # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) + raise click.ClickException( + f'[{exc.__class__.__name__}] ' + f'{" ".join(str(exc).split())}' + ) from exc - error = False - tracked = {} - for section, model in yaml_sections: + # flush session to show/count all changes + if dry_run or verbose >= 1: + db.session.flush() - items = new_config.get(section) - if items is None: - if delete_objects: - out(f'[ERROR] Invalid yaml: Section "{section}" is missing') - error = True - break - else: - continue - - del new_config[section] - - if type(items) is not list: - out(f'[ERROR] Section "{section}" must be a list, not {items.__class__.__name__}') - error = True - break - elif not items: - continue - - # create items - for data in items: - - if verbose: - out(f'Handling {model.__table__} data: {data!r}') - - try: - changed = model.from_dict(data, delete_objects) - except Exception as reason: - out(f'[ERROR] {reason.args[0]} in data: {data}') - error = True - break - - for item, created in changed: - - if created is True: - # flush newly created item - db.session.add(item) - db.session.flush() - if verbose: - out(f'Added {item!r}: {item.to_dict()}') - else: - out(f'Added {item!r}') - - elif len(created): - # modified instance - if verbose: - for key, old, new in created: - out(f'Updated {key!r} of {item!r}: {old!r} -> {new!r}') - else: - out(f'Updated {item!r}: {", ".join(sorted([kon[0] for kon in created]))}') - - # track primary key of all items - tracked.setdefault(item.__class__, set()).update(set([item._dict_pval()])) - - if error: - break - - # on error: stop early - if error: - out('An error occured. Not committing changes.') - db.session.rollback() - sys.exit(1) - - # are there sections left in new_config? - if new_config: - out(f'[ERROR] Unknown section(s) in yaml: {", ".join(sorted(new_config.keys()))}') - error = True - - # test for conflicting domains - domains = set() - for model, items in tracked.items(): - if model in (models.Domain, models.Alternative, models.Relay): - if domains & items: - for domain in domains & items: - out(f'[ERROR] Duplicate domain name used: {domain}') - error = True - domains.update(items) - - # delete items not tracked - if delete_objects: - for model, items in tracked.items(): - for item in model.query.all(): - if not item._dict_pval() in items: - out(f'Deleted {item!r} {item}') - db.session.delete(item) + # check for duplicate domain names + dup = set() + for fqdn in chain( + db.session.query(models.Domain.name), + db.session.query(models.Alternative.name), + db.session.query(models.Relay.name) + ): + if fqdn in dup: + raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}') + dup.add(fqdn) # don't commit when running dry if dry_run: + if not quiet: + print(*format_changes('Dry run. Not commiting changes.')) db.session.rollback() else: + if not quiet: + print(*format_changes('Committing changes.')) db.session.commit() @mailu.command() -@click.option('-f', '--full', is_flag=True, help='Include default attributes') -@click.option('-s', '--secrets', is_flag=True, help='Include secrets (dkim-key, plain-text / not hashed)') -@click.option('-d', '--dns', is_flag=True, help='Include dns records') -@click.argument('sections', nargs=-1) -@flask_cli.with_appcontext -def config_dump(full=False, secrets=False, dns=False, sections=None): - """dump configuration as YAML-formatted data to stdout - - SECTIONS can be: domains, relays, users, aliases +@click.option('-f', '--full', is_flag=True, help='Include attributes with default value.') +@click.option('-s', '--secrets', is_flag=True, + help='Include secret attributes (dkim-key, passwords).') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') +@click.option('-d', '--dns', is_flag=True, help='Include dns records.') +@click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), + help='Save configuration to file.') +@click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.') +@click.argument('only', metavar='[FILTER]...', nargs=-1) +@with_appcontext +def config_export(full=False, secrets=False, color=False, dns=False, output=None, as_json=False, only=None): + """ Export configuration as YAML or JSON to stdout or file """ - class spacedDumper(yaml.Dumper): + if only: + for spec in only: + if spec.split('.', 1)[0] not in MailuSchema.Meta.order: + raise click.ClickException(f'[ValidationError] Unknown section: {spec}') + else: + only = MailuSchema.Meta.order - def write_line_break(self, data=None): - super().write_line_break(data) - if len(self.indents) == 1: - super().write_line_break() + context = { + 'full': full, + 'secrets': secrets, + 'dns': dns, + } - def increase_indent(self, flow=False, indentless=False): - return super().increase_indent(flow, False) + schema = MailuSchema(only=only, context=context) + color_cfg = {'color': color or output.isatty()} - if sections: - check = dict(yaml_sections) - for section in sections: - if section not in check: - print(f'[ERROR] Invalid section: {section}') - return 1 + if as_json: + schema.opts.render_module = RenderJSON + color_cfg['lexer'] = 'json' + color_cfg['strip'] = True - extra = [] - if dns: - extra.append('dns') - - config = {} - for section, model in yaml_sections: - if not sections or section in sections: - dump = [item.to_dict(full, secrets, extra) for item in model.query.all()] - if len(dump): - config[section] = dump - - yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True) + try: + print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) + except ValueError as exc: + if spec := get_fieldspec(exc): + raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc + raise @mailu.command() @click.argument('email') -@flask_cli.with_appcontext +@with_appcontext def user_delete(email): """delete user""" user = models.User.query.get(email) @@ -354,7 +626,7 @@ def user_delete(email): @mailu.command() @click.argument('email') -@flask_cli.with_appcontext +@with_appcontext def alias_delete(email): """delete alias""" alias = models.Alias.query.get(email) @@ -368,7 +640,7 @@ def alias_delete(email): @click.argument('domain_name') @click.argument('destination') @click.option('-w', '--wildcard', is_flag=True) -@flask_cli.with_appcontext +@with_appcontext def alias(localpart, domain_name, destination, wildcard=False): """ Create an alias """ @@ -381,7 +653,7 @@ def alias(localpart, domain_name, destination, wildcard=False): domain=domain, wildcard=wildcard, destination=destination.split(','), - email="%s@%s" % (localpart, domain_name) + email=f'{localpart}@{domain_name}' ) db.session.add(alias) db.session.commit() @@ -392,7 +664,7 @@ def alias(localpart, domain_name, destination, wildcard=False): @click.argument('max_users') @click.argument('max_aliases') @click.argument('max_quota_bytes') -@flask_cli.with_appcontext +@with_appcontext def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): """ Set domain limits """ @@ -407,12 +679,12 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): @mailu.command() @click.argument('domain_name') @click.argument('user_name') -@flask_cli.with_appcontext +@with_appcontext def setmanager(domain_name, user_name='manager'): """ Make a user manager of a domain """ domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) + manageruser = models.User.query.get(f'{user_name}@{domain_name}') domain.managers.append(manageruser) db.session.add(domain) db.session.commit() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 13ebce60..4c119984 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,23 +1,29 @@ -from mailu import dkim +""" Mailu config storage model +""" -from sqlalchemy.ext import declarative -from passlib import context, hash -from datetime import datetime, date +import re +import os +import smtplib +import json + +from datetime import date from email.mime import text -from flask import current_app as app -from textwrap import wrap +from itertools import chain import flask_sqlalchemy import sqlalchemy -import re -import time -import os -import glob -import smtplib +import passlib.context +import passlib.hash import idna import dns -import json -import itertools + +from flask import current_app as app +from sqlalchemy.ext import declarative +from sqlalchemy.ext.hybrid import hybrid_property +from sqlalchemy.inspection import inspect +from werkzeug.utils import cached_property + +from . import dkim db = flask_sqlalchemy.SQLAlchemy() @@ -27,12 +33,15 @@ class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ + # TODO: use db.String(255)? impl = db.String(80) def process_bind_param(self, value, dialect): - return idna.encode(value).decode('ascii').lower() + """ encode unicode domain name to punycode """ + return idna.encode(value.lower()).decode('ascii') def process_result_value(self, value, dialect): + """ decode punycode domain name to unicode """ return idna.decode(value) python_type = str @@ -41,24 +50,21 @@ class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ + # TODO: use db.String(254)? impl = db.String(255) def process_bind_param(self, value, dialect): - try: - localpart, domain_name = value.split('@') - return '{0}@{1}'.format( - localpart, - idna.encode(domain_name).decode('ascii'), - ).lower() - except ValueError: - pass + """ encode unicode domain part of email address to punycode """ + localpart, domain_name = value.rsplit('@', 1) + if '@' in localpart: + raise ValueError('email local part must not contain "@"') + domain_name = domain_name.lower() + return f'{localpart}@{idna.encode(domain_name).decode("ascii")}' def process_result_value(self, value, dialect): - localpart, domain_name = value.split('@') - return '{0}@{1}'.format( - localpart, - idna.decode(domain_name), - ) + """ decode punycode domain part of email to unicode """ + localpart, domain_name = value.rsplit('@', 1) + return f'{localpart}@{idna.decode(domain_name)}' python_type = str @@ -69,15 +75,17 @@ class CommaSeparatedList(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): - if not isinstance(value, (list, set)): - raise TypeError('Must be a list') + """ join list of items to comma separated string """ + if not isinstance(value, (list, tuple, set)): + raise TypeError('Must be a list of strings') for item in value: if ',' in item: - raise ValueError('Item must not contain a comma') - return ','.join(sorted(value)) + raise ValueError('list item must not contain ","') + return ','.join(sorted(set(value))) def process_result_value(self, value, dialect): - return list(filter(bool, value.split(','))) if value else [] + """ split comma separated string to list """ + return list(filter(bool, [item.strip() for item in value.split(',')])) if value else [] python_type = list @@ -88,9 +96,11 @@ class JSONEncoded(db.TypeDecorator): impl = db.String def process_bind_param(self, value, dialect): + """ encode data as json """ return json.dumps(value) if value else None def process_result_value(self, value, dialect): + """ decode json to data """ return json.loads(value) if value else None python_type = str @@ -110,248 +120,37 @@ class Base(db.Model): created_at = db.Column(db.Date, nullable=False, default=date.today) updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) - comment = db.Column(db.String(255), nullable=True) + comment = db.Column(db.String(255), nullable=True, default='') - @classmethod - def _dict_pkey(model): - return model.__mapper__.primary_key[0].name + def __str__(self): + pkey = self.__table__.primary_key.columns.values()[0].name + if pkey == 'email': + # ugly hack for email declared attr. _email is not always up2date + return str(f'{self.localpart}@{self.domain_name}') + elif pkey in {'name', 'email'}: + return str(getattr(self, pkey, None)) + else: + return self.__repr__() + return str(getattr(self, self.__table__.primary_key.columns.values()[0].name)) - def _dict_pval(self): - return getattr(self, self._dict_pkey()) + def __repr__(self): + return f'<{self.__class__.__name__} {str(self)!r}>' - def to_dict(self, full=False, include_secrets=False, include_extra=None, recursed=False, hide=None): - """ Return a dictionary representation of this model. - """ + def __eq__(self, other): + if isinstance(other, self.__class__): + pkey = self.__table__.primary_key.columns.values()[0].name + this = getattr(self, pkey, None) + other = getattr(other, pkey, None) + return this is not None and other is not None and str(this) == str(other) + else: + return NotImplemented - if recursed and not getattr(self, '_dict_recurse', False): - return str(self) - - hide = set(hide or []) | {'created_at', 'updated_at'} - if hasattr(self, '_dict_hide'): - hide |= self._dict_hide - - secret = set() - if not include_secrets and hasattr(self, '_dict_secret'): - secret |= self._dict_secret - - convert = getattr(self, '_dict_output', {}) - - extra_keys = getattr(self, '_dict_extra', {}) - if include_extra is None: - include_extra = [] - - res = {} - - for key in itertools.chain( - self.__table__.columns.keys(), - getattr(self, '_dict_show', []), - *[extra_keys.get(extra, []) for extra in include_extra] - ): - if key in hide: - continue - if key in self.__table__.columns: - default = self.__table__.columns[key].default - if isinstance(default, sqlalchemy.sql.schema.ColumnDefault): - default = default.arg - else: - default = None - value = getattr(self, key) - if full or ((default or value) and value != default): - if key in secret: - value = '' - elif value is not None and key in convert: - value = convert[key](value) - res[key] = value - - for key in self.__mapper__.relationships.keys(): - if key in hide: - continue - if self.__mapper__.relationships[key].uselist: - items = getattr(self, key) - if self.__mapper__.relationships[key].query_class is not None: - if hasattr(items, 'all'): - items = items.all() - if full or len(items): - if key in secret: - res[key] = '' - else: - res[key] = [item.to_dict(full, include_secrets, include_extra, True) for item in items] - else: - value = getattr(self, key) - if full or value is not None: - if key in secret: - res[key] = '' - else: - res[key] = value.to_dict(full, include_secrets, include_extra, True) - - return res - - @classmethod - def from_dict(model, data, delete=False): - - changed = [] - - pkey = model._dict_pkey() - - # handle "primary key" only - if type(data) is not dict: - data = {pkey: data} - - # modify input data - if hasattr(model, '_dict_input'): - try: - model._dict_input(data) - except Exception as reason: - raise ValueError(f'{reason}', model, None, data) - - # check for primary key (if not recursed) - if not getattr(model, '_dict_recurse', False): - if not pkey in data: - raise KeyError(f'primary key {model.__table__}.{pkey} is missing', model, pkey, data) - - # check data keys and values - for key in list(data.keys()): - - # check key - if not hasattr(model, key) and not key in model.__mapper__.relationships: - raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data) - - # check value type - value = data[key] - col = model.__mapper__.columns.get(key) - if col is not None: - if not ((value is None and col.nullable) or (type(value) is col.type.python_type)): - raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) - else: - rel = model.__mapper__.relationships.get(key) - if rel is None: - itype = getattr(model, '_dict_types', {}).get(key) - if itype is not None: - if itype is False: # ignore value. TODO: emit warning? - del data[key] - continue - elif not isinstance(value, itype): - raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data) - else: - raise NotImplementedError(f'type not defined for {model.__table__}.{key}') - - # handle relationships - if key in model.__mapper__.relationships: - rel_model = model.__mapper__.relationships[key].argument - if not isinstance(rel_model, sqlalchemy.orm.Mapper): - add = rel_model.from_dict(value, delete) - assert len(add) == 1 - rel_item, updated = add[0] - changed.append((rel_item, updated)) - data[key] = rel_item - - # create item if necessary - created = False - item = model.query.get(data[pkey]) if pkey in data else None - if item is None: - - # check for mandatory keys - missing = getattr(model, '_dict_mandatory', set()) - set(data.keys()) - if missing: - raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {model.__table__} missing', model, missing, data) - - # remove mapped relationships from data - mapped = {} - for key in list(data.keys()): - if key in model.__mapper__.relationships: - if isinstance(model.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper): - mapped[key] = data[key] - del data[key] - - # create new item - item = model(**data) - created = True - - # and update mapped relationships (below) - data = mapped - - # update item - updated = [] - for key, value in data.items(): - - # skip primary key - if key == pkey: - continue - - if key in model.__mapper__.relationships: - # update relationship - rel_model = model.__mapper__.relationships[key].argument - if isinstance(rel_model, sqlalchemy.orm.Mapper): - rel_model = rel_model.class_ - # add (and create) referenced items - cur = getattr(item, key) - old = sorted(cur, key=lambda i:id(i)) - new = [] - for rel_data in value: - # get or create related item - add = rel_model.from_dict(rel_data, delete) - assert len(add) == 1 - rel_item, rel_updated = add[0] - changed.append((rel_item, rel_updated)) - if rel_item not in cur: - cur.append(rel_item) - new.append(rel_item) - - # delete referenced items missing in yaml - rel_pkey = rel_model._dict_pkey() - new_data = list([i.to_dict(True, True, None, True, [rel_pkey]) for i in new]) - for rel_item in old: - if rel_item not in new: - # check if item with same data exists to stabilze import without primary key - rel_data = rel_item.to_dict(True, True, None, True, [rel_pkey]) - try: - same_idx = new_data.index(rel_data) - except ValueError: - same = None - else: - same = new[same_idx] - - if same is None: - # delete items missing in new - if delete: - cur.remove(rel_item) - else: - new.append(rel_item) - else: - # swap found item with same data with newly created item - new.append(rel_item) - new_data.append(rel_data) - new.remove(same) - del new_data[same_idx] - for i, (ch_item, _) in enumerate(changed): - if ch_item is same: - changed[i] = (rel_item, []) - db.session.flush() - db.session.delete(ch_item) - break - - # remember changes - new = sorted(new, key=lambda i:id(i)) - if new != old: - updated.append((key, old, new)) - - else: - # update key - old = getattr(item, key) - if type(old) is list: - # deduplicate list value - assert type(value) is list - value = set(value) - old = set(old) - if not delete: - value = old | value - if value != old: - updated.append((key, old, value)) - setattr(item, key, value) - - changed.append((item, created if created else updated)) - - return changed + def __hash__(self): + primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) + if primary is None: + return NotImplemented + else: + return hash(primary) # Many-to-many association table for domain managers @@ -369,20 +168,11 @@ class Config(Base): value = db.Column(JSONEncoded) -@sqlalchemy.event.listens_for(db.session, 'after_commit') -def store_dkim_key(session): - """ Store DKIM key on commit - """ - +def _save_dkim_keys(session): + """ store DKIM keys after commit """ for obj in session.identity_map.values(): if isinstance(obj, Domain): - if obj._dkim_key_changed: - file_path = obj._dkim_file() - if obj._dkim_key: - with open(file_path, 'wb') as handle: - handle.write(obj._dkim_key) - elif os.path.exists(file_path): - os.unlink(file_path) + obj.save_dkim_key() class Domain(Base): """ A DNS domain that has mail addresses associated to it. @@ -390,147 +180,122 @@ class Domain(Base): __tablename__ = 'domain' - _dict_hide = {'users', 'managers', 'aliases'} - _dict_show = {'dkim_key'} - _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} - _dict_secret = {'dkim_key'} - _dict_types = { - 'dkim_key': (bytes, type(None)), - 'dkim_publickey': False, - 'dns_mx': False, - 'dns_spf': False, - 'dns_dkim': False, - 'dns_dmarc': False, - } - _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} - @staticmethod - def _dict_input(data): - if 'dkim_key' in data: - key = data['dkim_key'] - if key is not None: - if type(key) is list: - key = ''.join(key) - if type(key) is str: - key = ''.join(key.strip().split()) # removes all whitespace - if key == 'generate': - data['dkim_key'] = dkim.gen_key() - elif key: - m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) - if m is not None: - key = key[m.end():] - m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) - if m is not None: - key = key[:m.start()] - key = '\n'.join(wrap(key, 64)) - key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') - try: - dkim.strip_key(key) - except: - raise ValueError('invalid dkim key') - else: - data['dkim_key'] = key - else: - data['dkim_key'] = None - name = db.Column(IdnaDomain, primary_key=True, nullable=False) managers = db.relationship('User', secondary=managers, backref=db.backref('manager_of'), lazy='dynamic') max_users = db.Column(db.Integer, nullable=False, default=-1) max_aliases = db.Column(db.Integer, nullable=False, default=-1) - max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) - signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) - + max_quota_bytes = db.Column(db.BigInteger, nullable=False, default=0) + signup_enabled = db.Column(db.Boolean, nullable=False, default=False) + _dkim_key = None - _dkim_key_changed = False + _dkim_key_on_disk = None def _dkim_file(self): + """ return filename for active DKIM key """ return app.config['DKIM_PATH'].format( - domain=self.name, selector=app.config['DKIM_SELECTOR']) + domain=self.name, + selector=app.config['DKIM_SELECTOR'] + ) + + def save_dkim_key(self): + """ save changed DKIM key to disk """ + if self._dkim_key != self._dkim_key_on_disk: + file_path = self._dkim_file() + if self._dkim_key: + with open(file_path, 'wb') as handle: + handle.write(self._dkim_key) + elif os.path.exists(file_path): + os.unlink(file_path) + self._dkim_key_on_disk = self._dkim_key @property def dns_mx(self): - hostname = app.config['HOSTNAMES'].split(',')[0] + """ return MX record for domain """ + hostname = app.config['HOSTNAMES'].split(',', 1)[0] return f'{self.name}. 600 IN MX 10 {hostname}.' - + @property def dns_spf(self): - hostname = app.config['HOSTNAMES'].split(',')[0] + """ return SPF record for domain """ + hostname = app.config['HOSTNAMES'].split(',', 1)[0] return f'{self.name}. 600 IN TXT "v=spf1 mx a:{hostname} ~all"' - + @property def dns_dkim(self): - if os.path.exists(self._dkim_file()): + """ return DKIM record for domain """ + if self.dkim_key: selector = app.config['DKIM_SELECTOR'] - return f'{selector}._domainkey.{self.name}. 600 IN TXT "v=DKIM1; k=rsa; p={self.dkim_publickey}"' + return ( + f'{selector}._domainkey.{self.name}. 600 IN TXT' + f'"v=DKIM1; k=rsa; p={self.dkim_publickey}"' + ) @property def dns_dmarc(self): - if os.path.exists(self._dkim_file()): + """ return DMARC record for domain """ + if self.dkim_key: domain = app.config['DOMAIN'] rua = app.config['DMARC_RUA'] rua = f' rua=mailto:{rua}@{domain};' if rua else '' ruf = app.config['DMARC_RUF'] ruf = f' ruf=mailto:{ruf}@{domain};' if ruf else '' return f'_dmarc.{self.name}. 600 IN TXT "v=DMARC1; p=reject;{rua}{ruf} adkim=s; aspf=s"' - + @property def dkim_key(self): + """ return private DKIM key """ if self._dkim_key is None: file_path = self._dkim_file() if os.path.exists(file_path): with open(file_path, 'rb') as handle: - self._dkim_key = handle.read() + self._dkim_key = self._dkim_key_on_disk = handle.read() else: - self._dkim_key = b'' + self._dkim_key = self._dkim_key_on_disk = b'' return self._dkim_key if self._dkim_key else None @dkim_key.setter def dkim_key(self, value): + """ set private DKIM key """ old_key = self.dkim_key - if value is None: - value = b'' - self._dkim_key_changed = value != old_key - self._dkim_key = value + self._dkim_key = value if value is not None else b'' + if self._dkim_key != old_key: + if not sqlalchemy.event.contains(db.session, 'after_commit', _save_dkim_keys): + sqlalchemy.event.listen(db.session, 'after_commit', _save_dkim_keys) @property def dkim_publickey(self): + """ return public part of DKIM key """ dkim_key = self.dkim_key if dkim_key: return dkim.strip_key(dkim_key).decode('utf8') def generate_dkim_key(self): + """ generate and activate new DKIM key """ self.dkim_key = dkim.gen_key() def has_email(self, localpart): - for email in self.users + self.aliases: + """ checks if localpart is configured for domain """ + for email in chain(self.users, self.aliases): if email.localpart == localpart: return True - else: - return False + return False def check_mx(self): + """ checks if MX record for domain points to mailu host """ try: - hostnames = app.config['HOSTNAMES'].split(',') + hostnames = set(app.config['HOSTNAMES'].split(',')) return any( - str(rset).split()[-1][:-1] in hostnames + rset.exchange.to_text().rstrip('.') in hostnames for rset in dns.resolver.query(self.name, 'MX') ) - except Exception: - return False - - def __str__(self): - return self.name - - def __eq__(self, other): - try: - return self.name == other.name - except AttributeError: + except dns.exception.DNSException: return False class Alternative(Base): """ Alternative name for a served domain. - The name "domain alias" was avoided to prevent some confusion. + The name "domain alias" was avoided to prevent some confusion. """ __tablename__ = 'alternative' @@ -540,9 +305,6 @@ class Alternative(Base): domain = db.relationship(Domain, backref=db.backref('alternatives', cascade='all, delete-orphan')) - def __str__(self): - return self.name - class Relay(Base): """ Relayed mail domain. @@ -551,33 +313,21 @@ class Relay(Base): __tablename__ = 'relay' - _dict_mandatory = {'smtp'} - name = db.Column(IdnaDomain, primary_key=True, nullable=False) + # TODO: use db.String(266)? transport(8):(1)[nexthop(255)](2) smtp = db.Column(db.String(80), nullable=True) - def __str__(self): - return self.name - class Email(object): """ Abstraction for an email address (localpart and domain). """ + # TODO: use db.String(64)? localpart = db.Column(db.String(80), nullable=False) - @staticmethod - def _dict_input(data): - if 'email' in data: - if 'localpart' in data or 'domain' in data: - raise ValueError('ambigous key email and localpart/domain') - elif type(data['email']) is str: - data['localpart'], data['domain'] = data['email'].rsplit('@', 1) - else: - data['email'] = f'{data["localpart"]}@{data["domain"]}' - @declarative.declared_attr def domain_name(cls): + """ the domain part of the email address """ return db.Column(IdnaDomain, db.ForeignKey(Domain.name), nullable=False, default=IdnaDomain) @@ -585,36 +335,49 @@ class Email(object): # It is however very useful for quick lookups without joining tables, # especially when the mail server is reading the database. @declarative.declared_attr - def email(cls): - updater = lambda context: '{0}@{1}'.format( - context.current_parameters['localpart'], - context.current_parameters['domain_name'], - ) - return db.Column(IdnaEmail, - primary_key=True, nullable=False, - default=updater) + def _email(cls): + """ the complete email address (localpart@domain) """ + + def updater(ctx): + key = f'{cls.__tablename__}_email' + if key in ctx.current_parameters: + return ctx.current_parameters[key] + return '{localpart}@{domain_name}'.format(**ctx.current_parameters) + + return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater) + + # We need to keep email, localpart and domain_name in sync. + # But IMHO using email as primary key was not a good idea in the first place. + @hybrid_property + def email(self): + """ getter for email - gets _email """ + return self._email + + @email.setter + def email(self, value): + """ setter for email - sets _email, localpart and domain_name at once """ + self.localpart, self.domain_name = value.rsplit('@', 1) + self._email = value + + # hack for email declared attr - when _email is not updated yet + def __str__(self): + return str(f'{self.localpart}@{self.domain_name}') def sendmail(self, subject, body): - """ Send an email to the address. - """ - from_address = '{0}@{1}'.format( - app.config['POSTMASTER'], - idna.encode(app.config['DOMAIN']).decode('ascii'), - ) + """ send an email to the address """ + f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: - to_address = '{0}@{1}'.format( - self.localpart, - idna.encode(self.domain_name).decode('ascii'), - ) + to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' msg = text.MIMEText(body) msg['Subject'] = subject - msg['From'] = from_address + msg['From'] = f_addr msg['To'] = to_address - smtp.sendmail(from_address, [to_address], msg.as_string()) + smtp.sendmail(f_addr, [to_address], msg.as_string()) @classmethod def resolve_domain(cls, email): - localpart, domain_name = email.split('@', 1) if '@' in email else (None, email) + """ resolves domain alternative to real domain """ + localpart, domain_name = email.rsplit('@', 1) if '@' in email else (None, email) alternative = Alternative.query.get(domain_name) if alternative: domain_name = alternative.domain_name @@ -622,17 +385,19 @@ class Email(object): @classmethod def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False): + """ return destination for email address localpart@domain_name """ + localpart_stripped = None stripped_alias = None if os.environ.get('RECIPIENT_DELIMITER') in localpart: localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0] - user = User.query.get('{}@{}'.format(localpart, domain_name)) + user = User.query.get(f'{localpart}@{domain_name}') if not user and localpart_stripped: - user = User.query.get('{}@{}'.format(localpart_stripped, domain_name)) + user = User.query.get(f'{localpart_stripped}@{domain_name}') if user: - email = '{}@{}'.format(localpart, domain_name) + email = f'{localpart}@{domain_name}' if user.forward_enabled: destination = user.forward_destination @@ -647,13 +412,14 @@ class Email(object): if pure_alias and not pure_alias.wildcard: return pure_alias.destination - elif stripped_alias: + + if stripped_alias: return stripped_alias.destination - elif pure_alias: + + if pure_alias: return pure_alias.destination - def __str__(self): - return self.email + return None class User(Base, Email): @@ -662,49 +428,25 @@ class User(Base, Email): __tablename__ = 'user' - _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} - _dict_mandatory = {'localpart', 'domain', 'password'} - @classmethod - def _dict_input(cls, data): - Email._dict_input(data) - # handle password - if 'password' in data: - if 'password_hash' in data or 'hash_scheme' in data: - raise ValueError('ambigous key password and password_hash/hash_scheme') - # check (hashed) password - password = data['password'] - if password.startswith('{') and '}' in password: - scheme = password[1:password.index('}')] - if scheme not in cls.scheme_dict: - raise ValueError(f'invalid password scheme {scheme!r}') - else: - raise ValueError(f'invalid hashed password {password!r}') - elif 'password_hash' in data and 'hash_scheme' in data: - if data['hash_scheme'] not in cls.scheme_dict: - raise ValueError(f'invalid password scheme {scheme!r}') - data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] - del data['hash_scheme'] - del data['password_hash'] - domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) - quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) - global_admin = db.Column(db.Boolean(), nullable=False, default=False) - enabled = db.Column(db.Boolean(), nullable=False, default=True) + quota_bytes = db.Column(db.BigInteger, nullable=False, default=10**9) + quota_bytes_used = db.Column(db.BigInteger, nullable=False, default=0) + global_admin = db.Column(db.Boolean, nullable=False, default=False) + enabled = db.Column(db.Boolean, nullable=False, default=True) # Features - enable_imap = db.Column(db.Boolean(), nullable=False, default=True) - enable_pop = db.Column(db.Boolean(), nullable=False, default=True) + enable_imap = db.Column(db.Boolean, nullable=False, default=True) + enable_pop = db.Column(db.Boolean, nullable=False, default=True) # Filters - forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) - forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) - forward_keep = db.Column(db.Boolean(), nullable=False, default=True) - reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) + forward_enabled = db.Column(db.Boolean, nullable=False, default=False) + forward_destination = db.Column(CommaSeparatedList, nullable=True, default=list) + forward_keep = db.Column(db.Boolean, nullable=False, default=True) + reply_enabled = db.Column(db.Boolean, nullable=False, default=False) reply_subject = db.Column(db.String(255), nullable=True, default=None) - reply_body = db.Column(db.Text(), nullable=True, default=None) + reply_body = db.Column(db.Text, nullable=True, default=None) reply_startdate = db.Column(db.Date, nullable=False, default=date(1900, 1, 1)) reply_enddate = db.Column(db.Date, nullable=False, @@ -712,8 +454,8 @@ class User(Base, Email): # Settings displayed_name = db.Column(db.String(160), nullable=False, default='') - spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) - spam_threshold = db.Column(db.Integer(), nullable=False, default=80) + spam_enabled = db.Column(db.Boolean, nullable=False, default=True) + spam_threshold = db.Column(db.Integer, nullable=False, default=80) # Flask-login attributes is_authenticated = True @@ -721,10 +463,12 @@ class User(Base, Email): is_anonymous = False def get_id(self): + """ return users email address """ return self.email @property def destination(self): + """ returns comma separated string of destinations """ if self.forward_enabled: result = list(self.forward_destination) if self.forward_keep: @@ -735,6 +479,7 @@ class User(Base, Email): @property def reply_active(self): + """ returns status of autoreply function """ now = date.today() return ( self.reply_enabled and @@ -742,48 +487,57 @@ class User(Base, Email): self.reply_enddate > now ) - scheme_dict = {'PBKDF2': 'pbkdf2_sha512', - 'BLF-CRYPT': 'bcrypt', - 'SHA512-CRYPT': 'sha512_crypt', - 'SHA256-CRYPT': 'sha256_crypt', - 'MD5-CRYPT': 'md5_crypt', - 'CRYPT': 'des_crypt'} + scheme_dict = { + 'PBKDF2': 'pbkdf2_sha512', + 'BLF-CRYPT': 'bcrypt', + 'SHA512-CRYPT': 'sha512_crypt', + 'SHA256-CRYPT': 'sha256_crypt', + 'MD5-CRYPT': 'md5_crypt', + 'CRYPT': 'des_crypt', + } - def get_password_context(self): - return context.CryptContext( - schemes=self.scheme_dict.values(), - default=self.scheme_dict[app.config['PASSWORD_SCHEME']], + @classmethod + def get_password_context(cls): + """ Create password context for hashing and verification + """ + return passlib.context.CryptContext( + schemes=cls.scheme_dict.values(), + default=cls.scheme_dict[app.config['PASSWORD_SCHEME']], ) - def check_password(self, password): + def check_password(self, plain): + """ Check password against stored hash + Update hash when default scheme has changed + """ context = self.get_password_context() - reference = re.match('({[^}]+})?(.*)', self.password).group(2) - result = context.verify(password, reference) - if result and context.identify(reference) != context.default_scheme(): - self.set_password(password) + hashed = re.match('^({[^}]+})?(.*)$', self.password).group(2) + result = context.verify(plain, hashed) + if result and context.identify(hashed) != context.default_scheme(): + self.set_password(plain) db.session.add(self) db.session.commit() return result - def set_password(self, password, hash_scheme=None, raw=False): - """Set password for user with specified encryption scheme - @password: plain text password to encrypt (if raw == True the hash itself) + def set_password(self, new, hash_scheme=None, raw=False): + """ Set password for user with specified encryption scheme + @new: plain text password to encrypt (or, if raw is True: the hash itself) """ + # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes if hash_scheme is None: hash_scheme = app.config['PASSWORD_SCHEME'] - # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes - if raw: - self.password = '{'+hash_scheme+'}' + password - else: - self.password = '{'+hash_scheme+'}' + self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) + if not raw: + new = self.get_password_context().encrypt(new, self.scheme_dict[hash_scheme]) + self.password = f'{{{hash_scheme}}}{new}' def get_managed_domains(self): + """ return list of domains this user can manage """ if self.global_admin: return Domain.query.all() else: return self.manager_of def get_managed_emails(self, include_aliases=True): + """ returns list of email addresses this user can manage """ emails = [] for domain in self.get_managed_domains(): emails.extend(domain.users) @@ -792,16 +546,18 @@ class User(Base, Email): return emails def send_welcome(self): + """ send welcome email to user """ if app.config['WELCOME']: - self.sendmail(app.config['WELCOME_SUBJECT'], - app.config['WELCOME_BODY']) + self.sendmail(app.config['WELCOME_SUBJECT'], app.config['WELCOME_BODY']) @classmethod def get(cls, email): + """ find user object for email address """ return cls.query.get(email) @classmethod def login(cls, email, password): + """ login user when enabled and password is valid """ user = cls.query.get(email) return user if (user and user.enabled and user.check_password(password)) else None @@ -812,30 +568,23 @@ class Alias(Base, Email): __tablename__ = 'alias' - _dict_hide = {'domain_name', 'domain', 'localpart'} - @staticmethod - def _dict_input(data): - Email._dict_input(data) - # handle comma delimited string for backwards compability - dst = data.get('destination') - if type(dst) is str: - data['destination'] = list([adr.strip() for adr in dst.split(',')]) - domain = db.relationship(Domain, backref=db.backref('aliases', cascade='all, delete-orphan')) - wildcard = db.Column(db.Boolean(), nullable=False, default=False) - destination = db.Column(CommaSeparatedList, nullable=False, default=[]) + wildcard = db.Column(db.Boolean, nullable=False, default=False) + destination = db.Column(CommaSeparatedList, nullable=False, default=list) @classmethod def resolve(cls, localpart, domain_name): + """ find aliases matching email address localpart@domain_name """ + alias_preserve_case = cls.query.filter( sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( sqlalchemy.and_( - cls.wildcard == False, + cls.wildcard is False, cls.localpart == localpart ), sqlalchemy.and_( - cls.wildcard == True, + cls.wildcard is True, sqlalchemy.bindparam('l', localpart).like(cls.localpart) ) ) @@ -847,27 +596,29 @@ class Alias(Base, Email): sqlalchemy.and_(cls.domain_name == domain_name, sqlalchemy.or_( sqlalchemy.and_( - cls.wildcard == False, + cls.wildcard is False, sqlalchemy.func.lower(cls.localpart) == localpart_lower ), sqlalchemy.and_( - cls.wildcard == True, - sqlalchemy.bindparam('l', localpart_lower).like(sqlalchemy.func.lower(cls.localpart)) + cls.wildcard is True, + sqlalchemy.bindparam('l', localpart_lower).like( + sqlalchemy.func.lower(cls.localpart)) ) ) ) - ).order_by(cls.wildcard, sqlalchemy.func.char_length(sqlalchemy.func.lower(cls.localpart)).desc()).first() + ).order_by(cls.wildcard, sqlalchemy.func.char_length( + sqlalchemy.func.lower(cls.localpart)).desc()).first() if alias_preserve_case and alias_lower_case: - if alias_preserve_case.wildcard: - return alias_lower_case - else: - return alias_preserve_case - elif alias_preserve_case and not alias_lower_case: + return alias_lower_case if alias_preserve_case.wildcard else alias_preserve_case + + if alias_preserve_case and not alias_lower_case: return alias_preserve_case - elif alias_lower_case and not alias_preserve_case: + + if alias_lower_case and not alias_preserve_case: return alias_lower_case - else: - return None + + return None + class Token(Base): """ A token is an application password for a given user. @@ -875,26 +626,25 @@ class Token(Base): __tablename__ = 'token' - _dict_recurse = True - _dict_hide = {'user', 'user_email'} - _dict_mandatory = {'password'} - - id = db.Column(db.Integer(), primary_key=True) + id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) + # TODO: use db.String(32)? ip = db.Column(db.String(255)) def check_password(self, password): - return hash.sha256_crypt.verify(password, self.password) + """ verifies password against stored hash """ + return passlib.hash.sha256_crypt.verify(password, self.password) def set_password(self, password): - self.password = hash.sha256_crypt.using(rounds=1000).hash(password) + """ sets password using sha256_crypt(rounds=1000) """ + self.password = passlib.hash.sha256_crypt.using(rounds=1000).hash(password) - def __str__(self): - return self.comment or self.ip + def __repr__(self): + return f'' class Fetch(Base): @@ -904,25 +654,219 @@ class Fetch(Base): __tablename__ = 'fetch' - _dict_recurse = True - _dict_hide = {'user_email', 'user', 'last_check', 'error'} - _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} - _dict_secret = {'password'} - - id = db.Column(db.Integer(), primary_key=True) + id = db.Column(db.Integer, primary_key=True) user_email = db.Column(db.String(255), db.ForeignKey(User.email), nullable=False) user = db.relationship(User, backref=db.backref('fetches', cascade='all, delete-orphan')) protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) host = db.Column(db.String(255), nullable=False) - port = db.Column(db.Integer(), nullable=False) - tls = db.Column(db.Boolean(), nullable=False, default=False) + port = db.Column(db.Integer, nullable=False) + tls = db.Column(db.Boolean, nullable=False, default=False) username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) - keep = db.Column(db.Boolean(), nullable=False, default=False) + keep = db.Column(db.Boolean, nullable=False, default=False) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) - def __str__(self): - return f'{self.protocol}{"s" if self.tls else ""}://{self.username}@{self.host}:{self.port}' + def __repr__(self): + return ( + f'' + ) + + +class MailuConfig: + """ Class which joins whole Mailu config for dumping + and loading + """ + + class MailuCollection: + """ Provides dict- and list-like access to instances + of a sqlalchemy model + """ + + def __init__(self, model : db.Model): + self.model = model + + def __repr__(self): + return f'<{self.model.__name__}-Collection>' + + @cached_property + def _items(self): + return { + inspect(item).identity: item + for item in self.model.query.all() + } + + def __len__(self): + return len(self._items) + + def __iter__(self): + return iter(self._items.values()) + + def __getitem__(self, key): + return self._items[key] + + def __setitem__(self, key, item): + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items[key] = item + + def __delitem__(self, key): + del self._items[key] + + def append(self, item, update=False): + """ list-like append """ + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + key = inspect(item).identity + if key in self._items: + if not update: + raise ValueError(f'item {key!r} already present in collection') + self._items[key] = item + + def extend(self, items, update=False): + """ list-like extend """ + add = {} + for item in items: + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + key = inspect(item).identity + if not update and key in self._items: + raise ValueError(f'item {key!r} already present in collection') + add[key] = item + self._items.update(add) + + def pop(self, *args): + """ list-like (no args) and dict-like (1 or 2 args) pop """ + if args: + if len(args) > 2: + raise TypeError(f'pop expected at most 2 arguments, got {len(args)}') + return self._items.pop(*args) + else: + return self._items.popitem()[1] + + def popitem(self): + """ dict-like popitem """ + return self._items.popitem() + + def remove(self, item): + """ list-like remove """ + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + key = inspect(item).identity + if not key in self._items: + raise ValueError(f'item {key!r} not found in collection') + del self._items[key] + + def clear(self): + """ dict-like clear """ + while True: + try: + self.pop() + except IndexError: + break + + def update(self, items): + """ dict-like update """ + for key, item in items: + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items.update(items) + + def setdefault(self, key, item=None): + """ dict-like setdefault """ + if key in self._items: + return self._items[key] + if item is None: + return None + if not isinstance(item, self.model): + raise TypeError(f'expected {self.model.name}') + if key != inspect(item).identity: + raise ValueError(f'item identity != key {key!r}') + self._items[key] = item + return item + + def __init__(self): + + # section-name -> attr + self._sections = { + name: getattr(self, name) + for name in dir(self) + if isinstance(getattr(self, name), self.MailuCollection) + } + + # known models + self._models = tuple(section.model for section in self._sections.values()) + + # model -> attr + self._sections.update({ + section.model: section for section in self._sections.values() + }) + + def _get_model(self, section): + if section is None: + return None + model = self._sections.get(section) + if model is None: + raise ValueError(f'Invalid section: {section!r}') + if isinstance(model, self.MailuCollection): + return model.model + return model + + def _add(self, items, section, update): + + model = self._get_model(section) + if isinstance(items, self._models): + items = [items] + elif not hasattr(items, '__iter__'): + raise ValueError(f'{items!r} is not iterable') + + for item in items: + if model is not None and not isinstance(item, model): + what = item.__class__.__name__.capitalize() + raise ValueError(f'{what} can not be added to section {section!r}') + self._sections[type(item)].append(item, update=update) + + def add(self, items, section=None): + """ add item to config """ + self._add(items, section, update=False) + + def update(self, items, section=None): + """ add or replace item in config """ + self._add(items, section, update=True) + + def remove(self, items, section=None): + """ remove item from config """ + model = self._get_model(section) + if isinstance(items, self._models): + items = [items] + elif not hasattr(items, '__iter__'): + raise ValueError(f'{items!r} is not iterable') + + for item in items: + if isinstance(item, str): + if section is None: + raise ValueError(f'Cannot remove key {item!r} without section') + del self._sections[model][item] + elif model is not None and not isinstance(item, model): + what = item.__class__.__name__.capitalize() + raise ValueError(f'{what} can not be removed from section {section!r}') + self._sections[type(item)].remove(item,) + + def clear(self, models=None): + """ remove complete configuration """ + for model in self._models: + if models is None or model in models: + db.session.query(model).delete() + + domain = MailuCollection(Domain) + user = MailuCollection(User) + alias = MailuCollection(Alias) + relay = MailuCollection(Relay) + config = MailuCollection(Config) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py new file mode 100644 index 00000000..7d0393f0 --- /dev/null +++ b/core/admin/mailu/schemas.py @@ -0,0 +1,945 @@ +""" Mailu marshmallow fields and schema +""" + +from copy import deepcopy +from textwrap import wrap + +import re +import json +import yaml + +import sqlalchemy + +from marshmallow import pre_load, post_load, post_dump, fields, Schema +from marshmallow.utils import ensure_text_type +from marshmallow.exceptions import ValidationError +from marshmallow_sqlalchemy import SQLAlchemyAutoSchemaOpts +from marshmallow_sqlalchemy.fields import RelatedList + +from flask_marshmallow import Marshmallow + +from OpenSSL import crypto + +try: + from pygments import highlight + from pygments.token import Token + from pygments.lexers import get_lexer_by_name + from pygments.lexers.data import YamlLexer + from pygments.formatters import get_formatter_by_name +except ModuleNotFoundError: + COLOR_SUPPORTED = False +else: + COLOR_SUPPORTED = True + +from . import models, dkim + + +ma = Marshmallow() + +# TODO: how and where to mark keys as "required" while deserializing in api? +# - when modifying, nothing is required (only the primary key, but this key is in the uri) +# - the primary key from post data must not differ from the key in the uri +# - when creating all fields without default or auto-increment are required +# TODO: validate everything! + + +### class for hidden values ### + +class _Hidden: + def __bool__(self): + return False + def __copy__(self): + return self + def __deepcopy__(self, _): + return self + def __eq__(self, other): + return str(other) == '' + def __repr__(self): + return '' + __str__ = __repr__ + +HIDDEN = _Hidden() + + +### map model to schema ### + +_model2schema = {} + +def get_schema(model=None): + """ return schema class for model or instance of model """ + if model is None: + return _model2schema.values() + else: + return _model2schema.get(model) or _model2schema.get(model.__class__) + +def mapped(cls): + """ register schema in model2schema map """ + _model2schema[cls.Meta.model] = cls + return cls + + +### helper functions ### + +def get_fieldspec(exc): + """ walk traceback to extract spec of invalid field from marshmallow """ + path = [] + tbck = exc.__traceback__ + while tbck: + if tbck.tb_frame.f_code.co_name == '_serialize': + if 'attr' in tbck.tb_frame.f_locals: + path.append(tbck.tb_frame.f_locals['attr']) + elif tbck.tb_frame.f_code.co_name == '_init_fields': + path = '.'.join(path) + spec = ', '.join([f'{path}.{key}' for key in tbck.tb_frame.f_locals['invalid_fields']]) + return spec + tbck = tbck.tb_next + return None + +def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): + """ add ANSI color to data """ + if color is None: + # autodetect colorize + color = COLOR_SUPPORTED + if not color: + # no color wanted + return data + if not COLOR_SUPPORTED: + # want color, but not supported + raise ValueError('Please install pygments to colorize output') + + scheme = { + Token: ('', ''), + Token.Name.Tag: ('cyan', 'brightcyan'), + Token.Literal.Scalar: ('green', 'green'), + Token.Literal.String: ('green', 'green'), + Token.Keyword.Constant: ('magenta', 'brightmagenta'), + Token.Literal.Number: ('magenta', 'brightmagenta'), + Token.Error: ('red', 'brightred'), + Token.Name: ('red', 'brightred'), + Token.Operator: ('red', 'brightred'), + } + + class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + + res = highlight( + data, + MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer), + get_formatter_by_name(formatter, colorscheme=scheme) + ) + + return res.rstrip('\n') if strip else res + + +### render modules ### + +# allow yaml to represent hidden attributes +yaml.add_representer( + _Hidden, + lambda cls, data: cls.represent_data(str(data)) +) + +class RenderYAML: + """ Marshmallow YAML Render Module + """ + + class SpacedDumper(yaml.Dumper): + """ YAML Dumper to add a newline between main sections + and double the indent used + """ + + def write_line_break(self, data=None): + super().write_line_break(data) + if len(self.indents) == 1: + super().write_line_break() + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + @staticmethod + def _augment(kwargs, defaults): + """ add default kv's to kwargs if missing + """ + for key, value in defaults.items(): + if key not in kwargs: + kwargs[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load yaml data from string + """ + cls._augment(kwargs, cls._load_defaults) + return yaml.safe_load(*args, **kwargs) + + _dump_defaults = { + 'Dumper': SpacedDumper, + 'default_flow_style': False, + 'allow_unicode': True, + 'sort_keys': False, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump data to yaml string + """ + cls._augment(kwargs, cls._dump_defaults) + return yaml.dump(*args, **kwargs) + +class JSONEncoder(json.JSONEncoder): + """ JSONEncoder supporting serialization of HIDDEN """ + def default(self, o): + """ serialize HIDDEN """ + if isinstance(o, _Hidden): + return str(o) + return json.JSONEncoder.default(self, o) + +class RenderJSON: + """ Marshmallow JSON Render Module + """ + + @staticmethod + def _augment(kwargs, defaults): + """ add default kv's to kwargs if missing + """ + for key, value in defaults.items(): + if key not in kwargs: + kwargs[key] = value + + _load_defaults = {} + @classmethod + def loads(cls, *args, **kwargs): + """ load json data from string + """ + cls._augment(kwargs, cls._load_defaults) + return json.loads(*args, **kwargs) + + _dump_defaults = { + 'separators': (',',':'), + 'cls': JSONEncoder, + } + @classmethod + def dumps(cls, *args, **kwargs): + """ dump data to json string + """ + cls._augment(kwargs, cls._dump_defaults) + return json.dumps(*args, **kwargs) + + +### custom fields ### + +class LazyStringField(fields.String): + """ Field that serializes a "false" value to the empty string + """ + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize None to the empty string + """ + return value if value else '' + +class CommaSeparatedListField(fields.Raw): + """ Deserialize a string containing comma-separated values to + a list of strings + """ + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize comma separated string to list of strings + """ + + # empty + if not value: + return [] + + # split string + if isinstance(value, str): + return list([item.strip() for item in value.split(',') if item.strip()]) + else: + return value + + +class DkimKeyField(fields.String): + """ Serialize a dkim key to a list of strings (lines) and + Deserialize a string or list of strings to a valid dkim key + """ + + default_error_messages = { + "invalid": "Not a valid string or list.", + "invalid_utf8": "Not a valid utf-8 string or list.", + } + + _clean_re = re.compile( + r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', + flags=re.UNICODE + ) + + def _serialize(self, value, attr, obj, **kwargs): + """ serialize dkim key to a list of strings (lines) + """ + + # map empty string and None to None + if not value: + return None + + # return list of key lines without header/footer + return value.decode('utf-8').strip().split('\n')[1:-1] + + def _deserialize(self, value, attr, data, **kwargs): + """ deserialize a string or list of strings to dkim key data + with verification + """ + + # convert list to str + if isinstance(value, list): + try: + value = ''.join([ensure_text_type(item) for item in value]) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + + # only text is allowed + else: + if not isinstance(value, (str, bytes)): + raise self.make_error("invalid") + try: + value = ensure_text_type(value) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + + # clean value (remove whitespace and header/footer) + value = self._clean_re.sub('', value.strip()) + + # map empty string/list to None + if not value: + return None + + # handle special value 'generate' + elif value == 'generate': + return dkim.gen_key() + + # remember some keydata for error message + keydata = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value + + # wrap value into valid pem layout and check validity + value = ( + '-----BEGIN PRIVATE KEY-----\n' + + '\n'.join(wrap(value, 64)) + + '\n-----END PRIVATE KEY-----\n' + ).encode('ascii') + try: + crypto.load_privatekey(crypto.FILETYPE_PEM, value) + except crypto.Error as exc: + raise ValidationError(f'invalid dkim key {keydata!r}') from exc + else: + return value + +class PasswordField(fields.Str): + """ Serialize a hashed password hash by stripping the obsolete {SCHEME} + Deserialize a plain password or hashed password into a hashed password + """ + + _hashes = {'PBKDF2', 'BLF-CRYPT', 'SHA512-CRYPT', 'SHA256-CRYPT', 'MD5-CRYPT', 'CRYPT'} + + def _serialize(self, value, attr, obj, **kwargs): + """ strip obsolete {password-hash} when serializing """ + # strip scheme spec if in database - it's obsolete + if value.startswith('{') and (end := value.find('}', 1)) >= 0: + if value[1:end] in self._hashes: + return value[end+1:] + return value + + def _deserialize(self, value, attr, data, **kwargs): + """ hashes plain password or checks hashed password + also strips obsolete {password-hash} when deserializing + """ + + # when hashing is requested: use model instance to hash plain password + if data.get('hash_password'): + # hash password using model instance + inst = self.metadata['model']() + inst.set_password(value) + value = inst.password + del inst + + # strip scheme spec when specified - it's obsolete + if value.startswith('{') and (end := value.find('}', 1)) >= 0: + if value[1:end] in self._hashes: + value = value[end+1:] + + # check if algorithm is supported + inst = self.metadata['model'](password=value) + try: + # just check against empty string to see if hash is valid + inst.check_password('') + except ValueError as exc: + # ValueError: hash could not be identified + raise ValidationError(f'invalid password hash {value!r}') from exc + del inst + + return value + + +### base schema ### + +class BaseOpts(SQLAlchemyAutoSchemaOpts): + """ Option class with sqla session + """ + def __init__(self, meta, ordered=False): + if not hasattr(meta, 'sqla_session'): + meta.sqla_session = models.db.session + if not hasattr(meta, 'sibling'): + meta.sibling = False + super(BaseOpts, self).__init__(meta, ordered=ordered) + +class BaseSchema(ma.SQLAlchemyAutoSchema): + """ Marshmallow base schema with custom exclude logic + and option to hide sqla defaults + """ + + OPTIONS_CLASS = BaseOpts + + class Meta: + """ Schema config """ + include_by_context = {} + exclude_by_value = {} + hide_by_context = {} + order = [] + sibling = False + + def __init__(self, *args, **kwargs): + + # get context + context = kwargs.get('context', {}) + flags = {key for key, value in context.items() if value is True} + + # compile excludes + exclude = set(kwargs.get('exclude', [])) + + # always exclude + exclude.update({'created_at', 'updated_at'}) + + # add include_by_context + if context is not None: + for need, what in getattr(self.Meta, 'include_by_context', {}).items(): + if not flags & set(need): + exclude |= set(what) + + # update excludes + kwargs['exclude'] = exclude + + # init SQLAlchemyAutoSchema + super().__init__(*args, **kwargs) + + # exclude_by_value + self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) + + # exclude default values + if not context.get('full'): + for column in self.opts.model.__table__.columns: + if column.name not in exclude: + self._exclude_by_value.setdefault(column.name, []).append( + None if column.default is None else column.default.arg + ) + + # hide by context + self._hide_by_context = set() + if context is not None: + for need, what in getattr(self.Meta, 'hide_by_context', {}).items(): + if not flags & set(need): + self._hide_by_context |= set(what) + + # remember primary keys + self._primary = str(self.opts.model.__table__.primary_key.columns.values()[0].name) + + # determine attribute order + if hasattr(self.Meta, 'order'): + # use user-defined order + order = self.Meta.order + else: + # default order is: primary_key + other keys alphabetically + order = list(sorted(self.fields.keys())) + if self._primary in order: + order.remove(self._primary) + order.insert(0, self._primary) + + # order dump_fields + for field in order: + if field in self.dump_fields: + self.dump_fields[field] = self.dump_fields.pop(field) + + # move pre_load hook "_track_import" to the front + hooks = self._hooks[('pre_load', False)] + hooks.remove('_track_import') + hooks.insert(0, '_track_import') + # move pre_load hook "_add_instance" to the end + hooks.remove('_add_required') + hooks.append('_add_required') + + # move post_load hook "_add_instance" to the end + hooks = self._hooks[('post_load', False)] + hooks.remove('_add_instance') + hooks.append('_add_instance') + + def hide(self, data): + """ helper method to hide input data for logging """ + # always returns a copy of data + return { + key: HIDDEN if key in self._hide_by_context else deepcopy(value) + for key, value in data.items() + } + + def _call_and_store(self, *args, **kwargs): + """ track curent parent field for pruning """ + self.context['parent_field'] = kwargs['field_name'] + return super()._call_and_store(*args, **kwargs) + + # this is only needed to work around the declared attr "email" primary key in model + def get_instance(self, data): + """ lookup item by defined primary key instead of key(s) from model """ + if self.transient: + return None + if keys := getattr(self.Meta, 'primary_keys', None): + filters = {key: data.get(key) for key in keys} + if None not in filters.values(): + return self.session.query(self.opts.model).filter_by(**filters).first() + return super().get_instance(data) + + @pre_load(pass_many=True) + def _patch_input(self, items, many, **kwargs): # pylint: disable=unused-argument + """ - flush sqla session before serializing a section when requested + (make sure all objects that could be referred to later are created) + - when in update mode: patch input data before deserialization + - handle "prune" and "delete" items + - replace values in keys starting with '-' with default + """ + + # flush sqla session + if not self.Meta.sibling: + self.opts.sqla_session.flush() + + # stop early when not updating + if not self.context.get('update'): + return items + + # patch "delete", "prune" and "default" + want_prune = [] + def patch(count, data, prune): + + # don't allow __delete__ coming from input + if '__delete__' in data: + raise ValidationError('Unknown field.', f'{count}.__delete__') + + # handle "prune list" and "delete item" (-pkey: none and -pkey: id) + for key in data: + if key.startswith('-'): + if key[1:] == self._primary: + # delete or prune + if data[key] is None: + # prune + prune.append(True) + return None + # mark item for deletion + return {key[1:]: data[key], '__delete__': True} + + # handle "set to default value" (-key: none) + def set_default(key, value): + if not key.startswith('-'): + return (key, value) + key = key[1:] + if not key in self.opts.model.__table__.columns: + return (key, None) + if value is not None: + raise ValidationError( + 'When resetting to default value must be null.', + f'{count}.{key}' + ) + value = self.opts.model.__table__.columns[key].default + if value is None: + raise ValidationError( + 'Field has no default value.', + f'{count}.{key}' + ) + return (key, value.arg) + + return dict([set_default(key, value) for key, value in data.items()]) + + # convert items to "delete" and filter "prune" item + items = [ + item for item in [ + patch(count, item, want_prune) for count, item in enumerate(items) + ] if item + ] + + # prune: determine if existing items in db need to be added or marked for deletion + add_items = False + del_items = False + if self.Meta.sibling: + # parent prunes automatically + if not want_prune: + # no prune requested => add old items + add_items = True + else: + # parent does not prune automatically + if want_prune: + # prune requested => mark old items for deletion + del_items = True + + if add_items or del_items: + existing = {item[self._primary] for item in items if self._primary in item} + for item in getattr(self.context['parent'], self.context['parent_field']): + key = getattr(item, self._primary) + if key not in existing: + if add_items: + items.append({self._primary: key}) + else: + items.append({self._primary: key, '__delete__': True}) + + return items + + @pre_load + def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument + """ call callback function to track import + """ + # callback + if callback := self.context.get('callback'): + callback(self, data) + + return data + + @pre_load + def _add_required(self, data, many, **kwargs): # pylint: disable=unused-argument + """ when updating: + allow modification of existing items having required attributes + by loading existing value from db + """ + + if not self.opts.load_instance or not self.context.get('update'): + return data + + # stabilize import of auto-increment primary keys (not required), + # by matching import data to existing items and setting primary key + if not self._primary in data: + for item in getattr(self.context['parent'], self.context['parent_field']): + existing = self.dump(item, many=False) + this = existing.pop(self._primary) + if data == existing: + instance = item + data[self._primary] = this + break + + # try to load instance + instance = self.instance or self.get_instance(data) + if instance is None: + + if '__delete__' in data: + # deletion of non-existent item requested + raise ValidationError( + f'item to delete not found: {data[self._primary]!r}', + field_name=f'?.{self._primary}', + ) + + else: + + if self.context.get('update'): + # remember instance as parent for pruning siblings + if not self.Meta.sibling: + self.context['parent'] = instance + # delete instance when marked + if '__delete__' in data: + self.opts.sqla_session.delete(instance) + # delete item from lists or prune lists + # currently: domain.alternatives, user.forward_destination, + # user.manager_of, aliases.destination + for key, value in data.items(): + if not isinstance(self.fields[key], fields.Nested) and isinstance(value, list): + new_value = set(value) + # handle list pruning + if '-prune-' in value: + value.remove('-prune-') + new_value.remove('-prune-') + else: + for old in getattr(instance, key): + # using str() is okay for now (see above) + new_value.add(str(old)) + # handle item deletion + for item in value: + if item.startswith('-'): + new_value.remove(item) + try: + new_value.remove(item[1:]) + except KeyError as exc: + raise ValidationError( + f'item to delete not found: {item[1:]!r}', + field_name=f'?.{key}', + ) from exc + # deduplicate and sort list + data[key] = sorted(new_value) + # log backref modification not catched by hook + if isinstance(self.fields[key], RelatedList): + if callback := self.context.get('callback'): + callback(self, instance, { + 'key': key, + 'target': str(instance), + 'before': [str(v) for v in getattr(instance, key)], + 'after': data[key], + }) + + + + # add attributes required for validation from db + # TODO: this will cause validation errors if value from database does not validate + # but there should not be an invalid value in the database + for attr_name, field_obj in self.load_fields.items(): + if field_obj.required and attr_name not in data: + data[attr_name] = getattr(instance, attr_name) + + return data + + @post_load(pass_original=True) + def _add_instance(self, item, original, many, **kwargs): # pylint: disable=unused-argument + """ add new instances to sqla session """ + + if item in self.opts.sqla_session: + # item was modified + if 'hash_password' in original: + # stabilize import of passwords to be hashed, + # by not re-hashing an unchanged password + if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): + if attr.history.has_changes() and attr.history.deleted: + try: + # reset password hash, if password was not changed + inst = type(item)(password=attr.history.deleted[-1]) + if inst.check_password(original['password']): + item.password = inst.password + except ValueError: + # hash in db is invalid + pass + else: + del inst + else: + # new item + self.opts.sqla_session.add(item) + return item + + @post_dump + def _hide_values(self, data, many, **kwargs): # pylint: disable=unused-argument + """ hide secrets and order output """ + + # stop early when not excluding/hiding + if not self._exclude_by_value and not self._hide_by_context: + return data + + # exclude or hide values + full = self.context.get('full') + return type(data)([ + (key, HIDDEN if key in self._hide_by_context else value) + for key, value in data.items() + if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key] + ]) + + # this field is used to mark items for deletion + mark_delete = fields.Boolean(data_key='__delete__', load_only=True) + + # TODO: remove LazyStringField (when model was changed - IMHO comment should not be nullable) + comment = LazyStringField() + + +### schema definitions ### + +@mapped +class DomainSchema(BaseSchema): + """ Marshmallow schema for Domain model """ + class Meta: + """ Schema config """ + model = models.Domain + load_instance = True + include_relationships = True + exclude = ['users', 'managers', 'aliases'] + + include_by_context = { + ('dns',): {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}, + } + hide_by_context = { + ('secrets',): {'dkim_key'}, + } + exclude_by_value = { + 'alternatives': [[]], + 'dkim_key': [None], + 'dkim_publickey': [None], + 'dns_mx': [None], + 'dns_spf': [None], + 'dns_dkim': [None], + 'dns_dmarc': [None], + } + + dkim_key = DkimKeyField(allow_none=True) + dkim_publickey = fields.String(dump_only=True) + dns_mx = fields.String(dump_only=True) + dns_spf = fields.String(dump_only=True) + dns_dkim = fields.String(dump_only=True) + dns_dmarc = fields.String(dump_only=True) + + +@mapped +class TokenSchema(BaseSchema): + """ Marshmallow schema for Token model """ + class Meta: + """ Schema config """ + model = models.Token + load_instance = True + + sibling = True + + password = PasswordField(required=True, metadata={'model': models.User}) + hash_password = fields.Boolean(load_only=True, missing=False) + + +@mapped +class FetchSchema(BaseSchema): + """ Marshmallow schema for Fetch model """ + class Meta: + """ Schema config """ + model = models.Fetch + load_instance = True + + sibling = True + include_by_context = { + ('full', 'import'): {'last_check', 'error'}, + } + hide_by_context = { + ('secrets',): {'password'}, + } + + +@mapped +class UserSchema(BaseSchema): + """ Marshmallow schema for User model """ + class Meta: + """ Schema config """ + model = models.User + load_instance = True + include_relationships = True + exclude = ['_email', 'domain', 'localpart', 'domain_name', 'quota_bytes_used'] + + primary_keys = ['email'] + exclude_by_value = { + 'forward_destination': [[]], + 'tokens': [[]], + 'fetches': [[]], + 'manager_of': [[]], + 'reply_enddate': ['2999-12-31'], + 'reply_startdate': ['1900-01-01'], + } + + email = fields.String(required=True) + tokens = fields.Nested(TokenSchema, many=True) + fetches = fields.Nested(FetchSchema, many=True) + + password = PasswordField(required=True, metadata={'model': models.User}) + hash_password = fields.Boolean(load_only=True, missing=False) + + +@mapped +class AliasSchema(BaseSchema): + """ Marshmallow schema for Alias model """ + class Meta: + """ Schema config """ + model = models.Alias + load_instance = True + exclude = ['_email', 'domain', 'localpart', 'domain_name'] + + primary_keys = ['email'] + exclude_by_value = { + 'destination': [[]], + } + + email = fields.String(required=True) + destination = CommaSeparatedListField() + + +@mapped +class ConfigSchema(BaseSchema): + """ Marshmallow schema for Config model """ + class Meta: + """ Schema config """ + model = models.Config + load_instance = True + + +@mapped +class RelaySchema(BaseSchema): + """ Marshmallow schema for Relay model """ + class Meta: + """ Schema config """ + model = models.Relay + load_instance = True + + +class MailuSchema(Schema): + """ Marshmallow schema for complete Mailu config """ + class Meta: + """ Schema config """ + render_module = RenderYAML + + order = ['domain', 'user', 'alias', 'relay'] # 'config' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # order dump_fields + for field in self.Meta.order: + if field in self.dump_fields: + self.dump_fields[field] = self.dump_fields.pop(field) + + def _call_and_store(self, *args, **kwargs): + """ track current parent and field for pruning """ + self.context.update({ + 'parent': self.context.get('config'), + 'parent_field': kwargs['field_name'], + }) + return super()._call_and_store(*args, **kwargs) + + @pre_load + def _clear_config(self, data, many, **kwargs): # pylint: disable=unused-argument + """ create config object in context if missing + and clear it if requested + """ + if 'config' not in self.context: + self.context['config'] = models.MailuConfig() + if self.context.get('clear'): + self.context['config'].clear( + models = {field.nested.opts.model for field in self.fields.values()} + ) + return data + + @post_load + def _make_config(self, data, many, **kwargs): # pylint: disable=unused-argument + """ update and return config object """ + config = self.context['config'] + for section in self.Meta.order: + if section in data: + config.update(data[section], section) + + return config + + domain = fields.Nested(DomainSchema, many=True) + user = fields.Nested(UserSchema, many=True) + alias = fields.Nested(AliasSchema, many=True) + relay = fields.Nested(RelaySchema, many=True) +# config = fields.Nested(ConfigSchema, many=True) diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index a3c32855..8ad412cf 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -15,6 +15,7 @@ Flask-Bootstrap==3.3.7.1 Flask-DebugToolbar==0.10.1 Flask-Limiter==1.0.1 Flask-Login==0.4.1 +flask-marshmallow==0.14.0 Flask-Migrate==2.4.0 Flask-Script==2.0.6 Flask-SQLAlchemy==2.4.0 @@ -29,6 +30,8 @@ limits==1.3 Mako==1.0.9 MarkupSafe==1.1.1 mysqlclient==1.4.2.post1 +marshmallow==3.10.0 +marshmallow-sqlalchemy==0.24.1 passlib==1.7.1 psycopg2==2.8.2 pycparser==2.19 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index 9739ed3f..59383a07 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -23,3 +23,6 @@ mysqlclient psycopg2 idna srslib +marshmallow +flask-marshmallow +marshmallow-sqlalchemy diff --git a/docs/cli.rst b/docs/cli.rst index 1b2ed14f..6d48c576 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -10,8 +10,9 @@ Managing users and aliases can be done from CLI using commands: * user * user-import * user-delete -* config-dump * config-update +* config-export +* config-import alias ----- @@ -69,104 +70,175 @@ user-delete docker-compose exec admin flask mailu user-delete foo@example.net -config-dump ------------ - -The purpose of this command is to dump domain-, relay-, alias- and user-configuration to a YAML template. - -.. code-block:: bash - - # docker-compose exec admin flask mailu config-dump --help - - Usage: flask mailu config-dump [OPTIONS] [SECTIONS]... - - dump configuration as YAML-formatted data to stdout - - SECTIONS can be: domains, relays, users, aliases - - Options: - -f, --full Include default attributes - -s, --secrets Include secrets (dkim-key, plain-text / not hashed) - -d, --dns Include dns records - --help Show this message and exit. - -If you want to export secrets (dkim-keys, plain-text / not hashed) you have to add the ``--secrets`` option. -Only non-default attributes are dumped. If you want to dump all attributes use ``--full``. -To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. -Unless you specify some sections all sections are dumped by default. - -.. code-block:: bash - - docker-compose exec admin flask mailu config-dump > mail-config.yml - - docker-compose exec admin flask mailu config-dump --dns domains - config-update ------------- -The purpose of this command is for importing domain-, relay-, alias- and user-configuration in bulk and synchronizing DB entries with an external YAML template. +The sole purpose of this command is for importing users/aliases in bulk and synchronizing DB entries with external YAML template: .. code-block:: bash - # docker-compose exec admin flask mailu config-update --help + cat mail-config.yml | docker-compose exec -T admin flask mailu config-update --delete-objects - Usage: flask mailu config-update [OPTIONS] +where mail-config.yml looks like: - sync configuration with data from YAML-formatted stdin +.. code-block:: bash - Options: - -v, --verbose Increase verbosity - -d, --delete-objects Remove objects not included in yaml - -n, --dry-run Perform a trial run with no changes made - --help Show this message and exit. + users: + - localpart: foo + domain: example.com + password_hash: klkjhumnzxcjkajahsdqweqqwr + hash_scheme: MD5-CRYPT + aliases: + - localpart: alias1 + domain: example.com + destination: "user1@example.com,user2@example.com" + +without ``--delete-object`` option config-update will only add/update new values but will *not* remove any entries missing in provided YAML input. + +Users +^^^^^ + +following are additional parameters that could be defined for users: + +* comment +* quota_bytes +* global_admin +* enable_imap +* enable_pop +* forward_enabled +* forward_destination +* reply_enabled +* reply_subject +* reply_body +* displayed_name +* spam_enabled +* spam_threshold + +Alias +^^^^^ + +additional fields: + +* wildcard + +config-export +------------- + +The purpose of this command is to export the complete configuration in YAML or JSON format. + +.. code-block:: bash + + $ docker-compose exec admin flask mailu config-export --help + + Usage: flask mailu config-export [OPTIONS] [FILTER]... + + Export configuration as YAML or JSON to stdout or file + + Options: + -f, --full Include attributes with default value. + -s, --secrets Include secret attributes (dkim-key, passwords). + -c, --color Force colorized output. + -d, --dns Include dns records. + -o, --output-file FILENAME Save configuration to file. + -j, --json Export configuration in json format. + -?, -h, --help Show this message and exit. + +Only non-default attributes are exported. If you want to export all attributes use ``--full``. +If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option. +To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. +By default all configuration objects are exported (domain, user, alias, relay). You can specify +filters to export only some objects or attributes (try: ``user`` or ``domain.name``). + +.. code-block:: bash + + $ docker-compose exec admin flask mailu config-export -o mail-config.yml + + $ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf + +config-import +------------- + +This command imports configuration data from an external YAML or JSON source. + +.. code-block:: bash + + $ docker-compose exec admin flask mailu config-import --help + + Usage: flask mailu config-import [OPTIONS] [FILENAME|-] + + Import configuration as YAML or JSON from stdin or file + + Options: + -v, --verbose Increase verbosity. + -s, --secrets Show secret attributes in messages. + -q, --quiet Quiet mode - only show errors. + -c, --color Force colorized output. + -u, --update Update mode - merge input with existing config. + -n, --dry-run Perform a trial run with no changes made. + -?, -h, --help Show this message and exit. The current version of docker-compose exec does not pass stdin correctly, so you have to user docker exec instead: .. code-block:: bash - docker exec -i $(docker-compose ps -q admin) flask mailu config-update -nvd < mail-config.yml + docker exec -i $(docker-compose ps -q admin) flask mailu config-import -nv < mail-config.yml - -mail-config.yml looks like this: +mail-config.yml contains the configuration and looks like this: .. code-block:: yaml - - domains: + + domain: - name: example.com alternatives: - alternative.example.com - users: + user: - email: foo@example.com - password_hash: klkjhumnzxcjkajahsdqweqqwr + password_hash: '$2b$12$...' hash_scheme: MD5-CRYPT - aliases: + alias: - email: alias1@example.com - destination: "user1@example.com,user2@example.com" + destination: + - user1@example.com + - user2@example.com - relays: + relay: - name: relay.example.com comment: test smtp: mx.example.com -You can use ``--dry-run`` to test your YAML without comitting any changes to the database. -With ``--verbose`` config-update will show exactly what it changes in the database. -Without ``--delete-object`` option config-update will only add/update changed values but will *not* remove any entries missing in provided YAML input. +config-update shows the number of created/modified/deleted objects after import. +To suppress all messages except error messages use ``--quiet``. +By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed. +In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. +If you want to test what would be done when importing without committing any changes, use ``--dry-run``. -This is a complete YAML template with all additional parameters that could be defined: +By default config-update replaces the whole configuration. ``--update`` allows to modify the existing configuration instead. +New elements will be added and existing elements will be modified. +It is possible to delete a single element or prune all elements from lists and associative arrays using a special notation: + ++-----------------------------+------------------+--------------------------+ +| Delete what? | notation | example | ++=============================+==================+==========================+ +| specific array object | ``- -key: id`` | ``- -name: example.com`` | ++-----------------------------+------------------+--------------------------+ +| specific list item | ``- -id`` | ``- -user1@example.com`` | ++-----------------------------+------------------+--------------------------+ +| all remaining array objects | ``- -key: null`` | ``- -email: null`` | ++-----------------------------+------------------+--------------------------+ +| all remaining list items | ``- -prune-`` | ``- -prune-`` | ++-----------------------------+------------------+--------------------------+ + +The ``-key: null`` notation can also be used to reset an attribute to its default. +To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``. + +This is a complete YAML template with all additional parameters that can be defined: .. code-block:: yaml - aliases: - - email: email@example.com - comment: '' - destination: - - address@example.com - wildcard: false - - domains: + domain: - name: example.com alternatives: - alternative.tld @@ -176,13 +248,8 @@ This is a complete YAML template with all additional parameters that could be de max_quota_bytes: 0 max_users: -1 signup_enabled: false - - relays: - - name: relay.example.com - comment: '' - smtp: mx.example.com - - users: + + user: - email: postmaster@example.com comment: '' displayed_name: 'Postmaster' @@ -192,13 +259,16 @@ This is a complete YAML template with all additional parameters that could be de fetches: - id: 1 comment: 'test fetch' - username: fetch-user + error: null host: other.example.com + keep: true + last_check: '2020-12-29T17:09:48.200179' password: 'secret' + hash_password: true port: 993 protocol: imap tls: true - keep: true + username: fetch-user forward_destination: - address@remote.example.com forward_enabled: true @@ -206,12 +276,13 @@ This is a complete YAML template with all additional parameters that could be de global_admin: true manager_of: - example.com - password: '{BLF-CRYPT}$2b$12$...' + password: '$2b$12$...' + hash_password: true quota_bytes: 1000000000 reply_body: '' reply_enabled: false - reply_enddate: 2999-12-31 - reply_startdate: 1900-01-01 + reply_enddate: '2999-12-31' + reply_startdate: '1900-01-01' reply_subject: '' spam_enabled: true spam_threshold: 80 @@ -219,5 +290,16 @@ This is a complete YAML template with all additional parameters that could be de - id: 1 comment: email-client ip: 192.168.1.1 - password: '$5$rounds=1000$...' + password: '$5$rounds=1$...' + aliases: + - email: email@example.com + comment: '' + destination: + - address@example.com + wildcard: false + + relay: + - name: relay.example.com + comment: '' + smtp: mx.example.com diff --git a/tests/compose/core/02_forward_test.sh b/tests/compose/core/02_forward_test.sh index 651e027c..595820cf 100755 --- a/tests/compose/core/02_forward_test.sh +++ b/tests/compose/core/02_forward_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -10,7 +10,7 @@ EOF python3 tests/forward_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: forwardinguser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" diff --git a/tests/compose/core/03_alias_test.sh b/tests/compose/core/03_alias_test.sh index 2d40903a..dce1918a 100755 --- a/tests/compose/core/03_alias_test.sh +++ b/tests/compose/core/03_alias_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 aliases: - localpart: alltheusers domain: mailu.io @@ -7,6 +7,6 @@ EOF python3 tests/alias_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 aliases: [] EOF diff --git a/tests/compose/core/04_reply_test.sh b/tests/compose/core/04_reply_test.sh index 7615a0f8..83c114f6 100755 --- a/tests/compose/core/04_reply_test.sh +++ b/tests/compose/core/04_reply_test.sh @@ -1,4 +1,4 @@ -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" @@ -11,7 +11,7 @@ EOF python3 tests/reply_test.py -cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update --verbose +cat << EOF | docker-compose -f tests/compose/core/docker-compose.yml exec -T admin flask mailu config-update -v 1 users: - localpart: replyuser password_hash: "\$1\$F2OStvi1\$Q8hBIHkdJpJkJn/TrMIZ9/" diff --git a/towncrier/newsfragments/1604.feature b/towncrier/newsfragments/1604.feature index 06ee0beb..2b47791a 100644 --- a/towncrier/newsfragments/1604.feature +++ b/towncrier/newsfragments/1604.feature @@ -1 +1 @@ -Added cli command config-dump and enhanced config-update +Add cli commands config-import and config-export