diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index bef49faa..f9add0f4 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -4,22 +4,16 @@ import sys import os import socket -import logging import uuid -from collections import Counter -from itertools import chain - import click -import sqlalchemy import yaml from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext -from marshmallow.exceptions import ValidationError -from . import models -from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, canColorize, RenderJSON, HIDDEN +from mailu import models +from mailu.schemas import MailuSchema, Logger, RenderJSON db = models.db @@ -326,246 +320,53 @@ def config_update(verbose=False, delete_objects=False): @mailu.command() @click.option('-v', '--verbose', count=True, help='Increase verbosity.') @click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.') +@click.option('-d', '--debug', is_flag=True, help='Enable debug output.') @click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.') @click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.') @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @with_appcontext -def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None): +def config_import(verbose=0, secrets=False, debug=False, quiet=False, color=False, + update=False, dry_run=False, source=None): """ Import configuration as YAML or JSON from stdin or file """ - # verbose - # 0 : only show number of changes - # 1 : also show detailed changes - # 2 : also show input data - # 3 : also show sql queries (also needs -s, as sql may contain secrets) - # 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets) + log = Logger(want_color=color or None, can_color=sys.stdout.isatty(), secrets=secrets, debug=debug) + log.lexer = 'python' + log.strip = True + log.verbose = 0 if quiet else verbose + log.quiet = quiet - if quiet: - verbose = -1 - - if verbose > 2 and not secrets: - print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.') - verbose = 2 - - color_cfg = { - 'color': color or (canColorize and sys.stdout.isatty()), - 'lexer': 'python', - 'strip': True, - } - - counter = Counter() - logger = {} - - def format_errors(store, path=None): - - res = [] - if path is None: - path = [] - for key in sorted(store): - location = path + [str(key)] - value = store[key] - if isinstance(value, dict): - res.extend(format_errors(value, location)) - else: - for message in value: - res.append((".".join(location), message)) - - if path: - return res - - fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}' - res = [fmt.format(loc, msg) for loc, msg in res] - num = f'error{["s",""][len(res)==1]}' - res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation') - - return '\n'.join(res) - - def format_changes(*message): - if counter: - changes = [] - last = None - for (action, what), count in sorted(counter.items()): - if action != last: - if last: - changes.append('/') - changes.append(f'{action}:') - last = action - changes.append(f'{what}({count})') - else: - changes = ['No changes.'] - return chain(message, changes) - - def log(action, target, message=None): - - if message is None: - try: - message = logger[target.__class__].dump(target) - except KeyError: - message = target - if not isinstance(message, str): - message = repr(message) - print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}') - - def listen_insert(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - counter.update([('Created', target.__table__.name)]) - if verbose >= 1: - log('Created', target) - - def listen_update(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - - changed = {} - inspection = sqlalchemy.inspect(target) - for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: - history = getattr(inspection.attrs, attr.key).history - if history.has_changes() and history.deleted: - before = history.deleted[-1] - after = getattr(target, attr.key) - # TODO: remove special handling of "comment" after modifying model - if attr.key == 'comment' and not before and not after: - pass - # only remember changed keys - elif before != after: - if verbose >= 1: - changed[str(attr.key)] = (before, after) - else: - break - - if verbose >= 1: - # use schema with dump_context to hide secrets and sort keys - dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target) - for key, value in dumped.items(): - before, after = changed[key] - if value == HIDDEN: - before = HIDDEN if before else before - after = HIDDEN if after else after - else: - # TODO: need to use schema to "convert" before value? - after = value - log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') - - if changed: - counter.update([('Modified', target.__table__.name)]) - - def listen_delete(mapper, connection, target): # pylint: disable=unused-argument - """ callback function to track import """ - counter.update([('Deleted', target.__table__.name)]) - if verbose >= 1: - log('Deleted', target) - - # TODO: this listener will not be necessary, if dkim keys would be stored in database - _dedupe_dkim = set() - def listen_dkim(session, flush_context): # pylint: disable=unused-argument - """ callback function to track import """ - for target in session.identity_map.values(): - # look at Domains originally loaded from db - if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: - continue - before = target._dkim_key_on_disk - after = target._dkim_key - if before != after: - if secrets: - before = before.decode('ascii', 'ignore') - after = after.decode('ascii', 'ignore') - else: - before = HIDDEN if before else '' - after = HIDDEN if after else '' - # "de-dupe" messages; this event is fired at every flush - if not (target, before, after) in _dedupe_dkim: - _dedupe_dkim.add((target, before, after)) - counter.update([('Modified', target.__table__.name)]) - if verbose >= 1: - log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') - - def track_serialize(obj, item, backref=None): - """ callback function to track import """ - # called for backref modification? - if backref is not None: - log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref)) - return - # show input data? - if not verbose >= 2: - return - # hide secrets in data - data = logger[obj.opts.model].hide(item) - if 'hash_password' in data: - data['password'] = HIDDEN - if 'fetches' in data: - for fetch in data['fetches']: - fetch['password'] = HIDDEN - log('Handling', obj.opts.model, data) - - # configure contexts - diff_context = { - 'full': True, - 'secrets': secrets, - } - log_context = { - 'secrets': secrets, - } - load_context = { + context = { 'import': True, 'update': update, 'clear': not update, - 'callback': track_serialize, + 'callback': log.track_serialize, } - # register listeners - for schema in get_schema(): - model = schema.Meta.model - logger[model] = schema(context=log_context) - sqlalchemy.event.listen(model, 'after_insert', listen_insert) - sqlalchemy.event.listen(model, 'after_update', listen_update) - sqlalchemy.event.listen(model, 'after_delete', listen_delete) - - # special listener for dkim_key changes - sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim) - - if verbose >= 3: - logging.basicConfig() - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + schema = MailuSchema(only=MailuSchema.Meta.order, context=context) try: + # import source with models.db.session.no_autoflush: - config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source) - except ValidationError as exc: - raise click.ClickException(format_errors(exc.messages)) from exc + config = schema.loads(source) + # flush session to show/count all changes + if not quiet and (dry_run or verbose): + db.session.flush() + # check for duplicate domain names + config.check() except Exception as exc: - if verbose >= 3: - raise - # (yaml.scanner.ScannerError, UnicodeDecodeError, ...) - raise click.ClickException( - f'[{exc.__class__.__name__}] ' - f'{" ".join(str(exc).split())}' - ) from exc - - # flush session to show/count all changes - if dry_run or verbose >= 1: - db.session.flush() - - # check for duplicate domain names - dup = set() - for fqdn in chain( - db.session.query(models.Domain.name), - db.session.query(models.Alternative.name), - db.session.query(models.Relay.name) - ): - if fqdn in dup: - raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}') - dup.add(fqdn) + if msg := log.format_exception(exc): + raise click.ClickException(msg) from exc + raise # don't commit when running dry if dry_run: - if not quiet: - print(*format_changes('Dry run. Not commiting changes.')) + log.changes('Dry run. Not committing changes.') db.session.rollback() else: - if not quiet: - print(*format_changes('Committing changes.')) + log.changes('Committing changes.') db.session.commit() @@ -573,8 +374,8 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal @click.option('-f', '--full', is_flag=True, help='Include attributes with default value.') @click.option('-s', '--secrets', is_flag=True, help='Include secret attributes (dkim-key, passwords).') -@click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-d', '--dns', is_flag=True, help='Include dns records.') +@click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), help='Save configuration to file.') @click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.') @@ -584,32 +385,25 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None """ Export configuration as YAML or JSON to stdout or file """ - if only: - for spec in only: - if spec.split('.', 1)[0] not in MailuSchema.Meta.order: - raise click.ClickException(f'[ValidationError] Unknown section: {spec}') - else: - only = MailuSchema.Meta.order + only = only or MailuSchema.Meta.order context = { 'full': full, 'secrets': secrets, 'dns': dns, } - - schema = MailuSchema(only=only, context=context) - color_cfg = {'color': color or (canColorize and output.isatty())} - - if as_json: - schema.opts.render_module = RenderJSON - color_cfg['lexer'] = 'json' - color_cfg['strip'] = True + log = Logger(want_color=color or None, can_color=output.isatty()) try: - print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) - except ValueError as exc: - if spec := get_fieldspec(exc): - raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc + schema = MailuSchema(only=only, context=context) + if as_json: + schema.opts.render_module = RenderJSON + log.lexer = 'json' + log.strip = True + print(log.colorize(schema.dumps(models.MailuConfig())), file=output) + except Exception as exc: + if msg := log.format_exception(exc): + raise click.ClickException(msg) from exc raise diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 4c119984..1b3c787a 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -23,7 +23,7 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.inspection import inspect from werkzeug.utils import cached_property -from . import dkim +from mailu import dkim db = flask_sqlalchemy.SQLAlchemy() @@ -33,7 +33,6 @@ class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: use db.String(255)? impl = db.String(80) def process_bind_param(self, value, dialect): @@ -50,7 +49,6 @@ class IdnaEmail(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ - # TODO: use db.String(254)? impl = db.String(255) def process_bind_param(self, value, dialect): @@ -127,11 +125,7 @@ class Base(db.Model): if pkey == 'email': # ugly hack for email declared attr. _email is not always up2date return str(f'{self.localpart}@{self.domain_name}') - elif pkey in {'name', 'email'}: - return str(getattr(self, pkey, None)) - else: - return self.__repr__() - return str(getattr(self, self.__table__.primary_key.columns.values()[0].name)) + return str(getattr(self, pkey)) def __repr__(self): return f'<{self.__class__.__name__} {str(self)!r}>' @@ -145,12 +139,15 @@ class Base(db.Model): else: return NotImplemented + # we need hashable instances here for sqlalchemy to update collections + # in collections.bulk_replace, but auto-incrementing don't always have + # a valid primary key, in this case we use the object's id + __hashed = None def __hash__(self): - primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) - if primary is None: - return NotImplemented - else: - return hash(primary) + if self.__hashed is None: + primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) + self.__hashed = id(self) if primary is None else hash(primary) + return self.__hashed # Many-to-many association table for domain managers @@ -314,7 +311,6 @@ class Relay(Base): __tablename__ = 'relay' name = db.Column(IdnaDomain, primary_key=True, nullable=False) - # TODO: use db.String(266)? transport(8):(1)[nexthop(255)](2) smtp = db.Column(db.String(80), nullable=True) @@ -322,7 +318,6 @@ class Email(object): """ Abstraction for an email address (localpart and domain). """ - # TODO: use db.String(64)? localpart = db.Column(db.String(80), nullable=False) @declarative.declared_attr @@ -342,7 +337,7 @@ class Email(object): key = f'{cls.__tablename__}_email' if key in ctx.current_parameters: return ctx.current_parameters[key] - return '{localpart}@{domain_name}'.format(**ctx.current_parameters) + return '{localpart}@{domain_name}'.format_map(ctx.current_parameters) return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater) @@ -632,7 +627,6 @@ class Token(Base): user = db.relationship(User, backref=db.backref('tokens', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - # TODO: use db.String(32)? ip = db.Column(db.String(255)) def check_password(self, password): @@ -865,6 +859,18 @@ class MailuConfig: if models is None or model in models: db.session.query(model).delete() + def check(self): + """ check for duplicate domain names """ + dup = set() + for fqdn in chain( + db.session.query(Domain.name), + db.session.query(Alternative.name), + db.session.query(Relay.name) + ): + if fqdn in dup: + raise ValueError(f'Duplicate domain name: {fqdn}') + dup.add(fqdn) + domain = MailuCollection(Domain) user = MailuCollection(User) alias = MailuCollection(Alias) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 6a8303c5..4c5042ea 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -2,10 +2,10 @@ """ from copy import deepcopy -from textwrap import wrap +from collections import Counter -import re import json +import logging import yaml import sqlalchemy @@ -27,24 +27,309 @@ try: from pygments.lexers.data import YamlLexer from pygments.formatters import get_formatter_by_name except ModuleNotFoundError: - canColorize = False + COLOR_SUPPORTED = False else: - canColorize = True + COLOR_SUPPORTED = True -from . import models, dkim +from mailu import models, dkim ma = Marshmallow() -# TODO: how and where to mark keys as "required" while deserializing in api? -# - when modifying, nothing is required (only the primary key, but this key is in the uri) -# - the primary key from post data must not differ from the key in the uri -# - when creating all fields without default or auto-increment are required -# TODO: validate everything! + +### import logging and schema colorization ### + +_model2schema = {} + +def get_schema(cls=None): + """ return schema class for model """ + if cls is None: + return _model2schema.values() + return _model2schema.get(cls) + +def mapped(cls): + """ register schema in model2schema map """ + _model2schema[cls.Meta.model] = cls + return cls + +class MyYamlLexer(YamlLexer): + """ colorize yaml constants and integers """ + def get_tokens(self, text, unfiltered=False): + for typ, value in super().get_tokens(text, unfiltered): + if typ is Token.Literal.Scalar.Plain: + if value in {'true', 'false', 'null'}: + typ = Token.Keyword.Constant + elif value == HIDDEN: + typ = Token.Error + else: + try: + int(value, 10) + except ValueError: + try: + float(value) + except ValueError: + pass + else: + typ = Token.Literal.Number.Float + else: + typ = Token.Literal.Number.Integer + yield typ, value + +class Logger: + + def __init__(self, want_color=None, can_color=False, debug=False, secrets=False): + + self.lexer = 'yaml' + self.formatter = 'terminal' + self.strip = False + self.verbose = 0 + self.quiet = False + self.secrets = secrets + self.debug = debug + self.print = print + + if want_color and not COLOR_SUPPORTED: + raise ValueError('Please install pygments to colorize output') + + self.color = want_color or (can_color and COLOR_SUPPORTED) + + self._counter = Counter() + self._schemas = {} + + # log contexts + self._diff_context = { + 'full': True, + 'secrets': secrets, + } + log_context = { + 'secrets': secrets, + } + + # register listeners + for schema in get_schema(): + model = schema.Meta.model + self._schemas[model] = schema(context=log_context) + sqlalchemy.event.listen(model, 'after_insert', self._listen_insert) + sqlalchemy.event.listen(model, 'after_update', self._listen_update) + sqlalchemy.event.listen(model, 'after_delete', self._listen_delete) + + # special listener for dkim_key changes + # TODO: _listen_dkim can be removed when dkim keys are stored in database + self._dedupe_dkim = set() + sqlalchemy.event.listen(models.db.session, 'after_flush', self._listen_dkim) + + # register debug logger for sqlalchemy + if self.debug: + logging.basicConfig() + logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + def _log(self, action, target, message=None): + if message is None: + try: + message = self._schemas[target.__class__].dump(target) + except KeyError: + message = target + if not isinstance(message, str): + message = repr(message) + self.print(f'{action} {target.__table__}: {self.colorize(message)}') + + def _listen_insert(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + self._counter.update([('Created', target.__table__.name)]) + if self.verbose: + self._log('Created', target) + + def _listen_update(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + + changes = {} + inspection = sqlalchemy.inspect(target) + for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs: + history = getattr(inspection.attrs, attr.key).history + if history.has_changes() and history.deleted: + before = history.deleted[-1] + after = getattr(target, attr.key) + # TODO: this can be removed when comment is not nullable in model + if attr.key == 'comment' and not before and not after: + pass + # only remember changed keys + elif before != after: + if self.verbose: + changes[str(attr.key)] = (before, after) + else: + break + + if self.verbose: + # use schema to log changed attributes + schema = get_schema(target.__class__) + only = set(changes.keys()) & set(schema().fields.keys()) + if only: + for key, value in schema( + only=only, + context=self._diff_context + ).dump(target).items(): + before, after = changes[key] + if value == HIDDEN: + before = HIDDEN if before else before + after = HIDDEN if after else after + else: + # also hide this + after = value + self._log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}') + + if changes: + self._counter.update([('Modified', target.__table__.name)]) + + def _listen_delete(self, mapper, connection, target): # pylint: disable=unused-argument + """ callback method to track import """ + self._counter.update([('Deleted', target.__table__.name)]) + if self.verbose: + self._log('Deleted', target) + + # TODO: _listen_dkim can be removed when dkim keys are stored in database + def _listen_dkim(self, session, flush_context): # pylint: disable=unused-argument + """ callback method to track import """ + for target in session.identity_map.values(): + # look at Domains originally loaded from db + if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path: + continue + before = target._dkim_key_on_disk + after = target._dkim_key + if before != after: + # "de-dupe" messages; this event is fired at every flush + if not (target, before, after) in self._dedupe_dkim: + self._dedupe_dkim.add((target, before, after)) + self._counter.update([('Modified', target.__table__.name)]) + if self.verbose: + if self.secrets: + before = before.decode('ascii', 'ignore') + after = after.decode('ascii', 'ignore') + else: + before = HIDDEN if before else '' + after = HIDDEN if after else '' + self._log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}') + + def track_serialize(self, obj, item, backref=None): + """ callback method to track import """ + # called for backref modification? + if backref is not None: + self._log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format_map(backref)) + return + # show input data? + if self.verbose < 2: + return + # hide secrets in data + if not self.secrets: + item = self._schemas[obj.opts.model].hide(item) + if 'hash_password' in item: + item['password'] = HIDDEN + if 'fetches' in item: + for fetch in item['fetches']: + fetch['password'] = HIDDEN + self._log('Handling', obj.opts.model, item) + + def changes(self, *messages, **kwargs): + """ show changes gathered in counter """ + if self.quiet: + return + if self._counter: + changes = [] + last = None + for (action, what), count in sorted(self._counter.items()): + if action != last: + if last: + changes.append('/') + changes.append(f'{action}:') + last = action + changes.append(f'{what}({count})') + else: + changes = ['No changes.'] + self.print(*messages, *changes, **kwargs) + + def _format_errors(self, store, path=None): + + res = [] + if path is None: + path = [] + for key in sorted(store): + location = path + [str(key)] + value = store[key] + if isinstance(value, dict): + res.extend(self._format_errors(value, location)) + else: + for message in value: + res.append((".".join(location), message)) + + if path: + return res + + maxlen = max([len(loc) for loc, msg in res]) + res = [f' - {loc.ljust(maxlen)} : {msg}' for loc, msg in res] + errors = f'{len(res)} error{["s",""][len(res)==1]}' + res.insert(0, f'[ValidationError] {errors} occurred during input validation') + + return '\n'.join(res) + + def _is_validation_error(self, exc): + """ walk traceback to extract invalid field from marshmallow """ + path = [] + trace = exc.__traceback__ + while trace: + if trace.tb_frame.f_code.co_name == '_serialize': + if 'attr' in trace.tb_frame.f_locals: + path.append(trace.tb_frame.f_locals['attr']) + elif trace.tb_frame.f_code.co_name == '_init_fields': + spec = ', '.join(['.'.join(path + [key]) for key in trace.tb_frame.f_locals['invalid_fields']]) + return f'Invalid filter: {spec}' + trace = trace.tb_next + return None + + def format_exception(self, exc): + """ format ValidationErrors and other exceptions when not debugging """ + if isinstance(exc, ValidationError): + return self._format_errors(exc.messages) + if isinstance(exc, ValueError): + if msg := self._is_validation_error(exc): + return msg + if self.debug: + return None + msg = ' '.join(str(exc).split()) + return f'[{exc.__class__.__name__}] {msg}' + + colorscheme = { + Token: ('', ''), + Token.Name.Tag: ('cyan', 'cyan'), + Token.Literal.Scalar: ('green', 'green'), + Token.Literal.String: ('green', 'green'), + Token.Name.Constant: ('green', 'green'), # multiline strings + Token.Keyword.Constant: ('magenta', 'magenta'), + Token.Literal.Number: ('magenta', 'magenta'), + Token.Error: ('red', 'red'), + Token.Name: ('red', 'red'), + Token.Operator: ('red', 'red'), + } + + def colorize(self, data, lexer=None, formatter=None, color=None, strip=None): + """ add ANSI color to data """ + + if color is False or not self.color: + return data + + lexer = lexer or self.lexer + lexer = MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer) + formatter = get_formatter_by_name(formatter or self.formatter, colorscheme=self.colorscheme) + if strip is None: + strip = self.strip + + res = highlight(data, lexer, formatter) + if strip: + return res.rstrip('\n') + return res -### class for hidden values ### +### marshmallow render modules ### +# hidden attributes class _Hidden: def __bool__(self): return False @@ -58,107 +343,24 @@ class _Hidden: return '' __str__ = __repr__ -HIDDEN = _Hidden() - - -### map model to schema ### - -_model2schema = {} - -def get_schema(model=None): - """ return schema class for model or instance of model """ - if model is None: - return _model2schema.values() - else: - return _model2schema.get(model) or _model2schema.get(model.__class__) - -def mapped(cls): - """ register schema in model2schema map """ - _model2schema[cls.Meta.model] = cls - return cls - - -### helper functions ### - -def get_fieldspec(exc): - """ walk traceback to extract spec of invalid field from marshmallow """ - path = [] - tbck = exc.__traceback__ - while tbck: - if tbck.tb_frame.f_code.co_name == '_serialize': - if 'attr' in tbck.tb_frame.f_locals: - path.append(tbck.tb_frame.f_locals['attr']) - elif tbck.tb_frame.f_code.co_name == '_init_fields': - path = '.'.join(path) - spec = ', '.join([f'{path}.{key}' for key in tbck.tb_frame.f_locals['invalid_fields']]) - return spec - tbck = tbck.tb_next - return None - -def colorize(data, lexer='yaml', formatter='terminal', color=None, strip=False): - """ add ANSI color to data """ - if color is None: - # autodetect colorize - color = canColorize - if not color: - # no color wanted - return data - if not canColorize: - # want color, but not supported - raise ValueError('Please install pygments to colorize output') - - scheme = { - Token: ('', ''), - Token.Name.Tag: ('cyan', 'brightcyan'), - Token.Literal.Scalar: ('green', 'green'), - Token.Literal.String: ('green', 'green'), - Token.Keyword.Constant: ('magenta', 'brightmagenta'), - Token.Literal.Number: ('magenta', 'brightmagenta'), - Token.Error: ('red', 'brightred'), - Token.Name: ('red', 'brightred'), - Token.Operator: ('red', 'brightred'), - } - - class MyYamlLexer(YamlLexer): - """ colorize yaml constants and integers """ - def get_tokens(self, text, unfiltered=False): - for typ, value in super().get_tokens(text, unfiltered): - if typ is Token.Literal.Scalar.Plain: - if value in {'true', 'false', 'null'}: - typ = Token.Keyword.Constant - elif value == HIDDEN: - typ = Token.Error - else: - try: - int(value, 10) - except ValueError: - try: - float(value) - except ValueError: - pass - else: - typ = Token.Literal.Number.Float - else: - typ = Token.Literal.Number.Integer - yield typ, value - - res = highlight( - data, - MyYamlLexer() if lexer == 'yaml' else get_lexer_by_name(lexer), - get_formatter_by_name(formatter, colorscheme=scheme) - ) - - return res.rstrip('\n') if strip else res - - -### render modules ### - -# allow yaml to represent hidden attributes yaml.add_representer( _Hidden, - lambda cls, data: cls.represent_data(str(data)) + lambda dumper, data: dumper.represent_data(str(data)) ) +HIDDEN = _Hidden() + +# multiline attributes +class _Multiline(str): + pass + +yaml.add_representer( + _Multiline, + lambda dumper, data: dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|') + +) + +# yaml render module class RenderYAML: """ Marshmallow YAML Render Module """ @@ -178,7 +380,7 @@ class RenderYAML: @staticmethod def _augment(kwargs, defaults): - """ add default kv's to kwargs if missing + """ add defaults to kwargs if missing """ for key, value in defaults.items(): if key not in kwargs: @@ -205,6 +407,7 @@ class RenderYAML: cls._augment(kwargs, cls._dump_defaults) return yaml.dump(*args, **kwargs) +# json encoder class JSONEncoder(json.JSONEncoder): """ JSONEncoder supporting serialization of HIDDEN """ def default(self, o): @@ -213,13 +416,14 @@ class JSONEncoder(json.JSONEncoder): return str(o) return json.JSONEncoder.default(self, o) +# json render module class RenderJSON: """ Marshmallow JSON Render Module """ @staticmethod def _augment(kwargs, defaults): - """ add default kv's to kwargs if missing + """ add defaults to kwargs if missing """ for key, value in defaults.items(): if key not in kwargs: @@ -245,7 +449,7 @@ class RenderJSON: return json.dumps(*args, **kwargs) -### custom fields ### +### schema: custom fields ### class LazyStringField(fields.String): """ Field that serializes a "false" value to the empty string @@ -261,6 +465,11 @@ class CommaSeparatedListField(fields.Raw): a list of strings """ + default_error_messages = { + "invalid": "Not a valid string or list.", + "invalid_utf8": "Not a valid utf-8 string or list.", + } + def _deserialize(self, value, attr, data, **kwargs): """ deserialize comma separated string to list of strings """ @@ -269,16 +478,31 @@ class CommaSeparatedListField(fields.Raw): if not value: return [] - # split string - if isinstance(value, str): - return list([item.strip() for item in value.split(',') if item.strip()]) + # handle list + if isinstance(value, list): + try: + value = [ensure_text_type(item) for item in value] + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + + # handle text else: - return value + if not isinstance(value, (str, bytes)): + raise self.make_error("invalid") + try: + value = ensure_text_type(value) + except UnicodeDecodeError as exc: + raise self.make_error("invalid_utf8") from exc + else: + value = filter(None, [item.strip() for item in value.split(',')]) + + return list(value) class DkimKeyField(fields.String): - """ Serialize a dkim key to a list of strings (lines) and - Deserialize a string or list of strings to a valid dkim key + """ Serialize a dkim key to a multiline string and + deserialize a dkim key data as string or list of strings + to a valid dkim key """ default_error_messages = { @@ -286,21 +510,26 @@ class DkimKeyField(fields.String): "invalid_utf8": "Not a valid utf-8 string or list.", } - _clean_re = re.compile( - r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)', - flags=re.UNICODE - ) - def _serialize(self, value, attr, obj, **kwargs): - """ serialize dkim key to a list of strings (lines) + """ serialize dkim key as multiline string """ # map empty string and None to None if not value: - return None + return '' - # return list of key lines without header/footer - return value.decode('utf-8').strip().split('\n')[1:-1] + # return multiline string + return _Multiline(value.decode('utf-8')) + + def _wrap_key(self, begin, data, end): + """ generator to wrap key into RFC 7468 format """ + yield begin + pos = 0 + while pos < len(data): + yield data[pos:pos+64] + pos += 64 + yield end + yield '' def _deserialize(self, value, attr, data, **kwargs): """ deserialize a string or list of strings to dkim key data @@ -310,7 +539,7 @@ class DkimKeyField(fields.String): # convert list to str if isinstance(value, list): try: - value = ''.join([ensure_text_type(item) for item in value]) + value = ''.join([ensure_text_type(item) for item in value]).strip() except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc @@ -319,34 +548,53 @@ class DkimKeyField(fields.String): if not isinstance(value, (str, bytes)): raise self.make_error("invalid") try: - value = ensure_text_type(value) + value = ensure_text_type(value).strip() except UnicodeDecodeError as exc: raise self.make_error("invalid_utf8") from exc - # clean value (remove whitespace and header/footer) - value = self._clean_re.sub('', value.strip()) + # generate new key? + if value.lower() == '-generate-': + return dkim.gen_key() - # map empty string/list to None + # no key? if not value: return None - # handle special value 'generate' - elif value == 'generate': - return dkim.gen_key() + # remember part of value for ValidationError + bad_key = value - # remember some keydata for error message - keydata = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value + # strip header and footer, clean whitespace and wrap to 64 characters + try: + if value.startswith('-----BEGIN '): + end = value.index('-----', 11) + 5 + header = value[:end] + value = value[end:] + else: + header = '-----BEGIN PRIVATE KEY-----' - # wrap value into valid pem layout and check validity - value = ( - '-----BEGIN PRIVATE KEY-----\n' + - '\n'.join(wrap(value, 64)) + - '\n-----END PRIVATE KEY-----\n' - ).encode('ascii') + if (pos := value.find('-----END ')) >= 0: + end = value.index('-----', pos+9) + 5 + footer = value[pos:end] + value = value[:pos] + else: + footer = '-----END PRIVATE KEY-----' + except ValueError: + raise ValidationError(f'invalid dkim key {bad_key!r}') from exc + + # remove whitespace from key data + value = ''.join(value.split()) + + # remember part of value for ValidationError + bad_key = f'{value[:25]}...{value[-10:]}' if len(value) > 40 else value + + # wrap key according to RFC 7468 + value = ('\n'.join(self._wrap_key(header, value, footer))).encode('ascii') + + # check key validity try: crypto.load_privatekey(crypto.FILETYPE_PEM, value) except crypto.Error as exc: - raise ValidationError(f'invalid dkim key {keydata!r}') from exc + raise ValidationError(f'invalid dkim key {bad_key!r}') from exc else: return value @@ -398,6 +646,27 @@ class PasswordField(fields.Str): ### base schema ### +class Storage: + """ Storage class to save information in context + """ + + context = {} + + def _bind(self, key, bind): + if bind is True: + return (self.__class__, key) + if isinstance(bind, str): + return (get_schema(self.recall(bind).__class__), key) + return (bind, key) + + def store(self, key, value, bind=None): + """ store value under key """ + self.context.setdefault('_track', {})[self._bind(key, bind)]= value + + def recall(self, key, bind=None): + """ recall value from key """ + return self.context['_track'][self._bind(key, bind)] + class BaseOpts(SQLAlchemyAutoSchemaOpts): """ Option class with sqla session """ @@ -408,7 +677,7 @@ class BaseOpts(SQLAlchemyAutoSchemaOpts): meta.sibling = False super(BaseOpts, self).__init__(meta, ordered=ordered) -class BaseSchema(ma.SQLAlchemyAutoSchema): +class BaseSchema(ma.SQLAlchemyAutoSchema, Storage): """ Marshmallow base schema with custom exclude logic and option to hide sqla defaults """ @@ -425,6 +694,9 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): def __init__(self, *args, **kwargs): + # prepare only to auto-include explicitly specified attributes + only = set(kwargs.get('only') or []) + # get context context = kwargs.get('context', {}) flags = {key for key, value in context.items() if value is True} @@ -433,13 +705,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): exclude = set(kwargs.get('exclude', [])) # always exclude - exclude.update({'created_at', 'updated_at'}) + exclude.update({'created_at', 'updated_at'} - only) # add include_by_context if context is not None: for need, what in getattr(self.Meta, 'include_by_context', {}).items(): if not flags & set(need): - exclude |= set(what) + exclude |= what - only # update excludes kwargs['exclude'] = exclude @@ -448,12 +720,15 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): super().__init__(*args, **kwargs) # exclude_by_value - self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {}) + self._exclude_by_value = { + key: values for key, values in getattr(self.Meta, 'exclude_by_value', {}).items() + if key not in only + } # exclude default values if not context.get('full'): for column in self.opts.model.__table__.columns: - if column.name not in exclude: + if column.name not in exclude and column.name not in only: self._exclude_by_value.setdefault(column.name, []).append( None if column.default is None else column.default.arg ) @@ -463,7 +738,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if context is not None: for need, what in getattr(self.Meta, 'hide_by_context', {}).items(): if not flags & set(need): - self._hide_by_context |= set(what) + self._hide_by_context |= what - only # remember primary keys self._primary = str(self.opts.model.__table__.primary_key.columns.values()[0].name) @@ -479,20 +754,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): order.remove(self._primary) order.insert(0, self._primary) - # order dump_fields - for field in order: - if field in self.dump_fields: - self.dump_fields[field] = self.dump_fields.pop(field) + # order fieldlists + for fieldlist in (self.fields, self.load_fields, self.dump_fields): + for field in order: + if field in fieldlist: + fieldlist[field] = fieldlist.pop(field) - # move pre_load hook "_track_import" to the front - hooks = self._hooks[('pre_load', False)] - hooks.remove('_track_import') - hooks.insert(0, '_track_import') - # move pre_load hook "_add_instance" to the end - hooks.remove('_add_required') - hooks.append('_add_required') - - # move post_load hook "_add_instance" to the end + # move post_load hook "_add_instance" to the end (after load_instance mixin) hooks = self._hooks[('post_load', False)] hooks.remove('_add_instance') hooks.append('_add_instance') @@ -506,8 +774,8 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): } def _call_and_store(self, *args, **kwargs): - """ track curent parent field for pruning """ - self.context['parent_field'] = kwargs['field_name'] + """ track current parent field for pruning """ + self.store('field', kwargs['field_name'], True) return super()._call_and_store(*args, **kwargs) # this is only needed to work around the declared attr "email" primary key in model @@ -518,11 +786,13 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if keys := getattr(self.Meta, 'primary_keys', None): filters = {key: data.get(key) for key in keys} if None not in filters.values(): - return self.session.query(self.opts.model).filter_by(**filters).first() - return super().get_instance(data) + res= self.session.query(self.opts.model).filter_by(**filters).first() + return res + res= super().get_instance(data) + return res @pre_load(pass_many=True) - def _patch_input(self, items, many, **kwargs): # pylint: disable=unused-argument + def _patch_many(self, items, many, **kwargs): # pylint: disable=unused-argument """ - flush sqla session before serializing a section when requested (make sure all objects that could be referred to later are created) - when in update mode: patch input data before deserialization @@ -540,12 +810,19 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # patch "delete", "prune" and "default" want_prune = [] - def patch(count, data, prune): + def patch(count, data): # don't allow __delete__ coming from input if '__delete__' in data: raise ValidationError('Unknown field.', f'{count}.__delete__') + # fail when hash_password is specified without password + if 'hash_password' in data and not 'password' in data: + raise ValidationError( + 'Nothing to hash. Field "password" is missing.', + field_name = f'{count}.hash_password', + ) + # handle "prune list" and "delete item" (-pkey: none and -pkey: id) for key in data: if key.startswith('-'): @@ -553,10 +830,10 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # delete or prune if data[key] is None: # prune - prune.append(True) + want_prune.append(True) return None # mark item for deletion - return {key[1:]: data[key], '__delete__': True} + return {key[1:]: data[key], '__delete__': count} # handle "set to default value" (-key: none) def set_default(key, value): @@ -567,7 +844,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): return (key, None) if value is not None: raise ValidationError( - 'When resetting to default value must be null.', + 'Value must be "null" when resetting to default.', f'{count}.{key}' ) value = self.opts.model.__table__.columns[key].default @@ -583,10 +860,128 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # convert items to "delete" and filter "prune" item items = [ item for item in [ - patch(count, item, want_prune) for count, item in enumerate(items) + patch(count, item) for count, item in enumerate(items) ] if item ] + # remember if prune was requested for _prune_items@post_load + self.store('prune', bool(want_prune), True) + + # remember original items to stabilize password-changes in _add_instance@post_load + self.store('original', items, True) + + return items + + @pre_load + def _patch_item(self, data, many, **kwargs): # pylint: disable=unused-argument + """ - call callback function to track import + - stabilize import of items with auto-increment primary key + - delete items + - delete/prune list attributes + - add missing required attributes + """ + + # callback + if callback := self.context.get('callback'): + callback(self, data) + + # stop early when not updating + if not self.opts.load_instance or not self.context.get('update'): + return data + + # stabilize import of auto-increment primary keys (not required), + # by matching import data to existing items and setting primary key + if not self._primary in data: + for item in getattr(self.recall('parent'), self.recall('field', 'parent')): + existing = self.dump(item, many=False) + this = existing.pop(self._primary) + if data == existing: + instance = item + data[self._primary] = this + break + + # try to load instance + instance = self.instance or self.get_instance(data) + if instance is None: + + if '__delete__' in data: + # deletion of non-existent item requested + raise ValidationError( + f'Item to delete not found: {data[self._primary]!r}.', + field_name = f'{data["__delete__"]}.{self._primary}', + ) + + else: + + if self.context.get('update'): + # remember instance as parent for pruning siblings + if not self.Meta.sibling: + self.store('parent', instance) + # delete instance from session when marked + if '__delete__' in data: + self.opts.sqla_session.delete(instance) + # delete item from lists or prune lists + # currently: domain.alternatives, user.forward_destination, + # user.manager_of, aliases.destination + for key, value in data.items(): + if not isinstance(self.fields.get(key), ( + RelatedList, CommaSeparatedListField, fields.Raw) + ) or not isinstance(value, list): + continue + # deduplicate new value + new_value = set(value) + # handle list pruning + if '-prune-' in value: + value.remove('-prune-') + new_value.remove('-prune-') + else: + for old in getattr(instance, key): + # using str() is okay for now (see above) + new_value.add(str(old)) + # handle item deletion + for item in value: + if item.startswith('-'): + new_value.remove(item) + try: + new_value.remove(item[1:]) + except KeyError as exc: + raise ValidationError( + f'Item to delete not found: {item[1:]!r}.', + field_name=f'?.{key}', + ) from exc + # sort list of new values + data[key] = sorted(new_value) + # log backref modification not catched by modify hook + if isinstance(self.fields[key], RelatedList): + if callback := self.context.get('callback'): + before = {str(v) for v in getattr(instance, key)} + after = set(data[key]) + if before != after: + callback(self, instance, { + 'key': key, + 'target': str(instance), + 'before': before, + 'after': after, + }) + + # add attributes required for validation from db + for attr_name, field_obj in self.load_fields.items(): + if field_obj.required and attr_name not in data: + data[attr_name] = getattr(instance, attr_name) + + return data + + @post_load(pass_many=True) + def _prune_items(self, items, many, **kwargs): # pylint: disable=unused-argument + """ handle list pruning """ + + # stop early when not updating + if not self.context.get('update'): + return items + + # get prune flag from _patch_many@pre_load + want_prune = self.recall('prune', True) + # prune: determine if existing items in db need to be added or marked for deletion add_items = False del_items = False @@ -603,144 +998,60 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): if add_items or del_items: existing = {item[self._primary] for item in items if self._primary in item} - for item in getattr(self.context['parent'], self.context['parent_field']): + for item in getattr(self.recall('parent'), self.recall('field', 'parent')): key = getattr(item, self._primary) if key not in existing: if add_items: items.append({self._primary: key}) else: - items.append({self._primary: key, '__delete__': True}) + items.append({self._primary: key, '__delete__': '?'}) return items - @pre_load - def _track_import(self, data, many, **kwargs): # pylint: disable=unused-argument - """ call callback function to track import - """ - # callback - if callback := self.context.get('callback'): - callback(self, data) - - return data - - @pre_load - def _add_required(self, data, many, **kwargs): # pylint: disable=unused-argument - """ when updating: - allow modification of existing items having required attributes - by loading existing value from db + @post_load + def _add_instance(self, item, many, **kwargs): # pylint: disable=unused-argument + """ - undo password change in existing instances when plain password did not change + - add new instances to sqla session """ - if not self.opts.load_instance or not self.context.get('update'): - return data - - # stabilize import of auto-increment primary keys (not required), - # by matching import data to existing items and setting primary key - if not self._primary in data: - for item in getattr(self.context['parent'], self.context['parent_field']): - existing = self.dump(item, many=False) - this = existing.pop(self._primary) - if data == existing: - instance = item - data[self._primary] = this - break - - # try to load instance - instance = self.instance or self.get_instance(data) - if instance is None: - - if '__delete__' in data: - # deletion of non-existent item requested - raise ValidationError( - f'item to delete not found: {data[self._primary]!r}', - field_name=f'?.{self._primary}', - ) - - else: - - if self.context.get('update'): - # remember instance as parent for pruning siblings - if not self.Meta.sibling: - self.context['parent'] = instance - # delete instance when marked - if '__delete__' in data: - self.opts.sqla_session.delete(instance) - # delete item from lists or prune lists - # currently: domain.alternatives, user.forward_destination, - # user.manager_of, aliases.destination - for key, value in data.items(): - if not isinstance(self.fields[key], fields.Nested) and isinstance(value, list): - new_value = set(value) - # handle list pruning - if '-prune-' in value: - value.remove('-prune-') - new_value.remove('-prune-') - else: - for old in getattr(instance, key): - # using str() is okay for now (see above) - new_value.add(str(old)) - # handle item deletion - for item in value: - if item.startswith('-'): - new_value.remove(item) - try: - new_value.remove(item[1:]) - except KeyError as exc: - raise ValidationError( - f'item to delete not found: {item[1:]!r}', - field_name=f'?.{key}', - ) from exc - # deduplicate and sort list - data[key] = sorted(new_value) - # log backref modification not catched by hook - if isinstance(self.fields[key], RelatedList): - if callback := self.context.get('callback'): - callback(self, instance, { - 'key': key, - 'target': str(instance), - 'before': [str(v) for v in getattr(instance, key)], - 'after': data[key], - }) - - - - # add attributes required for validation from db - # TODO: this will cause validation errors if value from database does not validate - # but there should not be an invalid value in the database - for attr_name, field_obj in self.load_fields.items(): - if field_obj.required and attr_name not in data: - data[attr_name] = getattr(instance, attr_name) - - return data - - @post_load(pass_original=True) - def _add_instance(self, item, original, many, **kwargs): # pylint: disable=unused-argument - """ add new instances to sqla session """ - - if item in self.opts.sqla_session: - # item was modified - if 'hash_password' in original: - # stabilize import of passwords to be hashed, - # by not re-hashing an unchanged password - if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): - if attr.history.has_changes() and attr.history.deleted: - try: - # reset password hash, if password was not changed - inst = type(item)(password=attr.history.deleted[-1]) - if inst.check_password(original['password']): - item.password = inst.password - except ValueError: - # hash in db is invalid - pass - else: - del inst - else: - # new item + if not item in self.opts.sqla_session: self.opts.sqla_session.add(item) + return item + + # stop early if item has no password attribute + if not hasattr(item, 'password'): + return item + + # did we hash a new plaintext password? + original = None + pkey = getattr(item, self._primary) + for data in self.recall('original', True): + if 'hash_password' in data and data.get(self._primary) == pkey: + original = data['password'] + break + if original is None: + # password was hashed by us + return item + + # reset hash if plain password matches hash from db + if attr := getattr(sqlalchemy.inspect(item).attrs, 'password', None): + if attr.history.has_changes() and attr.history.deleted: + try: + # reset password hash + inst = type(item)(password=attr.history.deleted[-1]) + if inst.check_password(original): + item.password = inst.password + except ValueError: + # hash in db is invalid + pass + else: + del inst + return item @post_dump def _hide_values(self, data, many, **kwargs): # pylint: disable=unused-argument - """ hide secrets and order output """ + """ hide secrets """ # stop early when not excluding/hiding if not self._exclude_by_value and not self._hide_by_context: @@ -757,7 +1068,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema): # this field is used to mark items for deletion mark_delete = fields.Boolean(data_key='__delete__', load_only=True) - # TODO: remove LazyStringField (when model was changed - IMHO comment should not be nullable) + # TODO: this can be removed when comment is not nullable in model comment = LazyStringField() @@ -892,27 +1203,28 @@ class RelaySchema(BaseSchema): load_instance = True -class MailuSchema(Schema): +@mapped +class MailuSchema(Schema, Storage): """ Marshmallow schema for complete Mailu config """ class Meta: """ Schema config """ + model = models.MailuConfig render_module = RenderYAML order = ['domain', 'user', 'alias', 'relay'] # 'config' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - # order dump_fields - for field in self.Meta.order: - if field in self.dump_fields: - self.dump_fields[field] = self.dump_fields.pop(field) + # order fieldlists + for fieldlist in (self.fields, self.load_fields, self.dump_fields): + for field in self.Meta.order: + if field in fieldlist: + fieldlist[field] = fieldlist.pop(field) def _call_and_store(self, *args, **kwargs): """ track current parent and field for pruning """ - self.context.update({ - 'parent': self.context.get('config'), - 'parent_field': kwargs['field_name'], - }) + self.store('field', kwargs['field_name'], True) + self.store('parent', self.context.get('config')) return super()._call_and_store(*args, **kwargs) @pre_load diff --git a/docs/cli.rst b/docs/cli.rst index 6d48c576..891db152 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -138,8 +138,8 @@ The purpose of this command is to export the complete configuration in YAML or J Options: -f, --full Include attributes with default value. -s, --secrets Include secret attributes (dkim-key, passwords). - -c, --color Force colorized output. -d, --dns Include dns records. + -c, --color Force colorized output. -o, --output-file FILENAME Save configuration to file. -j, --json Export configuration in json format. -?, -h, --help Show this message and exit. @@ -147,14 +147,18 @@ The purpose of this command is to export the complete configuration in YAML or J Only non-default attributes are exported. If you want to export all attributes use ``--full``. If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option. To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. + By default all configuration objects are exported (domain, user, alias, relay). You can specify filters to export only some objects or attributes (try: ``user`` or ``domain.name``). +Attributes explicitly specified in filters are automatically exported: there is no need to add ``--secrets`` or ``--full``. .. code-block:: bash - $ docker-compose exec admin flask mailu config-export -o mail-config.yml + $ docker-compose exec admin flask mailu config-export --output mail-config.yml - $ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf + $ docker-compose exec admin flask mailu config-export domain.dns_mx domain.dns_spf + + $ docker-compose exec admin flask mailu config-export user.spam_threshold config-import ------------- @@ -211,7 +215,7 @@ mail-config.yml contains the configuration and looks like this: config-update shows the number of created/modified/deleted objects after import. To suppress all messages except error messages use ``--quiet``. -By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed. +By adding the ``--verbose`` switch the import gets more detailed and shows exactly what attributes changed. In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. If you want to test what would be done when importing without committing any changes, use ``--dry-run``. @@ -234,6 +238,9 @@ It is possible to delete a single element or prune all elements from lists and a The ``-key: null`` notation can also be used to reset an attribute to its default. To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``. +A new dkim key can be generated when adding or modifying a domain, by using the special value +``dkim_key: -generate-``. + This is a complete YAML template with all additional parameters that can be defined: .. code-block:: yaml