lub
/
mailu
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

moved import logging to schema

Browse Source 
- yaml-import is now logged via schema.Logger
- iremoved relative imports - not used in other mailu modules
- removed develepment comments
- added Mailconfig.check method to check for duplicate domain names
- converted .format() to .format_map() where possible
- switched to yaml multiline dump for dkim_key
- converted dkim_key import from regex to string functions
- automatically unhide/unexclude explicitly specified attributes on dump
- use field order when loading to stabilize import
- fail when using 'hash_password' without 'password'
- fixed logging of dkim_key
- fixed pruning and deleting of lists
- modified error messages
- added debug flag and two verbosity levels
master
Alexander Graf 4 years ago
parent e4c83e162d
commit bde7a2b6c4
4 changed files with 688 additions and 569 deletions
Show Stats Download Patch File Download Diff File

280
core/admin/mailu/manage.py
Unescape Escape View File

@ -4,22 +4,16 @@
import sys import sys
import os import os
import socket import socket
import logging
import uuid import uuid
from collections import Counter
from itertools import chain
import click import click
import sqlalchemy
import yaml import yaml
from flask import current_app as app from flask import current_app as app
from flask.cli import FlaskGroup, with_appcontext from flask.cli import FlaskGroup, with_appcontext
from marshmallow.exceptions import ValidationError
from . import models from mailu import models
from .schemas import MailuSchema, get_schema, get_fieldspec, colorize, canColorize, RenderJSON, HIDDEN from mailu.schemas import MailuSchema, Logger, RenderJSON
db = models.db db = models.db
@ -326,246 +320,53 @@ def config_update(verbose=False, delete_objects=False):
@mailu.command() @mailu.command()
@click.option('-v', '--verbose', count=True, help='Increase verbosity.') @click.option('-v', '--verbose', count=True, help='Increase verbosity.')
@click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.') @click.option('-s', '--secrets', is_flag=True, help='Show secret attributes in messages.')
@click.option('-d', '--debug', is_flag=True, help='Enable debug output.')
@click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.') @click.option('-q', '--quiet', is_flag=True, help='Quiet mode - only show errors.')
@click.option('-c', '--color', is_flag=True, help='Force colorized output.') @click.option('-c', '--color', is_flag=True, help='Force colorized output.')
@click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.') @click.option('-u', '--update', is_flag=True, help='Update mode - merge input with existing config.')
@click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.') @click.option('-n', '--dry-run', is_flag=True, help='Perform a trial run with no changes made.')
@click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin) @click.argument('source', metavar='[FILENAME|-]', type=click.File(mode='r'), default=sys.stdin)
@with_appcontext @with_appcontext
def config_import(verbose=0, secrets=False, quiet=False, color=False, update=False, dry_run=False, source=None): def config_import(verbose=0, secrets=False, debug=False, quiet=False, color=False,
update=False, dry_run=False, source=None):
""" Import configuration as YAML or JSON from stdin or file """ Import configuration as YAML or JSON from stdin or file
""" """
# verbose log = Logger(want_color=color or None, can_color=sys.stdout.isatty(), secrets=secrets, debug=debug)
# 0 : only show number of changes log.lexer = 'python'
# 1 : also show detailed changes log.strip = True
# 2 : also show input data log.verbose = 0 if quiet else verbose
# 3 : also show sql queries (also needs -s, as sql may contain secrets) log.quiet = quiet
# 4 : also show tracebacks (also needs -s, as tracebacks may contain secrets)
if quiet: context = {
verbose = -1
if verbose > 2 and not secrets:
print('[Warning] Verbosity level capped to 2. Specify --secrets to log sql and tracebacks.')
verbose = 2
color_cfg = {
'color': color or (canColorize and sys.stdout.isatty()),
'lexer': 'python',
'strip': True,
}
counter = Counter()
logger = {}
def format_errors(store, path=None):
res = []
if path is None:
path = []
for key in sorted(store):
location = path + [str(key)]
value = store[key]
if isinstance(value, dict):
res.extend(format_errors(value, location))
else:
for message in value:
res.append((".".join(location), message))
if path:
return res
fmt = f' - {{:<{max([len(loc) for loc, msg in res])}}} : {{}}'
res = [fmt.format(loc, msg) for loc, msg in res]
num = f'error{["s",""][len(res)==1]}'
res.insert(0, f'[ValidationError] {len(res)} {num} occurred during input validation')
return '\n'.join(res)
def format_changes(*message):
if counter:
changes = []
last = None
for (action, what), count in sorted(counter.items()):
if action != last:
if last:
changes.append('/')
changes.append(f'{action}:')
last = action
changes.append(f'{what}({count})')
else:
changes = ['No changes.']
return chain(message, changes)
def log(action, target, message=None):
if message is None:
try:
message = logger[target.__class__].dump(target)
except KeyError:
message = target
if not isinstance(message, str):
message = repr(message)
print(f'{action} {target.__table__}: {colorize(message, **color_cfg)}')
def listen_insert(mapper, connection, target): # pylint: disable=unused-argument
""" callback function to track import """
counter.update([('Created', target.__table__.name)])
if verbose >= 1:
log('Created', target)
def listen_update(mapper, connection, target): # pylint: disable=unused-argument
""" callback function to track import """
changed = {}
inspection = sqlalchemy.inspect(target)
for attr in sqlalchemy.orm.class_mapper(target.__class__).column_attrs:
history = getattr(inspection.attrs, attr.key).history
if history.has_changes() and history.deleted:
before = history.deleted[-1]
after = getattr(target, attr.key)
# TODO: remove special handling of "comment" after modifying model
if attr.key == 'comment' and not before and not after:
pass
# only remember changed keys
elif before != after:
if verbose >= 1:
changed[str(attr.key)] = (before, after)
else:
break
if verbose >= 1:
# use schema with dump_context to hide secrets and sort keys
dumped = get_schema(target)(only=changed.keys(), context=diff_context).dump(target)
for key, value in dumped.items():
before, after = changed[key]
if value == HIDDEN:
before = HIDDEN if before else before
after = HIDDEN if after else after
else:
# TODO: need to use schema to "convert" before value?
after = value
log('Modified', target, f'{str(target)!r} {key}: {before!r} -> {after!r}')
if changed:
counter.update([('Modified', target.__table__.name)])
def listen_delete(mapper, connection, target): # pylint: disable=unused-argument
""" callback function to track import """
counter.update([('Deleted', target.__table__.name)])
if verbose >= 1:
log('Deleted', target)
# TODO: this listener will not be necessary, if dkim keys would be stored in database
_dedupe_dkim = set()
def listen_dkim(session, flush_context): # pylint: disable=unused-argument
""" callback function to track import """
for target in session.identity_map.values():
# look at Domains originally loaded from db
if not isinstance(target, models.Domain) or not target._sa_instance_state.load_path:
continue
before = target._dkim_key_on_disk
after = target._dkim_key
if before != after:
if secrets:
before = before.decode('ascii', 'ignore')
after = after.decode('ascii', 'ignore')
else:
before = HIDDEN if before else ''
after = HIDDEN if after else ''
# "de-dupe" messages; this event is fired at every flush
if not (target, before, after) in _dedupe_dkim:
_dedupe_dkim.add((target, before, after))
counter.update([('Modified', target.__table__.name)])
if verbose >= 1:
log('Modified', target, f'{str(target)!r} dkim_key: {before!r} -> {after!r}')
def track_serialize(obj, item, backref=None):
""" callback function to track import """
# called for backref modification?
if backref is not None:
log('Modified', item, '{target!r} {key}: {before!r} -> {after!r}'.format(**backref))
return
# show input data?
if not verbose >= 2:
return
# hide secrets in data
data = logger[obj.opts.model].hide(item)
if 'hash_password' in data:
data['password'] = HIDDEN
if 'fetches' in data:
for fetch in data['fetches']:
fetch['password'] = HIDDEN
log('Handling', obj.opts.model, data)
# configure contexts
diff_context = {
'full': True,
'secrets': secrets,
}
log_context = {
'secrets': secrets,
}
load_context = {
'import': True, 'import': True,
'update': update, 'update': update,
'clear': not update, 'clear': not update,
'callback': track_serialize, 'callback': log.track_serialize,
} }
# register listeners schema = MailuSchema(only=MailuSchema.Meta.order, context=context)
for schema in get_schema():
model = schema.Meta.model
logger[model] = schema(context=log_context)
sqlalchemy.event.listen(model, 'after_insert', listen_insert)
sqlalchemy.event.listen(model, 'after_update', listen_update)
sqlalchemy.event.listen(model, 'after_delete', listen_delete)
# special listener for dkim_key changes
sqlalchemy.event.listen(db.session, 'after_flush', listen_dkim)
if verbose >= 3:
logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
try: try:
# import source
with models.db.session.no_autoflush: with models.db.session.no_autoflush:
config = MailuSchema(only=MailuSchema.Meta.order, context=load_context).loads(source) config = schema.loads(source)
except ValidationError as exc: # flush session to show/count all changes
raise click.ClickException(format_errors(exc.messages)) from exc if not quiet and (dry_run or verbose):
db.session.flush()
# check for duplicate domain names
config.check()
except Exception as exc: except Exception as exc:
if verbose >= 3: if msg := log.format_exception(exc):
raise raise click.ClickException(msg) from exc
# (yaml.scanner.ScannerError, UnicodeDecodeError, ...) raise
raise click.ClickException(
f'[{exc.__class__.__name__}] '
f'{" ".join(str(exc).split())}'
) from exc
# flush session to show/count all changes
if dry_run or verbose >= 1:
db.session.flush()
# check for duplicate domain names
dup = set()
for fqdn in chain(
db.session.query(models.Domain.name),
db.session.query(models.Alternative.name),
db.session.query(models.Relay.name)
):
if fqdn in dup:
raise click.ClickException(f'[ValidationError] Duplicate domain name: {fqdn}')
dup.add(fqdn)
# don't commit when running dry # don't commit when running dry
if dry_run: if dry_run:
if not quiet: log.changes('Dry run. Not committing changes.')
print(*format_changes('Dry run. Not commiting changes.'))
db.session.rollback() db.session.rollback()
else: else:
if not quiet: log.changes('Committing changes.')
print(*format_changes('Committing changes.'))
db.session.commit() db.session.commit()
@ -573,8 +374,8 @@ def config_import(verbose=0, secrets=False, quiet=False, color=False, update=Fal
@click.option('-f', '--full', is_flag=True, help='Include attributes with default value.') @click.option('-f', '--full', is_flag=True, help='Include attributes with default value.')
@click.option('-s', '--secrets', is_flag=True, @click.option('-s', '--secrets', is_flag=True,
help='Include secret attributes (dkim-key, passwords).') help='Include secret attributes (dkim-key, passwords).')
@click.option('-c', '--color', is_flag=True, help='Force colorized output.')
@click.option('-d', '--dns', is_flag=True, help='Include dns records.') @click.option('-d', '--dns', is_flag=True, help='Include dns records.')
@click.option('-c', '--color', is_flag=True, help='Force colorized output.')
@click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'), @click.option('-o', '--output-file', 'output', default=sys.stdout, type=click.File(mode='w'),
help='Save configuration to file.') help='Save configuration to file.')
@click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.') @click.option('-j', '--json', 'as_json', is_flag=True, help='Export configuration in json format.')
@ -584,32 +385,25 @@ def config_export(full=False, secrets=False, color=False, dns=False, output=None
""" Export configuration as YAML or JSON to stdout or file """ Export configuration as YAML or JSON to stdout or file
""" """
if only: only = only or MailuSchema.Meta.order
for spec in only:
if spec.split('.', 1)[0] not in MailuSchema.Meta.order:
raise click.ClickException(f'[ValidationError] Unknown section: {spec}')
else:
only = MailuSchema.Meta.order
context = { context = {
'full': full, 'full': full,
'secrets': secrets, 'secrets': secrets,
'dns': dns, 'dns': dns,
} }
log = Logger(want_color=color or None, can_color=output.isatty())
schema = MailuSchema(only=only, context=context)
color_cfg = {'color': color or (canColorize and output.isatty())}
if as_json:
schema.opts.render_module = RenderJSON
color_cfg['lexer'] = 'json'
color_cfg['strip'] = True
try: try:
print(colorize(schema.dumps(models.MailuConfig()), **color_cfg), file=output) schema = MailuSchema(only=only, context=context)
except ValueError as exc: if as_json:
if spec := get_fieldspec(exc): schema.opts.render_module = RenderJSON
raise click.ClickException(f'[ValidationError] Invalid filter: {spec}') from exc log.lexer = 'json'
log.strip = True
print(log.colorize(schema.dumps(models.MailuConfig())), file=output)
except Exception as exc:
if msg := log.format_exception(exc):
raise click.ClickException(msg) from exc
raise raise

40
core/admin/mailu/models.py
Unescape Escape View File

@ -23,7 +23,7 @@ from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.inspection import inspect from sqlalchemy.inspection import inspect
from werkzeug.utils import cached_property from werkzeug.utils import cached_property
from . import dkim from mailu import dkim
db = flask_sqlalchemy.SQLAlchemy() db = flask_sqlalchemy.SQLAlchemy()
@ -33,7 +33,6 @@ class IdnaDomain(db.TypeDecorator):
""" Stores a Unicode string in it's IDNA representation (ASCII only) """ Stores a Unicode string in it's IDNA representation (ASCII only)
""" """
# TODO: use db.String(255)?
impl = db.String(80) impl = db.String(80)
def process_bind_param(self, value, dialect): def process_bind_param(self, value, dialect):
@ -50,7 +49,6 @@ class IdnaEmail(db.TypeDecorator):
""" Stores a Unicode string in it's IDNA representation (ASCII only) """ Stores a Unicode string in it's IDNA representation (ASCII only)
""" """
# TODO: use db.String(254)?
impl = db.String(255) impl = db.String(255)
def process_bind_param(self, value, dialect): def process_bind_param(self, value, dialect):
@ -127,11 +125,7 @@ class Base(db.Model):
if pkey == 'email': if pkey == 'email':
# ugly hack for email declared attr. _email is not always up2date # ugly hack for email declared attr. _email is not always up2date
return str(f'{self.localpart}@{self.domain_name}') return str(f'{self.localpart}@{self.domain_name}')
elif pkey in {'name', 'email'}: return str(getattr(self, pkey))
return str(getattr(self, pkey, None))
else:
return self.__repr__()
return str(getattr(self, self.__table__.primary_key.columns.values()[0].name))
def __repr__(self): def __repr__(self):
return f'<{self.__class__.__name__} {str(self)!r}>' return f'<{self.__class__.__name__} {str(self)!r}>'
@ -145,12 +139,15 @@ class Base(db.Model):
else: else:
return NotImplemented return NotImplemented
# we need hashable instances here for sqlalchemy to update collections
# in collections.bulk_replace, but auto-incrementing don't always have
# a valid primary key, in this case we use the object's id
__hashed = None
def __hash__(self): def __hash__(self):
primary = getattr(self, self.__table__.primary_key.columns.values()[0].name) if self.__hashed is None:
if primary is None: primary = getattr(self, self.__table__.primary_key.columns.values()[0].name)
return NotImplemented self.__hashed = id(self) if primary is None else hash(primary)
else: return self.__hashed
return hash(primary)
# Many-to-many association table for domain managers # Many-to-many association table for domain managers
@ -314,7 +311,6 @@ class Relay(Base):
__tablename__ = 'relay' __tablename__ = 'relay'
name = db.Column(IdnaDomain, primary_key=True, nullable=False) name = db.Column(IdnaDomain, primary_key=True, nullable=False)
# TODO: use db.String(266)? transport(8):(1)[nexthop(255)](2)
smtp = db.Column(db.String(80), nullable=True) smtp = db.Column(db.String(80), nullable=True)
@ -322,7 +318,6 @@ class Email(object):
""" Abstraction for an email address (localpart and domain). """ Abstraction for an email address (localpart and domain).
""" """
# TODO: use db.String(64)?
localpart = db.Column(db.String(80), nullable=False) localpart = db.Column(db.String(80), nullable=False)
@declarative.declared_attr @declarative.declared_attr
@ -342,7 +337,7 @@ class Email(object):
key = f'{cls.__tablename__}_email' key = f'{cls.__tablename__}_email'
if key in ctx.current_parameters: if key in ctx.current_parameters:
return ctx.current_parameters[key] return ctx.current_parameters[key]
return '{localpart}@{domain_name}'.format(**ctx.current_parameters) return '{localpart}@{domain_name}'.format_map(ctx.current_parameters)
return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater) return db.Column('email', IdnaEmail, primary_key=True, nullable=False, onupdate=updater)
@ -632,7 +627,6 @@ class Token(Base):
user = db.relationship(User, user = db.relationship(User,
backref=db.backref('tokens', cascade='all, delete-orphan')) backref=db.backref('tokens', cascade='all, delete-orphan'))
password = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False)
# TODO: use db.String(32)?
ip = db.Column(db.String(255)) ip = db.Column(db.String(255))
def check_password(self, password): def check_password(self, password):
@ -865,6 +859,18 @@ class MailuConfig:
if models is None or model in models: if models is None or model in models:
db.session.query(model).delete() db.session.query(model).delete()
def check(self):
""" check for duplicate domain names """
dup = set()
for fqdn in chain(
db.session.query(Domain.name),
db.session.query(Alternative.name),
db.session.query(Relay.name)
):
if fqdn in dup:
raise ValueError(f'Duplicate domain name: {fqdn}')
dup.add(fqdn)
domain = MailuCollection(Domain) domain = MailuCollection(Domain)
user = MailuCollection(User) user = MailuCollection(User)
alias = MailuCollection(Alias) alias = MailuCollection(Alias)

922
core/admin/mailu/schemas.py
View File

File diff suppressed because it is too large Load Diff

15
docs/cli.rst
Unescape Escape View File

@ -138,8 +138,8 @@ The purpose of this command is to export the complete configuration in YAML or J
Options: Options:
-f, --full Include attributes with default value. -f, --full Include attributes with default value.
-s, --secrets Include secret attributes (dkim-key, passwords). -s, --secrets Include secret attributes (dkim-key, passwords).
-c, --color Force colorized output.
-d, --dns Include dns records. -d, --dns Include dns records.
-c, --color Force colorized output.
-o, --output-file FILENAME Save configuration to file. -o, --output-file FILENAME Save configuration to file.
-j, --json Export configuration in json format. -j, --json Export configuration in json format.
-?, -h, --help Show this message and exit. -?, -h, --help Show this message and exit.
@ -147,14 +147,18 @@ The purpose of this command is to export the complete configuration in YAML or J
Only non-default attributes are exported. If you want to export all attributes use ``--full``. Only non-default attributes are exported. If you want to export all attributes use ``--full``.
If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option. If you want to export plain-text secrets (dkim-keys, passwords) you have to add the ``--secrets`` option.
To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option. To include dns records (mx, spf, dkim and dmarc) add the ``--dns`` option.
By default all configuration objects are exported (domain, user, alias, relay). You can specify By default all configuration objects are exported (domain, user, alias, relay). You can specify
filters to export only some objects or attributes (try: ``user`` or ``domain.name``). filters to export only some objects or attributes (try: ``user`` or ``domain.name``).
Attributes explicitly specified in filters are automatically exported: there is no need to add ``--secrets`` or ``--full``.
.. code-block:: bash .. code-block:: bash
$ docker-compose exec admin flask mailu config-export -o mail-config.yml $ docker-compose exec admin flask mailu config-export --output mail-config.yml
$ docker-compose exec admin flask mailu config-export --dns domain.dns_mx domain.dns_spf $ docker-compose exec admin flask mailu config-export domain.dns_mx domain.dns_spf
$ docker-compose exec admin flask mailu config-export user.spam_threshold
config-import config-import
------------- -------------
@ -211,7 +215,7 @@ mail-config.yml contains the configuration and looks like this:
config-update shows the number of created/modified/deleted objects after import. config-update shows the number of created/modified/deleted objects after import.
To suppress all messages except error messages use ``--quiet``. To suppress all messages except error messages use ``--quiet``.
By adding the ``--verbose`` switch (up to two times) the import gets more detailed and shows exactly what attributes changed. By adding the ``--verbose`` switch the import gets more detailed and shows exactly what attributes changed.
In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets. In all log messages plain-text secrets (dkim-keys, passwords) are hidden by default. Use ``--secrets`` to log secrets.
If you want to test what would be done when importing without committing any changes, use ``--dry-run``. If you want to test what would be done when importing without committing any changes, use ``--dry-run``.
@ -234,6 +238,9 @@ It is possible to delete a single element or prune all elements from lists and a
The ``-key: null`` notation can also be used to reset an attribute to its default. The ``-key: null`` notation can also be used to reset an attribute to its default.
To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``. To reset *spam_threshold* to it's default *80* use ``-spam_threshold: null``.
A new dkim key can be generated when adding or modifying a domain, by using the special value
``dkim_key: -generate-``.
This is a complete YAML template with all additional parameters that can be defined: This is a complete YAML template with all additional parameters that can be defined:
.. code-block:: yaml .. code-block:: yaml

Write Preview
Loading…
Cancel
Save