config_dump now using marshmallow

core/admin/mailu/manage.py

@@ -1,5 +1,5 @@
 from mailu import models
-from .schemas import schemas
+from .schemas import MailuConfig, MailuSchema
 
 from flask import current_app as app
 from flask.cli import FlaskGroup, with_appcontext
@@ -310,41 +310,17 @@ def config_dump(full=False, secrets=False, dns=False, sections=None):
     SECTIONS can be: domains, relays, users, aliases
     """
 
-    class spacedDumper(yaml.Dumper):
+    try:
-
+        config = MailuConfig(sections)
-        def write_line_break(self, data=None):
+    except ValueError as reason:
-            super().write_line_break(data)
+        print(f'[ERROR] {reason}')
-            if len(self.indents) == 1:
-                super().write_line_break()
-
-        def increase_indent(self, flow=False, indentless=False):
-            return super().increase_indent(flow, False)
-
-    if sections:
-        for section in sections:
-            if section not in schemas:
-                print(f'[ERROR] Invalid section: {section}')
         return 1
-    else:
-        sections = sorted(schemas.keys())
 
-# TODO: create World Schema and dump only this with Word.dumps ?
+    MailuSchema(context={
-
-    for section in sections:
-        schema = schemas[section](many=True)
-        schema.context.update({
         'full': full,
         'secrets': secrets,
         'dns': dns,
-        })
+    }).dumps(config, sys.stdout)
-        yaml.dump(
-            {section: schema.dump(schema.Meta.model.query.all())},
-            sys.stdout,
-            Dumper=spacedDumper,
-            default_flow_style=False,
-            allow_unicode=True
-        )
-        sys.stdout.write('\n')
 
 
 @mailu.command()

core/admin/mailu/models.py

@@ -110,7 +110,7 @@ class Base(db.Model):
 
     created_at = db.Column(db.Date, nullable=False, default=date.today)
     updated_at = db.Column(db.Date, nullable=True, onupdate=date.today)
-    comment = db.Column(db.String(255), nullable=True)
+    comment = db.Column(db.String(255), nullable=True, default='')
 
     @classmethod
     def _dict_pkey(cls):
@@ -171,7 +171,7 @@ class Base(db.Model):
                 if self.__mapper__.relationships[key].query_class is not None:
                     if hasattr(items, 'all'):
                         items = items.all()
-                if full or len(items):
+                if full or items:
                     if key in secret:
                         res[key] = '<hidden>'
                     else:

core/admin/mailu/schemas.py

@@ -1,63 +1,252 @@
-import marshmallow
+"""
-import sqlalchemy
+Mailu marshmallow schema
-import flask_marshmallow
+"""
 
-from . import models
+from textwrap import wrap
+
+import re
+import yaml
+
+from marshmallow import post_dump, fields, Schema
+from flask_marshmallow import Marshmallow
+from OpenSSL import crypto
+
+from . import models, dkim
 
 
-ma = flask_marshmallow.Marshmallow()
+ma = Marshmallow()
+# TODO:
+# how to mark keys as "required" while unserializing (in certain use cases/API)?
+# - fields withoud default => required
+# - fields which are the primary key => unchangeable when updating
+
+
+### yaml render module ###
+
+class RenderYAML:
+    """ Marshmallow YAML Render Module
+    """
+
+    class SpacedDumper(yaml.Dumper):
+        """ YAML Dumper to add a newline between main sections
+            and double the indent used
+        """
+
+        def write_line_break(self, data=None):
+            super().write_line_break(data)
+            if len(self.indents) == 1:
+                super().write_line_break()
+
+        def increase_indent(self, flow=False, indentless=False):
+            return super().increase_indent(flow, False)
+
+    @staticmethod
+    def _update_dict(dict1, dict2):
+        """ sets missing keys in dict1 to values of dict2
+        """
+        for key, value in dict2.items():
+            if key not in dict1:
+                dict1[key] = value
+
+    _load_defaults = {}
+    @classmethod
+    def loads(cls, *args, **kwargs):
+        """ load yaml data from string
+        """
+        cls._update_dict(kwargs, cls._load_defaults)
+        return yaml.load(*args, **kwargs)
+
+    _dump_defaults = {
+        'Dumper': SpacedDumper,
+        'default_flow_style': False,
+        'allow_unicode': True,
+    }
+    @classmethod
+    def dumps(cls, *args, **kwargs):
+        """ dump yaml data to string
+        """
+        cls._update_dict(kwargs, cls._dump_defaults)
+        return yaml.dump(*args, **kwargs)
+
+
+### field definitions ###
+
+class LazyString(fields.String):
+    """ Field that serializes a "false" value to the empty string
+    """
+
+    def _serialize(self, value, attr, obj, **kwargs):
+        """ serialize None to the empty string
+        """
+        return value if value else ''
+
+
+class CommaSeparatedList(fields.Raw):
+    """ Field that deserializes a string containing comma-separated values to
+        a list of strings
+    """
+    # TODO: implement this
+
+
+class DkimKey(fields.String):
+    """ Field that serializes a dkim key to a list of strings (lines) and
+        deserializes a string or list of strings.
+    """
+
+    _clean_re = re.compile(
+        r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)',
+        flags=re.UNICODE
+    )
+
+    def _serialize(self, value, attr, obj, **kwargs):
+        """ serialize dkim key to a list of strings (lines)
+        """
+
+        # map empty string and None to None
+        if not value:
+            return None
+
+        # return list of key lines without header/footer
+        return value.decode('utf-8').strip().split('\n')[1:-1]
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        """ deserialize a string or list of strings to dkim key data
+            with verification
+        """
+
+        # convert list to str
+        if isinstance(value, list):
+            value = ''.join(value)
+
+        # only strings are allowed
+        if not isinstance(value, str):
+            raise TypeError(f'invalid type: {type(value).__name__!r}')
+
+        # clean value (remove whitespace and header/footer)
+        value = self._clean_re.sub('', value.strip())
+
+        # map empty string/list to None
+        if not value:
+            return None
+
+        # handle special value 'generate'
+        elif value == 'generate':
+            return dkim.gen_key()
+
+        # wrap value into valid pem layout and check validity
+        value = (
+            '-----BEGIN PRIVATE KEY-----\n' +
+            '\n'.join(wrap(value, 64)) +
+            '\n-----END PRIVATE KEY-----\n'
+        ).encode('ascii')
+        try:
+            crypto.load_privatekey(crypto.FILETYPE_PEM, value)
+        except crypto.Error as exc:
+            raise ValueError('invalid dkim key') from exc
+        else:
+            return value
+
+
+### schema definitions ###
 
-import collections
 class BaseSchema(ma.SQLAlchemyAutoSchema):
+    """ Marshmallow base schema with custom exclude logic
+        and option to hide sqla defaults
+    """
 
     class Meta:
-        base_hide_always  = {'created_at', 'updated_at'}
+        """ Schema config """
-        base_hide_secrets = set()
+        model = None
-        base_hide_by_value = {
-#            'comment': {'', None}
-        }
 
-    @marshmallow.post_dump
+    def __init__(self, *args, **kwargs):
-    def remove_skip_values(self, data, many, **kwargs):
-#        print(repr(data), self.context)
 
-        # always hide
+        # get and remove config from kwargs
-        hide_by_key = self.Meta.base_hide_always | set(getattr(self.Meta, 'hide_always', ()))
+        context = kwargs.get('context', {})
 
-        # hide secrets
+        # compile excludes
-        if not self.context.get('secrets'):
+        exclude = set(kwargs.get('exclude', []))
-            hide_by_key |= self.Meta.base_hide_secrets
-            hide_by_key |= set(getattr(self.Meta, 'hide_secrets', ()))
 
-        # hide by value
+        # always exclude
-        hide_by_value = self.Meta.base_hide_by_value | getattr(self.Meta, 'hide_by_value', {})
+        exclude.update({'created_at', 'updated_at'})
 
-        # hide defaults
+        # add include_by_context
-        if not self.context.get('full'):
+        if context is not None:
+            for ctx, what in getattr(self.Meta, 'include_by_context', {}).items():
+                if not context.get(ctx):
+                    exclude |= set(what)
+
+        # update excludes
+        kwargs['exclude'] = exclude
+
+        # exclude_by_value
+        self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {})
+
+        # exclude default values
+        if not context.get('full'):
             for column in self.Meta.model.__table__.columns:
-#                print(column.name, column.default.arg if isinstance(column.default, sqlalchemy.sql.schema.ColumnDefault) else column.default)
+                if column.name not in exclude:
-# alias.destiantion has default [] - is this okay. how to check it?
+                    self._exclude_by_value.setdefault(column.name, []).append(
-                if column.name not in hide_by_key:
+                        None if column.default is None else column.default.arg
-                    hide_by_value.setdefault(column.name, set()).add(None if column.default is None else column.default.arg)
+                    )
 
+        # hide by context
+        self._hide_by_context = set()
+        if context is not None:
+            for ctx, what in getattr(self.Meta, 'hide_by_context', {}).items():
+                if not context.get(ctx):
+                    self._hide_by_context |= set(what)
+
+        # init SQLAlchemyAutoSchema
+        super().__init__(*args, **kwargs)
+
+    @post_dump
+    def _remove_skip_values(self, data, many, **kwargs): # pylint: disable=unused-argument
+
+        if not self._exclude_by_value and not self._hide_by_context:
+            return data
+
+        full = self.context.get('full')
         return {
-            key: value for key, value in data.items()
+            key: '<hidden>' if key in self._hide_by_context else value
-            if
+            for key, value in data.items()
-                not isinstance(value, collections.Hashable)
+            if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key]
-            or(
-                key not in hide_by_key
-            and
-                (key not in hide_by_value or value not in hide_by_value[key]))
         }
 
+    # TODO: remove LazyString and fix model definition (comment should not be nullable)
+    comment = LazyString()
+
 class DomainSchema(BaseSchema):
-    class Meta(BaseSchema.Meta):
+    """ Marshmallow schema for Domain model """
+    class Meta:
+        """ Schema config """
         model = models.Domain
+        include_relationships = True
+        #include_fk = True
+        exclude = ['users', 'managers', 'aliases']
+
+        include_by_context = {
+            'dns': {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'},
+        }
+        hide_by_context = {
+            'secrets': {'dkim_key'},
+        }
+        exclude_by_value = {
+            'alternatives': [[]],
+            'dkim_key': [None],
+            'dkim_publickey': [None],
+            'dns_mx': [None],
+            'dns_spf': [None],
+            'dns_dkim': [None],
+            'dns_dmarc': [None],
+        }
+
+    dkim_key = DkimKey()
+    dkim_publickey = fields.String(dump_only=True)
+    dns_mx = fields.String(dump_only=True)
+    dns_spf = fields.String(dump_only=True)
+    dns_dkim = fields.String(dump_only=True)
+    dns_dmarc = fields.String(dump_only=True)
 
-    # _dict_hide = {'users', 'managers', 'aliases'}
-    # _dict_show = {'dkim_key'}
-    # _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}}
-    # _dict_secret = {'dkim_key'}
     # _dict_types = {
     #     'dkim_key': (bytes, type(None)),
     #     'dkim_publickey': False,
@@ -66,50 +255,62 @@ class DomainSchema(BaseSchema):
     #     'dns_dkim': False,
     #     'dns_dmarc': False,
     # }
-    # _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]}
-    # @staticmethod
-    # def _dict_input(data):
-    #     if 'dkim_key' in data:
-    #         key = data['dkim_key']
-    #         if key is not None:
-    #             if type(key) is list:
-    #                 key = ''.join(key)
-    #             if type(key) is str:
-    #                 key = ''.join(key.strip().split()) # removes all whitespace
-    #                 if key == 'generate':
-    #                     data['dkim_key'] = dkim.gen_key()
-    #                 elif key:
-    #                     m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key)
-    #                     if m is not None:
-    #                         key = key[m.end():]
-    #                     m = re.search('-----END (RSA )?PRIVATE KEY-----$', key)
-    #                     if m is not None:
-    #                         key = key[:m.start()]
-    #                     key = '\n'.join(wrap(key, 64))
-    #                     key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii')
-    #                     try:
-    #                         dkim.strip_key(key)
-    #                     except:
-    #                         raise ValueError('invalid dkim key')
-    #                     else:
-    #                       data['dkim_key'] = key
-    #                 else:
-    #                     data['dkim_key'] = None
 
-    # name = db.Column(IdnaDomain, primary_key=True, nullable=False)
+
-    # managers = db.relationship('User', secondary=managers,
+class TokenSchema(BaseSchema):
-    #     backref=db.backref('manager_of'), lazy='dynamic')
+    """ Marshmallow schema for Token model """
-    # max_users = db.Column(db.Integer, nullable=False, default=-1)
+    class Meta:
-    # max_aliases = db.Column(db.Integer, nullable=False, default=-1)
+        """ Schema config """
-    # max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0)
+        model = models.Token
-    # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False)
+
+    # _dict_recurse = True
+    # _dict_hide = {'user', 'user_email'}
+    # _dict_mandatory = {'password'}
+
+    # id = db.Column(db.Integer(), primary_key=True)
+    # user_email = db.Column(db.String(255), db.ForeignKey(User.email),
+    #     nullable=False)
+    # user = db.relationship(User,
+    #     backref=db.backref('tokens', cascade='all, delete-orphan'))
+    # password = db.Column(db.String(255), nullable=False)
+    # ip = db.Column(db.String(255))
+
+
+class FetchSchema(BaseSchema):
+    """ Marshmallow schema for Fetch model """
+    class Meta:
+        """ Schema config """
+        model = models.Fetch
+        include_by_context = {
+            'full': {'last_check', 'error'},
+        }
+        hide_by_context = {
+            'secrets': {'password'},
+        }
+
+# TODO: What about mandatory keys?
+    # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'}
 
 
 class UserSchema(BaseSchema):
-    class Meta(BaseSchema.Meta):
+    """ Marshmallow schema for User model """
+    class Meta:
+        """ Schema config """
         model = models.User
+        include_relationships = True
+        exclude = ['localpart', 'domain', 'quota_bytes_used']
 
-    # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'}
+        exclude_by_value = {
+            'forward_destination': [[]],
+            'tokens': [[]],
+            'reply_enddate': ['2999-12-31'],
+            'reply_startdate': ['1900-01-01'],
+        }
+
+    tokens = fields.Nested(TokenSchema, many=True)
+    fetches = fields.Nested(FetchSchema, many=True)
+
+# TODO: deserialize password/password_hash! What about mandatory keys?
     # _dict_mandatory = {'localpart', 'domain', 'password'}
     # @classmethod
     # def _dict_input(cls, data):
@@ -133,44 +334,19 @@ class UserSchema(BaseSchema):
     #         del data['hash_scheme']
     #         del data['password_hash']
 
-    # domain = db.relationship(Domain,
-    #     backref=db.backref('users', cascade='all, delete-orphan'))
-    # password = db.Column(db.String(255), nullable=False)
-    # quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9)
-    # quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0)
-    # global_admin = db.Column(db.Boolean(), nullable=False, default=False)
-    # enabled = db.Column(db.Boolean(), nullable=False, default=True)
-
-    # # Features
-    # enable_imap = db.Column(db.Boolean(), nullable=False, default=True)
-    # enable_pop = db.Column(db.Boolean(), nullable=False, default=True)
-
-    # # Filters
-    # forward_enabled = db.Column(db.Boolean(), nullable=False, default=False)
-    # forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[])
-    # forward_keep = db.Column(db.Boolean(), nullable=False, default=True)
-    # reply_enabled = db.Column(db.Boolean(), nullable=False, default=False)
-    # reply_subject = db.Column(db.String(255), nullable=True, default=None)
-    # reply_body = db.Column(db.Text(), nullable=True, default=None)
-    # reply_startdate = db.Column(db.Date, nullable=False,
-    #     default=date(1900, 1, 1))
-    # reply_enddate = db.Column(db.Date, nullable=False,
-    #     default=date(2999, 12, 31))
-
-    # # Settings
-    # displayed_name = db.Column(db.String(160), nullable=False, default='')
-    # spam_enabled = db.Column(db.Boolean(), nullable=False, default=True)
-    # spam_threshold = db.Column(db.Integer(), nullable=False, default=80)
 
 class AliasSchema(BaseSchema):
-    class Meta(BaseSchema.Meta):
+    """ Marshmallow schema for Alias model """
+    class Meta:
+        """ Schema config """
         model = models.Alias
-        hide_always  = {'localpart'}
+        exclude  = ['localpart']
-        hide_secrets = {'wildcard'}
+
-        hide_by_value = {
+        exclude_by_value = {
-            'destination': set([]) # always hide empty lists?!
+            'destination': [[]],
         }
 
+# TODO: deserialize destination!
     # @staticmethod
     # def _dict_input(data):
     #     Email._dict_input(data)
@@ -180,65 +356,57 @@ class AliasSchema(BaseSchema):
     #         data['destination'] = list([adr.strip() for adr in dst.split(',')])
 
 
-class TokenSchema(BaseSchema):
-    class Meta(BaseSchema.Meta):
-        model = models.Token
-
-    # _dict_recurse = True
-    # _dict_hide = {'user', 'user_email'}
-    # _dict_mandatory = {'password'}
-
-    # id = db.Column(db.Integer(), primary_key=True)
-    # user_email = db.Column(db.String(255), db.ForeignKey(User.email),
-    #     nullable=False)
-    # user = db.relationship(User,
-    #     backref=db.backref('tokens', cascade='all, delete-orphan'))
-    # password = db.Column(db.String(255), nullable=False)
-    # ip = db.Column(db.String(255))
-
-
-class FetchSchema(BaseSchema):
-    class Meta(BaseSchema.Meta):
-        model = models.Fetch
-
-    # _dict_recurse = True
-    # _dict_hide = {'user_email', 'user', 'last_check', 'error'}
-    # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'}
-    # _dict_secret = {'password'}
-
-    # id = db.Column(db.Integer(), primary_key=True)
-    # user_email = db.Column(db.String(255), db.ForeignKey(User.email),
-    #     nullable=False)
-    # user = db.relationship(User,
-    #     backref=db.backref('fetches', cascade='all, delete-orphan'))
-    # protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False)
-    # host = db.Column(db.String(255), nullable=False)
-    # port = db.Column(db.Integer(), nullable=False)
-    # tls = db.Column(db.Boolean(), nullable=False, default=False)
-    # username = db.Column(db.String(255), nullable=False)
-    # password = db.Column(db.String(255), nullable=False)
-    # keep = db.Column(db.Boolean(), nullable=False, default=False)
-    # last_check = db.Column(db.DateTime, nullable=True)
-    # error = db.Column(db.String(1023), nullable=True)
-
-
 class ConfigSchema(BaseSchema):
-    class Meta(BaseSchema.Meta):
+    """ Marshmallow schema for Config model """
+    class Meta:
+        """ Schema config """
         model = models.Config
-# TODO: how to mark keys as "required" while unserializing (in certain use cases/API)
-    name = ma.auto_field(required=True)
-    value = ma.auto_field(required=True)
 
 
 class RelaySchema(BaseSchema):
-    class Meta(BaseSchema.Meta):
+    """ Marshmallow schema for Relay model """
+    class Meta:
+        """ Schema config """
         model = models.Relay
 
 
-schemas = {
+class MailuSchema(Schema):
-    'domains': DomainSchema,
+    """ Marshmallow schema for Mailu config """
-    'relays': RelaySchema,
+    class Meta:
-    'users': UserSchema,
+        """ Schema config """
-    'aliases': AliasSchema,
+        render_module = RenderYAML
-#    'config': ConfigSchema,
+    domains = fields.Nested(DomainSchema, many=True)
-}
+    relays = fields.Nested(RelaySchema, many=True)
+    users = fields.Nested(UserSchema, many=True)
+    aliases = fields.Nested(AliasSchema, many=True)
+    config = fields.Nested(ConfigSchema, many=True)
+
+
+### config class ###
+
+class MailuConfig:
+    """ Class which joins whole Mailu config for dumping
+    """
+
+    _models = {
+        'domains': models.Domain,
+        'relays': models.Relay,
+        'users': models.User,
+        'aliases': models.Alias,
+#       'config': models.Config,
+    }
+
+    def __init__(self, sections):
+        if sections:
+            for section in sections:
+                if section not in self._models:
+                    raise ValueError(f'Unknown section: {section!r}')
+            self._sections = set(sections)
+        else:
+            self._sections = set(self._models.keys())
+
+    def __getattr__(self, section):
+        if section in self._sections:
+            return self._models[section].query.all()
+        else:
+            raise AttributeError