lub
/
mailu
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

config_dump now using marshmallow

Browse Source
master
Alexander Graf 4 years ago
parent dc42d375e2
commit 7413f9b7b4
3 changed files with 346 additions and 202 deletions
Show Stats Download Patch File Download Diff File

38
core/admin/mailu/manage.py
Unescape Escape View File

@ -1,5 +1,5 @@
from mailu import models
from .schemas import schemas
from .schemas import MailuConfig, MailuSchema
from flask import current_app as app
from flask.cli import FlaskGroup, with_appcontext
@ -310,41 +310,17 @@ def config_dump(full=False, secrets=False, dns=False, sections=None):
SECTIONS can be: domains, relays, users, aliases
"""
class spacedDumper(yaml.Dumper):
def write_line_break(self, data=None):
super().write_line_break(data)
if len(self.indents) == 1:
super().write_line_break()
def increase_indent(self, flow=False, indentless=False):
return super().increase_indent(flow, False)
if sections:
for section in sections:
if section not in schemas:
print(f'[ERROR] Invalid section: {section}')
try:
config = MailuConfig(sections)
except ValueError as reason:
print(f'[ERROR] {reason}')
return 1
else:
sections = sorted(schemas.keys())
# TODO: create World Schema and dump only this with Word.dumps ?
for section in sections:
schema = schemas[section](many=True)
schema.context.update({
MailuSchema(context={
'full': full,
'secrets': secrets,
'dns': dns,
})
yaml.dump(
{section: schema.dump(schema.Meta.model.query.all())},
sys.stdout,
Dumper=spacedDumper,
default_flow_style=False,
allow_unicode=True
)
sys.stdout.write('\n')
}).dumps(config, sys.stdout)
@mailu.command()

4
core/admin/mailu/models.py
Unescape Escape View File

@ -110,7 +110,7 @@ class Base(db.Model):
created_at = db.Column(db.Date, nullable=False, default=date.today)
updated_at = db.Column(db.Date, nullable=True, onupdate=date.today)
comment = db.Column(db.String(255), nullable=True)
comment = db.Column(db.String(255), nullable=True, default='')
@classmethod
def _dict_pkey(cls):
@ -171,7 +171,7 @@ class Base(db.Model):
if self.__mapper__.relationships[key].query_class is not None:
if hasattr(items, 'all'):
items = items.all()
if full or len(items):
if full or items:
if key in secret:
res[key] = '<hidden>'
else:

496
core/admin/mailu/schemas.py
Unescape Escape View File

@ -1,63 +1,252 @@
import marshmallow
import sqlalchemy
import flask_marshmallow
"""
Mailu marshmallow schema
"""
from . import models
from textwrap import wrap
import re
import yaml
from marshmallow import post_dump, fields, Schema
from flask_marshmallow import Marshmallow
from OpenSSL import crypto
from . import models, dkim
ma = flask_marshmallow.Marshmallow()
ma = Marshmallow()
# TODO:
# how to mark keys as "required" while unserializing (in certain use cases/API)?
# - fields withoud default => required
# - fields which are the primary key => unchangeable when updating
### yaml render module ###
class RenderYAML:
""" Marshmallow YAML Render Module
"""
class SpacedDumper(yaml.Dumper):
""" YAML Dumper to add a newline between main sections
and double the indent used
"""
def write_line_break(self, data=None):
super().write_line_break(data)
if len(self.indents) == 1:
super().write_line_break()
def increase_indent(self, flow=False, indentless=False):
return super().increase_indent(flow, False)
@staticmethod
def _update_dict(dict1, dict2):
""" sets missing keys in dict1 to values of dict2
"""
for key, value in dict2.items():
if key not in dict1:
dict1[key] = value
_load_defaults = {}
@classmethod
def loads(cls, *args, **kwargs):
""" load yaml data from string
"""
cls._update_dict(kwargs, cls._load_defaults)
return yaml.load(*args, **kwargs)
_dump_defaults = {
'Dumper': SpacedDumper,
'default_flow_style': False,
'allow_unicode': True,
}
@classmethod
def dumps(cls, *args, **kwargs):
""" dump yaml data to string
"""
cls._update_dict(kwargs, cls._dump_defaults)
return yaml.dump(*args, **kwargs)
### field definitions ###
class LazyString(fields.String):
""" Field that serializes a "false" value to the empty string
"""
def _serialize(self, value, attr, obj, **kwargs):
""" serialize None to the empty string
"""
return value if value else ''
class CommaSeparatedList(fields.Raw):
""" Field that deserializes a string containing comma-separated values to
a list of strings
"""
# TODO: implement this
class DkimKey(fields.String):
""" Field that serializes a dkim key to a list of strings (lines) and
deserializes a string or list of strings.
"""
_clean_re = re.compile(
r'(^-----BEGIN (RSA )?PRIVATE KEY-----|-----END (RSA )?PRIVATE KEY-----$|\s+)',
flags=re.UNICODE
)
def _serialize(self, value, attr, obj, **kwargs):
""" serialize dkim key to a list of strings (lines)
"""
# map empty string and None to None
if not value:
return None
# return list of key lines without header/footer
return value.decode('utf-8').strip().split('\n')[1:-1]
def _deserialize(self, value, attr, data, **kwargs):
""" deserialize a string or list of strings to dkim key data
with verification
"""
# convert list to str
if isinstance(value, list):
value = ''.join(value)
# only strings are allowed
if not isinstance(value, str):
raise TypeError(f'invalid type: {type(value).__name__!r}')
# clean value (remove whitespace and header/footer)
value = self._clean_re.sub('', value.strip())
# map empty string/list to None
if not value:
return None
# handle special value 'generate'
elif value == 'generate':
return dkim.gen_key()
# wrap value into valid pem layout and check validity
value = (
'-----BEGIN PRIVATE KEY-----\n' +
'\n'.join(wrap(value, 64)) +
'\n-----END PRIVATE KEY-----\n'
).encode('ascii')
try:
crypto.load_privatekey(crypto.FILETYPE_PEM, value)
except crypto.Error as exc:
raise ValueError('invalid dkim key') from exc
else:
return value
### schema definitions ###
import collections
class BaseSchema(ma.SQLAlchemyAutoSchema):
""" Marshmallow base schema with custom exclude logic
and option to hide sqla defaults
"""
class Meta:
base_hide_always = {'created_at', 'updated_at'}
base_hide_secrets = set()
base_hide_by_value = {
# 'comment': {'', None}
}
""" Schema config """
model = None
@marshmallow.post_dump
def remove_skip_values(self, data, many, **kwargs):
# print(repr(data), self.context)
def __init__(self, *args, **kwargs):
# always hide
hide_by_key = self.Meta.base_hide_always | set(getattr(self.Meta, 'hide_always', ()))
# get and remove config from kwargs
context = kwargs.get('context', {})
# hide secrets
if not self.context.get('secrets'):
hide_by_key |= self.Meta.base_hide_secrets
hide_by_key |= set(getattr(self.Meta, 'hide_secrets', ()))
# compile excludes
exclude = set(kwargs.get('exclude', []))
# hide by value
hide_by_value = self.Meta.base_hide_by_value | getattr(self.Meta, 'hide_by_value', {})
# always exclude
exclude.update({'created_at', 'updated_at'})
# hide defaults
if not self.context.get('full'):
# add include_by_context
if context is not None:
for ctx, what in getattr(self.Meta, 'include_by_context', {}).items():
if not context.get(ctx):
exclude |= set(what)
# update excludes
kwargs['exclude'] = exclude
# exclude_by_value
self._exclude_by_value = getattr(self.Meta, 'exclude_by_value', {})
# exclude default values
if not context.get('full'):
for column in self.Meta.model.__table__.columns:
# print(column.name, column.default.arg if isinstance(column.default, sqlalchemy.sql.schema.ColumnDefault) else column.default)
# alias.destiantion has default [] - is this okay. how to check it?
if column.name not in hide_by_key:
hide_by_value.setdefault(column.name, set()).add(None if column.default is None else column.default.arg)
if column.name not in exclude:
self._exclude_by_value.setdefault(column.name, []).append(
None if column.default is None else column.default.arg
)
# hide by context
self._hide_by_context = set()
if context is not None:
for ctx, what in getattr(self.Meta, 'hide_by_context', {}).items():
if not context.get(ctx):
self._hide_by_context |= set(what)
# init SQLAlchemyAutoSchema
super().__init__(*args, **kwargs)
@post_dump
def _remove_skip_values(self, data, many, **kwargs): # pylint: disable=unused-argument
if not self._exclude_by_value and not self._hide_by_context:
return data
full = self.context.get('full')
return {
key: value for key, value in data.items()
if
not isinstance(value, collections.Hashable)
or(
key not in hide_by_key
and
(key not in hide_by_value or value not in hide_by_value[key]))
key: '<hidden>' if key in self._hide_by_context else value
for key, value in data.items()
if full or key not in self._exclude_by_value or value not in self._exclude_by_value[key]
}
# TODO: remove LazyString and fix model definition (comment should not be nullable)
comment = LazyString()
class DomainSchema(BaseSchema):
class Meta(BaseSchema.Meta):
""" Marshmallow schema for Domain model """
class Meta:
""" Schema config """
model = models.Domain
include_relationships = True
#include_fk = True
exclude = ['users', 'managers', 'aliases']
include_by_context = {
'dns': {'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'},
}
hide_by_context = {
'secrets': {'dkim_key'},
}
exclude_by_value = {
'alternatives': [[]],
'dkim_key': [None],
'dkim_publickey': [None],
'dns_mx': [None],
'dns_spf': [None],
'dns_dkim': [None],
'dns_dmarc': [None],
}
dkim_key = DkimKey()
dkim_publickey = fields.String(dump_only=True)
dns_mx = fields.String(dump_only=True)
dns_spf = fields.String(dump_only=True)
dns_dkim = fields.String(dump_only=True)
dns_dmarc = fields.String(dump_only=True)
# _dict_hide = {'users', 'managers', 'aliases'}
# _dict_show = {'dkim_key'}
# _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}}
# _dict_secret = {'dkim_key'}
# _dict_types = {
# 'dkim_key': (bytes, type(None)),
# 'dkim_publickey': False,
@ -66,50 +255,62 @@ class DomainSchema(BaseSchema):
# 'dns_dkim': False,
# 'dns_dmarc': False,
# }
# _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]}
# @staticmethod
# def _dict_input(data):
# if 'dkim_key' in data:
# key = data['dkim_key']
# if key is not None:
# if type(key) is list:
# key = ''.join(key)
# if type(key) is str:
# key = ''.join(key.strip().split()) # removes all whitespace
# if key == 'generate':
# data['dkim_key'] = dkim.gen_key()
# elif key:
# m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key)
# if m is not None:
# key = key[m.end():]
# m = re.search('-----END (RSA )?PRIVATE KEY-----$', key)
# if m is not None:
# key = key[:m.start()]
# key = '\n'.join(wrap(key, 64))
# key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii')
# try:
# dkim.strip_key(key)
# except:
# raise ValueError('invalid dkim key')
# else:
# data['dkim_key'] = key
# else:
# data['dkim_key'] = None
# name = db.Column(IdnaDomain, primary_key=True, nullable=False)
# managers = db.relationship('User', secondary=managers,
# backref=db.backref('manager_of'), lazy='dynamic')
# max_users = db.Column(db.Integer, nullable=False, default=-1)
# max_aliases = db.Column(db.Integer, nullable=False, default=-1)
# max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0)
# signup_enabled = db.Column(db.Boolean(), nullable=False, default=False)
class TokenSchema(BaseSchema):
""" Marshmallow schema for Token model """
class Meta:
""" Schema config """
model = models.Token
# _dict_recurse = True
# _dict_hide = {'user', 'user_email'}
# _dict_mandatory = {'password'}
# id = db.Column(db.Integer(), primary_key=True)
# user_email = db.Column(db.String(255), db.ForeignKey(User.email),
# nullable=False)
# user = db.relationship(User,
# backref=db.backref('tokens', cascade='all, delete-orphan'))
# password = db.Column(db.String(255), nullable=False)
# ip = db.Column(db.String(255))
class FetchSchema(BaseSchema):
""" Marshmallow schema for Fetch model """
class Meta:
""" Schema config """
model = models.Fetch
include_by_context = {
'full': {'last_check', 'error'},
}
hide_by_context = {
'secrets': {'password'},
}
# TODO: What about mandatory keys?
# _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'}
class UserSchema(BaseSchema):
class Meta(BaseSchema.Meta):
""" Marshmallow schema for User model """
class Meta:
""" Schema config """
model = models.User
include_relationships = True
exclude = ['localpart', 'domain', 'quota_bytes_used']
# _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'}
exclude_by_value = {
'forward_destination': [[]],
'tokens': [[]],
'reply_enddate': ['2999-12-31'],
'reply_startdate': ['1900-01-01'],
}
tokens = fields.Nested(TokenSchema, many=True)
fetches = fields.Nested(FetchSchema, many=True)
# TODO: deserialize password/password_hash! What about mandatory keys?
# _dict_mandatory = {'localpart', 'domain', 'password'}
# @classmethod
# def _dict_input(cls, data):
@ -133,44 +334,19 @@ class UserSchema(BaseSchema):
# del data['hash_scheme']
# del data['password_hash']
# domain = db.relationship(Domain,
# backref=db.backref('users', cascade='all, delete-orphan'))
# password = db.Column(db.String(255), nullable=False)
# quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9)
# quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0)
# global_admin = db.Column(db.Boolean(), nullable=False, default=False)
# enabled = db.Column(db.Boolean(), nullable=False, default=True)
# # Features
# enable_imap = db.Column(db.Boolean(), nullable=False, default=True)
# enable_pop = db.Column(db.Boolean(), nullable=False, default=True)
# # Filters
# forward_enabled = db.Column(db.Boolean(), nullable=False, default=False)
# forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[])
# forward_keep = db.Column(db.Boolean(), nullable=False, default=True)
# reply_enabled = db.Column(db.Boolean(), nullable=False, default=False)
# reply_subject = db.Column(db.String(255), nullable=True, default=None)
# reply_body = db.Column(db.Text(), nullable=True, default=None)
# reply_startdate = db.Column(db.Date, nullable=False,
# default=date(1900, 1, 1))
# reply_enddate = db.Column(db.Date, nullable=False,
# default=date(2999, 12, 31))
# # Settings
# displayed_name = db.Column(db.String(160), nullable=False, default='')
# spam_enabled = db.Column(db.Boolean(), nullable=False, default=True)
# spam_threshold = db.Column(db.Integer(), nullable=False, default=80)
class AliasSchema(BaseSchema):
class Meta(BaseSchema.Meta):
""" Marshmallow schema for Alias model """
class Meta:
""" Schema config """
model = models.Alias
hide_always = {'localpart'}
hide_secrets = {'wildcard'}
hide_by_value = {
'destination': set([]) # always hide empty lists?!
exclude = ['localpart']
exclude_by_value = {
'destination': [[]],
}
# TODO: deserialize destination!
# @staticmethod
# def _dict_input(data):
# Email._dict_input(data)
@ -180,65 +356,57 @@ class AliasSchema(BaseSchema):
# data['destination'] = list([adr.strip() for adr in dst.split(',')])
class TokenSchema(BaseSchema):
class Meta(BaseSchema.Meta):
model = models.Token
# _dict_recurse = True
# _dict_hide = {'user', 'user_email'}
# _dict_mandatory = {'password'}
# id = db.Column(db.Integer(), primary_key=True)
# user_email = db.Column(db.String(255), db.ForeignKey(User.email),
# nullable=False)
# user = db.relationship(User,
# backref=db.backref('tokens', cascade='all, delete-orphan'))
# password = db.Column(db.String(255), nullable=False)
# ip = db.Column(db.String(255))
class FetchSchema(BaseSchema):
class Meta(BaseSchema.Meta):
model = models.Fetch
# _dict_recurse = True
# _dict_hide = {'user_email', 'user', 'last_check', 'error'}
# _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'}
# _dict_secret = {'password'}
# id = db.Column(db.Integer(), primary_key=True)
# user_email = db.Column(db.String(255), db.ForeignKey(User.email),
# nullable=False)
# user = db.relationship(User,
# backref=db.backref('fetches', cascade='all, delete-orphan'))
# protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False)
# host = db.Column(db.String(255), nullable=False)
# port = db.Column(db.Integer(), nullable=False)
# tls = db.Column(db.Boolean(), nullable=False, default=False)
# username = db.Column(db.String(255), nullable=False)
# password = db.Column(db.String(255), nullable=False)
# keep = db.Column(db.Boolean(), nullable=False, default=False)
# last_check = db.Column(db.DateTime, nullable=True)
# error = db.Column(db.String(1023), nullable=True)
class ConfigSchema(BaseSchema):
class Meta(BaseSchema.Meta):
""" Marshmallow schema for Config model """
class Meta:
""" Schema config """
model = models.Config
# TODO: how to mark keys as "required" while unserializing (in certain use cases/API)
name = ma.auto_field(required=True)
value = ma.auto_field(required=True)
class RelaySchema(BaseSchema):
class Meta(BaseSchema.Meta):
""" Marshmallow schema for Relay model """
class Meta:
""" Schema config """
model = models.Relay
schemas = {
'domains': DomainSchema,
'relays': RelaySchema,
'users': UserSchema,
'aliases': AliasSchema,
# 'config': ConfigSchema,
class MailuSchema(Schema):
""" Marshmallow schema for Mailu config """
class Meta:
""" Schema config """
render_module = RenderYAML
domains = fields.Nested(DomainSchema, many=True)
relays = fields.Nested(RelaySchema, many=True)
users = fields.Nested(UserSchema, many=True)
aliases = fields.Nested(AliasSchema, many=True)
config = fields.Nested(ConfigSchema, many=True)
### config class ###
class MailuConfig:
""" Class which joins whole Mailu config for dumping
"""
_models = {
'domains': models.Domain,
'relays': models.Relay,
'users': models.User,
'aliases': models.Alias,
# 'config': models.Config,
}
def __init__(self, sections):
if sections:
for section in sections:
if section not in self._models:
raise ValueError(f'Unknown section: {section!r}')
self._sections = set(sections)
else:
self._sections = set(self._models.keys())
def __getattr__(self, section):
if section in self._sections:
return self._models[section].query.all()
else:
raise AttributeError

Write Preview
Loading…
Cancel
Save