From 6629aa3ff824c57b1c752fdf7a7e7ea87423bc5e Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 6 Jan 2021 17:05:21 +0100 Subject: [PATCH] first try at api using flask-restx & marshmallow --- core/admin/mailu/__init__.py | 6 +- core/admin/mailu/api/__init__.py | 38 +++++ core/admin/mailu/api/common.py | 8 + core/admin/mailu/api/v1/__init__.py | 27 ++++ core/admin/mailu/api/v1/domains.py | 183 +++++++++++++++++++++++ core/admin/mailu/manage.py | 33 +++-- core/admin/mailu/schemas.py | 222 ++++++++++++++++++++++++++++ 7 files changed, 502 insertions(+), 15 deletions(-) create mode 100644 core/admin/mailu/api/__init__.py create mode 100644 core/admin/mailu/api/common.py create mode 100644 core/admin/mailu/api/v1/__init__.py create mode 100644 core/admin/mailu/api/v1/domains.py create mode 100644 core/admin/mailu/schemas.py diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 4de3e580..9ab90add 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -41,15 +41,17 @@ def create_app_from_config(config): ) # Import views - from mailu import ui, internal + from mailu import ui, internal, api app.register_blueprint(ui.ui, url_prefix='/ui') app.register_blueprint(internal.internal, url_prefix='/internal') + if app.config.get('API'): + api.register(app) return app def create_app(): - """ Create a new application based on the config module + """ Create a new application based on the config module """ config = configuration.ConfigManager() return create_app_from_config(config) diff --git a/core/admin/mailu/api/__init__.py b/core/admin/mailu/api/__init__.py new file mode 100644 index 00000000..6e7d6386 --- /dev/null +++ b/core/admin/mailu/api/__init__.py @@ -0,0 +1,38 @@ +from flask import redirect, url_for + +# import api version(s) +from . import v1 + +# api +ROOT='/api' +ACTIVE=v1 + +# patch url for swaggerui static assets +from flask_restx.apidoc import apidoc +apidoc.static_url_path = f'{ROOT}/swaggerui' + +def register(app): + + # register api bluprint(s) + app.register_blueprint(v1.blueprint, url_prefix=f'{ROOT}/v{int(v1.VERSION)}') + + # add redirect to current api version + @app.route(f'{ROOT}/') + def redir(): + return redirect(url_for(f'{ACTIVE.blueprint.name}.root')) + + # swagger ui config + app.config.SWAGGER_UI_DOC_EXPANSION = 'list' + app.config.SWAGGER_UI_OPERATION_ID = True + app.config.SWAGGER_UI_REQUEST_DURATION = True + + # TODO: remove patch of static assets for debugging + import os + if 'STATIC_ASSETS' in os.environ: + app.blueprints['ui'].static_folder = os.environ['STATIC_ASSETS'] + +# TODO: authentication via username + password +# TODO: authentication via api token +# TODO: api access for all users (via token) +# TODO: use permissions from "manager_of" +# TODO: switch to marshmallow, as parser is deprecated. use flask_accepts? diff --git a/core/admin/mailu/api/common.py b/core/admin/mailu/api/common.py new file mode 100644 index 00000000..700835ac --- /dev/null +++ b/core/admin/mailu/api/common.py @@ -0,0 +1,8 @@ +from .. import models + +def fqdn_in_use(*names): + for name in names: + for model in models.Domain, models.Alternative, models.Relay: + if model.query.get(name): + return model + return None diff --git a/core/admin/mailu/api/v1/__init__.py b/core/admin/mailu/api/v1/__init__.py new file mode 100644 index 00000000..c6de6fa4 --- /dev/null +++ b/core/admin/mailu/api/v1/__init__.py @@ -0,0 +1,27 @@ +from flask import Blueprint +from flask_restx import Api, fields + +VERSION = 1.0 + +blueprint = Blueprint(f'api_v{int(VERSION)}', __name__) + +api = Api( + blueprint, version=f'{VERSION:.1f}', + title='Mailu API', default_label='Mailu', + validate=True +) + +response_fields = api.model('Response', { + 'code': fields.Integer, + 'message': fields.String, +}) + +error_fields = api.model('Error', { + 'errors': fields.Nested(api.model('Error_Key', { + 'key': fields.String, + 'message':fields.String + })), + 'message': fields.String, +}) + +from . import domains diff --git a/core/admin/mailu/api/v1/domains.py b/core/admin/mailu/api/v1/domains.py new file mode 100644 index 00000000..7afd26b9 --- /dev/null +++ b/core/admin/mailu/api/v1/domains.py @@ -0,0 +1,183 @@ +from flask_restx import Resource, fields, abort + +from . import api, response_fields, error_fields +from .. import common +from ... import models + +db = models.db + +dom = api.namespace('domain', description='Domain operations') +alt = api.namespace('alternative', description='Alternative operations') + +domain_fields = api.model('Domain', { + 'name': fields.String(description='FQDN', example='example.com', required=True), + 'comment': fields.String(description='a comment'), + 'max_users': fields.Integer(description='maximum number of users', min=-1, default=-1), + 'max_aliases': fields.Integer(description='maximum number of aliases', min=-1, default=-1), + 'max_quota_bytes': fields.Integer(description='maximum quota for mailbox', min=0), + 'signup_enabled': fields.Boolean(description='allow signup'), +# 'dkim_key': fields.String, + 'alternatives': fields.List(fields.String(attribute='name', description='FQDN', example='example.com')), +}) +# TODO - name ist required on creation but immutable on change +# TODO - name and alteranatives need to be checked to be a fqdn (regex) + +domain_parser = api.parser() +domain_parser.add_argument('max_users', type=int, help='maximum number of users') +# TODO ... add more (or use marshmallow) + +alternative_fields = api.model('Domain', { + 'name': fields.String(description='alternative FQDN', example='example.com', required=True), + 'domain': fields.String(description='domain FQDN', example='example.com', required=True), + 'dkim_key': fields.String, +}) +# TODO: domain and name are not always required and can't be changed + + +@dom.route('') +class Domains(Resource): + + @dom.doc('list_domain') + @dom.marshal_with(domain_fields, as_list=True, skip_none=True, mask=['dkim_key']) + def get(self): + """ List domains """ + return models.Domain.query.all() + + @dom.doc('create_domain') + @dom.expect(domain_fields) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(409, 'Duplicate domain name', error_fields) + def post(self): + """ Create a new domain """ + data = api.payload + if common.fqdn_in_use(data['name']): + abort(409, f'Duplicate domain name {data["name"]!r}', errors={ + 'name': data['name'], + }) + for item, created in models.Domain.from_dict(data): + if not created: + abort(409, f'Duplicate domain name {item.name!r}', errors={ + 'alternatives': item.name, + }) + db.session.add(item) + db.session.commit() + +@dom.route('/') +class Domain(Resource): + + @dom.doc('get_domain') + @dom.response(200, 'Success', domain_fields) + @dom.response(404, 'Domain not found') + @dom.marshal_with(domain_fields) + def get(self, name): + """ Find domain by name """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + return domain + + @dom.doc('update_domain') + @dom.expect(domain_fields) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(404, 'Domain not found') + def put(self, name): + """ Update an existing domain """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + data = api.payload + data['name'] = name + for item, created in models.Domain.from_dict(data): + if created is True: + db.session.add(item) + db.session.commit() + + @dom.doc('modify_domain') + @dom.expect(domain_parser) + @dom.response(200, 'Success', response_fields) + @dom.response(400, 'Input validation exception', error_fields) + @dom.response(404, 'Domain not found') + def post(self, name=None): + """ Updates domain with form data """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + data = dict(domain_parser.parse_args()) + data['name'] = name + for item, created in models.Domain.from_dict(data): + if created is True: + db.session.add(item) + # TODO: flush? + db.session.commit() + + @dom.doc('delete_domain') + @dom.response(200, 'Success', response_fields) + @dom.response(404, 'Domain not found') + def delete(self, name=None): + """ Delete domain """ + domain = models.Domain.query.get(name) + if not domain: + abort(404) + db.session.delete(domain) + db.session.commit() + + +# @dom.route('//alternative') +# @alt.route('') +# class Alternatives(Resource): + +# @alt.doc('alternatives_list') +# @alt.marshal_with(alternative_fields, as_list=True, skip_none=True, mask=['dkim_key']) +# def get(self, name=None): +# """ List alternatives (of domain) """ +# if name is None: +# return models.Alternative.query.all() +# else: +# return models.Alternative.query.filter_by(domain_name = name).all() + +# @alt.doc('alternative_create') +# @alt.expect(alternative_fields) +# @alt.response(200, 'Success', response_fields) +# @alt.response(400, 'Input validation exception', error_fields) +# @alt.response(404, 'Domain not found') +# @alt.response(409, 'Duplicate domain name', error_fields) +# def post(self, name=None): +# """ Create new alternative (for domain) """ +# # abort(501) +# data = api.payload +# if name is not None: +# data['name'] = name +# domain = models.Domain.query.get(name) +# if not domain: +# abort(404) +# if common.fqdn_in_use(data['name']): +# abort(409, f'Duplicate domain name {data["name"]!r}', errors={ +# 'name': data['name'], +# }) +# for item, created in models.Alternative.from_dict(data): +# # TODO: handle creation of domain +# if not created: +# abort(409, f'Duplicate domain name {item.name!r}', errors={ +# 'alternatives': item.name, +# }) +# # db.session.add(item) +# # db.session.commit() + +# @dom.route('//alternative/') +# @alt.route('/') +# class Alternative(Resource): +# def get(self, name, alt=None): +# """ Find alternative (of domain) """ +# abort(501) +# def put(self, name, alt=None): +# """ Update alternative (of domain) """ +# abort(501) +# def post(self, name, alt=None): +# """ Update alternative (of domain) with form data """ +# abort(501) +# def delete(self, name, alt=None): +# """ Delete alternative (for domain) """ +# abort(501) + diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index c07ca2b7..569c161c 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,4 +1,5 @@ from mailu import models +from .schemas import schemas from flask import current_app as app from flask.cli import FlaskGroup, with_appcontext @@ -320,24 +321,30 @@ def config_dump(full=False, secrets=False, dns=False, sections=None): return super().increase_indent(flow, False) if sections: - check = dict(yaml_sections) for section in sections: - if section not in check: + if section not in schemas: print(f'[ERROR] Invalid section: {section}') return 1 + else: + sections = sorted(schemas.keys()) - extra = [] - if dns: - extra.append('dns') +# TODO: create World Schema and dump only this with Word.dumps ? - config = {} - for section, model in yaml_sections: - if not sections or section in sections: - dump = [item.to_dict(full, secrets, extra) for item in model.query.all()] - if len(dump): - config[section] = dump - - yaml.dump(config, sys.stdout, Dumper=spacedDumper, default_flow_style=False, allow_unicode=True) + for section in sections: + schema = schemas[section](many=True) + schema.context.update({ + 'full': full, + 'secrets': secrets, + 'dns': dns, + }) + yaml.dump( + {section: schema.dump(schema.Meta.model.query.all())}, + sys.stdout, + Dumper=spacedDumper, + default_flow_style=False, + allow_unicode=True + ) + sys.stdout.write('\n') @mailu.command() diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py new file mode 100644 index 00000000..20aa98c9 --- /dev/null +++ b/core/admin/mailu/schemas.py @@ -0,0 +1,222 @@ +import marshmallow +import flask_marshmallow + +from . import models + + +ma = flask_marshmallow.Marshmallow() + + +class BaseSchema(ma.SQLAlchemyAutoSchema): + + SKIP_IF = { + 'comment': {'', None} + } + + @marshmallow.post_dump + def remove_skip_values(self, data, many, **kwargs): + print(repr(data), self.context) + return { + key: value for key, value in data.items() + if key not in self.SKIP_IF or value not in self.SKIP_IF[key] + } + +class BaseMeta: + exclude = ['created_at', 'updated_at'] + + +class DomainSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Domain + + # _dict_hide = {'users', 'managers', 'aliases'} + # _dict_show = {'dkim_key'} + # _dict_extra = {'dns':{'dkim_publickey', 'dns_mx', 'dns_spf', 'dns_dkim', 'dns_dmarc'}} + # _dict_secret = {'dkim_key'} + # _dict_types = { + # 'dkim_key': (bytes, type(None)), + # 'dkim_publickey': False, + # 'dns_mx': False, + # 'dns_spf': False, + # 'dns_dkim': False, + # 'dns_dmarc': False, + # } + # _dict_output = {'dkim_key': lambda key: key.decode('utf-8').strip().split('\n')[1:-1]} + # @staticmethod + # def _dict_input(data): + # if 'dkim_key' in data: + # key = data['dkim_key'] + # if key is not None: + # if type(key) is list: + # key = ''.join(key) + # if type(key) is str: + # key = ''.join(key.strip().split()) # removes all whitespace + # if key == 'generate': + # data['dkim_key'] = dkim.gen_key() + # elif key: + # m = re.match('^-----BEGIN (RSA )?PRIVATE KEY-----', key) + # if m is not None: + # key = key[m.end():] + # m = re.search('-----END (RSA )?PRIVATE KEY-----$', key) + # if m is not None: + # key = key[:m.start()] + # key = '\n'.join(wrap(key, 64)) + # key = f'-----BEGIN PRIVATE KEY-----\n{key}\n-----END PRIVATE KEY-----\n'.encode('ascii') + # try: + # dkim.strip_key(key) + # except: + # raise ValueError('invalid dkim key') + # else: + # data['dkim_key'] = key + # else: + # data['dkim_key'] = None + + # name = db.Column(IdnaDomain, primary_key=True, nullable=False) + # managers = db.relationship('User', secondary=managers, + # backref=db.backref('manager_of'), lazy='dynamic') + # max_users = db.Column(db.Integer, nullable=False, default=-1) + # max_aliases = db.Column(db.Integer, nullable=False, default=-1) + # max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) + # signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) + + +class UserSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.User + + # _dict_hide = {'domain_name', 'domain', 'localpart', 'quota_bytes_used'} + # _dict_mandatory = {'localpart', 'domain', 'password'} + # @classmethod + # def _dict_input(cls, data): + # Email._dict_input(data) + # # handle password + # if 'password' in data: + # if 'password_hash' in data or 'hash_scheme' in data: + # raise ValueError('ambigous key password and password_hash/hash_scheme') + # # check (hashed) password + # password = data['password'] + # if password.startswith('{') and '}' in password: + # scheme = password[1:password.index('}')] + # if scheme not in cls.scheme_dict: + # raise ValueError(f'invalid password scheme {scheme!r}') + # else: + # raise ValueError(f'invalid hashed password {password!r}') + # elif 'password_hash' in data and 'hash_scheme' in data: + # if data['hash_scheme'] not in cls.scheme_dict: + # raise ValueError(f'invalid password scheme {scheme!r}') + # data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash'] + # del data['hash_scheme'] + # del data['password_hash'] + + # domain = db.relationship(Domain, + # backref=db.backref('users', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) + # quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) + # global_admin = db.Column(db.Boolean(), nullable=False, default=False) + # enabled = db.Column(db.Boolean(), nullable=False, default=True) + + # # Features + # enable_imap = db.Column(db.Boolean(), nullable=False, default=True) + # enable_pop = db.Column(db.Boolean(), nullable=False, default=True) + + # # Filters + # forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) + # forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[]) + # forward_keep = db.Column(db.Boolean(), nullable=False, default=True) + # reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) + # reply_subject = db.Column(db.String(255), nullable=True, default=None) + # reply_body = db.Column(db.Text(), nullable=True, default=None) + # reply_startdate = db.Column(db.Date, nullable=False, + # default=date(1900, 1, 1)) + # reply_enddate = db.Column(db.Date, nullable=False, + # default=date(2999, 12, 31)) + + # # Settings + # displayed_name = db.Column(db.String(160), nullable=False, default='') + # spam_enabled = db.Column(db.Boolean(), nullable=False, default=True) + # spam_threshold = db.Column(db.Integer(), nullable=False, default=80) + +class AliasSchema(BaseSchema): + class Meta(BaseMeta): + model = models.Alias + exclude = BaseMeta.exclude + ['localpart'] + # TODO look for good way to exclude secrets, unverbose and defaults + + # _dict_hide = {'domain_name', 'domain', 'localpart'} + # @staticmethod + # def _dict_input(data): + # Email._dict_input(data) + # # handle comma delimited string for backwards compability + # dst = data.get('destination') + # if type(dst) is str: + # data['destination'] = list([adr.strip() for adr in dst.split(',')]) + + +class TokenSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Token + + # _dict_recurse = True + # _dict_hide = {'user', 'user_email'} + # _dict_mandatory = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('tokens', cascade='all, delete-orphan')) + # password = db.Column(db.String(255), nullable=False) + # ip = db.Column(db.String(255)) + + +class FetchSchema(ma.SQLAlchemyAutoSchema): + class Meta: + model = models.Fetch + + # _dict_recurse = True + # _dict_hide = {'user_email', 'user', 'last_check', 'error'} + # _dict_mandatory = {'protocol', 'host', 'port', 'username', 'password'} + # _dict_secret = {'password'} + + # id = db.Column(db.Integer(), primary_key=True) + # user_email = db.Column(db.String(255), db.ForeignKey(User.email), + # nullable=False) + # user = db.relationship(User, + # backref=db.backref('fetches', cascade='all, delete-orphan')) + # protocol = db.Column(db.Enum('imap', 'pop3'), nullable=False) + # host = db.Column(db.String(255), nullable=False) + # port = db.Column(db.Integer(), nullable=False) + # tls = db.Column(db.Boolean(), nullable=False, default=False) + # username = db.Column(db.String(255), nullable=False) + # password = db.Column(db.String(255), nullable=False) + # keep = db.Column(db.Boolean(), nullable=False, default=False) + # last_check = db.Column(db.DateTime, nullable=True) + # error = db.Column(db.String(1023), nullable=True) + + +class ConfigSchema(ma.SQLAlchemySchema): + class Meta: + model = models.Config +# created_at = ma.auto_field(dump_only=True) +# updated_at = ma.auto_field(dump_only=True) + comment = ma.auto_field() + name = ma.auto_field(required=True) + value = ma.auto_field(required=True) + +class RelaySchema(BaseSchema): + class Meta(BaseMeta): + model = models.Relay +# created_at = ma.auto_field(dump_only=True) +# updated_at = ma.auto_field(dump_only=True) +# comment = ma.auto_field() +# name = ma.auto_field(required=True) +# smtp = ma.auto_field(required=True) + +schemas = { + 'domains': DomainSchema, + 'relays': RelaySchema, + 'users': UserSchema, + 'aliases': AliasSchema, +# 'config': ConfigSchema, +}