From fc244262919116dd0289e9046d227164eb180d5f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 18 Oct 2018 15:57:43 +0200 Subject: [PATCH 01/13] First batch of refactoring, using the app factory pattern --- core/admin/mailu/__init__.py | 154 +++-------- core/admin/mailu/configuration.py | 70 +++++ core/admin/mailu/debug.py | 17 ++ core/admin/mailu/internal/__init__.py | 4 +- core/admin/mailu/internal/nginx.py | 3 +- core/admin/mailu/internal/views/auth.py | 5 +- core/admin/mailu/internal/views/dovecot.py | 5 +- core/admin/mailu/internal/views/fetch.py | 6 +- core/admin/mailu/internal/views/postfix.py | 2 +- core/admin/mailu/manage.py | 305 +++++++++++++++++++++ core/admin/mailu/models.py | 30 +- core/admin/mailu/ui/access.py | 2 +- core/admin/mailu/ui/views/admins.py | 6 +- core/admin/mailu/ui/views/aliases.py | 12 +- core/admin/mailu/ui/views/alternatives.py | 8 +- core/admin/mailu/ui/views/base.py | 4 +- core/admin/mailu/ui/views/domains.py | 19 +- core/admin/mailu/ui/views/fetches.py | 12 +- core/admin/mailu/ui/views/managers.py | 6 +- core/admin/mailu/ui/views/relays.py | 12 +- core/admin/mailu/ui/views/tokens.py | 10 +- core/admin/mailu/ui/views/users.py | 25 +- core/admin/mailu/utils.py | 46 ++++ 23 files changed, 577 insertions(+), 186 deletions(-) create mode 100644 core/admin/mailu/configuration.py create mode 100644 core/admin/mailu/debug.py create mode 100644 core/admin/mailu/manage.py create mode 100644 core/admin/mailu/utils.py diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 167f04ae..874fa8e0 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -1,132 +1,58 @@ import flask -import flask_sqlalchemy import flask_bootstrap -import flask_login -import flask_script -import flask_migrate -import flask_babel -import flask_limiter import os import docker import socket import uuid -from werkzeug.contrib import fixers +from mailu import utils, debug, db -# Create application -app = flask.Flask(__name__) -default_config = { - # Specific to the admin UI - 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', - 'SQLALCHEMY_TRACK_MODIFICATIONS': False, - 'DOCKER_SOCKET': 'unix:///var/run/docker.sock', - 'BABEL_DEFAULT_LOCALE': 'en', - 'BABEL_DEFAULT_TIMEZONE': 'UTC', - 'BOOTSTRAP_SERVE_LOCAL': True, - 'RATELIMIT_STORAGE_URL': 'redis://redis/2', - 'QUOTA_STORAGE_URL': 'redis://redis/1', - 'DEBUG': False, - 'DOMAIN_REGISTRATION': False, - # Statistics management - 'INSTANCE_ID_PATH': '/data/instance', - 'STATS_ENDPOINT': '0.{}.stats.mailu.io', - # Common configuration variables - 'SECRET_KEY': 'changeMe', - 'DOMAIN': 'mailu.io', - 'HOSTNAMES': 'mail.mailu.io,alternative.mailu.io,yetanother.mailu.io', - 'POSTMASTER': 'postmaster', - 'TLS_FLAVOR': 'cert', - 'AUTH_RATELIMIT': '10/minute;1000/hour', - 'DISABLE_STATISTICS': 'False', - # Mail settings - 'DMARC_RUA': None, - 'DMARC_RUF': None, - 'WELCOME': 'False', - 'WELCOME_SUBJECT': 'Dummy welcome topic', - 'WELCOME_BODY': 'Dummy welcome body', - 'DKIM_SELECTOR': 'dkim', - 'DKIM_PATH': '/dkim/{domain}.{selector}.key', - 'DEFAULT_QUOTA': 1000000000, - # Web settings - 'SITENAME': 'Mailu', - 'WEBSITE': 'https://mailu.io', - 'WEB_ADMIN': '/admin', - 'WEB_WEBMAIL': '/webmail', - 'RECAPTCHA_PUBLIC_KEY': '', - 'RECAPTCHA_PRIVATE_KEY': '', - # Advanced settings - 'PASSWORD_SCHEME': 'BLF-CRYPT', - # Host settings - 'HOST_IMAP': 'imap', - 'HOST_POP3': 'imap', - 'HOST_SMTP': 'smtp', - 'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'), -} +def create_app_from_config(config): + """ Create a new application based on the given configuration + """ + app = flask.Flask(__name__) + app.config = config -# Load configuration from the environment if available -for key, value in default_config.items(): - app.config[key] = os.environ.get(key, value) + # Bootstrap is used for basic JS and CSS loading + # TODO: remove this and use statically generated assets instead + app.bootstrap = flask_bootstrap.Bootstrap(app) -# Base application -flask_bootstrap.Bootstrap(app) -db = flask_sqlalchemy.SQLAlchemy(app) -migrate = flask_migrate.Migrate(app, db) -limiter = flask_limiter.Limiter(app, key_func=lambda: current_user.username) + # Initialize application extensions + models.db.init_app(app) + utils.limiter.init_app(app) + utils.babel.init_app(app) + utils.login.init_app(app) + utils.proxy.init_app(app) + manage.migrate.init_app(app) + manage.manager.init_app(app) -# Debugging toolbar -if app.config.get("DEBUG"): - import flask_debugtoolbar - toolbar = flask_debugtoolbar.DebugToolbarExtension(app) + # Initialize debugging tools + if app.config.get("app.debug"): + debug.toolbar.init_app(app) + debug.profiler.init_app(app) -# Manager commnad -manager = flask_script.Manager(app) -manager.add_command('db', flask_migrate.MigrateCommand) + # Inject the default variables in the Jinja parser + @app.context_processor + def inject_defaults(): + signup_domains = models.Domain.query.filter_by(signup_enabled=True).all() + return dict( + current_user=utils.login.current_user, + signup_domains=signup_domains, + config=app.config + ) -# Babel configuration -babel = flask_babel.Babel(app) -translations = list(map(str, babel.list_translations())) + # Import views + from mailu import ui, internal + app.register_blueprint(ui.ui, url_prefix='/ui') + app.register_blueprint(internal.internal, url_prefix='/internal') -@babel.localeselector -def get_locale(): - return flask.request.accept_languages.best_match(translations) + return app -# Login configuration -login_manager = flask_login.LoginManager() -login_manager.init_app(app) -login_manager.login_view = "ui.login" -@login_manager.unauthorized_handler -def handle_needs_login(): - return flask.redirect( - flask.url_for('ui.login', next=flask.request.endpoint) - ) - -@app.context_processor -def inject_defaults(): - signup_domains = models.Domain.query.filter_by(signup_enabled=True).all() - return dict( - current_user=flask_login.current_user, - signup_domains=signup_domains, - config=app.config - ) - -# Import views -from mailu import ui, internal -app.register_blueprint(ui.ui, url_prefix='/ui') -app.register_blueprint(internal.internal, url_prefix='/internal') - -# Create the prefix middleware -class PrefixMiddleware(object): - - def __init__(self, app): - self.app = app - - def __call__(self, environ, start_response): - prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '') - if prefix: - environ['SCRIPT_NAME'] = prefix - return self.app(environ, start_response) - -app.wsgi_app = PrefixMiddleware(fixers.ProxyFix(app.wsgi_app)) +def create_app(): + """ Create a new application based on the config module + """ + config = configuration.ConfigManager() + return create_app_from_config(config) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py new file mode 100644 index 00000000..43184bad --- /dev/null +++ b/core/admin/mailu/configuration.py @@ -0,0 +1,70 @@ +import os + + +DEFAULT_CONFIG = { + # Specific to the admin UI + 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', + 'SQLALCHEMY_TRACK_MODIFICATIONS': False, + 'DOCKER_SOCKET': 'unix:///var/run/docker.sock', + 'BABEL_DEFAULT_LOCALE': 'en', + 'BABEL_DEFAULT_TIMEZONE': 'UTC', + 'BOOTSTRAP_SERVE_LOCAL': True, + 'RATELIMIT_STORAGE_URL': 'redis://redis/2', + 'QUOTA_STORAGE_URL': 'redis://redis/1', + 'DEBUG': False, + 'DOMAIN_REGISTRATION': False, + # Statistics management + 'INSTANCE_ID_PATH': '/data/instance', + 'STATS_ENDPOINT': '0.{}.stats.mailu.io', + # Common configuration variables + 'SECRET_KEY': 'changeMe', + 'DOMAIN': 'mailu.io', + 'HOSTNAMES': 'mail.mailu.io,alternative.mailu.io,yetanother.mailu.io', + 'POSTMASTER': 'postmaster', + 'TLS_FLAVOR': 'cert', + 'AUTH_RATELIMIT': '10/minute;1000/hour', + 'DISABLE_STATISTICS': 'False', + # Mail settings + 'DMARC_RUA': None, + 'DMARC_RUF': None, + 'WELCOME': 'False', + 'WELCOME_SUBJECT': 'Dummy welcome topic', + 'WELCOME_BODY': 'Dummy welcome body', + 'DKIM_SELECTOR': 'dkim', + 'DKIM_PATH': '/dkim/{domain}.{selector}.key', + 'DEFAULT_QUOTA': 1000000000, + # Web settings + 'SITENAME': 'Mailu', + 'WEBSITE': 'https://mailu.io', + 'WEB_ADMIN': '/admin', + 'WEB_WEBMAIL': '/webmail', + 'RECAPTCHA_PUBLIC_KEY': '', + 'RECAPTCHA_PRIVATE_KEY': '', + # Advanced settings + 'PASSWORD_SCHEME': 'BLF-CRYPT', + # Host settings + 'HOST_IMAP': 'imap', + 'HOST_POP3': 'imap', + 'HOST_SMTP': 'smtp', + 'HOST_WEBMAIL': 'webmail', + 'HOST_FRONT': 'front', + 'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'), + 'POD_ADDRESS_RANGE': None +} + + +class ConfigManager(object): + """ Naive configuration manager that uses environment only + """ + + def __init__(self): + self.config = { + os.environ.get(key, value) + for key, value in DEFAULT_CONFIG.items() + } + + def get(self, *args): + return self.config.get(*args) + + def __getitem__(self, key): + return self.get(key) diff --git a/core/admin/mailu/debug.py b/core/admin/mailu/debug.py new file mode 100644 index 00000000..69e88cf8 --- /dev/null +++ b/core/admin/mailu/debug.py @@ -0,0 +1,17 @@ +import flask_debugtoolbar + +from werkzeug.contrib import profiler as werkzeug_profiler + + +# Debugging toolbar +toolbar = flask_debugtoolbar.DebugToolbarExtension() + + +# Profiler +class Profiler(object): + def init_app(self): + app.wsgi_app = werkzeug_profiler.ProfilerMiddleware( + app.wsgi_app, restrictions=[30] + ) + +profiler = Profiler() diff --git a/core/admin/mailu/internal/__init__.py b/core/admin/mailu/internal/__init__.py index 80a3c754..fcb19692 100644 --- a/core/admin/mailu/internal/__init__.py +++ b/core/admin/mailu/internal/__init__.py @@ -1,6 +1,6 @@ from flask_limiter import RateLimitExceeded -from mailu import limiter +from mailu import utils import socket import flask @@ -19,7 +19,7 @@ def rate_limit_handler(e): return response -@limiter.request_filter +@utils.limiter.request_filter def whitelist_webmail(): try: return flask.request.headers["Client-Ip"] ==\ diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index cb6bc9cb..460719f2 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -1,4 +1,5 @@ -from mailu import db, models, app +from mailu import models +from flask import current_app as app import re import socket diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index 823fbd40..459a8e57 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -1,5 +1,6 @@ -from mailu import db, models, app, limiter +from mailu import models, utils from mailu.internal import internal, nginx +from flask import current_app as app import flask import flask_login @@ -7,7 +8,7 @@ import base64 @internal.route("/auth/email") -@limiter.limit( +@utils.limiter.limit( app.config["AUTH_RATELIMIT"], lambda: flask.request.headers["Client-Ip"] ) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index c2f53794..99ec0f6b 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -1,5 +1,6 @@ -from mailu import db, models +from mailu import models from mailu.internal import internal +from flask import current_app as app import flask @@ -25,7 +26,7 @@ def dovecot_quota(ns, user_email): user = models.User.query.get(user_email) or flask.abort(404) if ns == "storage": user.quota_bytes_used = flask.request.get_json() - db.session.commit() + models.db.session.commit() return flask.jsonify(None) diff --git a/core/admin/mailu/internal/views/fetch.py b/core/admin/mailu/internal/views/fetch.py index 60249c4b..ccd3a159 100644 --- a/core/admin/mailu/internal/views/fetch.py +++ b/core/admin/mailu/internal/views/fetch.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.internal import internal import flask @@ -27,6 +27,6 @@ def fetch_done(fetch_id): fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) fetch.last_check = datetime.datetime.now() fetch.error_message = str(flask.request.get_json()) - db.session.add(fetch) - db.session.commit() + models.db.session.add(fetch) + models.db.session.commit() return "" diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index 79fbdb8a..8c690b73 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.internal import internal import flask diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py new file mode 100644 index 00000000..bc2473eb --- /dev/null +++ b/core/admin/mailu/manage.py @@ -0,0 +1,305 @@ +from mailu import models + +from flask import current_app as app + +import flask +import os +import socket +import uuid + + +manager = flask_script.Manager() +db = models.db + + +@manager.command +def advertise(): + """ Advertise this server against statistic services. + """ + if os.path.isfile(app.config["INSTANCE_ID_PATH"]): + with open(app.config["INSTANCE_ID_PATH"], "r") as handle: + instance_id = handle.read() + else: + instance_id = str(uuid.uuid4()) + with open(app.config["INSTANCE_ID_PATH"], "w") as handle: + handle.write(instance_id) + if app.config["DISABLE_STATISTICS"].lower() != "true": + try: + socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) + except: + pass + + +@manager.command +def admin(localpart, domain_name, password): + """ Create an admin user + """ + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User( + localpart=localpart, + domain=domain, + global_admin=True + ) + user.set_password(password) + db.session.add(user) + db.session.commit() + + +@manager.command +def user(localpart, domain_name, password, + hash_scheme=app.config['PASSWORD_SCHEME']): + """ Create a user + """ + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User( + localpart=localpart, + domain=domain, + global_admin=False + ) + user.set_password(password, hash_scheme=hash_scheme) + db.session.add(user) + db.session.commit() + + +@manager.option('-n', '--domain_name', dest='domain_name') +@manager.option('-u', '--max_users', dest='max_users') +@manager.option('-a', '--max_aliases', dest='max_aliases') +@manager.option('-q', '--max_quota_bytes', dest='max_quota_bytes') +def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + db.session.commit() + + +@manager.command +def user_import(localpart, domain_name, password_hash, + hash_scheme=app.config['PASSWORD_SCHEME']): + """ Import a user along with password hash. Available hashes: + 'SHA512-CRYPT' + 'SHA256-CRYPT' + 'MD5-CRYPT' + 'CRYPT' + """ + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User( + localpart=localpart, + domain=domain, + global_admin=False + ) + user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) + db.session.add(user) + db.session.commit() + + +@manager.command +def config_update(verbose=False, delete_objects=False): + """sync configuration with data from YAML-formatted stdin""" + import yaml + import sys + new_config = yaml.load(sys.stdin) + # print new_config + domains = new_config.get('domains', []) + tracked_domains = set() + for domain_config in domains: + if verbose: + print(str(domain_config)) + domain_name = domain_config['name'] + max_users = domain_config.get('max_users', 0) + max_aliases = domain_config.get('max_aliases', 0) + max_quota_bytes = domain_config.get('max_quota_bytes', 0) + tracked_domains.add(domain_name) + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name, + max_users=max_users, + max_aliases=max_aliases, + max_quota_bytes=max_quota_bytes) + db.session.add(domain) + print("Added " + str(domain_config)) + else: + domain.max_users = max_users + domain.max_aliases = max_aliases + domain.max_quota_bytes = max_quota_bytes + db.session.add(domain) + print("Updated " + str(domain_config)) + + users = new_config.get('users', []) + tracked_users = set() + user_optional_params = ('comment', 'quota_bytes', 'global_admin', + 'enable_imap', 'enable_pop', 'forward_enabled', + 'forward_destination', 'reply_enabled', + 'reply_subject', 'reply_body', 'displayed_name', + 'spam_enabled', 'email', 'spam_threshold') + for user_config in users: + if verbose: + print(str(user_config)) + localpart = user_config['localpart'] + domain_name = user_config['domain'] + password_hash = user_config.get('password_hash', None) + hash_scheme = user_config.get('hash_scheme', None) + domain = models.Domain.query.get(domain_name) + email = '{0}@{1}'.format(localpart, domain_name) + optional_params = {} + for k in user_optional_params: + if k in user_config: + optional_params[k] = user_config[k] + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User.query.get(email) + tracked_users.add(email) + tracked_domains.add(domain_name) + if not user: + user = models.User( + localpart=localpart, + domain=domain, + **optional_params + ) + else: + for k in optional_params: + setattr(user, k, optional_params[k]) + user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) + db.session.add(user) + + aliases = new_config.get('aliases', []) + tracked_aliases = set() + for alias_config in aliases: + if verbose: + print(str(alias_config)) + localpart = alias_config['localpart'] + domain_name = alias_config['domain'] + if type(alias_config['destination']) is str: + destination = alias_config['destination'].split(',') + else: + destination = alias_config['destination'] + wildcard = alias_config.get('wildcard', False) + domain = models.Domain.query.get(domain_name) + email = '{0}@{1}'.format(localpart, domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + alias = models.Alias.query.get(email) + tracked_aliases.add(email) + tracked_domains.add(domain_name) + if not alias: + alias = models.Alias( + localpart=localpart, + domain=domain, + wildcard=wildcard, + destination=destination, + email=email + ) + else: + alias.destination = destination + alias.wildcard = wildcard + db.session.add(alias) + + db.session.commit() + + managers = new_config.get('managers', []) + # tracked_managers=set() + for manager_config in managers: + if verbose: + print(str(manager_config)) + domain_name = manager_config['domain'] + user_name = manager_config['user'] + domain = models.Domain.query.get(domain_name) + manageruser = models.User.query.get(user_name + '@' + domain_name) + if manageruser not in domain.managers: + domain.managers.append(manageruser) + db.session.add(domain) + + db.session.commit() + + if delete_objects: + for user in db.session.query(models.User).all(): + if not (user.email in tracked_users): + if verbose: + print("Deleting user: " + str(user.email)) + db.session.delete(user) + for alias in db.session.query(models.Alias).all(): + if not (alias.email in tracked_aliases): + if verbose: + print("Deleting alias: " + str(alias.email)) + db.session.delete(alias) + for domain in db.session.query(models.Domain).all(): + if not (domain.name in tracked_domains): + if verbose: + print("Deleting domain: " + str(domain.name)) + db.session.delete(domain) + db.session.commit() + + +@manager.command +def user_delete(email): + """delete user""" + user = models.User.query.get(email) + if user: + db.session.delete(user) + db.session.commit() + + +@manager.command +def alias_delete(email): + """delete alias""" + alias = models.Alias.query.get(email) + if alias: + db.session.delete(alias) + db.session.commit() + + +@manager.command +def alias(localpart, domain_name, destination): + """ Create an alias + """ + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + alias = models.Alias( + localpart=localpart, + domain=domain, + destination=destination.split(','), + email="%s@%s" % (localpart, domain_name) + ) + db.session.add(alias) + db.session.commit() + +# Set limits to a domain + + +@manager.command +def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): + domain = models.Domain.query.get(domain_name) + domain.max_users = max_users + domain.max_aliases = max_aliases + domain.max_quota_bytes = max_quota_bytes + + db.session.add(domain) + db.session.commit() + +# Make the user manager of a domain + + +@manager.command +def setmanager(domain_name, user_name='manager'): + domain = models.Domain.query.get(domain_name) + manageruser = models.User.query.get(user_name + '@' + domain_name) + domain.managers.append(manageruser) + db.session.add(domain) + db.session.commit() + + +if __name__ == "__main__": + manager.run() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 0c80fd4f..43565c55 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,10 +1,12 @@ -from mailu import app, db, dkim, login_manager +from mailu import dkim from sqlalchemy.ext import declarative from passlib import context, hash from datetime import datetime, date from email.mime import text +from flask import current_app as app +import flask_sqlalchemy import sqlalchemy import re import time @@ -15,6 +17,9 @@ import idna import dns +db = flask_sqlalchemy.SQLAlchemy() + + class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ @@ -67,6 +72,27 @@ class CommaSeparatedList(db.TypeDecorator): return filter(bool, value.split(",")) +class JSONEncoded(db.TypeDecorator): + """Represents an immutable structure as a json-encoded string. + """ + + impl = db.String + + def process_bind_param(self, value, dialect): + return json.dumps(value) if value else None + + def process_result_value(self, value, dialect): + return json.loads(value) if value else None + + +class Config(db.Model): + """ In-database configuration values + """ + + name = db.Column(db.String(255), primary_key=True, nullable=False) + value = db.Column(JSONEncoded) + + # Many-to-many association table for domain managers managers = db.Table('manager', db.Column('domain_name', IdnaDomain, db.ForeignKey('domain.name')), @@ -324,8 +350,6 @@ class User(Base, Email): user = cls.query.get(email) return user if (user and user.enabled and user.check_password(password)) else None -login_manager.user_loader(User.query.get) - class Alias(Base, Email): """ An alias is an email address that redirects to some destination. diff --git a/core/admin/mailu/ui/access.py b/core/admin/mailu/ui/access.py index 2b6a3767..6a923c8e 100644 --- a/core/admin/mailu/ui/access.py +++ b/core/admin/mailu/ui/access.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import forms import flask diff --git a/core/admin/mailu/ui/views/admins.py b/core/admin/mailu/ui/views/admins.py index e9771ec6..7462c545 100644 --- a/core/admin/mailu/ui/views/admins.py +++ b/core/admin/mailu/ui/views/admins.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -25,7 +25,7 @@ def admin_create(): user = models.User.query.get(form.admin.data) if user: user.global_admin = True - db.session.commit() + models.db.session.commit() flask.flash('User %s is now admin' % user) return flask.redirect(flask.url_for('.admin_list')) else: @@ -40,7 +40,7 @@ def admin_delete(admin): user = models.User.query.get(admin) if user: user.global_admin = False - db.session.commit() + models.db.session.commit() flask.flash('User %s is no longer admin' % user) return flask.redirect(flask.url_for('.admin_list')) else: diff --git a/core/admin/mailu/ui/views/aliases.py b/core/admin/mailu/ui/views/aliases.py index 7fc03266..eaab5cad 100644 --- a/core/admin/mailu/ui/views/aliases.py +++ b/core/admin/mailu/ui/views/aliases.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -27,8 +27,8 @@ def alias_create(domain_name): else: alias = models.Alias(domain=domain) form.populate_obj(alias) - db.session.add(alias) - db.session.commit() + models.db.session.add(alias) + models.db.session.commit() flask.flash('Alias %s created' % alias) return flask.redirect( flask.url_for('.alias_list', domain_name=domain.name)) @@ -45,7 +45,7 @@ def alias_edit(alias): form.localpart.validators = [] if form.validate_on_submit(): form.populate_obj(alias) - db.session.commit() + models.db.session.commit() flask.flash('Alias %s updated' % alias) return flask.redirect( flask.url_for('.alias_list', domain_name=alias.domain.name)) @@ -59,8 +59,8 @@ def alias_edit(alias): def alias_delete(alias): alias = models.Alias.query.get(alias) or flask.abort(404) domain = alias.domain - db.session.delete(alias) - db.session.commit() + models.db.session.delete(alias) + models.db.session.commit() flask.flash('Alias %s deleted' % alias) return flask.redirect( flask.url_for('.alias_list', domain_name=domain.name)) diff --git a/core/admin/mailu/ui/views/alternatives.py b/core/admin/mailu/ui/views/alternatives.py index 79acb074..e5761c83 100644 --- a/core/admin/mailu/ui/views/alternatives.py +++ b/core/admin/mailu/ui/views/alternatives.py @@ -26,8 +26,8 @@ def alternative_create(domain_name): else: alternative = models.Alternative(domain=domain) form.populate_obj(alternative) - db.session.add(alternative) - db.session.commit() + models.db.session.add(alternative) + models.db.session.commit() flask.flash('Alternative domain %s created' % alternative) return flask.redirect( flask.url_for('.alternative_list', domain_name=domain.name)) @@ -41,8 +41,8 @@ def alternative_create(domain_name): def alternative_delete(alternative): alternative = models.Alternative.query.get(alternative) or flask.abort(404) domain = alternative.domain - db.session.delete(alternative) - db.session.commit() + models.db.session.delete(alternative) + models.db.session.commit() flask.flash('Alternative %s deleted' % alternative) return flask.redirect( flask.url_for('.alternative_list', domain_name=domain.name)) diff --git a/core/admin/mailu/ui/views/base.py b/core/admin/mailu/ui/views/base.py index 1ea87401..7501a883 100644 --- a/core/admin/mailu/ui/views/base.py +++ b/core/admin/mailu/ui/views/base.py @@ -1,11 +1,9 @@ -from mailu import dockercli, app, db, models +from mailu import models from mailu.ui import ui, forms, access import flask import flask_login -from urllib import parse - @ui.route('/', methods=["GET"]) @access.authenticated diff --git a/core/admin/mailu/ui/views/domains.py b/core/admin/mailu/ui/views/domains.py index 7afc1746..719d3844 100644 --- a/core/admin/mailu/ui/views/domains.py +++ b/core/admin/mailu/ui/views/domains.py @@ -1,5 +1,6 @@ -from mailu import app, db, models +from mailu import models from mailu.ui import ui, forms, access +from flask import current_app as app import flask import flask_login @@ -26,8 +27,8 @@ def domain_create(): else: domain = models.Domain() form.populate_obj(domain) - db.session.add(domain) - db.session.commit() + models.db.session.add(domain) + models.db.session.commit() flask.flash('Domain %s created' % domain) return flask.redirect(flask.url_for('.domain_list')) return flask.render_template('domain/create.html', form=form) @@ -42,7 +43,7 @@ def domain_edit(domain_name): form.name.validators = [] if form.validate_on_submit(): form.populate_obj(domain) - db.session.commit() + models.db.session.commit() flask.flash('Domain %s saved' % domain) return flask.redirect(flask.url_for('.domain_list')) return flask.render_template('domain/edit.html', form=form, @@ -54,8 +55,8 @@ def domain_edit(domain_name): @access.confirmation_required("delete {domain_name}") def domain_delete(domain_name): domain = models.Domain.query.get(domain_name) or flask.abort(404) - db.session.delete(domain) - db.session.commit() + models.db.session.delete(domain) + models.db.session.commit() flask.flash('Domain %s deleted' % domain) return flask.redirect(flask.url_for('.domain_list')) @@ -99,7 +100,7 @@ def domain_signup(domain_name=None): domain.max_users = 10 domain.max_aliases = 10 if domain.check_mx(): - db.session.add(domain) + models.db.session.add(domain) if flask_login.current_user.is_authenticated: user = models.User.query.get(flask_login.current_user.email) else: @@ -108,9 +109,9 @@ def domain_signup(domain_name=None): form.populate_obj(user) user.set_password(form.pw.data) user.quota_bytes = domain.max_quota_bytes - db.session.add(user) + models.db.session.add(user) domain.managers.append(user) - db.session.commit() + models.db.session.commit() flask.flash('Domain %s created' % domain) return flask.redirect(flask.url_for('.domain_list')) else: diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index b4ddb824..c0421146 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -24,8 +24,8 @@ def fetch_create(user_email): if form.validate_on_submit(): fetch = models.Fetch(user=user) form.populate_obj(fetch) - db.session.add(fetch) - db.session.commit() + models.db.session.add(fetch) + models.db.session.commit() flask.flash('Fetch configuration created') return flask.redirect( flask.url_for('.fetch_list', user_email=user.email)) @@ -39,7 +39,7 @@ def fetch_edit(fetch_id): form = forms.FetchForm(obj=fetch) if form.validate_on_submit(): form.populate_obj(fetch) - db.session.commit() + models.db.session.commit() flask.flash('Fetch configuration updated') return flask.redirect( flask.url_for('.fetch_list', user_email=fetch.user.email)) @@ -53,8 +53,8 @@ def fetch_edit(fetch_id): def fetch_delete(fetch_id): fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) user = fetch.user - db.session.delete(fetch) - db.session.commit() + models.db.session.delete(fetch) + models.db.session.commit() flask.flash('Fetch configuration delete') return flask.redirect( flask.url_for('.fetch_list', user_email=user.email)) diff --git a/core/admin/mailu/ui/views/managers.py b/core/admin/mailu/ui/views/managers.py index 15ebcd50..45974f94 100644 --- a/core/admin/mailu/ui/views/managers.py +++ b/core/admin/mailu/ui/views/managers.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -30,7 +30,7 @@ def manager_create(domain_name): flask.flash('User %s is already manager' % user, 'error') else: domain.managers.append(user) - db.session.commit() + models.db.session.commit() flask.flash('User %s can now manage %s' % (user, domain.name)) return flask.redirect( flask.url_for('.manager_list', domain_name=domain.name)) @@ -46,7 +46,7 @@ def manager_delete(domain_name, user_email): user = models.User.query.get(user_email) or flask.abort(404) if user in domain.managers: domain.managers.remove(user) - db.session.commit() + models.db.session.commit() flask.flash('User %s can no longer manager %s' % (user, domain)) else: flask.flash('User %s is not manager' % user, 'error') diff --git a/core/admin/mailu/ui/views/relays.py b/core/admin/mailu/ui/views/relays.py index a0ddc614..f1847798 100644 --- a/core/admin/mailu/ui/views/relays.py +++ b/core/admin/mailu/ui/views/relays.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -25,8 +25,8 @@ def relay_create(): else: relay = models.Relay() form.populate_obj(relay) - db.session.add(relay) - db.session.commit() + models.db.session.add(relay) + models.db.session.commit() flask.flash('Relayed domain %s created' % relay) return flask.redirect(flask.url_for('.relay_list')) return flask.render_template('relay/create.html', form=form) @@ -41,7 +41,7 @@ def relay_edit(relay_name): form.name.validators = [] if form.validate_on_submit(): form.populate_obj(relay) - db.session.commit() + models.db.session.commit() flask.flash('Relayed domain %s saved' % relay) return flask.redirect(flask.url_for('.relay_list')) return flask.render_template('relay/edit.html', form=form, @@ -53,8 +53,8 @@ def relay_edit(relay_name): @access.confirmation_required("delete {relay_name}") def relay_delete(relay_name): relay = models.Relay.query.get(relay_name) or flask.abort(404) - db.session.delete(relay) - db.session.commit() + models.db.session.delete(relay) + models.db.session.commit() flask.flash('Relayed domain %s deleted' % relay) return flask.redirect(flask.url_for('.relay_list')) diff --git a/core/admin/mailu/ui/views/tokens.py b/core/admin/mailu/ui/views/tokens.py index 4b9881af..3864617b 100644 --- a/core/admin/mailu/ui/views/tokens.py +++ b/core/admin/mailu/ui/views/tokens.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access from passlib import pwd @@ -32,8 +32,8 @@ def token_create(user_email): token = models.Token(user=user) token.set_password(form.raw_password.data) form.populate_obj(token) - db.session.add(token) - db.session.commit() + models.db.session.add(token) + models.db.session.commit() flask.flash('Authentication token created') return flask.redirect( flask.url_for('.token_list', user_email=user.email)) @@ -46,8 +46,8 @@ def token_create(user_email): def token_delete(token_id): token = models.Token.query.get(token_id) or flask.abort(404) user = token.user - db.session.delete(token) - db.session.commit() + models.db.session.delete(token) + models.db.session.commit() flask.flash('Authentication token deleted') return flask.redirect( flask.url_for('.token_list', user_email=user.email)) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index c54d22ec..ac572c2e 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -1,5 +1,6 @@ -from mailu import db, models, app +from mailu import models from mailu.ui import ui, access, forms +from flask import current_app as app import flask import flask_login @@ -33,8 +34,8 @@ def user_create(domain_name): user = models.User(domain=domain) form.populate_obj(user) user.set_password(form.pw.data) - db.session.add(user) - db.session.commit() + models.db.session.add(user) + models.db.session.commit() user.send_welcome() flask.flash('User %s created' % user) return flask.redirect( @@ -63,7 +64,7 @@ def user_edit(user_email): form.populate_obj(user) if form.pw.data: user.set_password(form.pw.data) - db.session.commit() + models.db.session.commit() flask.flash('User %s updated' % user) return flask.redirect( flask.url_for('.user_list', domain_name=user.domain.name)) @@ -77,8 +78,8 @@ def user_edit(user_email): def user_delete(user_email): user = models.User.query.get(user_email) or flask.abort(404) domain = user.domain - db.session.delete(user) - db.session.commit() + models.db.session.delete(user) + models.db.session.commit() flask.flash('User %s deleted' % user) return flask.redirect( flask.url_for('.user_list', domain_name=domain.name)) @@ -93,7 +94,7 @@ def user_settings(user_email): form = forms.UserSettingsForm(obj=user) if form.validate_on_submit(): form.populate_obj(user) - db.session.commit() + models.db.session.commit() flask.flash('Settings updated for %s' % user) if user_email: return flask.redirect( @@ -113,7 +114,7 @@ def user_password(user_email): flask.flash('Passwords do not match', 'error') else: user.set_password(form.pw.data) - db.session.commit() + models.db.session.commit() flask.flash('Password updated for %s' % user) if user_email: return flask.redirect(flask.url_for('.user_list', @@ -130,7 +131,7 @@ def user_forward(user_email): form = forms.UserForwardForm(obj=user) if form.validate_on_submit(): form.populate_obj(user) - db.session.commit() + models.db.session.commit() flask.flash('Forward destination updated for %s' % user) if user_email: return flask.redirect( @@ -147,7 +148,7 @@ def user_reply(user_email): form = forms.UserReplyForm(obj=user) if form.validate_on_submit(): form.populate_obj(user) - db.session.commit() + models.db.session.commit() flask.flash('Auto-reply message updated for %s' % user) if user_email: return flask.redirect( @@ -179,8 +180,8 @@ def user_signup(domain_name=None): form.populate_obj(user) user.set_password(form.pw.data) user.quota_bytes = quota_bytes - db.session.add(user) - db.session.commit() + models.db.session.add(user) + models.db.session.commit() user.send_welcome() flask.flash('Successfully signed up %s' % user) return flask.redirect(flask.url_for('.index')) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py new file mode 100644 index 00000000..60902951 --- /dev/null +++ b/core/admin/mailu/utils.py @@ -0,0 +1,46 @@ +import flask +import flask_login +import flask_script +import flask_migrate +import flask_babel +import flask_limiter + + +# Login configuration +login = flask_login.LoginManager() +login.login_view = "ui.login" +login.user_loader(models.User.query.get) + +@login_manager.unauthorized_handler +def handle_needs_login(): + return flask.redirect( + flask.url_for('ui.login', next=flask.request.endpoint) + ) + + +# Request rate limitation +limiter = flask_limiter.Limiter(key_func=lambda: current_user.username) + + +# Application translation +babel = flask_babel.Babel() + +@babel.localeselector +def get_locale(): + translations = list(map(str, babel.list_translations())) + return flask.request.accept_languages.best_match(translations) + + +# Proxy fixer +class PrefixMiddleware(object): + def __call__(self, environ, start_response): + prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '') + if prefix: + environ['SCRIPT_NAME'] = prefix + return self.app(environ, start_response) + + def init_app(self, app): + self.app = fixers.ProxyFix(app.wsgi_app) + app.wsgi_app = self + +proxy = PrefixMiddleware() From f40fcd7ac03e5d3a2693cfdb30456afd548e28d8 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 18 Oct 2018 16:20:56 +0200 Subject: [PATCH 02/13] Use click for the manager command --- core/admin/mailu/manage.py | 82 ++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index bc2473eb..2defa9d6 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,18 +1,25 @@ -from mailu import models +from mailu import models, create_app from flask import current_app as app +from flask import cli as flask_cli import flask import os import socket import uuid +import click -manager = flask_script.Manager() db = models.db -@manager.command +@click.group() +def cli(cls=flask_cli.FlaskGroup, create_app=mailu.create_app): + """ Main command group + """ + + +@cli.command() def advertise(): """ Advertise this server against statistic services. """ @@ -30,7 +37,10 @@ def advertise(): pass -@manager.command +@cli.command() +@cli.argument('localpart', help='localpart for the new admin') +@cli.argument('domain_name', help='domain name for the new admin') +@cli.argument('password', help='plain password for the new admin') def admin(localpart, domain_name, password): """ Create an admin user """ @@ -48,7 +58,11 @@ def admin(localpart, domain_name, password): db.session.commit() -@manager.command +@cli.command() +@cli.argument('localpart', help='localpart for the new user') +@cli.argument('domain_name', help='domain name for the new user') +@cli.argument('password', help='plain password for the new user') +@cli.argument('hash_scheme', help='password hashing scheme') def user(localpart, domain_name, password, hash_scheme=app.config['PASSWORD_SCHEME']): """ Create a user @@ -67,10 +81,11 @@ def user(localpart, domain_name, password, db.session.commit() -@manager.option('-n', '--domain_name', dest='domain_name') -@manager.option('-u', '--max_users', dest='max_users') -@manager.option('-a', '--max_aliases', dest='max_aliases') -@manager.option('-q', '--max_quota_bytes', dest='max_quota_bytes') +@cli.command() +@cli.option('-n', '--domain_name', dest='domain_name') +@cli.option('-u', '--max_users', dest='max_users') +@cli.option('-a', '--max_aliases', dest='max_aliases') +@cli.option('-q', '--max_quota_bytes', dest='max_quota_bytes') def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): domain = models.Domain.query.get(domain_name) if not domain: @@ -79,14 +94,14 @@ def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): db.session.commit() -@manager.command +@cli.command() +@cli.argument('localpart', help='localpart for the new user') +@cli.argument('domain_name', help='domain name for the new user') +@cli.argument('password_hash', help='password hash for the new user') +@cli.argument('hash_scheme', help='password hashing scheme') def user_import(localpart, domain_name, password_hash, hash_scheme=app.config['PASSWORD_SCHEME']): - """ Import a user along with password hash. Available hashes: - 'SHA512-CRYPT' - 'SHA256-CRYPT' - 'MD5-CRYPT' - 'CRYPT' + """ Import a user along with password hash. """ domain = models.Domain.query.get(domain_name) if not domain: @@ -102,7 +117,9 @@ def user_import(localpart, domain_name, password_hash, db.session.commit() -@manager.command +@cli.command() +@cli.option('-v', dest='verbose') +@cli.option('-d', dest='delete_objects') def config_update(verbose=False, delete_objects=False): """sync configuration with data from YAML-formatted stdin""" import yaml @@ -241,7 +258,8 @@ def config_update(verbose=False, delete_objects=False): db.session.commit() -@manager.command +@cli.command() +@cli.argument('email', help='email address to be deleted') def user_delete(email): """delete user""" user = models.User.query.get(email) @@ -250,7 +268,8 @@ def user_delete(email): db.session.commit() -@manager.command +@cli.command() +@cli.argument('email', help='email alias to be deleted') def alias_delete(email): """delete alias""" alias = models.Alias.query.get(email) @@ -259,7 +278,10 @@ def alias_delete(email): db.session.commit() -@manager.command +@cli.command() +@cli.argument('localpart', help='localpart for the new alias') +@cli.argument('domain_name', help='domain name for the new alias') +@cli.argument('destination', help='destination for the new alias') def alias(localpart, domain_name, destination): """ Create an alias """ @@ -276,30 +298,32 @@ def alias(localpart, domain_name, destination): db.session.add(alias) db.session.commit() -# Set limits to a domain - -@manager.command +@cli.command() +@cli.argument('domain_name', help='domain to be updated') +@cli.argument('max_users', help='maximum user count') +@cli.argument('max_aliases', help='maximum alias count') +@cli.argument('max_quota_bytes', help='maximum quota bytes par user') def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): + """ Set domain limits + """ domain = models.Domain.query.get(domain_name) domain.max_users = max_users domain.max_aliases = max_aliases domain.max_quota_bytes = max_quota_bytes - db.session.add(domain) db.session.commit() -# Make the user manager of a domain - -@manager.command +@cli.command() +@cli.argument('domain_name', help='target domain name') +@cli.argument('user_name', help='username inside the target domain') def setmanager(domain_name, user_name='manager'): + """ Make a user manager of a domain + """ domain = models.Domain.query.get(domain_name) manageruser = models.User.query.get(user_name + '@' + domain_name) domain.managers.append(manageruser) db.session.add(domain) db.session.commit() - -if __name__ == "__main__": - manager.run() From 82069ea3f009dabf9cfefaa606edadea5edb97f1 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 18 Oct 2018 17:55:07 +0200 Subject: [PATCH 03/13] Clean most of the refactored code --- core/admin/mailu/__init__.py | 20 +- core/admin/mailu/configuration.py | 27 +- core/admin/mailu/debug.py | 2 +- core/admin/mailu/manage.py | 82 +++--- core/admin/mailu/models.py | 18 +- core/admin/mailu/ui/views/alternatives.py | 2 +- core/admin/mailu/utils.py | 7 +- core/admin/manage.py | 298 ---------------------- 8 files changed, 97 insertions(+), 359 deletions(-) delete mode 100644 core/admin/manage.py diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 874fa8e0..8996fa20 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -1,44 +1,40 @@ import flask import flask_bootstrap -import os -import docker -import socket -import uuid - -from mailu import utils, debug, db +from mailu import utils, debug, models, configuration def create_app_from_config(config): """ Create a new application based on the given configuration """ app = flask.Flask(__name__) - app.config = config + app.app_context().push() # Bootstrap is used for basic JS and CSS loading # TODO: remove this and use statically generated assets instead app.bootstrap = flask_bootstrap.Bootstrap(app) # Initialize application extensions + config.init_app(app) models.db.init_app(app) utils.limiter.init_app(app) utils.babel.init_app(app) utils.login.init_app(app) + utils.login.user_loader(models.User.query.get) utils.proxy.init_app(app) - manage.migrate.init_app(app) - manage.manager.init_app(app) # Initialize debugging tools - if app.config.get("app.debug"): + if app.config.get("DEBUG"): debug.toolbar.init_app(app) - debug.profiler.init_app(app) + # TODO: add a specific configuration variable for profiling + # debug.profiler.init_app(app) # Inject the default variables in the Jinja parser + # TODO: move this to blueprints when needed @app.context_processor def inject_defaults(): signup_domains = models.Domain.query.filter_by(signup_enabled=True).all() return dict( - current_user=utils.login.current_user, signup_domains=signup_domains, config=app.config ) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 43184bad..05a3d570 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -4,7 +4,7 @@ import os DEFAULT_CONFIG = { # Specific to the admin UI 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', - 'SQLALCHEMY_TRACK_MODIFICATIONS': False, + 'SQLALCHEMY_TRACK_MODIFICATIONS': True, 'DOCKER_SOCKET': 'unix:///var/run/docker.sock', 'BABEL_DEFAULT_LOCALE': 'en', 'BABEL_DEFAULT_TIMEZONE': 'UTC', @@ -13,6 +13,7 @@ DEFAULT_CONFIG = { 'QUOTA_STORAGE_URL': 'redis://redis/1', 'DEBUG': False, 'DOMAIN_REGISTRATION': False, + 'TEMPLATES_AUTO_RELOAD': True, # Statistics management 'INSTANCE_ID_PATH': '/data/instance', 'STATS_ENDPOINT': '0.{}.stats.mailu.io', @@ -58,13 +59,29 @@ class ConfigManager(object): """ def __init__(self): - self.config = { - os.environ.get(key, value) + self.config = dict() + + def init_app(self, app): + self.config.update(app.config) + self.config.update({ + key: os.environ.get(key, value) for key, value in DEFAULT_CONFIG.items() - } + }) + app.config = self + + def setdefault(self, key, value): + if key not in self.config: + self.config[key] = value + return self.config[key] def get(self, *args): return self.config.get(*args) def __getitem__(self, key): - return self.get(key) + return self.config.get(key) + + def __setitem__(self, key, value): + self.config[key] = value + + def __contains__(self, key): + return key in self.config diff --git a/core/admin/mailu/debug.py b/core/admin/mailu/debug.py index 69e88cf8..7677901b 100644 --- a/core/admin/mailu/debug.py +++ b/core/admin/mailu/debug.py @@ -9,7 +9,7 @@ toolbar = flask_debugtoolbar.DebugToolbarExtension() # Profiler class Profiler(object): - def init_app(self): + def init_app(self, app): app.wsgi_app = werkzeug_profiler.ProfilerMiddleware( app.wsgi_app, restrictions=[30] ) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 2defa9d6..e0aa6286 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -14,12 +14,13 @@ db = models.db @click.group() -def cli(cls=flask_cli.FlaskGroup, create_app=mailu.create_app): +def cli(cls=flask_cli.FlaskGroup, create_app=create_app): """ Main command group """ @cli.command() +@flask_cli.with_appcontext def advertise(): """ Advertise this server against statistic services. """ @@ -38,9 +39,10 @@ def advertise(): @cli.command() -@cli.argument('localpart', help='localpart for the new admin') -@cli.argument('domain_name', help='domain name for the new admin') -@cli.argument('password', help='plain password for the new admin') +@click.argument('localpart') +@click.argument('domain_name') +@click.argument('password') +@flask_cli.with_appcontext def admin(localpart, domain_name, password): """ Create an admin user """ @@ -59,14 +61,16 @@ def admin(localpart, domain_name, password): @cli.command() -@cli.argument('localpart', help='localpart for the new user') -@cli.argument('domain_name', help='domain name for the new user') -@cli.argument('password', help='plain password for the new user') -@cli.argument('hash_scheme', help='password hashing scheme') -def user(localpart, domain_name, password, - hash_scheme=app.config['PASSWORD_SCHEME']): +@click.argument('localpart') +@click.argument('domain_name') +@click.argument('password') +@click.argument('hash_scheme') +@flask_cli.with_appcontext +def user(localpart, domain_name, password, hash_scheme=None): """ Create a user """ + if hash_scheme is None: + hash_scheme = app.config['PASSWORD_SCHEME'] domain = models.Domain.query.get(domain_name) if not domain: domain = models.Domain(name=domain_name) @@ -82,10 +86,11 @@ def user(localpart, domain_name, password, @cli.command() -@cli.option('-n', '--domain_name', dest='domain_name') -@cli.option('-u', '--max_users', dest='max_users') -@cli.option('-a', '--max_aliases', dest='max_aliases') -@cli.option('-q', '--max_quota_bytes', dest='max_quota_bytes') +@click.option('-n', '--domain_name') +@click.option('-u', '--max_users') +@click.option('-a', '--max_aliases') +@click.option('-q', '--max_quota_bytes') +@flask_cli.with_appcontext def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): domain = models.Domain.query.get(domain_name) if not domain: @@ -95,14 +100,16 @@ def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): @cli.command() -@cli.argument('localpart', help='localpart for the new user') -@cli.argument('domain_name', help='domain name for the new user') -@cli.argument('password_hash', help='password hash for the new user') -@cli.argument('hash_scheme', help='password hashing scheme') -def user_import(localpart, domain_name, password_hash, - hash_scheme=app.config['PASSWORD_SCHEME']): +@click.argument('localpart') +@click.argument('domain_name') +@click.argument('password_hash') +@click.argument('hash_scheme') +@flask_cli.with_appcontext +def user_import(localpart, domain_name, password_hash, hash_scheme = None): """ Import a user along with password hash. """ + if hash_scheme is None: + hash_scheme = app.config['PASSWORD_SCHEME'] domain = models.Domain.query.get(domain_name) if not domain: domain = models.Domain(name=domain_name) @@ -118,8 +125,9 @@ def user_import(localpart, domain_name, password_hash, @cli.command() -@cli.option('-v', dest='verbose') -@cli.option('-d', dest='delete_objects') +@click.option('-v', '--verbose') +@click.option('-d', '--delete_objects') +@flask_cli.with_appcontext def config_update(verbose=False, delete_objects=False): """sync configuration with data from YAML-formatted stdin""" import yaml @@ -259,7 +267,8 @@ def config_update(verbose=False, delete_objects=False): @cli.command() -@cli.argument('email', help='email address to be deleted') +@click.argument('email') +@flask_cli.with_appcontext def user_delete(email): """delete user""" user = models.User.query.get(email) @@ -269,7 +278,8 @@ def user_delete(email): @cli.command() -@cli.argument('email', help='email alias to be deleted') +@click.argument('email') +@flask_cli.with_appcontext def alias_delete(email): """delete alias""" alias = models.Alias.query.get(email) @@ -279,9 +289,10 @@ def alias_delete(email): @cli.command() -@cli.argument('localpart', help='localpart for the new alias') -@cli.argument('domain_name', help='domain name for the new alias') -@cli.argument('destination', help='destination for the new alias') +@click.argument('localpart') +@click.argument('domain_name') +@click.argument('destination') +@flask_cli.with_appcontext def alias(localpart, domain_name, destination): """ Create an alias """ @@ -300,10 +311,11 @@ def alias(localpart, domain_name, destination): @cli.command() -@cli.argument('domain_name', help='domain to be updated') -@cli.argument('max_users', help='maximum user count') -@cli.argument('max_aliases', help='maximum alias count') -@cli.argument('max_quota_bytes', help='maximum quota bytes par user') +@click.argument('domain_name') +@click.argument('max_users') +@click.argument('max_aliases') +@click.argument('max_quota_bytes') +@flask_cli.with_appcontext def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): """ Set domain limits """ @@ -316,8 +328,9 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): @cli.command() -@cli.argument('domain_name', help='target domain name') -@cli.argument('user_name', help='username inside the target domain') +@click.argument('domain_name') +@click.argument('user_name') +@flask_cli.with_appcontext def setmanager(domain_name, user_name='manager'): """ Make a user manager of a domain """ @@ -327,3 +340,6 @@ def setmanager(domain_name, user_name='manager'): db.session.add(domain) db.session.commit() + +if __name__ == '__main__': + cli() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 43565c55..bc1044db 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -307,24 +307,28 @@ class User(Base, Email): 'SHA256-CRYPT': "sha256_crypt", 'MD5-CRYPT': "md5_crypt", 'CRYPT': "des_crypt"} - pw_context = context.CryptContext( - schemes = scheme_dict.values(), - default=scheme_dict[app.config['PASSWORD_SCHEME']], - ) + + def get_password_context(self): + return context.CryptContext( + schemes=self.scheme_dict.values(), + default=self.scheme_dict[app.config['PASSWORD_SCHEME']], + ) def check_password(self, password): reference = re.match('({[^}]+})?(.*)', self.password).group(2) - return User.pw_context.verify(password, reference) + return self.get_password_context().verify(password, reference) - def set_password(self, password, hash_scheme=app.config['PASSWORD_SCHEME'], raw=False): + def set_password(self, password, hash_scheme=None, raw=False): """Set password for user with specified encryption scheme @password: plain text password to encrypt (if raw == True the hash itself) """ + if hash_scheme is None: + hash_scheme = app.config['PASSWORD_SCHEME'] # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes if raw: self.password = '{'+hash_scheme+'}' + password else: - self.password = '{'+hash_scheme+'}' + User.pw_context.encrypt(password, self.scheme_dict[hash_scheme]) + self.password = '{'+hash_scheme+'}' + self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) def get_managed_domains(self): if self.global_admin: diff --git a/core/admin/mailu/ui/views/alternatives.py b/core/admin/mailu/ui/views/alternatives.py index e5761c83..2e6e580b 100644 --- a/core/admin/mailu/ui/views/alternatives.py +++ b/core/admin/mailu/ui/views/alternatives.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 60902951..18292041 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -1,3 +1,5 @@ +from mailu import models + import flask import flask_login import flask_script @@ -5,13 +7,14 @@ import flask_migrate import flask_babel import flask_limiter +from werkzeug.contrib import fixers + # Login configuration login = flask_login.LoginManager() login.login_view = "ui.login" -login.user_loader(models.User.query.get) -@login_manager.unauthorized_handler +@login.unauthorized_handler def handle_needs_login(): return flask.redirect( flask.url_for('ui.login', next=flask.request.endpoint) diff --git a/core/admin/manage.py b/core/admin/manage.py deleted file mode 100644 index f7e9e17d..00000000 --- a/core/admin/manage.py +++ /dev/null @@ -1,298 +0,0 @@ -from mailu import app, manager, db, models - -import os -import socket -import uuid - - -@manager.command -def advertise(): - """ Advertise this server against statistic services. - """ - if os.path.isfile(app.config["INSTANCE_ID_PATH"]): - with open(app.config["INSTANCE_ID_PATH"], "r") as handle: - instance_id = handle.read() - else: - instance_id = str(uuid.uuid4()) - with open(app.config["INSTANCE_ID_PATH"], "w") as handle: - handle.write(instance_id) - if app.config["DISABLE_STATISTICS"].lower() != "true": - try: - socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) - except: - pass - - -@manager.command -def admin(localpart, domain_name, password): - """ Create an admin user - """ - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User( - localpart=localpart, - domain=domain, - global_admin=True - ) - user.set_password(password) - db.session.add(user) - db.session.commit() - - -@manager.command -def user(localpart, domain_name, password, - hash_scheme=app.config['PASSWORD_SCHEME']): - """ Create a user - """ - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User( - localpart=localpart, - domain=domain, - global_admin=False - ) - user.set_password(password, hash_scheme=hash_scheme) - db.session.add(user) - db.session.commit() - - -@manager.option('-n', '--domain_name', dest='domain_name') -@manager.option('-u', '--max_users', dest='max_users') -@manager.option('-a', '--max_aliases', dest='max_aliases') -@manager.option('-q', '--max_quota_bytes', dest='max_quota_bytes') -def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - db.session.commit() - - -@manager.command -def user_import(localpart, domain_name, password_hash, - hash_scheme=app.config['PASSWORD_SCHEME']): - """ Import a user along with password hash. Available hashes: - 'SHA512-CRYPT' - 'SHA256-CRYPT' - 'MD5-CRYPT' - 'CRYPT' - """ - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User( - localpart=localpart, - domain=domain, - global_admin=False - ) - user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) - db.session.add(user) - db.session.commit() - - -@manager.command -def config_update(verbose=False, delete_objects=False): - """sync configuration with data from YAML-formatted stdin""" - import yaml - import sys - new_config = yaml.load(sys.stdin) - # print new_config - domains = new_config.get('domains', []) - tracked_domains = set() - for domain_config in domains: - if verbose: - print(str(domain_config)) - domain_name = domain_config['name'] - max_users = domain_config.get('max_users', 0) - max_aliases = domain_config.get('max_aliases', 0) - max_quota_bytes = domain_config.get('max_quota_bytes', 0) - tracked_domains.add(domain_name) - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name, - max_users=max_users, - max_aliases=max_aliases, - max_quota_bytes=max_quota_bytes) - db.session.add(domain) - print("Added " + str(domain_config)) - else: - domain.max_users = max_users - domain.max_aliases = max_aliases - domain.max_quota_bytes = max_quota_bytes - db.session.add(domain) - print("Updated " + str(domain_config)) - - users = new_config.get('users', []) - tracked_users = set() - user_optional_params = ('comment', 'quota_bytes', 'global_admin', - 'enable_imap', 'enable_pop', 'forward_enabled', - 'forward_destination', 'reply_enabled', - 'reply_subject', 'reply_body', 'displayed_name', - 'spam_enabled', 'email', 'spam_threshold') - for user_config in users: - if verbose: - print(str(user_config)) - localpart = user_config['localpart'] - domain_name = user_config['domain'] - password_hash = user_config.get('password_hash', None) - hash_scheme = user_config.get('hash_scheme', None) - domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) - optional_params = {} - for k in user_optional_params: - if k in user_config: - optional_params[k] = user_config[k] - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User.query.get(email) - tracked_users.add(email) - tracked_domains.add(domain_name) - if not user: - user = models.User( - localpart=localpart, - domain=domain, - **optional_params - ) - else: - for k in optional_params: - setattr(user, k, optional_params[k]) - user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) - db.session.add(user) - - aliases = new_config.get('aliases', []) - tracked_aliases = set() - for alias_config in aliases: - if verbose: - print(str(alias_config)) - localpart = alias_config['localpart'] - domain_name = alias_config['domain'] - if type(alias_config['destination']) is str: - destination = alias_config['destination'].split(',') - else: - destination = alias_config['destination'] - wildcard = alias_config.get('wildcard', False) - domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - alias = models.Alias.query.get(email) - tracked_aliases.add(email) - tracked_domains.add(domain_name) - if not alias: - alias = models.Alias( - localpart=localpart, - domain=domain, - wildcard=wildcard, - destination=destination, - email=email - ) - else: - alias.destination = destination - alias.wildcard = wildcard - db.session.add(alias) - - db.session.commit() - - managers = new_config.get('managers', []) - # tracked_managers=set() - for manager_config in managers: - if verbose: - print(str(manager_config)) - domain_name = manager_config['domain'] - user_name = manager_config['user'] - domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) - if manageruser not in domain.managers: - domain.managers.append(manageruser) - db.session.add(domain) - - db.session.commit() - - if delete_objects: - for user in db.session.query(models.User).all(): - if not (user.email in tracked_users): - if verbose: - print("Deleting user: " + str(user.email)) - db.session.delete(user) - for alias in db.session.query(models.Alias).all(): - if not (alias.email in tracked_aliases): - if verbose: - print("Deleting alias: " + str(alias.email)) - db.session.delete(alias) - for domain in db.session.query(models.Domain).all(): - if not (domain.name in tracked_domains): - if verbose: - print("Deleting domain: " + str(domain.name)) - db.session.delete(domain) - db.session.commit() - - -@manager.command -def user_delete(email): - """delete user""" - user = models.User.query.get(email) - if user: - db.session.delete(user) - db.session.commit() - - -@manager.command -def alias_delete(email): - """delete alias""" - alias = models.Alias.query.get(email) - if alias: - db.session.delete(alias) - db.session.commit() - - -@manager.command -def alias(localpart, domain_name, destination): - """ Create an alias - """ - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - alias = models.Alias( - localpart=localpart, - domain=domain, - destination=destination.split(','), - email="%s@%s" % (localpart, domain_name) - ) - db.session.add(alias) - db.session.commit() - -# Set limits to a domain - - -@manager.command -def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): - domain = models.Domain.query.get(domain_name) - domain.max_users = max_users - domain.max_aliases = max_aliases - domain.max_quota_bytes = max_quota_bytes - - db.session.add(domain) - db.session.commit() - -# Make the user manager of a domain - - -@manager.command -def setmanager(domain_name, user_name='manager'): - domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) - domain.managers.append(manageruser) - db.session.add(domain) - db.session.commit() - - -if __name__ == "__main__": - manager.run() From 206cce0b47dec42d42442e2dd45ca83edc43990a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 20:29:52 +0100 Subject: [PATCH 04/13] Finish the configuration bits --- core/admin/mailu/__init__.py | 3 ++- core/admin/mailu/configuration.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 8996fa20..be216e59 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -20,8 +20,9 @@ def create_app_from_config(config): utils.limiter.init_app(app) utils.babel.init_app(app) utils.login.init_app(app) - utils.login.user_loader(models.User.query.get) + utils.login.user_loader(models.User.get) utils.proxy.init_app(app) + utils.migrate.init_app(app, models.db) # Initialize debugging tools if app.config.get("DEBUG"): diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 05a3d570..48599d5e 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -4,7 +4,7 @@ import os DEFAULT_CONFIG = { # Specific to the admin UI 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', - 'SQLALCHEMY_TRACK_MODIFICATIONS': True, + 'SQLALCHEMY_TRACK_MODIFICATIONS': False, 'DOCKER_SOCKET': 'unix:///var/run/docker.sock', 'BABEL_DEFAULT_LOCALE': 'en', 'BABEL_DEFAULT_TIMEZONE': 'UTC', @@ -54,7 +54,7 @@ DEFAULT_CONFIG = { } -class ConfigManager(object): +class ConfigManager(dict): """ Naive configuration manager that uses environment only """ @@ -77,6 +77,9 @@ class ConfigManager(object): def get(self, *args): return self.config.get(*args) + def keys(self): + return self.config.keys() + def __getitem__(self, key): return self.config.get(key) From f6013aa29fddf9883f8f0bea4b7733718b9d8846 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 20:30:20 +0100 Subject: [PATCH 05/13] Fix an old migration that was reading configuration before migrating --- core/admin/migrations/versions/3f6994568962_.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/admin/migrations/versions/3f6994568962_.py b/core/admin/migrations/versions/3f6994568962_.py index ffe8569c..65620c00 100644 --- a/core/admin/migrations/versions/3f6994568962_.py +++ b/core/admin/migrations/versions/3f6994568962_.py @@ -13,8 +13,6 @@ down_revision = '2335c80a6bc3' from alembic import op import sqlalchemy as sa -from mailu import app - fetch_table = sa.Table( 'fetch', @@ -24,13 +22,7 @@ fetch_table = sa.Table( def upgrade(): - connection = op.get_bind() op.add_column('fetch', sa.Column('keep', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false())) - # also apply the current config value if set - if app.config.get("FETCHMAIL_KEEP", "False") == "True": - connection.execute( - fetch_table.update().values(keep=True) - ) def downgrade(): From f57d4859f36069e967d8ee6a5530db6917f722c0 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 20:30:41 +0100 Subject: [PATCH 06/13] Provide an in-context wrapper for getting users --- core/admin/mailu/models.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index bc1044db..22839e3b 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -349,6 +349,10 @@ class User(Base, Email): self.sendmail(app.config["WELCOME_SUBJECT"], app.config["WELCOME_BODY"]) + @classmethod + def get(cls, email): + return cls.query.get(email) + @classmethod def login(cls, email, password): user = cls.query.get(email) From 2a8808bdec519912f0badb0c5a14d3205f1e2892 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 20:32:06 +0100 Subject: [PATCH 07/13] Add the configuration table migration --- core/admin/mailu/dockercli.py | 26 ------------------- .../migrations/versions/cd79ed46d9c2_.py | 25 ++++++++++++++++++ core/admin/run.py | 7 ----- 3 files changed, 25 insertions(+), 33 deletions(-) delete mode 100644 core/admin/mailu/dockercli.py create mode 100644 core/admin/migrations/versions/cd79ed46d9c2_.py delete mode 100644 core/admin/run.py diff --git a/core/admin/mailu/dockercli.py b/core/admin/mailu/dockercli.py deleted file mode 100644 index 94f71144..00000000 --- a/core/admin/mailu/dockercli.py +++ /dev/null @@ -1,26 +0,0 @@ -from mailu import app - -import docker -import signal - - -# Connect to the Docker socket -cli = docker.Client(base_url=app.config['DOCKER_SOCKET']) - - -def get(*names): - result = {} - all_containers = cli.containers(all=True) - for brief in all_containers: - if brief['Image'].startswith('mailu/'): - container = cli.inspect_container(brief['Id']) - container['Image'] = cli.inspect_image(container['Image']) - name = container['Config']['Labels']['com.docker.compose.service'] - if not names or name in names: - result[name] = container - return result - - -def reload(*names): - for name, container in get(*names).items(): - cli.kill(container["Id"], signal.SIGHUP.value) diff --git a/core/admin/migrations/versions/cd79ed46d9c2_.py b/core/admin/migrations/versions/cd79ed46d9c2_.py new file mode 100644 index 00000000..0e9da5d3 --- /dev/null +++ b/core/admin/migrations/versions/cd79ed46d9c2_.py @@ -0,0 +1,25 @@ +""" Add a configuration table + +Revision ID: cd79ed46d9c2 +Revises: 25fd6c7bcb4a +Create Date: 2018-10-17 21:44:48.924921 + +""" + +revision = 'cd79ed46d9c2' +down_revision = '25fd6c7bcb4a' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table('config', + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('value', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('name') + ) + + +def downgrade(): + op.drop_table('config') diff --git a/core/admin/run.py b/core/admin/run.py deleted file mode 100644 index a55ac1e2..00000000 --- a/core/admin/run.py +++ /dev/null @@ -1,7 +0,0 @@ -import os - - -if __name__ == "__main__": - os.environ["DEBUG"] = "True" - from mailu import app - app.run() From 4a7eb1eb6cd59e93edece14f17aeb05e8688f3e4 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 20:32:23 +0100 Subject: [PATCH 08/13] Explicitely declare flask migrate --- core/admin/mailu/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 18292041..b11b1689 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -47,3 +47,7 @@ class PrefixMiddleware(object): app.wsgi_app = self proxy = PrefixMiddleware() + + +# Data migrate +migrate = flask_migrate.Migrate() From f9e30bd87c3bd08307ada55d303598b01f54d90e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 21:29:11 +0100 Subject: [PATCH 09/13] Update the dockerfile and upgrade dependencies --- core/admin/Dockerfile | 2 +- core/admin/requirements-prod.txt | 47 ++++++++++++++------------------ core/admin/requirements.txt | 1 - core/admin/start.sh | 7 +++-- 4 files changed, 25 insertions(+), 32 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 2e637206..29eb69f6 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -11,12 +11,12 @@ RUN apk add --no-cache openssl \ COPY mailu ./mailu COPY migrations ./migrations -COPY manage.py . COPY start.sh /start.sh RUN pybabel compile -d mailu/translations EXPOSE 80/tcp VOLUME ["/data"] +ENV FLASK_APP mailu CMD ["/start.sh"] diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index e321a4d6..2ca59edc 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -1,53 +1,46 @@ -alembic==0.9.9 +alembic==1.0.2 asn1crypto==0.24.0 -Babel==2.5.3 +Babel==2.6.0 bcrypt==3.1.4 blinker==1.4 -certifi==2018.4.16 cffi==1.11.5 -chardet==3.0.4 -click==6.7 -cryptography==2.2.2 +Click==7.0 +cryptography==2.3.1 decorator==4.3.0 dnspython==1.15.0 -docker-py==1.10.6 -docker-pycreds==0.2.2 -dominate==2.3.1 -Flask==0.12.2 -Flask-Babel==0.11.2 +dominate==2.3.4 +Flask==1.0.2 +Flask-Babel==0.12.2 Flask-Bootstrap==3.3.7.1 Flask-DebugToolbar==0.10.1 Flask-Limiter==1.0.1 Flask-Login==0.4.1 -Flask-Migrate==2.1.1 +Flask-Migrate==2.3.0 Flask-Script==2.0.6 Flask-SQLAlchemy==2.3.2 Flask-WTF==0.14.2 -gunicorn==19.7.1 -idna==2.6 +gunicorn==19.9.0 +idna==2.7 infinity==1.4 intervals==0.8.1 -itsdangerous==0.24 +itsdangerous==1.1.0 Jinja2==2.10 limits==1.3 Mako==1.0.7 -MarkupSafe==1.0 +MarkupSafe==1.1.0 passlib==1.7.1 -pycparser==2.18 -pyOpenSSL==17.5.0 -python-dateutil==2.7.2 +pycparser==2.19 +pyOpenSSL==18.0.0 +python-dateutil==2.7.5 python-editor==1.0.3 -pytz==2018.4 -PyYAML==3.12 +pytz==2018.7 +PyYAML==3.13 redis==2.10.6 -requests==2.18.4 six==1.11.0 -SQLAlchemy==1.2.6 +SQLAlchemy==1.2.13 tabulate==0.8.2 -urllib3==1.22 -validators==0.12.1 +validators==0.12.2 visitor==0.1.3 -websocket-client==0.47.0 Werkzeug==0.14.1 -WTForms==2.1 +WTForms==2.2.1 WTForms-Components==0.10.3 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index d6e7adb1..95a65bbe 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -12,7 +12,6 @@ redis WTForms-Components passlib gunicorn -docker-py tabulate PyYAML PyOpenSSL diff --git a/core/admin/start.sh b/core/admin/start.sh index 8208e4a1..b3538204 100755 --- a/core/admin/start.sh +++ b/core/admin/start.sh @@ -1,5 +1,6 @@ #!/bin/sh -python manage.py advertise -python manage.py db upgrade -gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload mailu:app +flask mailu advertise +flask db upgrade + +gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload "$FLASK_APP:create_app()" From 02995f0a15d46272b219c01acec8b9f3ec938b8c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 21:29:30 +0100 Subject: [PATCH 10/13] Add a mailu command line to flask --- core/admin/mailu/__init__.py | 3 ++- core/admin/mailu/manage.py | 28 ++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index be216e59..3914e9f6 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -1,7 +1,7 @@ import flask import flask_bootstrap -from mailu import utils, debug, models, configuration +from mailu import utils, debug, models, manage, configuration def create_app_from_config(config): @@ -9,6 +9,7 @@ def create_app_from_config(config): """ app = flask.Flask(__name__) app.app_context().push() + app.cli.add_command(manage.mailu) # Bootstrap is used for basic JS and CSS loading # TODO: remove this and use statically generated assets instead diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index e0aa6286..7f32f6c8 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,4 +1,4 @@ -from mailu import models, create_app +from mailu import models from flask import current_app as app from flask import cli as flask_cli @@ -14,12 +14,12 @@ db = models.db @click.group() -def cli(cls=flask_cli.FlaskGroup, create_app=create_app): - """ Main command group +def mailu(cls=flask_cli.FlaskGroup): + """ Mailu command line """ -@cli.command() +@mailu.command() @flask_cli.with_appcontext def advertise(): """ Advertise this server against statistic services. @@ -38,7 +38,7 @@ def advertise(): pass -@cli.command() +@mailu.command() @click.argument('localpart') @click.argument('domain_name') @click.argument('password') @@ -60,7 +60,7 @@ def admin(localpart, domain_name, password): db.session.commit() -@cli.command() +@mailu.command() @click.argument('localpart') @click.argument('domain_name') @click.argument('password') @@ -85,7 +85,7 @@ def user(localpart, domain_name, password, hash_scheme=None): db.session.commit() -@cli.command() +@mailu.command() @click.option('-n', '--domain_name') @click.option('-u', '--max_users') @click.option('-a', '--max_aliases') @@ -99,7 +99,7 @@ def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): db.session.commit() -@cli.command() +@mailu.command() @click.argument('localpart') @click.argument('domain_name') @click.argument('password_hash') @@ -124,7 +124,7 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None): db.session.commit() -@cli.command() +@mailu.command() @click.option('-v', '--verbose') @click.option('-d', '--delete_objects') @flask_cli.with_appcontext @@ -266,7 +266,7 @@ def config_update(verbose=False, delete_objects=False): db.session.commit() -@cli.command() +@mailu.command() @click.argument('email') @flask_cli.with_appcontext def user_delete(email): @@ -277,7 +277,7 @@ def user_delete(email): db.session.commit() -@cli.command() +@mailu.command() @click.argument('email') @flask_cli.with_appcontext def alias_delete(email): @@ -288,7 +288,7 @@ def alias_delete(email): db.session.commit() -@cli.command() +@mailu.command() @click.argument('localpart') @click.argument('domain_name') @click.argument('destination') @@ -310,7 +310,7 @@ def alias(localpart, domain_name, destination): db.session.commit() -@cli.command() +@mailu.command() @click.argument('domain_name') @click.argument('max_users') @click.argument('max_aliases') @@ -327,7 +327,7 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): db.session.commit() -@cli.command() +@mailu.command() @click.argument('domain_name') @click.argument('user_name') @flask_cli.with_appcontext From 30716b8bdf0694ebdd7f06abb9a928c1a29a3513 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 21:35:41 +0100 Subject: [PATCH 11/13] Update docs with the new flask command --- docs/cli.rst | 12 ++++++------ docs/compose/setup.rst | 4 ++-- docs/contributors/database.rst | 4 ++-- docs/kubernetes/1.6/README.md | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index 4a5250a2..8cfb440b 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -15,7 +15,7 @@ alias .. code-block:: bash - docker-compose run --rm admin python manage.py alias foo example.net "mail1@example.com,mail2@example.com" + docker-compose exec admin flask mailu alias foo example.net "mail1@example.com,mail2@example.com" alias_delete @@ -23,14 +23,14 @@ alias_delete .. code-block:: bash - docker-compose run --rm admin python manage.py alias_delete foo@example.net + docker-compose exec admin flask mailu alias_delete foo@example.net user ---- .. code-block:: bash - docker-compose run --rm admin python manage.py user --hash_scheme='SHA512-CRYPT' myuser example.net 'password123' + docker-compose exec admin flask mailu user --hash_scheme='SHA512-CRYPT' myuser example.net 'password123' user_import ----------- @@ -39,14 +39,14 @@ primary difference with simple `user` command is that password is being imported .. code-block:: bash - docker-compose run --rm admin python manage.py user --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' + docker-compose exec admin flask mailu user --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' user_delete ------------ .. code-block:: bash - docker-compose run --rm admin python manage.py user_delete foo@example.net + docker-compose exec admin flask mailu user_delete foo@example.net config_update ------------- @@ -55,7 +55,7 @@ The sole purpose of this command is for importing users/aliases in bulk and sync .. code-block:: bash - cat mail-config.yml | docker-compose run --rm admin python manage.py config_update --delete_objects + cat mail-config.yml | docker-compose exec admin flask mailu config_update --delete_objects where mail-config.yml looks like: diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst index 64ad7b25..942a368e 100644 --- a/docs/compose/setup.rst +++ b/docs/compose/setup.rst @@ -151,6 +151,6 @@ Finally, you must create the initial admin user account: .. code-block:: bash - docker-compose run --rm admin python manage.py admin root example.net password + docker-compose exec admin flask mailu admin me example.net password -This will create a user named ``root@example.net`` with password ``password`` and administration privileges. Connect to the Web admin interface and change the password to a strong one. +This will create a user named ``me@example.net`` with password ``password`` and administration privileges. Connect to the Web admin interface and change the password to a strong one. diff --git a/docs/contributors/database.rst b/docs/contributors/database.rst index ca97f604..a987c980 100644 --- a/docs/contributors/database.rst +++ b/docs/contributors/database.rst @@ -17,7 +17,7 @@ migration script: .. code-block:: bash - python manage.py db migrate + flask db migrate This will generate a new script in ``migrations/versions`` that you must review before adding it for commit. @@ -54,7 +54,7 @@ At that point, to start working on the changed database structure, you will need .. code-block:: bash - python manage.py db upgrade + flask db upgrade If any error arises, restore the backup, fix the migration script and try again. diff --git a/docs/kubernetes/1.6/README.md b/docs/kubernetes/1.6/README.md index c0dd935b..21780a0c 100644 --- a/docs/kubernetes/1.6/README.md +++ b/docs/kubernetes/1.6/README.md @@ -82,7 +82,7 @@ And in the pod run the following command. The command uses following entries: - `password` the chosen password for the user ```bash -python manage.py admin root example.com password +flask mailu admin root example.com password ``` Now you should be able to login on the mail account: `https://mail.example.com/admin` @@ -154,4 +154,4 @@ Wait for the pod to recreate and you're online! Happy mailing! Wait for the pod to recreate and you're online! -Happy mailing! \ No newline at end of file +Happy mailing! From 72e1b444caaa5dd07700bd8f993a8ffa40f83a21 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Nov 2018 21:55:39 +0100 Subject: [PATCH 12/13] Merge alembic migrations --- core/admin/migrations/versions/cd79ed46d9c2_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/migrations/versions/cd79ed46d9c2_.py b/core/admin/migrations/versions/cd79ed46d9c2_.py index 0e9da5d3..ccf210fe 100644 --- a/core/admin/migrations/versions/cd79ed46d9c2_.py +++ b/core/admin/migrations/versions/cd79ed46d9c2_.py @@ -7,7 +7,7 @@ Create Date: 2018-10-17 21:44:48.924921 """ revision = 'cd79ed46d9c2' -down_revision = '25fd6c7bcb4a' +down_revision = '3b281286c7bd' from alembic import op import sqlalchemy as sa From 4783e6169371e6016260fb768942abf71bd724c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 9 Nov 2018 11:45:08 +0200 Subject: [PATCH 13/13] Fix password context Fixes the following error: ``` admin_1 | [2018-11-09 09:44:10,533] ERROR in app: Exception on /internal/auth/email [GET] admin_1 | Traceback (most recent call last): admin_1 | File "/usr/lib/python3.6/site-packages/flask/app.py", line 2292, in wsgi_app admin_1 | response = self.full_dispatch_request() admin_1 | File "/usr/lib/python3.6/site-packages/flask/app.py", line 1815, in full_dispatch_request admin_1 | rv = self.handle_user_exception(e) admin_1 | File "/usr/lib/python3.6/site-packages/flask/app.py", line 1718, in handle_user_exception admin_1 | reraise(exc_type, exc_value, tb) admin_1 | File "/usr/lib/python3.6/site-packages/flask/_compat.py", line 35, in reraise admin_1 | raise value admin_1 | File "/usr/lib/python3.6/site-packages/flask/app.py", line 1813, in full_dispatch_request admin_1 | rv = self.dispatch_request() admin_1 | File "/usr/lib/python3.6/site-packages/flask/app.py", line 1799, in dispatch_request admin_1 | return self.view_functions[rule.endpoint](**req.view_args) admin_1 | File "/usr/lib/python3.6/site-packages/flask_limiter/extension.py", line 544, in __inner admin_1 | return obj(*a, **k) admin_1 | File "/app/mailu/internal/views/auth.py", line 18, in nginx_authentication admin_1 | headers = nginx.handle_authentication(flask.request.headers) admin_1 | File "/app/mailu/internal/nginx.py", line 48, in handle_authentication admin_1 | if user.check_password(password): admin_1 | File "/app/mailu/models.py", line 333, in check_password admin_1 | context = User.pw_context admin_1 | AttributeError: type object 'User' has no attribute 'pw_context' ``` --- core/admin/mailu/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index d1c6b370..c25d30cb 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -330,7 +330,7 @@ class User(Base, Email): ) def check_password(self, password): - context = User.pw_context + context = self.get_password_context() reference = re.match('({[^}]+})?(.*)', self.password).group(2) result = context.verify(password, reference) if result and context.identify(reference) != context.default_scheme():