Merge pull request #670 from kaiyou/refactor-config

Refactor the admin architecture and configuration management

core/admin/Dockerfile

@@ -15,13 +15,13 @@ RUN apk add --no-cache openssl curl \
 
 COPY mailu ./mailu
 COPY migrations ./migrations
-COPY manage.py .
 COPY start.py /start.py
 
 RUN pybabel compile -d mailu/translations
 
 EXPOSE 80/tcp
 VOLUME ["/data"]
+ENV FLASK_APP mailu
 
 CMD /start.py
 

core/admin/mailu/__init__.py

@@ -1,140 +1,57 @@
 import flask
-import flask_sqlalchemy
 import flask_bootstrap
-import flask_login
-import flask_script
-import flask_migrate
-import flask_babel
-import flask_limiter
 
-import os
+from mailu import utils, debug, models, manage, configuration
-import docker
-import socket
-import uuid
 
-from werkzeug.contrib import fixers, profiler
 
-# Create application
+def create_app_from_config(config):
-app = flask.Flask(__name__)
+    """ Create a new application based on the given configuration
+    """
+    app = flask.Flask(__name__)
+    app.app_context().push()
+    app.cli.add_command(manage.mailu)
 
-default_config = {
+    # Bootstrap is used for basic JS and CSS loading
-    # Specific to the admin UI
+    # TODO: remove this and use statically generated assets instead
-    'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db',
+    app.bootstrap = flask_bootstrap.Bootstrap(app)
-    'SQLALCHEMY_TRACK_MODIFICATIONS': False,
-    'DOCKER_SOCKET': 'unix:///var/run/docker.sock',
-    'BABEL_DEFAULT_LOCALE': 'en',
-    'BABEL_DEFAULT_TIMEZONE': 'UTC',
-    'BOOTSTRAP_SERVE_LOCAL': True,
-    'RATELIMIT_STORAGE_URL': 'redis://redis/2',
-    'QUOTA_STORAGE_URL': 'redis://redis/1',
-    'DEBUG': False,
-    'DOMAIN_REGISTRATION': False,
-    # Statistics management
-    'INSTANCE_ID_PATH': '/data/instance',
-    'STATS_ENDPOINT': '0.{}.stats.mailu.io',
-    # Common configuration variables
-    'SECRET_KEY': 'changeMe',
-    'DOMAIN': 'mailu.io',
-    'HOSTNAMES': 'mail.mailu.io,alternative.mailu.io,yetanother.mailu.io',
-    'POSTMASTER': 'postmaster',
-    'TLS_FLAVOR': 'cert',
-    'AUTH_RATELIMIT': '10/minute;1000/hour',
-    'DISABLE_STATISTICS': 'False',
-    # Mail settings
-    'DMARC_RUA': None,
-    'DMARC_RUF': None,
-    'WELCOME': 'False',
-    'WELCOME_SUBJECT': 'Dummy welcome topic',
-    'WELCOME_BODY': 'Dummy welcome body',
-    'DKIM_SELECTOR': 'dkim',
-    'DKIM_PATH': '/dkim/{domain}.{selector}.key',
-    'DEFAULT_QUOTA': 1000000000,
-    # Web settings
-    'SITENAME': 'Mailu',
-    'WEBSITE': 'https://mailu.io',
-    'WEB_ADMIN': '/admin',
-    'WEB_WEBMAIL': '/webmail',
-    'RECAPTCHA_PUBLIC_KEY': '',
-    'RECAPTCHA_PRIVATE_KEY': '',
-    # Advanced settings
-    'PASSWORD_SCHEME': 'BLF-CRYPT',
-    # Host settings
-    'HOST_IMAP': 'imap',
-    'HOST_POP3': 'imap',
-    'HOST_SMTP': 'smtp',
-    'HOST_WEBMAIL': 'webmail',
-    'HOST_FRONT': 'front',
-    'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'),
-    'POD_ADDRESS_RANGE': None
-}
 
-# Load configuration from the environment if available
+    # Initialize application extensions
-for key, value in default_config.items():
+    config.init_app(app)
-    app.config[key] = os.environ.get(key, value)
+    models.db.init_app(app)
+    utils.limiter.init_app(app)
+    utils.babel.init_app(app)
+    utils.login.init_app(app)
+    utils.login.user_loader(models.User.get)
+    utils.proxy.init_app(app)
+    utils.migrate.init_app(app, models.db)
 
-# Base application
+    # Initialize debugging tools
-flask_bootstrap.Bootstrap(app)
+    if app.config.get("DEBUG"):
-db = flask_sqlalchemy.SQLAlchemy(app)
+        debug.toolbar.init_app(app)
-migrate = flask_migrate.Migrate(app, db)
+        # TODO: add a specific configuration variable for profiling
-limiter = flask_limiter.Limiter(app, key_func=lambda: current_user.username)
+        # debug.profiler.init_app(app)
 
-# Debugging toolbar
+    # Inject the default variables in the Jinja parser
-if app.config.get("DEBUG"):
+    # TODO: move this to blueprints when needed
-    import flask_debugtoolbar
+    @app.context_processor
-    toolbar = flask_debugtoolbar.DebugToolbarExtension(app)
+    def inject_defaults():
-
-# Profiler
-if app.config.get("DEBUG"):
-    app.wsgi_app = profiler.ProfilerMiddleware(app.wsgi_app, restrictions=[30])
-
-# Manager commnad
-manager = flask_script.Manager(app)
-manager.add_command('db', flask_migrate.MigrateCommand)
-
-# Babel configuration
-babel = flask_babel.Babel(app)
-translations = list(map(str, babel.list_translations()))
-
-@babel.localeselector
-def get_locale():
-    return flask.request.accept_languages.best_match(translations)
-
-# Login configuration
-login_manager = flask_login.LoginManager()
-login_manager.init_app(app)
-login_manager.login_view = "ui.login"
-
-@login_manager.unauthorized_handler
-def handle_needs_login():
-    return flask.redirect(
-        flask.url_for('ui.login', next=flask.request.endpoint)
-    )
-
-@app.context_processor
-def inject_defaults():
         signup_domains = models.Domain.query.filter_by(signup_enabled=True).all()
         return dict(
-        current_user=flask_login.current_user,
             signup_domains=signup_domains,
             config=app.config
         )
 
-# Import views
+    # Import views
-from mailu import ui, internal
+    from mailu import ui, internal
-app.register_blueprint(ui.ui, url_prefix='/ui')
+    app.register_blueprint(ui.ui, url_prefix='/ui')
-app.register_blueprint(internal.internal, url_prefix='/internal')
+    app.register_blueprint(internal.internal, url_prefix='/internal')
 
-# Create the prefix middleware
+    return app
-class PrefixMiddleware(object):
-
-    def __init__(self, app):
-        self.app = app
-
-    def __call__(self, environ, start_response):
-        prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '')
-        if prefix:
-            environ['SCRIPT_NAME'] = prefix
-        return self.app(environ, start_response)
 
 
-app.wsgi_app = PrefixMiddleware(fixers.ProxyFix(app.wsgi_app))
+def create_app():
+    """ Create a new application based on the config module 
+    """
+    config = configuration.ConfigManager()
+    return create_app_from_config(config)
+

core/admin/mailu/configuration.py

@@ -0,0 +1,90 @@
+import os
+
+
+DEFAULT_CONFIG = {
+    # Specific to the admin UI
+    'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db',
+    'SQLALCHEMY_TRACK_MODIFICATIONS': False,
+    'DOCKER_SOCKET': 'unix:///var/run/docker.sock',
+    'BABEL_DEFAULT_LOCALE': 'en',
+    'BABEL_DEFAULT_TIMEZONE': 'UTC',
+    'BOOTSTRAP_SERVE_LOCAL': True,
+    'RATELIMIT_STORAGE_URL': 'redis://redis/2',
+    'QUOTA_STORAGE_URL': 'redis://redis/1',
+    'DEBUG': False,
+    'DOMAIN_REGISTRATION': False,
+    'TEMPLATES_AUTO_RELOAD': True,
+    # Statistics management
+    'INSTANCE_ID_PATH': '/data/instance',
+    'STATS_ENDPOINT': '0.{}.stats.mailu.io',
+    # Common configuration variables
+    'SECRET_KEY': 'changeMe',
+    'DOMAIN': 'mailu.io',
+    'HOSTNAMES': 'mail.mailu.io,alternative.mailu.io,yetanother.mailu.io',
+    'POSTMASTER': 'postmaster',
+    'TLS_FLAVOR': 'cert',
+    'AUTH_RATELIMIT': '10/minute;1000/hour',
+    'DISABLE_STATISTICS': 'False',
+    # Mail settings
+    'DMARC_RUA': None,
+    'DMARC_RUF': None,
+    'WELCOME': 'False',
+    'WELCOME_SUBJECT': 'Dummy welcome topic',
+    'WELCOME_BODY': 'Dummy welcome body',
+    'DKIM_SELECTOR': 'dkim',
+    'DKIM_PATH': '/dkim/{domain}.{selector}.key',
+    'DEFAULT_QUOTA': 1000000000,
+    # Web settings
+    'SITENAME': 'Mailu',
+    'WEBSITE': 'https://mailu.io',
+    'WEB_ADMIN': '/admin',
+    'WEB_WEBMAIL': '/webmail',
+    'RECAPTCHA_PUBLIC_KEY': '',
+    'RECAPTCHA_PRIVATE_KEY': '',
+    # Advanced settings
+    'PASSWORD_SCHEME': 'BLF-CRYPT',
+    # Host settings
+    'HOST_IMAP': 'imap',
+    'HOST_POP3': 'imap',
+    'HOST_SMTP': 'smtp',
+    'HOST_WEBMAIL': 'webmail',
+    'HOST_FRONT': 'front',
+    'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'),
+    'POD_ADDRESS_RANGE': None
+}
+
+
+class ConfigManager(dict):
+    """ Naive configuration manager that uses environment only
+    """
+
+    def __init__(self):
+        self.config = dict()
+
+    def init_app(self, app):
+        self.config.update(app.config)
+        self.config.update({
+            key: os.environ.get(key, value)
+            for key, value in DEFAULT_CONFIG.items()
+        })
+        app.config = self
+
+    def setdefault(self, key, value):
+        if key not in self.config:
+            self.config[key] = value
+        return self.config[key]
+
+    def get(self, *args):
+        return self.config.get(*args)
+
+    def keys(self):
+        return self.config.keys()
+
+    def __getitem__(self, key):
+        return self.config.get(key)
+
+    def __setitem__(self, key, value):
+        self.config[key] = value
+
+    def __contains__(self, key):
+        return key in self.config

core/admin/mailu/debug.py

@@ -0,0 +1,17 @@
+import flask_debugtoolbar
+
+from werkzeug.contrib import profiler as werkzeug_profiler
+
+
+# Debugging toolbar
+toolbar = flask_debugtoolbar.DebugToolbarExtension()
+
+
+# Profiler
+class Profiler(object):
+    def init_app(self, app):
+        app.wsgi_app = werkzeug_profiler.ProfilerMiddleware(
+            app.wsgi_app, restrictions=[30]
+        )
+
+profiler = Profiler()

core/admin/mailu/dockercli.py

@@ -1,26 +0,0 @@
-from mailu import app
-
-import docker
-import signal
-
-
-# Connect to the Docker socket
-cli = docker.Client(base_url=app.config['DOCKER_SOCKET'])
-
-
-def get(*names):
-    result = {}
-    all_containers = cli.containers(all=True)
-    for brief in all_containers:
-        if brief['Image'].startswith('mailu/'):
-            container = cli.inspect_container(brief['Id'])
-            container['Image'] = cli.inspect_image(container['Image'])
-            name = container['Config']['Labels']['com.docker.compose.service']
-            if not names or name in names:
-                result[name] = container
-    return result
-
-
-def reload(*names):
-    for name, container in get(*names).items():
-        cli.kill(container["Id"], signal.SIGHUP.value)

core/admin/mailu/internal/__init__.py

@@ -1,6 +1,6 @@
 from flask_limiter import RateLimitExceeded
 
-from mailu import limiter
+from mailu import utils
 
 import socket
 import flask
@@ -19,7 +19,7 @@ def rate_limit_handler(e):
     return response
 
 
-@limiter.request_filter
+@utils.limiter.request_filter
 def whitelist_webmail():
     try:
         return flask.request.headers["Client-Ip"] ==\

core/admin/mailu/internal/nginx.py

@@ -1,4 +1,5 @@
-from mailu import db, models, app
+from mailu import models
+from flask import current_app as app
 
 import re
 import socket

core/admin/mailu/internal/views/auth.py

@@ -1,5 +1,6 @@
-from mailu import db, models, app, limiter
+from mailu import models, utils
 from mailu.internal import internal, nginx
+from flask import current_app as app
 
 import flask
 import flask_login
@@ -7,7 +8,7 @@ import base64
 
 
 @internal.route("/auth/email")
-@limiter.limit(
+@utils.limiter.limit(
     app.config["AUTH_RATELIMIT"],
     lambda: flask.request.headers["Client-Ip"]
 )

core/admin/mailu/internal/views/dovecot.py

@@ -1,5 +1,6 @@
-from mailu import db, models, app
+from mailu import models
 from mailu.internal import internal
+from flask import current_app as app
 
 import flask
 import socket
@@ -36,7 +37,7 @@ def dovecot_quota(ns, user_email):
     user = models.User.query.get(user_email) or flask.abort(404)
     if ns == "storage":
         user.quota_bytes_used = flask.request.get_json()
-        db.session.commit()
+        models.db.session.commit()
     return flask.jsonify(None)
 
 

core/admin/mailu/internal/views/fetch.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.internal import internal
 
 import flask
@@ -27,6 +27,6 @@ def fetch_done(fetch_id):
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     fetch.last_check = datetime.datetime.now()
     fetch.error_message = str(flask.request.get_json())
-    db.session.add(fetch)
+    models.db.session.add(fetch)
-    db.session.commit()
+    models.db.session.commit()
     return ""

core/admin/mailu/internal/views/postfix.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.internal import internal
 
 import flask

core/admin/mailu/manage.py

@@ -1,11 +1,26 @@
-from mailu import app, manager, db, models
+from mailu import models
 
+from flask import current_app as app
+from flask import cli as flask_cli
+
+import flask
 import os
 import socket
 import uuid
+import click
 
 
-@manager.command
+db = models.db
+
+
+@click.group()
+def mailu(cls=flask_cli.FlaskGroup):
+    """ Mailu command line
+    """
+
+
+@mailu.command()
+@flask_cli.with_appcontext
 def advertise():
     """ Advertise this server against statistic services.
     """
@@ -23,7 +38,11 @@ def advertise():
             pass
 
 
-@manager.command
+@mailu.command()
+@click.argument('localpart')
+@click.argument('domain_name')
+@click.argument('password')
+@flask_cli.with_appcontext
 def admin(localpart, domain_name, password):
     """ Create an admin user
     """
@@ -41,11 +60,17 @@ def admin(localpart, domain_name, password):
     db.session.commit()
 
 
-@manager.command
+@mailu.command()
-def user(localpart, domain_name, password,
+@click.argument('localpart')
-         hash_scheme=app.config['PASSWORD_SCHEME']):
+@click.argument('domain_name')
+@click.argument('password')
+@click.argument('hash_scheme')
+@flask_cli.with_appcontext
+def user(localpart, domain_name, password, hash_scheme=None):
     """ Create a user
     """
+    if hash_scheme is None:
+        hash_scheme = app.config['PASSWORD_SCHEME']
     domain = models.Domain.query.get(domain_name)
     if not domain:
         domain = models.Domain(name=domain_name)
@@ -60,10 +85,12 @@ def user(localpart, domain_name, password,
     db.session.commit()
 
 
-@manager.option('-n', '--domain_name', dest='domain_name')
+@mailu.command()
-@manager.option('-u', '--max_users', dest='max_users')
+@click.option('-n', '--domain_name')
-@manager.option('-a', '--max_aliases', dest='max_aliases')
+@click.option('-u', '--max_users')
-@manager.option('-q', '--max_quota_bytes', dest='max_quota_bytes')
+@click.option('-a', '--max_aliases')
+@click.option('-q', '--max_quota_bytes')
+@flask_cli.with_appcontext
 def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0):
     domain = models.Domain.query.get(domain_name)
     if not domain:
@@ -72,15 +99,17 @@ def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0):
         db.session.commit()
 
 
-@manager.command
+@mailu.command()
-def user_import(localpart, domain_name, password_hash,
+@click.argument('localpart')
-                hash_scheme=app.config['PASSWORD_SCHEME']):
+@click.argument('domain_name')
-    """ Import a user along with password hash. Available hashes:
+@click.argument('password_hash')
-                   'SHA512-CRYPT'
+@click.argument('hash_scheme')
-                   'SHA256-CRYPT'
+@flask_cli.with_appcontext
-                   'MD5-CRYPT'
+def user_import(localpart, domain_name, password_hash, hash_scheme = None):
-                   'CRYPT'
+    """ Import a user along with password hash.
     """
+    if hash_scheme is None:
+        hash_scheme = app.config['PASSWORD_SCHEME']
     domain = models.Domain.query.get(domain_name)
     if not domain:
         domain = models.Domain(name=domain_name)
@@ -95,7 +124,10 @@ def user_import(localpart, domain_name, password_hash,
     db.session.commit()
 
 
-@manager.command
+@mailu.command()
+@click.option('-v', '--verbose')
+@click.option('-d', '--delete_objects')
+@flask_cli.with_appcontext
 def config_update(verbose=False, delete_objects=False):
     """sync configuration with data from YAML-formatted stdin"""
     import yaml
@@ -234,7 +266,9 @@ def config_update(verbose=False, delete_objects=False):
     db.session.commit()
 
 
-@manager.command
+@mailu.command()
+@click.argument('email')
+@flask_cli.with_appcontext
 def user_delete(email):
     """delete user"""
     user = models.User.query.get(email)
@@ -243,7 +277,9 @@ def user_delete(email):
     db.session.commit()
 
 
-@manager.command
+@mailu.command()
+@click.argument('email')
+@flask_cli.with_appcontext
 def alias_delete(email):
     """delete alias"""
     alias = models.Alias.query.get(email)
@@ -252,7 +288,11 @@ def alias_delete(email):
     db.session.commit()
 
 
-@manager.command
+@mailu.command()
+@click.argument('localpart')
+@click.argument('domain_name')
+@click.argument('destination')
+@flask_cli.with_appcontext
 def alias(localpart, domain_name, destination):
     """ Create an alias
     """
@@ -269,24 +309,31 @@ def alias(localpart, domain_name, destination):
     db.session.add(alias)
     db.session.commit()
 
-# Set limits to a domain
 
-
+@mailu.command()
-@manager.command
+@click.argument('domain_name')
+@click.argument('max_users')
+@click.argument('max_aliases')
+@click.argument('max_quota_bytes')
+@flask_cli.with_appcontext
 def setlimits(domain_name, max_users, max_aliases, max_quota_bytes):
+    """ Set domain limits
+    """
     domain = models.Domain.query.get(domain_name)
     domain.max_users = max_users
     domain.max_aliases = max_aliases
     domain.max_quota_bytes = max_quota_bytes
-
     db.session.add(domain)
     db.session.commit()
 
-# Make the user manager of a domain
 
-
+@mailu.command()
-@manager.command
+@click.argument('domain_name')
+@click.argument('user_name')
+@flask_cli.with_appcontext
 def setmanager(domain_name, user_name='manager'):
+    """ Make a user manager of a domain
+    """
     domain = models.Domain.query.get(domain_name)
     manageruser = models.User.query.get(user_name + '@' + domain_name)
     domain.managers.append(manageruser)
@@ -294,5 +341,5 @@ def setmanager(domain_name, user_name='manager'):
     db.session.commit()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
-    manager.run()
+    cli()

core/admin/mailu/models.py

@@ -1,10 +1,12 @@
-from mailu import app, db, dkim, login_manager
+from mailu import dkim
 
 from sqlalchemy.ext import declarative
 from passlib import context, hash
 from datetime import datetime, date
 from email.mime import text
+from flask import current_app as app
 
+import flask_sqlalchemy
 import sqlalchemy
 import re
 import time
@@ -15,6 +17,9 @@ import idna
 import dns
 
 
+db = flask_sqlalchemy.SQLAlchemy()
+
+
 class IdnaDomain(db.TypeDecorator):
     """ Stores a Unicode string in it's IDNA representation (ASCII only)
     """
@@ -70,6 +75,27 @@ class CommaSeparatedList(db.TypeDecorator):
         return filter(bool, value.split(",")) if value else []
 
 
+class JSONEncoded(db.TypeDecorator):
+    """Represents an immutable structure as a json-encoded string.
+    """
+
+    impl = db.String
+
+    def process_bind_param(self, value, dialect):
+        return json.dumps(value) if value else None
+
+    def process_result_value(self, value, dialect):
+        return json.loads(value) if value else None
+
+
+class Config(db.Model):
+    """ In-database configuration values
+    """
+
+    name = db.Column(db.String(255), primary_key=True, nullable=False)
+    value = db.Column(JSONEncoded)
+
+
 # Many-to-many association table for domain managers
 managers = db.Table('manager',
     db.Column('domain_name', IdnaDomain, db.ForeignKey('domain.name')),
@@ -318,13 +344,15 @@ class User(Base, Email):
                    'SHA256-CRYPT': "sha256_crypt",
                    'MD5-CRYPT': "md5_crypt",
                    'CRYPT': "des_crypt"}
-    pw_context = context.CryptContext(
+
-        schemes = scheme_dict.values(),
+    def get_password_context(self):
-        default=scheme_dict[app.config['PASSWORD_SCHEME']],
+        return context.CryptContext(
+            schemes=self.scheme_dict.values(),
+            default=self.scheme_dict[app.config['PASSWORD_SCHEME']],
         )
 
     def check_password(self, password):
-        context = User.pw_context
+        context = self.get_password_context()
         reference = re.match('({[^}]+})?(.*)', self.password).group(2)
         result = context.verify(password, reference)
         if result and context.identify(reference) != context.default_scheme():
@@ -333,15 +361,17 @@ class User(Base, Email):
             db.session.commit()
         return result
 
-    def set_password(self, password, hash_scheme=app.config['PASSWORD_SCHEME'], raw=False):
+    def set_password(self, password, hash_scheme=None, raw=False):
         """Set password for user with specified encryption scheme
            @password: plain text password to encrypt (if raw == True the hash itself)
         """
+        if hash_scheme is None:
+            hash_scheme = app.config['PASSWORD_SCHEME']
         # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes
         if raw:
             self.password = '{'+hash_scheme+'}' + password
         else:
-            self.password = '{'+hash_scheme+'}' + User.pw_context.encrypt(password, self.scheme_dict[hash_scheme])
+            self.password = '{'+hash_scheme+'}' + self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme])
 
     def get_managed_domains(self):
         if self.global_admin:
@@ -362,13 +392,15 @@ class User(Base, Email):
             self.sendmail(app.config["WELCOME_SUBJECT"],
                 app.config["WELCOME_BODY"])
 
+    @classmethod
+    def get(cls, email):
+        return cls.query.get(email)
+
     @classmethod
     def login(cls, email, password):
         user = cls.query.get(email)
         return user if (user and user.enabled and user.check_password(password)) else None
 
-login_manager.user_loader(User.query.get)
-
 
 class Alias(Base, Email):
     """ An alias is an email address that redirects to some destination.

core/admin/mailu/ui/access.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.ui import forms
 
 import flask

core/admin/mailu/ui/views/admins.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.ui import ui, forms, access
 
 import flask
@@ -25,7 +25,7 @@ def admin_create():
         user = models.User.query.get(form.admin.data)
         if user:
             user.global_admin = True
-            db.session.commit()
+            models.db.session.commit()
             flask.flash('User %s is now admin' % user)
             return flask.redirect(flask.url_for('.admin_list'))
         else:
@@ -40,7 +40,7 @@ def admin_delete(admin):
     user = models.User.query.get(admin)
     if user:
         user.global_admin  = False
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('User %s is no longer admin' % user)
         return flask.redirect(flask.url_for('.admin_list'))
     else:

core/admin/mailu/ui/views/aliases.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.ui import ui, forms, access
 
 import flask
@@ -27,8 +27,8 @@ def alias_create(domain_name):
         else:
             alias = models.Alias(domain=domain)
             form.populate_obj(alias)
-            db.session.add(alias)
+            models.db.session.add(alias)
-            db.session.commit()
+            models.db.session.commit()
             flask.flash('Alias %s created' % alias)
             return flask.redirect(
                 flask.url_for('.alias_list', domain_name=domain.name))
@@ -45,7 +45,7 @@ def alias_edit(alias):
     form.localpart.validators = []
     if form.validate_on_submit():
         form.populate_obj(alias)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Alias %s updated' % alias)
         return flask.redirect(
             flask.url_for('.alias_list', domain_name=alias.domain.name))
@@ -59,8 +59,8 @@ def alias_edit(alias):
 def alias_delete(alias):
     alias = models.Alias.query.get(alias) or flask.abort(404)
     domain = alias.domain
-    db.session.delete(alias)
+    models.db.session.delete(alias)
-    db.session.commit()
+    models.db.session.commit()
     flask.flash('Alias %s deleted' % alias)
     return flask.redirect(
         flask.url_for('.alias_list', domain_name=domain.name))

core/admin/mailu/ui/views/alternatives.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.ui import ui, forms, access
 
 import flask
@@ -26,8 +26,8 @@ def alternative_create(domain_name):
         else:
             alternative = models.Alternative(domain=domain)
             form.populate_obj(alternative)
-            db.session.add(alternative)
+            models.db.session.add(alternative)
-            db.session.commit()
+            models.db.session.commit()
             flask.flash('Alternative domain %s created' % alternative)
             return flask.redirect(
                 flask.url_for('.alternative_list', domain_name=domain.name))
@@ -41,8 +41,8 @@ def alternative_create(domain_name):
 def alternative_delete(alternative):
     alternative = models.Alternative.query.get(alternative) or flask.abort(404)
     domain = alternative.domain
-    db.session.delete(alternative)
+    models.db.session.delete(alternative)
-    db.session.commit()
+    models.db.session.commit()
     flask.flash('Alternative %s deleted' % alternative)
     return flask.redirect(
         flask.url_for('.alternative_list', domain_name=domain.name))

core/admin/mailu/ui/views/base.py

@@ -1,11 +1,9 @@
-from mailu import dockercli, app, db, models
+from mailu import models
 from mailu.ui import ui, forms, access
 
 import flask
 import flask_login
 
-from urllib import parse
-
 
 @ui.route('/', methods=["GET"])
 @access.authenticated

core/admin/mailu/ui/views/domains.py

@@ -1,5 +1,6 @@
-from mailu import app, db, models
+from mailu import models
 from mailu.ui import ui, forms, access
+from flask import current_app as app
 
 import flask
 import flask_login
@@ -26,8 +27,8 @@ def domain_create():
         else:
             domain = models.Domain()
             form.populate_obj(domain)
-            db.session.add(domain)
+            models.db.session.add(domain)
-            db.session.commit()
+            models.db.session.commit()
             flask.flash('Domain %s created' % domain)
             return flask.redirect(flask.url_for('.domain_list'))
     return flask.render_template('domain/create.html', form=form)
@@ -42,7 +43,7 @@ def domain_edit(domain_name):
     form.name.validators = []
     if form.validate_on_submit():
         form.populate_obj(domain)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Domain %s saved' % domain)
         return flask.redirect(flask.url_for('.domain_list'))
     return flask.render_template('domain/edit.html', form=form,
@@ -54,8 +55,8 @@ def domain_edit(domain_name):
 @access.confirmation_required("delete {domain_name}")
 def domain_delete(domain_name):
     domain = models.Domain.query.get(domain_name) or flask.abort(404)
-    db.session.delete(domain)
+    models.db.session.delete(domain)
-    db.session.commit()
+    models.db.session.commit()
     flask.flash('Domain %s deleted' % domain)
     return flask.redirect(flask.url_for('.domain_list'))
 
@@ -99,7 +100,7 @@ def domain_signup(domain_name=None):
             domain.max_users = 10
             domain.max_aliases = 10
             if domain.check_mx():
-                db.session.add(domain)
+                models.db.session.add(domain)
                 if flask_login.current_user.is_authenticated:
                     user = models.User.query.get(flask_login.current_user.email)
                 else:
@@ -108,9 +109,9 @@ def domain_signup(domain_name=None):
                     form.populate_obj(user)
                     user.set_password(form.pw.data)
                     user.quota_bytes = domain.max_quota_bytes
-                db.session.add(user)
+                models.db.session.add(user)
                 domain.managers.append(user)
-                db.session.commit()
+                models.db.session.commit()
                 flask.flash('Domain %s created' % domain)
                 return flask.redirect(flask.url_for('.domain_list'))
             else:

core/admin/mailu/ui/views/fetches.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.ui import ui, forms, access
 
 import flask
@@ -24,8 +24,8 @@ def fetch_create(user_email):
     if form.validate_on_submit():
         fetch = models.Fetch(user=user)
         form.populate_obj(fetch)
-        db.session.add(fetch)
+        models.db.session.add(fetch)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Fetch configuration created')
         return flask.redirect(
             flask.url_for('.fetch_list', user_email=user.email))
@@ -39,7 +39,7 @@ def fetch_edit(fetch_id):
     form = forms.FetchForm(obj=fetch)
     if form.validate_on_submit():
         form.populate_obj(fetch)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Fetch configuration updated')
         return flask.redirect(
             flask.url_for('.fetch_list', user_email=fetch.user.email))
@@ -53,8 +53,8 @@ def fetch_edit(fetch_id):
 def fetch_delete(fetch_id):
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     user = fetch.user
-    db.session.delete(fetch)
+    models.db.session.delete(fetch)
-    db.session.commit()
+    models.db.session.commit()
     flask.flash('Fetch configuration delete')
     return flask.redirect(
         flask.url_for('.fetch_list', user_email=user.email))

core/admin/mailu/ui/views/managers.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.ui import ui, forms, access
 
 import flask
@@ -30,7 +30,7 @@ def manager_create(domain_name):
             flask.flash('User %s is already manager' % user, 'error')
         else:
             domain.managers.append(user)
-            db.session.commit()
+            models.db.session.commit()
             flask.flash('User %s can now manage %s' % (user, domain.name))
             return flask.redirect(
                 flask.url_for('.manager_list', domain_name=domain.name))
@@ -46,7 +46,7 @@ def manager_delete(domain_name, user_email):
     user = models.User.query.get(user_email) or flask.abort(404)
     if user in domain.managers:
         domain.managers.remove(user)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('User %s can no longer manager %s' % (user, domain))
     else:
         flask.flash('User %s is not manager' % user, 'error')

core/admin/mailu/ui/views/relays.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.ui import ui, forms, access
 
 import flask
@@ -25,8 +25,8 @@ def relay_create():
         else:
             relay = models.Relay()
             form.populate_obj(relay)
-            db.session.add(relay)
+            models.db.session.add(relay)
-            db.session.commit()
+            models.db.session.commit()
             flask.flash('Relayed domain %s created' % relay)
             return flask.redirect(flask.url_for('.relay_list'))
     return flask.render_template('relay/create.html', form=form)
@@ -41,7 +41,7 @@ def relay_edit(relay_name):
     form.name.validators = []
     if form.validate_on_submit():
         form.populate_obj(relay)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Relayed domain %s saved' % relay)
         return flask.redirect(flask.url_for('.relay_list'))
     return flask.render_template('relay/edit.html', form=form,
@@ -53,8 +53,8 @@ def relay_edit(relay_name):
 @access.confirmation_required("delete {relay_name}")
 def relay_delete(relay_name):
     relay = models.Relay.query.get(relay_name) or flask.abort(404)
-    db.session.delete(relay)
+    models.db.session.delete(relay)
-    db.session.commit()
+    models.db.session.commit()
     flask.flash('Relayed domain %s deleted' % relay)
     return flask.redirect(flask.url_for('.relay_list'))
 

core/admin/mailu/ui/views/tokens.py

@@ -1,4 +1,4 @@
-from mailu import db, models
+from mailu import models
 from mailu.ui import ui, forms, access
 
 from passlib import pwd
@@ -32,8 +32,8 @@ def token_create(user_email):
         token = models.Token(user=user)
         token.set_password(form.raw_password.data)
         form.populate_obj(token)
-        db.session.add(token)
+        models.db.session.add(token)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Authentication token created')
         return flask.redirect(
             flask.url_for('.token_list', user_email=user.email))
@@ -46,8 +46,8 @@ def token_create(user_email):
 def token_delete(token_id):
     token = models.Token.query.get(token_id) or flask.abort(404)
     user = token.user
-    db.session.delete(token)
+    models.db.session.delete(token)
-    db.session.commit()
+    models.db.session.commit()
     flask.flash('Authentication token deleted')
     return flask.redirect(
         flask.url_for('.token_list', user_email=user.email))

core/admin/mailu/ui/views/users.py

@@ -1,5 +1,6 @@
-from mailu import db, models, app
+from mailu import models
 from mailu.ui import ui, access, forms
+from flask import current_app as app
 
 import flask
 import flask_login
@@ -33,8 +34,8 @@ def user_create(domain_name):
             user = models.User(domain=domain)
             form.populate_obj(user)
             user.set_password(form.pw.data)
-            db.session.add(user)
+            models.db.session.add(user)
-            db.session.commit()
+            models.db.session.commit()
             user.send_welcome()
             flask.flash('User %s created' % user)
             return flask.redirect(
@@ -63,7 +64,7 @@ def user_edit(user_email):
         form.populate_obj(user)
         if form.pw.data:
             user.set_password(form.pw.data)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('User %s updated' % user)
         return flask.redirect(
             flask.url_for('.user_list', domain_name=user.domain.name))
@@ -77,8 +78,8 @@ def user_edit(user_email):
 def user_delete(user_email):
     user = models.User.query.get(user_email) or flask.abort(404)
     domain = user.domain
-    db.session.delete(user)
+    models.db.session.delete(user)
-    db.session.commit()
+    models.db.session.commit()
     flask.flash('User %s deleted' % user)
     return flask.redirect(
         flask.url_for('.user_list', domain_name=domain.name))
@@ -93,7 +94,7 @@ def user_settings(user_email):
     form = forms.UserSettingsForm(obj=user)
     if form.validate_on_submit():
         form.populate_obj(user)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Settings updated for %s' % user)
         if user_email:
             return flask.redirect(
@@ -113,7 +114,7 @@ def user_password(user_email):
             flask.flash('Passwords do not match', 'error')
         else:
             user.set_password(form.pw.data)
-            db.session.commit()
+            models.db.session.commit()
             flask.flash('Password updated for %s' % user)
             if user_email:
                 return flask.redirect(flask.url_for('.user_list',
@@ -130,7 +131,7 @@ def user_forward(user_email):
     form = forms.UserForwardForm(obj=user)
     if form.validate_on_submit():
         form.populate_obj(user)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Forward destination updated for %s' % user)
         if user_email:
             return flask.redirect(
@@ -147,7 +148,7 @@ def user_reply(user_email):
     form = forms.UserReplyForm(obj=user)
     if form.validate_on_submit():
         form.populate_obj(user)
-        db.session.commit()
+        models.db.session.commit()
         flask.flash('Auto-reply message updated for %s' % user)
         if user_email:
             return flask.redirect(
@@ -183,8 +184,8 @@ def user_signup(domain_name=None):
             form.populate_obj(user)
             user.set_password(form.pw.data)
             user.quota_bytes = quota_bytes
-            db.session.add(user)
+            models.db.session.add(user)
-            db.session.commit()
+            models.db.session.commit()
             user.send_welcome()
             flask.flash('Successfully signed up %s' % user)
             return flask.redirect(flask.url_for('.index'))

core/admin/mailu/utils.py

@@ -0,0 +1,53 @@
+from mailu import models
+
+import flask
+import flask_login
+import flask_script
+import flask_migrate
+import flask_babel
+import flask_limiter
+
+from werkzeug.contrib import fixers
+
+
+# Login configuration
+login = flask_login.LoginManager()
+login.login_view = "ui.login"
+
+@login.unauthorized_handler
+def handle_needs_login():
+    return flask.redirect(
+        flask.url_for('ui.login', next=flask.request.endpoint)
+    )
+
+
+# Request rate limitation
+limiter = flask_limiter.Limiter(key_func=lambda: current_user.username)
+
+
+# Application translation
+babel = flask_babel.Babel()
+
+@babel.localeselector
+def get_locale():
+    translations = list(map(str, babel.list_translations()))
+    return flask.request.accept_languages.best_match(translations)
+
+
+# Proxy fixer
+class PrefixMiddleware(object):
+    def __call__(self, environ, start_response):
+        prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '')
+        if prefix:
+            environ['SCRIPT_NAME'] = prefix
+        return self.app(environ, start_response)
+
+    def init_app(self, app):
+        self.app = fixers.ProxyFix(app.wsgi_app)
+        app.wsgi_app = self
+
+proxy = PrefixMiddleware()
+
+
+# Data migrate
+migrate = flask_migrate.Migrate()

core/admin/migrations/versions/3f6994568962_.py

@@ -13,8 +13,6 @@ down_revision = '2335c80a6bc3'
 from alembic import op
 import sqlalchemy as sa
 
-from mailu import app
-
 
 fetch_table = sa.Table(
     'fetch',
@@ -24,13 +22,7 @@ fetch_table = sa.Table(
 
 
 def upgrade():
-    connection = op.get_bind()
     op.add_column('fetch', sa.Column('keep', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
-    # also apply the current config value if set
-    if app.config.get("FETCHMAIL_KEEP", "False") == "True":
-        connection.execute(
-            fetch_table.update().values(keep=True)
-        )
 
 
 def downgrade():

core/admin/migrations/versions/cd79ed46d9c2_.py

@@ -0,0 +1,25 @@
+""" Add a configuration table
+
+Revision ID: cd79ed46d9c2
+Revises: 25fd6c7bcb4a
+Create Date: 2018-10-17 21:44:48.924921
+
+"""
+
+revision = 'cd79ed46d9c2'
+down_revision = '3b281286c7bd'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('config',
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.Column('value', sa.String(length=255), nullable=True),
+    sa.PrimaryKeyConstraint('name')
+    )
+
+
+def downgrade():
+    op.drop_table('config')

core/admin/requirements-prod.txt

@@ -1,53 +1,46 @@
-alembic==0.9.9
+alembic==1.0.2
 asn1crypto==0.24.0
-Babel==2.5.3
+Babel==2.6.0
 bcrypt==3.1.4
 blinker==1.4
-certifi==2018.4.16
 cffi==1.11.5
-chardet==3.0.4
+Click==7.0
-click==6.7
+cryptography==2.3.1
-cryptography==2.2.2
 decorator==4.3.0
 dnspython==1.15.0
-docker-py==1.10.6
+dominate==2.3.4
-docker-pycreds==0.2.2
+Flask==1.0.2
-dominate==2.3.1
+Flask-Babel==0.12.2
-Flask==0.12.2
-Flask-Babel==0.11.2
 Flask-Bootstrap==3.3.7.1
 Flask-DebugToolbar==0.10.1
 Flask-Limiter==1.0.1
 Flask-Login==0.4.1
-Flask-Migrate==2.1.1
+Flask-Migrate==2.3.0
 Flask-Script==2.0.6
 Flask-SQLAlchemy==2.3.2
 Flask-WTF==0.14.2
-gunicorn==19.7.1
+gunicorn==19.9.0
-idna==2.6
+idna==2.7
 infinity==1.4
 intervals==0.8.1
-itsdangerous==0.24
+itsdangerous==1.1.0
 Jinja2==2.10
 limits==1.3
 Mako==1.0.7
-MarkupSafe==1.0
+MarkupSafe==1.1.0
 passlib==1.7.1
-pycparser==2.18
+pycparser==2.19
-pyOpenSSL==17.5.0
+pyOpenSSL==18.0.0
-python-dateutil==2.7.2
+python-dateutil==2.7.5
 python-editor==1.0.3
-pytz==2018.4
+pytz==2018.7
-PyYAML==3.12
+PyYAML==3.13
 redis==2.10.6
-requests==2.18.4
 six==1.11.0
-SQLAlchemy==1.2.6
+SQLAlchemy==1.2.13
 tabulate==0.8.2
-urllib3==1.22
+validators==0.12.2
-validators==0.12.1
 visitor==0.1.3
-websocket-client==0.47.0
 Werkzeug==0.14.1
-WTForms==2.1
+WTForms==2.2.1
 WTForms-Components==0.10.3

core/admin/requirements.txt

@@ -12,7 +12,6 @@ redis
 WTForms-Components
 passlib
 gunicorn
-docker-py
 tabulate
 PyYAML
 PyOpenSSL

core/admin/run.py

@@ -1,7 +0,0 @@
-import os
-
-
-if __name__ == "__main__":
-    os.environ["DEBUG"] = "True"
-    from mailu import app
-    app.run()

core/admin/start.py

@@ -2,6 +2,6 @@
 
 import os
 
-os.system("python3 manage.py advertise")
+os.system("flask mailu advertise")
-os.system("python3 manage.py db upgrade")
+os.system("flask db upgrade")
-os.system("gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload mailu:app")
+os.system("gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload 'mailu:create_app()'")

docs/cli.rst

@@ -15,7 +15,7 @@ alias
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py alias foo example.net "mail1@example.com,mail2@example.com"
+  docker-compose exec admin flask mailu alias foo example.net "mail1@example.com,mail2@example.com"
 
 
 alias_delete
@@ -23,14 +23,14 @@ alias_delete
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py alias_delete foo@example.net
+  docker-compose exec admin flask mailu alias_delete foo@example.net
 
 user
 ----
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py user --hash_scheme='SHA512-CRYPT' myuser example.net 'password123'
+  docker-compose exec admin flask mailu user --hash_scheme='SHA512-CRYPT' myuser example.net 'password123'
 
 user_import
 -----------
@@ -39,14 +39,14 @@ primary difference with simple `user` command is that password is being imported
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py user_import --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce'
+  docker-compose run --rm admin python manage.py user --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce'
 
 user_delete
 ------------
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py user_delete foo@example.net
+  docker-compose exec admin flask mailu user_delete foo@example.net
 
 config_update
 -------------
@@ -55,7 +55,7 @@ The sole purpose of this command is for importing users/aliases in bulk and sync
 
 .. code-block:: bash
 
-  cat mail-config.yml | docker-compose run --rm admin python manage.py config_update --delete_objects
+  cat mail-config.yml | docker-compose exec admin flask mailu config_update --delete_objects
 
 where mail-config.yml looks like:
 

docs/compose/setup.rst

@@ -151,6 +151,6 @@ Finally, you must create the initial admin user account:
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py admin root example.net password
+  docker-compose exec admin flask mailu admin me example.net password
 
-This will create a user named ``root@example.net`` with password ``password`` and administration privileges. Connect to the Web admin interface and change the password to a strong one.
+This will create a user named ``me@example.net`` with password ``password`` and administration privileges. Connect to the Web admin interface and change the password to a strong one.

docs/contributors/database.rst

@@ -17,7 +17,7 @@ migration script:
 
 .. code-block:: bash
 
-  python manage.py db migrate
+  flask db migrate
 
 This will generate a new script in ``migrations/versions`` that you must review
 before adding it for commit.
@@ -54,7 +54,7 @@ At that point, to start working on the changed database structure, you will need
 
 .. code-block:: bash
 
-  python manage.py db upgrade
+  flask db upgrade
 
 If any error arises, restore the backup, fix the migration script and try again.