From 8b189ed1452fd80a4fa166d0ba6cfc007cdd18e0 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 7 Oct 2018 16:23:53 +0200 Subject: [PATCH 01/62] Separate senderaccess and senderlogin maps --- core/postfix/conf/main.cf | 4 ++-- core/postfix/start.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index cd052d46..12541cc5 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -80,14 +80,14 @@ lmtp_host_lookup = native smtpd_delay_reject = yes # Allowed senders are: the user or one of the alias destinations -smtpd_sender_login_maps = $virtual_alias_maps +smtpd_sender_login_maps = ${podop}senderlogin # Restrictions for incoming SMTP, other restrictions are applied in master.cf smtpd_helo_required = yes smtpd_client_restrictions = permit_mynetworks, - check_sender_access ${podop}sender, + check_sender_access ${podop}senderaccess, reject_non_fqdn_sender, reject_unknown_sender_domain, reject_unknown_recipient_domain, diff --git a/core/postfix/start.py b/core/postfix/start.py index 251f5b05..44ab1b26 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -17,7 +17,8 @@ def start_podop(): ("alias", "url", "http://admin/internal/postfix/alias/§"), ("domain", "url", "http://admin/internal/postfix/domain/§"), ("mailbox", "url", "http://admin/internal/postfix/mailbox/§"), - ("sender", "url", "http://admin/internal/postfix/sender/§") + ("senderaccess", "url", "http://admin/internal/postfix/sender/access/§"), + ("senderlogin", "url", "http://admin/internal/postfix/sender/login/§") ]) convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) From 508e519a34f75260673d6ba62e21307ec38dae82 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 7 Oct 2018 16:24:48 +0200 Subject: [PATCH 02/62] Refactor the postfix views and implement sender checks --- core/admin/mailu/internal/views/postfix.py | 43 ++++++++++------------ core/admin/mailu/models.py | 34 +++++++++++------ 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index 79fbdb8a..f0fbaa5d 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -18,37 +18,32 @@ def postfix_mailbox_map(email): @internal.route("/postfix/alias/") def postfix_alias_map(alias): - localpart, domain = alias.split('@', 1) if '@' in alias else (None, alias) - alternative = models.Alternative.query.get(domain) - if alternative: - domain = alternative.domain_name - email = '{}@{}'.format(localpart, domain) + localpart, domain_name = models.Email.resolve_domain(alias) if localpart is None: - return flask.jsonify(domain) - else: - alias_obj = models.Alias.resolve(localpart, domain) - if alias_obj: - return flask.jsonify(",".join(alias_obj.destination)) - user_obj = models.User.query.get(email) - if user_obj: - return flask.jsonify(user_obj.destination) - return flask.abort(404) + return flask.jsonify(domain_name) + destination = models.Email.resolve_destination(localpart, domain_name) + return flask.jsonify(",".join(destination)) if destination else flask.abort(404) @internal.route("/postfix/transport/") def postfix_transport(email): - localpart, domain = email.split('@', 1) if '@' in email else (None, email) - relay = models.Relay.query.get(domain) or flask.abort(404) + localpart, domain = models.Email.resolve_domain(email) + relay = models.Relay.query.get(domain_name) or flask.abort(404) return flask.jsonify("smtp:[{}]".format(relay.smtp)) -@internal.route("/postfix/sender/") -def postfix_sender(sender): +@internal.route("/postfix/sender/login/") +def postfix_sender_login(sender): + localpart, domain_name = models.Email.resolve_domain(sender) + if localpart is None: + return flask.abort(404) + destination = models.Email.resolve_destination(localpart, domain_name, True) + return flask.jsonify(",".join(destination)) if destination else flask.abort(404) + + +@internal.route("/postfix/sender/access/") +def postfix_sender_access(sender): """ Simply reject any sender that pretends to be from a local domain """ - localpart, domain_name = sender.split('@', 1) if '@' in sender else (None, sender) - domain = models.Domain.query.get(domain_name) - alternative = models.Alternative.query.get(domain_name) - if domain or alternative: - return flask.jsonify("REJECT") - return flask.abort(404) + localpart, domain_name = models.Email.resolve_domain(sender) + return flask.jsonify("REJECT") if models.Domain.query.get(domain_name) else flask.abort(404) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 1bcc4e9f..6685fc60 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -221,6 +221,28 @@ class Email(object): msg['To'] = to_address smtp.sendmail(from_address, [to_address], msg.as_string()) + @classmethod + def resolve_domain(cls, email): + localpart, domain_name = email.split('@', 1) if '@' in email else (None, email) + alternative = models.Alternative.query.get(domain_name) + if alternative: + domain_name = alternative.domain_name + return (localpart, domain_name) + + @classmethod + def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False): + alias = models.Alias.resolve(localpart, domain_name) + if alias: + return alias.destination + user = models.User.query.get('{}@{}'.format(localpart, domain_name)) + if user: + if user.forward_enabled: + destination = user.forward_destination + if user.forward_keep or ignore_forward_keep: + destination.append(user.email) + else: + destination = [user.email] + return destination def __str__(self): return self.email @@ -245,7 +267,7 @@ class User(Base, Email): # Filters forward_enabled = db.Column(db.Boolean(), nullable=False, default=False) - forward_destination = db.Column(db.String(255), nullable=True, default=None) + forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=None) forward_keep = db.Column(db.Boolean(), nullable=False, default=True) reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) reply_subject = db.Column(db.String(255), nullable=True, default=None) @@ -266,16 +288,6 @@ class User(Base, Email): def get_id(self): return self.email - @property - def destination(self): - if self.forward_enabled: - result = self.self.forward_destination - if self.forward_keep: - result += ',' + self.email - return result - else: - return self.email - scheme_dict = {'SHA512-CRYPT': "sha512_crypt", 'SHA256-CRYPT': "sha256_crypt", 'MD5-CRYPT': "md5_crypt", From e784556330c385ae73c39c009f62eb619af7a11f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 16 Oct 2018 20:47:38 +0200 Subject: [PATCH 03/62] Fix an edge case with old values containing None for coma separated lists --- core/admin/mailu/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6685fc60..63d0e4f9 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -64,7 +64,7 @@ class CommaSeparatedList(db.TypeDecorator): return ",".join(value) def process_result_value(self, value, dialect): - return filter(bool, value.split(",")) + return filter(bool, value.split(",")) if value else [] # Many-to-many association table for domain managers From fc244262919116dd0289e9046d227164eb180d5f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 18 Oct 2018 15:57:43 +0200 Subject: [PATCH 04/62] First batch of refactoring, using the app factory pattern --- core/admin/mailu/__init__.py | 154 +++-------- core/admin/mailu/configuration.py | 70 +++++ core/admin/mailu/debug.py | 17 ++ core/admin/mailu/internal/__init__.py | 4 +- core/admin/mailu/internal/nginx.py | 3 +- core/admin/mailu/internal/views/auth.py | 5 +- core/admin/mailu/internal/views/dovecot.py | 5 +- core/admin/mailu/internal/views/fetch.py | 6 +- core/admin/mailu/internal/views/postfix.py | 2 +- core/admin/mailu/manage.py | 305 +++++++++++++++++++++ core/admin/mailu/models.py | 30 +- core/admin/mailu/ui/access.py | 2 +- core/admin/mailu/ui/views/admins.py | 6 +- core/admin/mailu/ui/views/aliases.py | 12 +- core/admin/mailu/ui/views/alternatives.py | 8 +- core/admin/mailu/ui/views/base.py | 4 +- core/admin/mailu/ui/views/domains.py | 19 +- core/admin/mailu/ui/views/fetches.py | 12 +- core/admin/mailu/ui/views/managers.py | 6 +- core/admin/mailu/ui/views/relays.py | 12 +- core/admin/mailu/ui/views/tokens.py | 10 +- core/admin/mailu/ui/views/users.py | 25 +- core/admin/mailu/utils.py | 46 ++++ 23 files changed, 577 insertions(+), 186 deletions(-) create mode 100644 core/admin/mailu/configuration.py create mode 100644 core/admin/mailu/debug.py create mode 100644 core/admin/mailu/manage.py create mode 100644 core/admin/mailu/utils.py diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 167f04ae..874fa8e0 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -1,132 +1,58 @@ import flask -import flask_sqlalchemy import flask_bootstrap -import flask_login -import flask_script -import flask_migrate -import flask_babel -import flask_limiter import os import docker import socket import uuid -from werkzeug.contrib import fixers +from mailu import utils, debug, db -# Create application -app = flask.Flask(__name__) -default_config = { - # Specific to the admin UI - 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', - 'SQLALCHEMY_TRACK_MODIFICATIONS': False, - 'DOCKER_SOCKET': 'unix:///var/run/docker.sock', - 'BABEL_DEFAULT_LOCALE': 'en', - 'BABEL_DEFAULT_TIMEZONE': 'UTC', - 'BOOTSTRAP_SERVE_LOCAL': True, - 'RATELIMIT_STORAGE_URL': 'redis://redis/2', - 'QUOTA_STORAGE_URL': 'redis://redis/1', - 'DEBUG': False, - 'DOMAIN_REGISTRATION': False, - # Statistics management - 'INSTANCE_ID_PATH': '/data/instance', - 'STATS_ENDPOINT': '0.{}.stats.mailu.io', - # Common configuration variables - 'SECRET_KEY': 'changeMe', - 'DOMAIN': 'mailu.io', - 'HOSTNAMES': 'mail.mailu.io,alternative.mailu.io,yetanother.mailu.io', - 'POSTMASTER': 'postmaster', - 'TLS_FLAVOR': 'cert', - 'AUTH_RATELIMIT': '10/minute;1000/hour', - 'DISABLE_STATISTICS': 'False', - # Mail settings - 'DMARC_RUA': None, - 'DMARC_RUF': None, - 'WELCOME': 'False', - 'WELCOME_SUBJECT': 'Dummy welcome topic', - 'WELCOME_BODY': 'Dummy welcome body', - 'DKIM_SELECTOR': 'dkim', - 'DKIM_PATH': '/dkim/{domain}.{selector}.key', - 'DEFAULT_QUOTA': 1000000000, - # Web settings - 'SITENAME': 'Mailu', - 'WEBSITE': 'https://mailu.io', - 'WEB_ADMIN': '/admin', - 'WEB_WEBMAIL': '/webmail', - 'RECAPTCHA_PUBLIC_KEY': '', - 'RECAPTCHA_PRIVATE_KEY': '', - # Advanced settings - 'PASSWORD_SCHEME': 'BLF-CRYPT', - # Host settings - 'HOST_IMAP': 'imap', - 'HOST_POP3': 'imap', - 'HOST_SMTP': 'smtp', - 'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'), -} +def create_app_from_config(config): + """ Create a new application based on the given configuration + """ + app = flask.Flask(__name__) + app.config = config -# Load configuration from the environment if available -for key, value in default_config.items(): - app.config[key] = os.environ.get(key, value) + # Bootstrap is used for basic JS and CSS loading + # TODO: remove this and use statically generated assets instead + app.bootstrap = flask_bootstrap.Bootstrap(app) -# Base application -flask_bootstrap.Bootstrap(app) -db = flask_sqlalchemy.SQLAlchemy(app) -migrate = flask_migrate.Migrate(app, db) -limiter = flask_limiter.Limiter(app, key_func=lambda: current_user.username) + # Initialize application extensions + models.db.init_app(app) + utils.limiter.init_app(app) + utils.babel.init_app(app) + utils.login.init_app(app) + utils.proxy.init_app(app) + manage.migrate.init_app(app) + manage.manager.init_app(app) -# Debugging toolbar -if app.config.get("DEBUG"): - import flask_debugtoolbar - toolbar = flask_debugtoolbar.DebugToolbarExtension(app) + # Initialize debugging tools + if app.config.get("app.debug"): + debug.toolbar.init_app(app) + debug.profiler.init_app(app) -# Manager commnad -manager = flask_script.Manager(app) -manager.add_command('db', flask_migrate.MigrateCommand) + # Inject the default variables in the Jinja parser + @app.context_processor + def inject_defaults(): + signup_domains = models.Domain.query.filter_by(signup_enabled=True).all() + return dict( + current_user=utils.login.current_user, + signup_domains=signup_domains, + config=app.config + ) -# Babel configuration -babel = flask_babel.Babel(app) -translations = list(map(str, babel.list_translations())) + # Import views + from mailu import ui, internal + app.register_blueprint(ui.ui, url_prefix='/ui') + app.register_blueprint(internal.internal, url_prefix='/internal') -@babel.localeselector -def get_locale(): - return flask.request.accept_languages.best_match(translations) + return app -# Login configuration -login_manager = flask_login.LoginManager() -login_manager.init_app(app) -login_manager.login_view = "ui.login" -@login_manager.unauthorized_handler -def handle_needs_login(): - return flask.redirect( - flask.url_for('ui.login', next=flask.request.endpoint) - ) - -@app.context_processor -def inject_defaults(): - signup_domains = models.Domain.query.filter_by(signup_enabled=True).all() - return dict( - current_user=flask_login.current_user, - signup_domains=signup_domains, - config=app.config - ) - -# Import views -from mailu import ui, internal -app.register_blueprint(ui.ui, url_prefix='/ui') -app.register_blueprint(internal.internal, url_prefix='/internal') - -# Create the prefix middleware -class PrefixMiddleware(object): - - def __init__(self, app): - self.app = app - - def __call__(self, environ, start_response): - prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '') - if prefix: - environ['SCRIPT_NAME'] = prefix - return self.app(environ, start_response) - -app.wsgi_app = PrefixMiddleware(fixers.ProxyFix(app.wsgi_app)) +def create_app(): + """ Create a new application based on the config module + """ + config = configuration.ConfigManager() + return create_app_from_config(config) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py new file mode 100644 index 00000000..43184bad --- /dev/null +++ b/core/admin/mailu/configuration.py @@ -0,0 +1,70 @@ +import os + + +DEFAULT_CONFIG = { + # Specific to the admin UI + 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', + 'SQLALCHEMY_TRACK_MODIFICATIONS': False, + 'DOCKER_SOCKET': 'unix:///var/run/docker.sock', + 'BABEL_DEFAULT_LOCALE': 'en', + 'BABEL_DEFAULT_TIMEZONE': 'UTC', + 'BOOTSTRAP_SERVE_LOCAL': True, + 'RATELIMIT_STORAGE_URL': 'redis://redis/2', + 'QUOTA_STORAGE_URL': 'redis://redis/1', + 'DEBUG': False, + 'DOMAIN_REGISTRATION': False, + # Statistics management + 'INSTANCE_ID_PATH': '/data/instance', + 'STATS_ENDPOINT': '0.{}.stats.mailu.io', + # Common configuration variables + 'SECRET_KEY': 'changeMe', + 'DOMAIN': 'mailu.io', + 'HOSTNAMES': 'mail.mailu.io,alternative.mailu.io,yetanother.mailu.io', + 'POSTMASTER': 'postmaster', + 'TLS_FLAVOR': 'cert', + 'AUTH_RATELIMIT': '10/minute;1000/hour', + 'DISABLE_STATISTICS': 'False', + # Mail settings + 'DMARC_RUA': None, + 'DMARC_RUF': None, + 'WELCOME': 'False', + 'WELCOME_SUBJECT': 'Dummy welcome topic', + 'WELCOME_BODY': 'Dummy welcome body', + 'DKIM_SELECTOR': 'dkim', + 'DKIM_PATH': '/dkim/{domain}.{selector}.key', + 'DEFAULT_QUOTA': 1000000000, + # Web settings + 'SITENAME': 'Mailu', + 'WEBSITE': 'https://mailu.io', + 'WEB_ADMIN': '/admin', + 'WEB_WEBMAIL': '/webmail', + 'RECAPTCHA_PUBLIC_KEY': '', + 'RECAPTCHA_PRIVATE_KEY': '', + # Advanced settings + 'PASSWORD_SCHEME': 'BLF-CRYPT', + # Host settings + 'HOST_IMAP': 'imap', + 'HOST_POP3': 'imap', + 'HOST_SMTP': 'smtp', + 'HOST_WEBMAIL': 'webmail', + 'HOST_FRONT': 'front', + 'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'), + 'POD_ADDRESS_RANGE': None +} + + +class ConfigManager(object): + """ Naive configuration manager that uses environment only + """ + + def __init__(self): + self.config = { + os.environ.get(key, value) + for key, value in DEFAULT_CONFIG.items() + } + + def get(self, *args): + return self.config.get(*args) + + def __getitem__(self, key): + return self.get(key) diff --git a/core/admin/mailu/debug.py b/core/admin/mailu/debug.py new file mode 100644 index 00000000..69e88cf8 --- /dev/null +++ b/core/admin/mailu/debug.py @@ -0,0 +1,17 @@ +import flask_debugtoolbar + +from werkzeug.contrib import profiler as werkzeug_profiler + + +# Debugging toolbar +toolbar = flask_debugtoolbar.DebugToolbarExtension() + + +# Profiler +class Profiler(object): + def init_app(self): + app.wsgi_app = werkzeug_profiler.ProfilerMiddleware( + app.wsgi_app, restrictions=[30] + ) + +profiler = Profiler() diff --git a/core/admin/mailu/internal/__init__.py b/core/admin/mailu/internal/__init__.py index 80a3c754..fcb19692 100644 --- a/core/admin/mailu/internal/__init__.py +++ b/core/admin/mailu/internal/__init__.py @@ -1,6 +1,6 @@ from flask_limiter import RateLimitExceeded -from mailu import limiter +from mailu import utils import socket import flask @@ -19,7 +19,7 @@ def rate_limit_handler(e): return response -@limiter.request_filter +@utils.limiter.request_filter def whitelist_webmail(): try: return flask.request.headers["Client-Ip"] ==\ diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index cb6bc9cb..460719f2 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -1,4 +1,5 @@ -from mailu import db, models, app +from mailu import models +from flask import current_app as app import re import socket diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index 823fbd40..459a8e57 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -1,5 +1,6 @@ -from mailu import db, models, app, limiter +from mailu import models, utils from mailu.internal import internal, nginx +from flask import current_app as app import flask import flask_login @@ -7,7 +8,7 @@ import base64 @internal.route("/auth/email") -@limiter.limit( +@utils.limiter.limit( app.config["AUTH_RATELIMIT"], lambda: flask.request.headers["Client-Ip"] ) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index c2f53794..99ec0f6b 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -1,5 +1,6 @@ -from mailu import db, models +from mailu import models from mailu.internal import internal +from flask import current_app as app import flask @@ -25,7 +26,7 @@ def dovecot_quota(ns, user_email): user = models.User.query.get(user_email) or flask.abort(404) if ns == "storage": user.quota_bytes_used = flask.request.get_json() - db.session.commit() + models.db.session.commit() return flask.jsonify(None) diff --git a/core/admin/mailu/internal/views/fetch.py b/core/admin/mailu/internal/views/fetch.py index 60249c4b..ccd3a159 100644 --- a/core/admin/mailu/internal/views/fetch.py +++ b/core/admin/mailu/internal/views/fetch.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.internal import internal import flask @@ -27,6 +27,6 @@ def fetch_done(fetch_id): fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) fetch.last_check = datetime.datetime.now() fetch.error_message = str(flask.request.get_json()) - db.session.add(fetch) - db.session.commit() + models.db.session.add(fetch) + models.db.session.commit() return "" diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index 79fbdb8a..8c690b73 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.internal import internal import flask diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py new file mode 100644 index 00000000..bc2473eb --- /dev/null +++ b/core/admin/mailu/manage.py @@ -0,0 +1,305 @@ +from mailu import models + +from flask import current_app as app + +import flask +import os +import socket +import uuid + + +manager = flask_script.Manager() +db = models.db + + +@manager.command +def advertise(): + """ Advertise this server against statistic services. + """ + if os.path.isfile(app.config["INSTANCE_ID_PATH"]): + with open(app.config["INSTANCE_ID_PATH"], "r") as handle: + instance_id = handle.read() + else: + instance_id = str(uuid.uuid4()) + with open(app.config["INSTANCE_ID_PATH"], "w") as handle: + handle.write(instance_id) + if app.config["DISABLE_STATISTICS"].lower() != "true": + try: + socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) + except: + pass + + +@manager.command +def admin(localpart, domain_name, password): + """ Create an admin user + """ + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User( + localpart=localpart, + domain=domain, + global_admin=True + ) + user.set_password(password) + db.session.add(user) + db.session.commit() + + +@manager.command +def user(localpart, domain_name, password, + hash_scheme=app.config['PASSWORD_SCHEME']): + """ Create a user + """ + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User( + localpart=localpart, + domain=domain, + global_admin=False + ) + user.set_password(password, hash_scheme=hash_scheme) + db.session.add(user) + db.session.commit() + + +@manager.option('-n', '--domain_name', dest='domain_name') +@manager.option('-u', '--max_users', dest='max_users') +@manager.option('-a', '--max_aliases', dest='max_aliases') +@manager.option('-q', '--max_quota_bytes', dest='max_quota_bytes') +def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + db.session.commit() + + +@manager.command +def user_import(localpart, domain_name, password_hash, + hash_scheme=app.config['PASSWORD_SCHEME']): + """ Import a user along with password hash. Available hashes: + 'SHA512-CRYPT' + 'SHA256-CRYPT' + 'MD5-CRYPT' + 'CRYPT' + """ + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User( + localpart=localpart, + domain=domain, + global_admin=False + ) + user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) + db.session.add(user) + db.session.commit() + + +@manager.command +def config_update(verbose=False, delete_objects=False): + """sync configuration with data from YAML-formatted stdin""" + import yaml + import sys + new_config = yaml.load(sys.stdin) + # print new_config + domains = new_config.get('domains', []) + tracked_domains = set() + for domain_config in domains: + if verbose: + print(str(domain_config)) + domain_name = domain_config['name'] + max_users = domain_config.get('max_users', 0) + max_aliases = domain_config.get('max_aliases', 0) + max_quota_bytes = domain_config.get('max_quota_bytes', 0) + tracked_domains.add(domain_name) + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name, + max_users=max_users, + max_aliases=max_aliases, + max_quota_bytes=max_quota_bytes) + db.session.add(domain) + print("Added " + str(domain_config)) + else: + domain.max_users = max_users + domain.max_aliases = max_aliases + domain.max_quota_bytes = max_quota_bytes + db.session.add(domain) + print("Updated " + str(domain_config)) + + users = new_config.get('users', []) + tracked_users = set() + user_optional_params = ('comment', 'quota_bytes', 'global_admin', + 'enable_imap', 'enable_pop', 'forward_enabled', + 'forward_destination', 'reply_enabled', + 'reply_subject', 'reply_body', 'displayed_name', + 'spam_enabled', 'email', 'spam_threshold') + for user_config in users: + if verbose: + print(str(user_config)) + localpart = user_config['localpart'] + domain_name = user_config['domain'] + password_hash = user_config.get('password_hash', None) + hash_scheme = user_config.get('hash_scheme', None) + domain = models.Domain.query.get(domain_name) + email = '{0}@{1}'.format(localpart, domain_name) + optional_params = {} + for k in user_optional_params: + if k in user_config: + optional_params[k] = user_config[k] + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + user = models.User.query.get(email) + tracked_users.add(email) + tracked_domains.add(domain_name) + if not user: + user = models.User( + localpart=localpart, + domain=domain, + **optional_params + ) + else: + for k in optional_params: + setattr(user, k, optional_params[k]) + user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) + db.session.add(user) + + aliases = new_config.get('aliases', []) + tracked_aliases = set() + for alias_config in aliases: + if verbose: + print(str(alias_config)) + localpart = alias_config['localpart'] + domain_name = alias_config['domain'] + if type(alias_config['destination']) is str: + destination = alias_config['destination'].split(',') + else: + destination = alias_config['destination'] + wildcard = alias_config.get('wildcard', False) + domain = models.Domain.query.get(domain_name) + email = '{0}@{1}'.format(localpart, domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + alias = models.Alias.query.get(email) + tracked_aliases.add(email) + tracked_domains.add(domain_name) + if not alias: + alias = models.Alias( + localpart=localpart, + domain=domain, + wildcard=wildcard, + destination=destination, + email=email + ) + else: + alias.destination = destination + alias.wildcard = wildcard + db.session.add(alias) + + db.session.commit() + + managers = new_config.get('managers', []) + # tracked_managers=set() + for manager_config in managers: + if verbose: + print(str(manager_config)) + domain_name = manager_config['domain'] + user_name = manager_config['user'] + domain = models.Domain.query.get(domain_name) + manageruser = models.User.query.get(user_name + '@' + domain_name) + if manageruser not in domain.managers: + domain.managers.append(manageruser) + db.session.add(domain) + + db.session.commit() + + if delete_objects: + for user in db.session.query(models.User).all(): + if not (user.email in tracked_users): + if verbose: + print("Deleting user: " + str(user.email)) + db.session.delete(user) + for alias in db.session.query(models.Alias).all(): + if not (alias.email in tracked_aliases): + if verbose: + print("Deleting alias: " + str(alias.email)) + db.session.delete(alias) + for domain in db.session.query(models.Domain).all(): + if not (domain.name in tracked_domains): + if verbose: + print("Deleting domain: " + str(domain.name)) + db.session.delete(domain) + db.session.commit() + + +@manager.command +def user_delete(email): + """delete user""" + user = models.User.query.get(email) + if user: + db.session.delete(user) + db.session.commit() + + +@manager.command +def alias_delete(email): + """delete alias""" + alias = models.Alias.query.get(email) + if alias: + db.session.delete(alias) + db.session.commit() + + +@manager.command +def alias(localpart, domain_name, destination): + """ Create an alias + """ + domain = models.Domain.query.get(domain_name) + if not domain: + domain = models.Domain(name=domain_name) + db.session.add(domain) + alias = models.Alias( + localpart=localpart, + domain=domain, + destination=destination.split(','), + email="%s@%s" % (localpart, domain_name) + ) + db.session.add(alias) + db.session.commit() + +# Set limits to a domain + + +@manager.command +def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): + domain = models.Domain.query.get(domain_name) + domain.max_users = max_users + domain.max_aliases = max_aliases + domain.max_quota_bytes = max_quota_bytes + + db.session.add(domain) + db.session.commit() + +# Make the user manager of a domain + + +@manager.command +def setmanager(domain_name, user_name='manager'): + domain = models.Domain.query.get(domain_name) + manageruser = models.User.query.get(user_name + '@' + domain_name) + domain.managers.append(manageruser) + db.session.add(domain) + db.session.commit() + + +if __name__ == "__main__": + manager.run() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 0c80fd4f..43565c55 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,10 +1,12 @@ -from mailu import app, db, dkim, login_manager +from mailu import dkim from sqlalchemy.ext import declarative from passlib import context, hash from datetime import datetime, date from email.mime import text +from flask import current_app as app +import flask_sqlalchemy import sqlalchemy import re import time @@ -15,6 +17,9 @@ import idna import dns +db = flask_sqlalchemy.SQLAlchemy() + + class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) """ @@ -67,6 +72,27 @@ class CommaSeparatedList(db.TypeDecorator): return filter(bool, value.split(",")) +class JSONEncoded(db.TypeDecorator): + """Represents an immutable structure as a json-encoded string. + """ + + impl = db.String + + def process_bind_param(self, value, dialect): + return json.dumps(value) if value else None + + def process_result_value(self, value, dialect): + return json.loads(value) if value else None + + +class Config(db.Model): + """ In-database configuration values + """ + + name = db.Column(db.String(255), primary_key=True, nullable=False) + value = db.Column(JSONEncoded) + + # Many-to-many association table for domain managers managers = db.Table('manager', db.Column('domain_name', IdnaDomain, db.ForeignKey('domain.name')), @@ -324,8 +350,6 @@ class User(Base, Email): user = cls.query.get(email) return user if (user and user.enabled and user.check_password(password)) else None -login_manager.user_loader(User.query.get) - class Alias(Base, Email): """ An alias is an email address that redirects to some destination. diff --git a/core/admin/mailu/ui/access.py b/core/admin/mailu/ui/access.py index 2b6a3767..6a923c8e 100644 --- a/core/admin/mailu/ui/access.py +++ b/core/admin/mailu/ui/access.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import forms import flask diff --git a/core/admin/mailu/ui/views/admins.py b/core/admin/mailu/ui/views/admins.py index e9771ec6..7462c545 100644 --- a/core/admin/mailu/ui/views/admins.py +++ b/core/admin/mailu/ui/views/admins.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -25,7 +25,7 @@ def admin_create(): user = models.User.query.get(form.admin.data) if user: user.global_admin = True - db.session.commit() + models.db.session.commit() flask.flash('User %s is now admin' % user) return flask.redirect(flask.url_for('.admin_list')) else: @@ -40,7 +40,7 @@ def admin_delete(admin): user = models.User.query.get(admin) if user: user.global_admin = False - db.session.commit() + models.db.session.commit() flask.flash('User %s is no longer admin' % user) return flask.redirect(flask.url_for('.admin_list')) else: diff --git a/core/admin/mailu/ui/views/aliases.py b/core/admin/mailu/ui/views/aliases.py index 7fc03266..eaab5cad 100644 --- a/core/admin/mailu/ui/views/aliases.py +++ b/core/admin/mailu/ui/views/aliases.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -27,8 +27,8 @@ def alias_create(domain_name): else: alias = models.Alias(domain=domain) form.populate_obj(alias) - db.session.add(alias) - db.session.commit() + models.db.session.add(alias) + models.db.session.commit() flask.flash('Alias %s created' % alias) return flask.redirect( flask.url_for('.alias_list', domain_name=domain.name)) @@ -45,7 +45,7 @@ def alias_edit(alias): form.localpart.validators = [] if form.validate_on_submit(): form.populate_obj(alias) - db.session.commit() + models.db.session.commit() flask.flash('Alias %s updated' % alias) return flask.redirect( flask.url_for('.alias_list', domain_name=alias.domain.name)) @@ -59,8 +59,8 @@ def alias_edit(alias): def alias_delete(alias): alias = models.Alias.query.get(alias) or flask.abort(404) domain = alias.domain - db.session.delete(alias) - db.session.commit() + models.db.session.delete(alias) + models.db.session.commit() flask.flash('Alias %s deleted' % alias) return flask.redirect( flask.url_for('.alias_list', domain_name=domain.name)) diff --git a/core/admin/mailu/ui/views/alternatives.py b/core/admin/mailu/ui/views/alternatives.py index 79acb074..e5761c83 100644 --- a/core/admin/mailu/ui/views/alternatives.py +++ b/core/admin/mailu/ui/views/alternatives.py @@ -26,8 +26,8 @@ def alternative_create(domain_name): else: alternative = models.Alternative(domain=domain) form.populate_obj(alternative) - db.session.add(alternative) - db.session.commit() + models.db.session.add(alternative) + models.db.session.commit() flask.flash('Alternative domain %s created' % alternative) return flask.redirect( flask.url_for('.alternative_list', domain_name=domain.name)) @@ -41,8 +41,8 @@ def alternative_create(domain_name): def alternative_delete(alternative): alternative = models.Alternative.query.get(alternative) or flask.abort(404) domain = alternative.domain - db.session.delete(alternative) - db.session.commit() + models.db.session.delete(alternative) + models.db.session.commit() flask.flash('Alternative %s deleted' % alternative) return flask.redirect( flask.url_for('.alternative_list', domain_name=domain.name)) diff --git a/core/admin/mailu/ui/views/base.py b/core/admin/mailu/ui/views/base.py index 1ea87401..7501a883 100644 --- a/core/admin/mailu/ui/views/base.py +++ b/core/admin/mailu/ui/views/base.py @@ -1,11 +1,9 @@ -from mailu import dockercli, app, db, models +from mailu import models from mailu.ui import ui, forms, access import flask import flask_login -from urllib import parse - @ui.route('/', methods=["GET"]) @access.authenticated diff --git a/core/admin/mailu/ui/views/domains.py b/core/admin/mailu/ui/views/domains.py index 7afc1746..719d3844 100644 --- a/core/admin/mailu/ui/views/domains.py +++ b/core/admin/mailu/ui/views/domains.py @@ -1,5 +1,6 @@ -from mailu import app, db, models +from mailu import models from mailu.ui import ui, forms, access +from flask import current_app as app import flask import flask_login @@ -26,8 +27,8 @@ def domain_create(): else: domain = models.Domain() form.populate_obj(domain) - db.session.add(domain) - db.session.commit() + models.db.session.add(domain) + models.db.session.commit() flask.flash('Domain %s created' % domain) return flask.redirect(flask.url_for('.domain_list')) return flask.render_template('domain/create.html', form=form) @@ -42,7 +43,7 @@ def domain_edit(domain_name): form.name.validators = [] if form.validate_on_submit(): form.populate_obj(domain) - db.session.commit() + models.db.session.commit() flask.flash('Domain %s saved' % domain) return flask.redirect(flask.url_for('.domain_list')) return flask.render_template('domain/edit.html', form=form, @@ -54,8 +55,8 @@ def domain_edit(domain_name): @access.confirmation_required("delete {domain_name}") def domain_delete(domain_name): domain = models.Domain.query.get(domain_name) or flask.abort(404) - db.session.delete(domain) - db.session.commit() + models.db.session.delete(domain) + models.db.session.commit() flask.flash('Domain %s deleted' % domain) return flask.redirect(flask.url_for('.domain_list')) @@ -99,7 +100,7 @@ def domain_signup(domain_name=None): domain.max_users = 10 domain.max_aliases = 10 if domain.check_mx(): - db.session.add(domain) + models.db.session.add(domain) if flask_login.current_user.is_authenticated: user = models.User.query.get(flask_login.current_user.email) else: @@ -108,9 +109,9 @@ def domain_signup(domain_name=None): form.populate_obj(user) user.set_password(form.pw.data) user.quota_bytes = domain.max_quota_bytes - db.session.add(user) + models.db.session.add(user) domain.managers.append(user) - db.session.commit() + models.db.session.commit() flask.flash('Domain %s created' % domain) return flask.redirect(flask.url_for('.domain_list')) else: diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index b4ddb824..c0421146 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -24,8 +24,8 @@ def fetch_create(user_email): if form.validate_on_submit(): fetch = models.Fetch(user=user) form.populate_obj(fetch) - db.session.add(fetch) - db.session.commit() + models.db.session.add(fetch) + models.db.session.commit() flask.flash('Fetch configuration created') return flask.redirect( flask.url_for('.fetch_list', user_email=user.email)) @@ -39,7 +39,7 @@ def fetch_edit(fetch_id): form = forms.FetchForm(obj=fetch) if form.validate_on_submit(): form.populate_obj(fetch) - db.session.commit() + models.db.session.commit() flask.flash('Fetch configuration updated') return flask.redirect( flask.url_for('.fetch_list', user_email=fetch.user.email)) @@ -53,8 +53,8 @@ def fetch_edit(fetch_id): def fetch_delete(fetch_id): fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) user = fetch.user - db.session.delete(fetch) - db.session.commit() + models.db.session.delete(fetch) + models.db.session.commit() flask.flash('Fetch configuration delete') return flask.redirect( flask.url_for('.fetch_list', user_email=user.email)) diff --git a/core/admin/mailu/ui/views/managers.py b/core/admin/mailu/ui/views/managers.py index 15ebcd50..45974f94 100644 --- a/core/admin/mailu/ui/views/managers.py +++ b/core/admin/mailu/ui/views/managers.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -30,7 +30,7 @@ def manager_create(domain_name): flask.flash('User %s is already manager' % user, 'error') else: domain.managers.append(user) - db.session.commit() + models.db.session.commit() flask.flash('User %s can now manage %s' % (user, domain.name)) return flask.redirect( flask.url_for('.manager_list', domain_name=domain.name)) @@ -46,7 +46,7 @@ def manager_delete(domain_name, user_email): user = models.User.query.get(user_email) or flask.abort(404) if user in domain.managers: domain.managers.remove(user) - db.session.commit() + models.db.session.commit() flask.flash('User %s can no longer manager %s' % (user, domain)) else: flask.flash('User %s is not manager' % user, 'error') diff --git a/core/admin/mailu/ui/views/relays.py b/core/admin/mailu/ui/views/relays.py index a0ddc614..f1847798 100644 --- a/core/admin/mailu/ui/views/relays.py +++ b/core/admin/mailu/ui/views/relays.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask @@ -25,8 +25,8 @@ def relay_create(): else: relay = models.Relay() form.populate_obj(relay) - db.session.add(relay) - db.session.commit() + models.db.session.add(relay) + models.db.session.commit() flask.flash('Relayed domain %s created' % relay) return flask.redirect(flask.url_for('.relay_list')) return flask.render_template('relay/create.html', form=form) @@ -41,7 +41,7 @@ def relay_edit(relay_name): form.name.validators = [] if form.validate_on_submit(): form.populate_obj(relay) - db.session.commit() + models.db.session.commit() flask.flash('Relayed domain %s saved' % relay) return flask.redirect(flask.url_for('.relay_list')) return flask.render_template('relay/edit.html', form=form, @@ -53,8 +53,8 @@ def relay_edit(relay_name): @access.confirmation_required("delete {relay_name}") def relay_delete(relay_name): relay = models.Relay.query.get(relay_name) or flask.abort(404) - db.session.delete(relay) - db.session.commit() + models.db.session.delete(relay) + models.db.session.commit() flask.flash('Relayed domain %s deleted' % relay) return flask.redirect(flask.url_for('.relay_list')) diff --git a/core/admin/mailu/ui/views/tokens.py b/core/admin/mailu/ui/views/tokens.py index 4b9881af..3864617b 100644 --- a/core/admin/mailu/ui/views/tokens.py +++ b/core/admin/mailu/ui/views/tokens.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access from passlib import pwd @@ -32,8 +32,8 @@ def token_create(user_email): token = models.Token(user=user) token.set_password(form.raw_password.data) form.populate_obj(token) - db.session.add(token) - db.session.commit() + models.db.session.add(token) + models.db.session.commit() flask.flash('Authentication token created') return flask.redirect( flask.url_for('.token_list', user_email=user.email)) @@ -46,8 +46,8 @@ def token_create(user_email): def token_delete(token_id): token = models.Token.query.get(token_id) or flask.abort(404) user = token.user - db.session.delete(token) - db.session.commit() + models.db.session.delete(token) + models.db.session.commit() flask.flash('Authentication token deleted') return flask.redirect( flask.url_for('.token_list', user_email=user.email)) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index c54d22ec..ac572c2e 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -1,5 +1,6 @@ -from mailu import db, models, app +from mailu import models from mailu.ui import ui, access, forms +from flask import current_app as app import flask import flask_login @@ -33,8 +34,8 @@ def user_create(domain_name): user = models.User(domain=domain) form.populate_obj(user) user.set_password(form.pw.data) - db.session.add(user) - db.session.commit() + models.db.session.add(user) + models.db.session.commit() user.send_welcome() flask.flash('User %s created' % user) return flask.redirect( @@ -63,7 +64,7 @@ def user_edit(user_email): form.populate_obj(user) if form.pw.data: user.set_password(form.pw.data) - db.session.commit() + models.db.session.commit() flask.flash('User %s updated' % user) return flask.redirect( flask.url_for('.user_list', domain_name=user.domain.name)) @@ -77,8 +78,8 @@ def user_edit(user_email): def user_delete(user_email): user = models.User.query.get(user_email) or flask.abort(404) domain = user.domain - db.session.delete(user) - db.session.commit() + models.db.session.delete(user) + models.db.session.commit() flask.flash('User %s deleted' % user) return flask.redirect( flask.url_for('.user_list', domain_name=domain.name)) @@ -93,7 +94,7 @@ def user_settings(user_email): form = forms.UserSettingsForm(obj=user) if form.validate_on_submit(): form.populate_obj(user) - db.session.commit() + models.db.session.commit() flask.flash('Settings updated for %s' % user) if user_email: return flask.redirect( @@ -113,7 +114,7 @@ def user_password(user_email): flask.flash('Passwords do not match', 'error') else: user.set_password(form.pw.data) - db.session.commit() + models.db.session.commit() flask.flash('Password updated for %s' % user) if user_email: return flask.redirect(flask.url_for('.user_list', @@ -130,7 +131,7 @@ def user_forward(user_email): form = forms.UserForwardForm(obj=user) if form.validate_on_submit(): form.populate_obj(user) - db.session.commit() + models.db.session.commit() flask.flash('Forward destination updated for %s' % user) if user_email: return flask.redirect( @@ -147,7 +148,7 @@ def user_reply(user_email): form = forms.UserReplyForm(obj=user) if form.validate_on_submit(): form.populate_obj(user) - db.session.commit() + models.db.session.commit() flask.flash('Auto-reply message updated for %s' % user) if user_email: return flask.redirect( @@ -179,8 +180,8 @@ def user_signup(domain_name=None): form.populate_obj(user) user.set_password(form.pw.data) user.quota_bytes = quota_bytes - db.session.add(user) - db.session.commit() + models.db.session.add(user) + models.db.session.commit() user.send_welcome() flask.flash('Successfully signed up %s' % user) return flask.redirect(flask.url_for('.index')) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py new file mode 100644 index 00000000..60902951 --- /dev/null +++ b/core/admin/mailu/utils.py @@ -0,0 +1,46 @@ +import flask +import flask_login +import flask_script +import flask_migrate +import flask_babel +import flask_limiter + + +# Login configuration +login = flask_login.LoginManager() +login.login_view = "ui.login" +login.user_loader(models.User.query.get) + +@login_manager.unauthorized_handler +def handle_needs_login(): + return flask.redirect( + flask.url_for('ui.login', next=flask.request.endpoint) + ) + + +# Request rate limitation +limiter = flask_limiter.Limiter(key_func=lambda: current_user.username) + + +# Application translation +babel = flask_babel.Babel() + +@babel.localeselector +def get_locale(): + translations = list(map(str, babel.list_translations())) + return flask.request.accept_languages.best_match(translations) + + +# Proxy fixer +class PrefixMiddleware(object): + def __call__(self, environ, start_response): + prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '') + if prefix: + environ['SCRIPT_NAME'] = prefix + return self.app(environ, start_response) + + def init_app(self, app): + self.app = fixers.ProxyFix(app.wsgi_app) + app.wsgi_app = self + +proxy = PrefixMiddleware() From f40fcd7ac03e5d3a2693cfdb30456afd548e28d8 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 18 Oct 2018 16:20:56 +0200 Subject: [PATCH 05/62] Use click for the manager command --- core/admin/mailu/manage.py | 82 ++++++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index bc2473eb..2defa9d6 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -1,18 +1,25 @@ -from mailu import models +from mailu import models, create_app from flask import current_app as app +from flask import cli as flask_cli import flask import os import socket import uuid +import click -manager = flask_script.Manager() db = models.db -@manager.command +@click.group() +def cli(cls=flask_cli.FlaskGroup, create_app=mailu.create_app): + """ Main command group + """ + + +@cli.command() def advertise(): """ Advertise this server against statistic services. """ @@ -30,7 +37,10 @@ def advertise(): pass -@manager.command +@cli.command() +@cli.argument('localpart', help='localpart for the new admin') +@cli.argument('domain_name', help='domain name for the new admin') +@cli.argument('password', help='plain password for the new admin') def admin(localpart, domain_name, password): """ Create an admin user """ @@ -48,7 +58,11 @@ def admin(localpart, domain_name, password): db.session.commit() -@manager.command +@cli.command() +@cli.argument('localpart', help='localpart for the new user') +@cli.argument('domain_name', help='domain name for the new user') +@cli.argument('password', help='plain password for the new user') +@cli.argument('hash_scheme', help='password hashing scheme') def user(localpart, domain_name, password, hash_scheme=app.config['PASSWORD_SCHEME']): """ Create a user @@ -67,10 +81,11 @@ def user(localpart, domain_name, password, db.session.commit() -@manager.option('-n', '--domain_name', dest='domain_name') -@manager.option('-u', '--max_users', dest='max_users') -@manager.option('-a', '--max_aliases', dest='max_aliases') -@manager.option('-q', '--max_quota_bytes', dest='max_quota_bytes') +@cli.command() +@cli.option('-n', '--domain_name', dest='domain_name') +@cli.option('-u', '--max_users', dest='max_users') +@cli.option('-a', '--max_aliases', dest='max_aliases') +@cli.option('-q', '--max_quota_bytes', dest='max_quota_bytes') def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): domain = models.Domain.query.get(domain_name) if not domain: @@ -79,14 +94,14 @@ def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): db.session.commit() -@manager.command +@cli.command() +@cli.argument('localpart', help='localpart for the new user') +@cli.argument('domain_name', help='domain name for the new user') +@cli.argument('password_hash', help='password hash for the new user') +@cli.argument('hash_scheme', help='password hashing scheme') def user_import(localpart, domain_name, password_hash, hash_scheme=app.config['PASSWORD_SCHEME']): - """ Import a user along with password hash. Available hashes: - 'SHA512-CRYPT' - 'SHA256-CRYPT' - 'MD5-CRYPT' - 'CRYPT' + """ Import a user along with password hash. """ domain = models.Domain.query.get(domain_name) if not domain: @@ -102,7 +117,9 @@ def user_import(localpart, domain_name, password_hash, db.session.commit() -@manager.command +@cli.command() +@cli.option('-v', dest='verbose') +@cli.option('-d', dest='delete_objects') def config_update(verbose=False, delete_objects=False): """sync configuration with data from YAML-formatted stdin""" import yaml @@ -241,7 +258,8 @@ def config_update(verbose=False, delete_objects=False): db.session.commit() -@manager.command +@cli.command() +@cli.argument('email', help='email address to be deleted') def user_delete(email): """delete user""" user = models.User.query.get(email) @@ -250,7 +268,8 @@ def user_delete(email): db.session.commit() -@manager.command +@cli.command() +@cli.argument('email', help='email alias to be deleted') def alias_delete(email): """delete alias""" alias = models.Alias.query.get(email) @@ -259,7 +278,10 @@ def alias_delete(email): db.session.commit() -@manager.command +@cli.command() +@cli.argument('localpart', help='localpart for the new alias') +@cli.argument('domain_name', help='domain name for the new alias') +@cli.argument('destination', help='destination for the new alias') def alias(localpart, domain_name, destination): """ Create an alias """ @@ -276,30 +298,32 @@ def alias(localpart, domain_name, destination): db.session.add(alias) db.session.commit() -# Set limits to a domain - -@manager.command +@cli.command() +@cli.argument('domain_name', help='domain to be updated') +@cli.argument('max_users', help='maximum user count') +@cli.argument('max_aliases', help='maximum alias count') +@cli.argument('max_quota_bytes', help='maximum quota bytes par user') def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): + """ Set domain limits + """ domain = models.Domain.query.get(domain_name) domain.max_users = max_users domain.max_aliases = max_aliases domain.max_quota_bytes = max_quota_bytes - db.session.add(domain) db.session.commit() -# Make the user manager of a domain - -@manager.command +@cli.command() +@cli.argument('domain_name', help='target domain name') +@cli.argument('user_name', help='username inside the target domain') def setmanager(domain_name, user_name='manager'): + """ Make a user manager of a domain + """ domain = models.Domain.query.get(domain_name) manageruser = models.User.query.get(user_name + '@' + domain_name) domain.managers.append(manageruser) db.session.add(domain) db.session.commit() - -if __name__ == "__main__": - manager.run() From 82069ea3f009dabf9cfefaa606edadea5edb97f1 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 18 Oct 2018 17:55:07 +0200 Subject: [PATCH 06/62] Clean most of the refactored code --- core/admin/mailu/__init__.py | 20 +- core/admin/mailu/configuration.py | 27 +- core/admin/mailu/debug.py | 2 +- core/admin/mailu/manage.py | 82 +++--- core/admin/mailu/models.py | 18 +- core/admin/mailu/ui/views/alternatives.py | 2 +- core/admin/mailu/utils.py | 7 +- core/admin/manage.py | 298 ---------------------- 8 files changed, 97 insertions(+), 359 deletions(-) delete mode 100644 core/admin/manage.py diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 874fa8e0..8996fa20 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -1,44 +1,40 @@ import flask import flask_bootstrap -import os -import docker -import socket -import uuid - -from mailu import utils, debug, db +from mailu import utils, debug, models, configuration def create_app_from_config(config): """ Create a new application based on the given configuration """ app = flask.Flask(__name__) - app.config = config + app.app_context().push() # Bootstrap is used for basic JS and CSS loading # TODO: remove this and use statically generated assets instead app.bootstrap = flask_bootstrap.Bootstrap(app) # Initialize application extensions + config.init_app(app) models.db.init_app(app) utils.limiter.init_app(app) utils.babel.init_app(app) utils.login.init_app(app) + utils.login.user_loader(models.User.query.get) utils.proxy.init_app(app) - manage.migrate.init_app(app) - manage.manager.init_app(app) # Initialize debugging tools - if app.config.get("app.debug"): + if app.config.get("DEBUG"): debug.toolbar.init_app(app) - debug.profiler.init_app(app) + # TODO: add a specific configuration variable for profiling + # debug.profiler.init_app(app) # Inject the default variables in the Jinja parser + # TODO: move this to blueprints when needed @app.context_processor def inject_defaults(): signup_domains = models.Domain.query.filter_by(signup_enabled=True).all() return dict( - current_user=utils.login.current_user, signup_domains=signup_domains, config=app.config ) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 43184bad..05a3d570 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -4,7 +4,7 @@ import os DEFAULT_CONFIG = { # Specific to the admin UI 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', - 'SQLALCHEMY_TRACK_MODIFICATIONS': False, + 'SQLALCHEMY_TRACK_MODIFICATIONS': True, 'DOCKER_SOCKET': 'unix:///var/run/docker.sock', 'BABEL_DEFAULT_LOCALE': 'en', 'BABEL_DEFAULT_TIMEZONE': 'UTC', @@ -13,6 +13,7 @@ DEFAULT_CONFIG = { 'QUOTA_STORAGE_URL': 'redis://redis/1', 'DEBUG': False, 'DOMAIN_REGISTRATION': False, + 'TEMPLATES_AUTO_RELOAD': True, # Statistics management 'INSTANCE_ID_PATH': '/data/instance', 'STATS_ENDPOINT': '0.{}.stats.mailu.io', @@ -58,13 +59,29 @@ class ConfigManager(object): """ def __init__(self): - self.config = { - os.environ.get(key, value) + self.config = dict() + + def init_app(self, app): + self.config.update(app.config) + self.config.update({ + key: os.environ.get(key, value) for key, value in DEFAULT_CONFIG.items() - } + }) + app.config = self + + def setdefault(self, key, value): + if key not in self.config: + self.config[key] = value + return self.config[key] def get(self, *args): return self.config.get(*args) def __getitem__(self, key): - return self.get(key) + return self.config.get(key) + + def __setitem__(self, key, value): + self.config[key] = value + + def __contains__(self, key): + return key in self.config diff --git a/core/admin/mailu/debug.py b/core/admin/mailu/debug.py index 69e88cf8..7677901b 100644 --- a/core/admin/mailu/debug.py +++ b/core/admin/mailu/debug.py @@ -9,7 +9,7 @@ toolbar = flask_debugtoolbar.DebugToolbarExtension() # Profiler class Profiler(object): - def init_app(self): + def init_app(self, app): app.wsgi_app = werkzeug_profiler.ProfilerMiddleware( app.wsgi_app, restrictions=[30] ) diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 2defa9d6..e0aa6286 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -14,12 +14,13 @@ db = models.db @click.group() -def cli(cls=flask_cli.FlaskGroup, create_app=mailu.create_app): +def cli(cls=flask_cli.FlaskGroup, create_app=create_app): """ Main command group """ @cli.command() +@flask_cli.with_appcontext def advertise(): """ Advertise this server against statistic services. """ @@ -38,9 +39,10 @@ def advertise(): @cli.command() -@cli.argument('localpart', help='localpart for the new admin') -@cli.argument('domain_name', help='domain name for the new admin') -@cli.argument('password', help='plain password for the new admin') +@click.argument('localpart') +@click.argument('domain_name') +@click.argument('password') +@flask_cli.with_appcontext def admin(localpart, domain_name, password): """ Create an admin user """ @@ -59,14 +61,16 @@ def admin(localpart, domain_name, password): @cli.command() -@cli.argument('localpart', help='localpart for the new user') -@cli.argument('domain_name', help='domain name for the new user') -@cli.argument('password', help='plain password for the new user') -@cli.argument('hash_scheme', help='password hashing scheme') -def user(localpart, domain_name, password, - hash_scheme=app.config['PASSWORD_SCHEME']): +@click.argument('localpart') +@click.argument('domain_name') +@click.argument('password') +@click.argument('hash_scheme') +@flask_cli.with_appcontext +def user(localpart, domain_name, password, hash_scheme=None): """ Create a user """ + if hash_scheme is None: + hash_scheme = app.config['PASSWORD_SCHEME'] domain = models.Domain.query.get(domain_name) if not domain: domain = models.Domain(name=domain_name) @@ -82,10 +86,11 @@ def user(localpart, domain_name, password, @cli.command() -@cli.option('-n', '--domain_name', dest='domain_name') -@cli.option('-u', '--max_users', dest='max_users') -@cli.option('-a', '--max_aliases', dest='max_aliases') -@cli.option('-q', '--max_quota_bytes', dest='max_quota_bytes') +@click.option('-n', '--domain_name') +@click.option('-u', '--max_users') +@click.option('-a', '--max_aliases') +@click.option('-q', '--max_quota_bytes') +@flask_cli.with_appcontext def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): domain = models.Domain.query.get(domain_name) if not domain: @@ -95,14 +100,16 @@ def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): @cli.command() -@cli.argument('localpart', help='localpart for the new user') -@cli.argument('domain_name', help='domain name for the new user') -@cli.argument('password_hash', help='password hash for the new user') -@cli.argument('hash_scheme', help='password hashing scheme') -def user_import(localpart, domain_name, password_hash, - hash_scheme=app.config['PASSWORD_SCHEME']): +@click.argument('localpart') +@click.argument('domain_name') +@click.argument('password_hash') +@click.argument('hash_scheme') +@flask_cli.with_appcontext +def user_import(localpart, domain_name, password_hash, hash_scheme = None): """ Import a user along with password hash. """ + if hash_scheme is None: + hash_scheme = app.config['PASSWORD_SCHEME'] domain = models.Domain.query.get(domain_name) if not domain: domain = models.Domain(name=domain_name) @@ -118,8 +125,9 @@ def user_import(localpart, domain_name, password_hash, @cli.command() -@cli.option('-v', dest='verbose') -@cli.option('-d', dest='delete_objects') +@click.option('-v', '--verbose') +@click.option('-d', '--delete_objects') +@flask_cli.with_appcontext def config_update(verbose=False, delete_objects=False): """sync configuration with data from YAML-formatted stdin""" import yaml @@ -259,7 +267,8 @@ def config_update(verbose=False, delete_objects=False): @cli.command() -@cli.argument('email', help='email address to be deleted') +@click.argument('email') +@flask_cli.with_appcontext def user_delete(email): """delete user""" user = models.User.query.get(email) @@ -269,7 +278,8 @@ def user_delete(email): @cli.command() -@cli.argument('email', help='email alias to be deleted') +@click.argument('email') +@flask_cli.with_appcontext def alias_delete(email): """delete alias""" alias = models.Alias.query.get(email) @@ -279,9 +289,10 @@ def alias_delete(email): @cli.command() -@cli.argument('localpart', help='localpart for the new alias') -@cli.argument('domain_name', help='domain name for the new alias') -@cli.argument('destination', help='destination for the new alias') +@click.argument('localpart') +@click.argument('domain_name') +@click.argument('destination') +@flask_cli.with_appcontext def alias(localpart, domain_name, destination): """ Create an alias """ @@ -300,10 +311,11 @@ def alias(localpart, domain_name, destination): @cli.command() -@cli.argument('domain_name', help='domain to be updated') -@cli.argument('max_users', help='maximum user count') -@cli.argument('max_aliases', help='maximum alias count') -@cli.argument('max_quota_bytes', help='maximum quota bytes par user') +@click.argument('domain_name') +@click.argument('max_users') +@click.argument('max_aliases') +@click.argument('max_quota_bytes') +@flask_cli.with_appcontext def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): """ Set domain limits """ @@ -316,8 +328,9 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): @cli.command() -@cli.argument('domain_name', help='target domain name') -@cli.argument('user_name', help='username inside the target domain') +@click.argument('domain_name') +@click.argument('user_name') +@flask_cli.with_appcontext def setmanager(domain_name, user_name='manager'): """ Make a user manager of a domain """ @@ -327,3 +340,6 @@ def setmanager(domain_name, user_name='manager'): db.session.add(domain) db.session.commit() + +if __name__ == '__main__': + cli() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 43565c55..bc1044db 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -307,24 +307,28 @@ class User(Base, Email): 'SHA256-CRYPT': "sha256_crypt", 'MD5-CRYPT': "md5_crypt", 'CRYPT': "des_crypt"} - pw_context = context.CryptContext( - schemes = scheme_dict.values(), - default=scheme_dict[app.config['PASSWORD_SCHEME']], - ) + + def get_password_context(self): + return context.CryptContext( + schemes=self.scheme_dict.values(), + default=self.scheme_dict[app.config['PASSWORD_SCHEME']], + ) def check_password(self, password): reference = re.match('({[^}]+})?(.*)', self.password).group(2) - return User.pw_context.verify(password, reference) + return self.get_password_context().verify(password, reference) - def set_password(self, password, hash_scheme=app.config['PASSWORD_SCHEME'], raw=False): + def set_password(self, password, hash_scheme=None, raw=False): """Set password for user with specified encryption scheme @password: plain text password to encrypt (if raw == True the hash itself) """ + if hash_scheme is None: + hash_scheme = app.config['PASSWORD_SCHEME'] # for the list of hash schemes see https://wiki2.dovecot.org/Authentication/PasswordSchemes if raw: self.password = '{'+hash_scheme+'}' + password else: - self.password = '{'+hash_scheme+'}' + User.pw_context.encrypt(password, self.scheme_dict[hash_scheme]) + self.password = '{'+hash_scheme+'}' + self.get_password_context().encrypt(password, self.scheme_dict[hash_scheme]) def get_managed_domains(self): if self.global_admin: diff --git a/core/admin/mailu/ui/views/alternatives.py b/core/admin/mailu/ui/views/alternatives.py index e5761c83..2e6e580b 100644 --- a/core/admin/mailu/ui/views/alternatives.py +++ b/core/admin/mailu/ui/views/alternatives.py @@ -1,4 +1,4 @@ -from mailu import db, models +from mailu import models from mailu.ui import ui, forms, access import flask diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 60902951..18292041 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -1,3 +1,5 @@ +from mailu import models + import flask import flask_login import flask_script @@ -5,13 +7,14 @@ import flask_migrate import flask_babel import flask_limiter +from werkzeug.contrib import fixers + # Login configuration login = flask_login.LoginManager() login.login_view = "ui.login" -login.user_loader(models.User.query.get) -@login_manager.unauthorized_handler +@login.unauthorized_handler def handle_needs_login(): return flask.redirect( flask.url_for('ui.login', next=flask.request.endpoint) diff --git a/core/admin/manage.py b/core/admin/manage.py deleted file mode 100644 index f7e9e17d..00000000 --- a/core/admin/manage.py +++ /dev/null @@ -1,298 +0,0 @@ -from mailu import app, manager, db, models - -import os -import socket -import uuid - - -@manager.command -def advertise(): - """ Advertise this server against statistic services. - """ - if os.path.isfile(app.config["INSTANCE_ID_PATH"]): - with open(app.config["INSTANCE_ID_PATH"], "r") as handle: - instance_id = handle.read() - else: - instance_id = str(uuid.uuid4()) - with open(app.config["INSTANCE_ID_PATH"], "w") as handle: - handle.write(instance_id) - if app.config["DISABLE_STATISTICS"].lower() != "true": - try: - socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) - except: - pass - - -@manager.command -def admin(localpart, domain_name, password): - """ Create an admin user - """ - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User( - localpart=localpart, - domain=domain, - global_admin=True - ) - user.set_password(password) - db.session.add(user) - db.session.commit() - - -@manager.command -def user(localpart, domain_name, password, - hash_scheme=app.config['PASSWORD_SCHEME']): - """ Create a user - """ - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User( - localpart=localpart, - domain=domain, - global_admin=False - ) - user.set_password(password, hash_scheme=hash_scheme) - db.session.add(user) - db.session.commit() - - -@manager.option('-n', '--domain_name', dest='domain_name') -@manager.option('-u', '--max_users', dest='max_users') -@manager.option('-a', '--max_aliases', dest='max_aliases') -@manager.option('-q', '--max_quota_bytes', dest='max_quota_bytes') -def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0): - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - db.session.commit() - - -@manager.command -def user_import(localpart, domain_name, password_hash, - hash_scheme=app.config['PASSWORD_SCHEME']): - """ Import a user along with password hash. Available hashes: - 'SHA512-CRYPT' - 'SHA256-CRYPT' - 'MD5-CRYPT' - 'CRYPT' - """ - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User( - localpart=localpart, - domain=domain, - global_admin=False - ) - user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) - db.session.add(user) - db.session.commit() - - -@manager.command -def config_update(verbose=False, delete_objects=False): - """sync configuration with data from YAML-formatted stdin""" - import yaml - import sys - new_config = yaml.load(sys.stdin) - # print new_config - domains = new_config.get('domains', []) - tracked_domains = set() - for domain_config in domains: - if verbose: - print(str(domain_config)) - domain_name = domain_config['name'] - max_users = domain_config.get('max_users', 0) - max_aliases = domain_config.get('max_aliases', 0) - max_quota_bytes = domain_config.get('max_quota_bytes', 0) - tracked_domains.add(domain_name) - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name, - max_users=max_users, - max_aliases=max_aliases, - max_quota_bytes=max_quota_bytes) - db.session.add(domain) - print("Added " + str(domain_config)) - else: - domain.max_users = max_users - domain.max_aliases = max_aliases - domain.max_quota_bytes = max_quota_bytes - db.session.add(domain) - print("Updated " + str(domain_config)) - - users = new_config.get('users', []) - tracked_users = set() - user_optional_params = ('comment', 'quota_bytes', 'global_admin', - 'enable_imap', 'enable_pop', 'forward_enabled', - 'forward_destination', 'reply_enabled', - 'reply_subject', 'reply_body', 'displayed_name', - 'spam_enabled', 'email', 'spam_threshold') - for user_config in users: - if verbose: - print(str(user_config)) - localpart = user_config['localpart'] - domain_name = user_config['domain'] - password_hash = user_config.get('password_hash', None) - hash_scheme = user_config.get('hash_scheme', None) - domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) - optional_params = {} - for k in user_optional_params: - if k in user_config: - optional_params[k] = user_config[k] - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - user = models.User.query.get(email) - tracked_users.add(email) - tracked_domains.add(domain_name) - if not user: - user = models.User( - localpart=localpart, - domain=domain, - **optional_params - ) - else: - for k in optional_params: - setattr(user, k, optional_params[k]) - user.set_password(password_hash, hash_scheme=hash_scheme, raw=True) - db.session.add(user) - - aliases = new_config.get('aliases', []) - tracked_aliases = set() - for alias_config in aliases: - if verbose: - print(str(alias_config)) - localpart = alias_config['localpart'] - domain_name = alias_config['domain'] - if type(alias_config['destination']) is str: - destination = alias_config['destination'].split(',') - else: - destination = alias_config['destination'] - wildcard = alias_config.get('wildcard', False) - domain = models.Domain.query.get(domain_name) - email = '{0}@{1}'.format(localpart, domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - alias = models.Alias.query.get(email) - tracked_aliases.add(email) - tracked_domains.add(domain_name) - if not alias: - alias = models.Alias( - localpart=localpart, - domain=domain, - wildcard=wildcard, - destination=destination, - email=email - ) - else: - alias.destination = destination - alias.wildcard = wildcard - db.session.add(alias) - - db.session.commit() - - managers = new_config.get('managers', []) - # tracked_managers=set() - for manager_config in managers: - if verbose: - print(str(manager_config)) - domain_name = manager_config['domain'] - user_name = manager_config['user'] - domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) - if manageruser not in domain.managers: - domain.managers.append(manageruser) - db.session.add(domain) - - db.session.commit() - - if delete_objects: - for user in db.session.query(models.User).all(): - if not (user.email in tracked_users): - if verbose: - print("Deleting user: " + str(user.email)) - db.session.delete(user) - for alias in db.session.query(models.Alias).all(): - if not (alias.email in tracked_aliases): - if verbose: - print("Deleting alias: " + str(alias.email)) - db.session.delete(alias) - for domain in db.session.query(models.Domain).all(): - if not (domain.name in tracked_domains): - if verbose: - print("Deleting domain: " + str(domain.name)) - db.session.delete(domain) - db.session.commit() - - -@manager.command -def user_delete(email): - """delete user""" - user = models.User.query.get(email) - if user: - db.session.delete(user) - db.session.commit() - - -@manager.command -def alias_delete(email): - """delete alias""" - alias = models.Alias.query.get(email) - if alias: - db.session.delete(alias) - db.session.commit() - - -@manager.command -def alias(localpart, domain_name, destination): - """ Create an alias - """ - domain = models.Domain.query.get(domain_name) - if not domain: - domain = models.Domain(name=domain_name) - db.session.add(domain) - alias = models.Alias( - localpart=localpart, - domain=domain, - destination=destination.split(','), - email="%s@%s" % (localpart, domain_name) - ) - db.session.add(alias) - db.session.commit() - -# Set limits to a domain - - -@manager.command -def setlimits(domain_name, max_users, max_aliases, max_quota_bytes): - domain = models.Domain.query.get(domain_name) - domain.max_users = max_users - domain.max_aliases = max_aliases - domain.max_quota_bytes = max_quota_bytes - - db.session.add(domain) - db.session.commit() - -# Make the user manager of a domain - - -@manager.command -def setmanager(domain_name, user_name='manager'): - domain = models.Domain.query.get(domain_name) - manageruser = models.User.query.get(user_name + '@' + domain_name) - domain.managers.append(manageruser) - db.session.add(domain) - db.session.commit() - - -if __name__ == "__main__": - manager.run() From 9881dd207480ddaa486718d1f26a2f8188013689 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Mon, 22 Oct 2018 12:13:22 +0300 Subject: [PATCH 07/62] Documentation for adding more steps/flavors --- setup/readme.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 setup/readme.md diff --git a/setup/readme.md b/setup/readme.md new file mode 100644 index 00000000..0f096734 --- /dev/null +++ b/setup/readme.md @@ -0,0 +1,46 @@ +## Adding more flavors/steps +(Everything will go under setup/ directory - using Kubernetes flavor as example) + +Until this point, the app is working as it follows: +- when accesing the setup page it will display the flavors selection step (`templates/steps/flavor.html`) +- after you choose your desired flavor it will iterare over the files in the flavor directory and building the page + (`templates/steps/config.html is general for all flavors`) +- when you complete all required fields and press "Setup Mailu" button it will redirect you to the setup page (`flavors/choosen-flavor/setup.html`) + +To add a new flavor you need to create a directory under `templates/steps/` in which you are adding actual steps. +Eg: Adding a WIP step we'll create `templates/steps/kubernetes/wip.html` + +*Note that wizard.html is iterating over files in this directory and building the page. Files are prefixed with a number for sorting purposes.* + +wip.html will start with + +``` +{% call macros.panel("info", "Step X - Work in progress") +``` + +and end with +``` +{% endcall %} +``` + +You store variable from front-page using the name attribute inside tag. +In the example below the string entered in the input field is stored in the variable `named var_test` +``` + +``` + +In order to user the variable furter you use it like `{{ var_test }}` + +In the setup page (`flavors/kubernetes/setup`) you cand add steps by importing macros + +``` +{% import "macros.html" as macros %} +``` + +and start and end every step with +``` +{% call macros.panel("info", "Step X - Title") %} +------------------- +{% endcall %} +``` + From 6a9e5c192116f84569761a29b983a5c5c4b448cd Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Mon, 22 Oct 2018 12:53:25 +0300 Subject: [PATCH 08/62] Create/generate file template --- setup/{readme.md => README.md} | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) rename setup/{readme.md => README.md} (72%) diff --git a/setup/readme.md b/setup/README.md similarity index 72% rename from setup/readme.md rename to setup/README.md index 0f096734..24c9cfa2 100644 --- a/setup/readme.md +++ b/setup/README.md @@ -15,7 +15,7 @@ Eg: Adding a WIP step we'll create `templates/steps/kubernetes/wip.html` wip.html will start with ``` -{% call macros.panel("info", "Step X - Work in progress") +{% call macros.panel("info", "Step X - Work in progress") %} ``` and end with @@ -31,7 +31,7 @@ In the example below the string entered in the input field is stored in the vari In order to user the variable furter you use it like `{{ var_test }}` -In the setup page (`flavors/kubernetes/setup`) you cand add steps by importing macros +In the setup page (`flavors/kubernetes/setup.html`) you cand add steps by importing macros ``` {% import "macros.html" as macros %} @@ -44,3 +44,16 @@ and start and end every step with {% endcall %} ``` +### Generating a file +Create the file template in `flavors/kubernetes/` (eg. file.txt) in which you save your variables +``` +ROOT = {{ root }} +MY_VAR = {{ var_test }} +``` + +When you submit to Setup Mailu the file will be generated. In order to get the file add the following command to setup.html + +``` +

curl {{ url_for('.file', uid=uid, filepath='file.txt', _external=True) }} > file.txt

+``` + From 3c9cae5d06c10e30a06d5ba0200415bcee643f35 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Mon, 22 Oct 2018 15:13:18 +0300 Subject: [PATCH 09/62] Added replicas variables for core containers - Added back restart: always on compose flavor - Moved Log driver from .env to docker-compose.yml --- setup/flavors/compose/docker-compose.yml | 12 +++++++++ setup/flavors/compose/mailu.env | 2 +- setup/flavors/stack/docker-compose.yml | 11 ++++---- setup/templates/steps/stack/03_expose.html | 3 ++- setup/templates/steps/stack/04_replicas.html | 28 ++++++++++++++++++++ 5 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 setup/templates/steps/stack/04_replicas.html diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index b01bb8fd..daea844e 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -10,13 +10,17 @@ services: # External dependencies redis: image: redis:alpine + restart: always volumes: - "{{ root }}/redis:/data" # Core services front: image: mailu/nginx:{{ version }} + restart: always env_file: {{ env }} + logging: + driver: {{ log_driver or 'json-file' }} ports: {% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %} {% if bind4 %} @@ -31,6 +35,7 @@ services: admin: image: mailu/admin:{{ version }} + restart: always env_file: {{ env }} {% if not admin_enabled %} ports: @@ -44,6 +49,7 @@ services: imap: image: mailu/dovecot:{{ version }} + restart: always env_file: {{ env }} volumes: - "{{ root }}/mail:/mail" @@ -53,6 +59,7 @@ services: smtp: image: mailu/postfix:{{ version }} + restart: always env_file: {{ env }} volumes: - "{{ root }}/overrides:/overrides" @@ -63,6 +70,7 @@ services: {% if antispam_enabled %} antispam: image: mailu/rspamd:{{ version }} + restart: always env_file: {{ env }} volumes: - "{{ root }}/filter:/var/lib/rspamd" @@ -75,6 +83,7 @@ services: {% if antivirus_enabled %} antivirus: image: mailu/clamav:{{ version }} + restart: always env_file: {{ env }} volumes: - "{{ root }}/filter:/data" @@ -83,6 +92,7 @@ services: {% if webdav_enabled %} webdav: image: mailu/radicale:{{ version }} + restart: always env_file: {{ env }} volumes: - "{{ root }}/dav:/data" @@ -91,6 +101,7 @@ services: {% if fetchmail_enabled %} fetchmail: image: mailu/fetchmail:{{ version }} + restart: always env_file: {{ env }} {% endif %} @@ -98,6 +109,7 @@ services: {% if webmail_type != 'none' %} webmail: image: mailu/{{ webmail_type }}:{{ version }} + restart: always env_file: {{ env }} volumes: - "{{ root }}/webmail:/data" diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 9fc1197d..9433ecce 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -138,7 +138,7 @@ DOMAIN_REGISTRATION=true # json-file (default) # journald (On systemd platforms, useful for Fail2Ban integration) # syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!) -LOG_DRIVER={{ log_driver or 'json-file' }} +# LOG_DRIVER={{ log_driver or 'json-file' }} # Docker-compose project name, this will prepended to containers names. COMPOSE_PROJECT_NAME={{ compose_project_name or 'mailu' }} diff --git a/setup/flavors/stack/docker-compose.yml b/setup/flavors/stack/docker-compose.yml index f27b661f..728135d5 100644 --- a/setup/flavors/stack/docker-compose.yml +++ b/setup/flavors/stack/docker-compose.yml @@ -10,7 +10,6 @@ services: # External dependencies redis: image: redis:alpine - restart: always volumes: - "{{ root }}/redis:/data" @@ -18,6 +17,8 @@ services: front: image: mailu/nginx:{{ version }} env_file: {{ env }} + logging: + driver: {{ log_driver or 'json-file' }} ports: {% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %} - target: {{ port }} @@ -27,7 +28,7 @@ services: volumes: - "{{ root }}/certs:/certs" deploy: - replicas: 1 + replicas: {{ front_replicas }} admin: image: mailu/admin:{{ version }} @@ -40,7 +41,7 @@ services: - "{{ root }}/data:/data" - "{{ root }}/dkim:/dkim" deploy: - replicas: 1 + replicas: {{ admin_replicas }} imap: image: mailu/dovecot:{{ version }} @@ -52,7 +53,7 @@ services: - "{{ root }}/mail:/mail" - "{{ root }}/overrides:/overrides" deploy: - replicas: 1 + replicas: {{ imap_replicas }} smtp: image: mailu/postfix:{{ version }} @@ -62,7 +63,7 @@ services: volumes: - "{{ root }}/overrides:/overrides" deploy: - replicas: 1 + replicas: {{ smtp_replicas }} # Optional services {% if antispam_enabled %} diff --git a/setup/templates/steps/stack/03_expose.html b/setup/templates/steps/stack/03_expose.html index a9cffc1c..bb61a35c 100644 --- a/setup/templates/steps/stack/03_expose.html +++ b/setup/templates/steps/stack/03_expose.html @@ -5,7 +5,8 @@ you expose it to the world.

- +

You server will be available under a main hostname but may expose multiple public diff --git a/setup/templates/steps/stack/04_replicas.html b/setup/templates/steps/stack/04_replicas.html new file mode 100644 index 00000000..785125cc --- /dev/null +++ b/setup/templates/steps/stack/04_replicas.html @@ -0,0 +1,28 @@ +{% call macros.panel("info", "Step 5 - Number of replicas for containers") %} +

Select number of replicas for containers

+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +{% endcall %} \ No newline at end of file From aed80a74faa241a0b37dea79e81c8cbd3f2420e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 23 Oct 2018 11:52:15 +0300 Subject: [PATCH 10/62] Rectify decleration of domain_name --- core/admin/mailu/internal/views/postfix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index f0fbaa5d..894532a3 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -27,7 +27,7 @@ def postfix_alias_map(alias): @internal.route("/postfix/transport/") def postfix_transport(email): - localpart, domain = models.Email.resolve_domain(email) + localpart, domain_name = models.Email.resolve_domain(email) relay = models.Relay.query.get(domain_name) or flask.abort(404) return flask.jsonify("smtp:[{}]".format(relay.smtp)) From ed81c076f2c635c342e21f08289aa203f00f0b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 23 Oct 2018 11:53:52 +0300 Subject: [PATCH 11/62] Take out "models" path, as we are already in it --- core/admin/mailu/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 2845334e..ffe1ad08 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -224,17 +224,17 @@ class Email(object): @classmethod def resolve_domain(cls, email): localpart, domain_name = email.split('@', 1) if '@' in email else (None, email) - alternative = models.Alternative.query.get(domain_name) + alternative = Alternative.query.get(domain_name) if alternative: domain_name = alternative.domain_name return (localpart, domain_name) @classmethod def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False): - alias = models.Alias.resolve(localpart, domain_name) + alias = Alias.resolve(localpart, domain_name) if alias: return alias.destination - user = models.User.query.get('{}@{}'.format(localpart, domain_name)) + user = User.query.get('{}@{}'.format(localpart, domain_name)) if user: if user.forward_enabled: destination = user.forward_destination From ae8f928fc0705adf1bde24c6c812ec45d964e6b9 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Tue, 23 Oct 2018 17:07:05 +0300 Subject: [PATCH 12/62] Added project name on docker commands --- setup/flavors/compose/setup.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup/flavors/compose/setup.html b/setup/flavors/compose/setup.html index 0379ba82..3d87a263 100644 --- a/setup/flavors/compose/setup.html +++ b/setup/flavors/compose/setup.html @@ -28,15 +28,15 @@ files before going any further.

{% call macros.panel("info", "Step 3 - Start the Compose project") %}

To start your compose project, simply run the Docker Compose up -command.

+command using -p mailu flag for project name.

cd {{ root }}
-docker-compose up -d
+docker-compose -p mailu up -d
Before you can use Mailu, you must create the primary administrator user account. This should be {{ postmaster }}@{{ domain }}. Use the following command, changing PASSWORD to your liking: -
docker-compose exec admin python manage.py admin {{ postmaster }} {{ domain }} PASSWORD
+
docker-compose -p mailu exec admin python manage.py admin {{ postmaster }} {{ domain }} PASSWORD
 

 
 
Login to the admin interface to change the password for a safe one, at

From e5268de0c750125b5d9621fed47f643e3b97237b Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Tue, 23 Oct 2018 17:55:44 +0300
Subject: [PATCH 13/62] Revert default value for subnet

---
 setup/templates/steps/stack/03_expose.html | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/setup/templates/steps/stack/03_expose.html b/setup/templates/steps/stack/03_expose.html
index bb61a35c..a9cffc1c 100644
--- a/setup/templates/steps/stack/03_expose.html
+++ b/setup/templates/steps/stack/03_expose.html
@@ -5,8 +5,7 @@ you expose it to the world.

 
 

   
-  
+  
 

 
 
You server will be available under a main hostname but may expose multiple public

From 238d4e7f208b13b245b1bb98e8e8b56ce50e229f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Wed, 24 Oct 2018 12:02:28 +0300
Subject: [PATCH 14/62] Provide test cases created with the new setup utility

---
 tests/compose/core.yml                |  74 ++++++++++++++++
 tests/compose/fetchmail.yml           |  77 +++++++++++++++++
 tests/compose/filters+dns.yml         | 119 ++++++++++++++++++++++++++
 tests/compose/{core.env => mailu.env} |  54 ++++++------
 tests/compose/rainloop.yml            |  84 ++++++++++++++++++
 tests/compose/roundcube.yml           |  84 ++++++++++++++++++
 tests/compose/run.yml                 | 101 ----------------------
 tests/compose/webdav.yml              |  79 +++++++++++++++++
 8 files changed, 545 insertions(+), 127 deletions(-)
 create mode 100644 tests/compose/core.yml
 create mode 100644 tests/compose/fetchmail.yml
 create mode 100644 tests/compose/filters+dns.yml
 rename tests/compose/{core.env => mailu.env} (72%)
 create mode 100644 tests/compose/rainloop.yml
 create mode 100644 tests/compose/roundcube.yml
 delete mode 100644 tests/compose/run.yml
 create mode 100644 tests/compose/webdav.yml

diff --git a/tests/compose/core.yml b/tests/compose/core.yml
new file mode 100644
index 00000000..460a6908
--- /dev/null
+++ b/tests/compose/core.yml
@@ -0,0 +1,74 @@
+# This file is auto-generated by the Mailu configuration wizard.
+# Please read the documentation before attempting any change.
+# Generated for compose flavor
+
+version: '3.6'
+
+services:
+
+  # External dependencies
+  redis:
+    image: redis:alpine
+    volumes:
+      - "/mailu/redis:/data"
+
+  # Core services
+  front:
+    image: mailu/nginx:master
+    env_file: mailu.env
+    ports:
+      - "127.0.0.1:80:80"
+      - "::1:80:80"
+      - "127.0.0.1:443:443"
+      - "::1:443:443"
+      - "127.0.0.1:25:25"
+      - "::1:25:25"
+      - "127.0.0.1:465:465"
+      - "::1:465:465"
+      - "127.0.0.1:587:587"
+      - "::1:587:587"
+      - "127.0.0.1:110:110"
+      - "::1:110:110"
+      - "127.0.0.1:995:995"
+      - "::1:995:995"
+      - "127.0.0.1:143:143"
+      - "::1:143:143"
+      - "127.0.0.1:993:993"
+      - "::1:993:993"
+    volumes:
+      - "/mailu/certs:/certs"
+      
+
+  admin:
+    image: mailu/admin:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/data:/data"
+      - "/mailu/dkim:/dkim"
+    depends_on:
+      - redis
+
+  imap:
+    image: mailu/dovecot:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/mail:/mail"
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  smtp:
+    image: mailu/postfix:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  # Optional services
+
+
+
+
+  # Webmail
+  
diff --git a/tests/compose/fetchmail.yml b/tests/compose/fetchmail.yml
new file mode 100644
index 00000000..8f40ba19
--- /dev/null
+++ b/tests/compose/fetchmail.yml
@@ -0,0 +1,77 @@
+# This file is auto-generated by the Mailu configuration wizard.
+# Please read the documentation before attempting any change.
+# Generated for compose flavor
+
+version: '3.6'
+
+services:
+
+  # External dependencies
+  redis:
+    image: redis:alpine
+    volumes:
+      - "/mailu/redis:/data"
+
+  # Core services
+  front:
+    image: mailu/nginx:master
+    env_file: mailu.env
+    ports:
+      - "127.0.0.1:80:80"
+      - "::1:80:80"
+      - "127.0.0.1:443:443"
+      - "::1:443:443"
+      - "127.0.0.1:25:25"
+      - "::1:25:25"
+      - "127.0.0.1:465:465"
+      - "::1:465:465"
+      - "127.0.0.1:587:587"
+      - "::1:587:587"
+      - "127.0.0.1:110:110"
+      - "::1:110:110"
+      - "127.0.0.1:995:995"
+      - "::1:995:995"
+      - "127.0.0.1:143:143"
+      - "::1:143:143"
+      - "127.0.0.1:993:993"
+      - "::1:993:993"
+    volumes:
+      - "/mailu/certs:/certs"
+      
+
+  admin:
+    image: mailu/admin:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/data:/data"
+      - "/mailu/dkim:/dkim"
+    depends_on:
+      - redis
+
+  imap:
+    image: mailu/dovecot:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/mail:/mail"
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  smtp:
+    image: mailu/postfix:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  # Optional services
+
+
+
+  fetchmail:
+    image: mailu/fetchmail:master
+    env_file: mailu.env
+
+  # Webmail
+  
diff --git a/tests/compose/filters+dns.yml b/tests/compose/filters+dns.yml
new file mode 100644
index 00000000..993ed786
--- /dev/null
+++ b/tests/compose/filters+dns.yml
@@ -0,0 +1,119 @@
+# This file is auto-generated by the Mailu configuration wizard.
+# Please read the documentation before attempting any change.
+# Generated for compose flavor
+
+version: '3.6'
+
+services:
+
+  # External dependencies
+  redis:
+    image: redis:alpine
+    volumes:
+      - "/mailu/redis:/data"
+
+  # Core services
+  front:
+    image: mailu/nginx:master
+    env_file: mailu.env
+    ports:
+      - "127.0.0.1:80:80"
+      - "::1:80:80"
+      - "127.0.0.1:443:443"
+      - "::1:443:443"
+      - "127.0.0.1:25:25"
+      - "::1:25:25"
+      - "127.0.0.1:465:465"
+      - "::1:465:465"
+      - "127.0.0.1:587:587"
+      - "::1:587:587"
+      - "127.0.0.1:110:110"
+      - "::1:110:110"
+      - "127.0.0.1:995:995"
+      - "::1:995:995"
+      - "127.0.0.1:143:143"
+      - "::1:143:143"
+      - "127.0.0.1:993:993"
+      - "::1:993:993"
+    volumes:
+      - "/mailu/certs:/certs"
+      
+  resolver:
+    image: mailu/unbound:master
+    env_file: mailu.env
+    restart: always
+    networks:
+      default:
+        ipv4_address: 192.168.0.254
+
+  admin:
+    image: mailu/admin:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/data:/data"
+      - "/mailu/dkim:/dkim"
+    depends_on:
+      - redis
+
+  imap:
+    image: mailu/dovecot:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/mail:/mail"
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  smtp:
+    image: mailu/postfix:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+      - resolver
+    dns:
+      - 192.168.0.254
+
+  # Optional services
+  antispam:
+    image: mailu/rspamd:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/filter:/var/lib/rspamd"
+      - "/mailu/dkim:/dkim"
+      - "/mailu/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
+      - resolver
+    dns:
+      - 192.168.0.254
+
+  antivirus:
+    image: mailu/clamav:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/filter:/data"
+    depends_on:
+      - resolver
+    dns:
+      - 192.168.0.254
+
+
+
+  # Webmail
+  webmail:
+    image: mailu/rainloop:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/webmail:/data"
+    depends_on:
+      - imap
+  
+networks:
+  default:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 192.168.0.0/24
diff --git a/tests/compose/core.env b/tests/compose/mailu.env
similarity index 72%
rename from tests/compose/core.env
rename to tests/compose/mailu.env
index 78c307c0..74c87118 100644
--- a/tests/compose/core.env
+++ b/tests/compose/mailu.env
@@ -1,31 +1,38 @@
 # Mailu main configuration file
 #
-# Most configuration variables can be modified through the Web interface,
-# these few settings must however be configured before starting the mail
-# server and require a restart upon change.
+# Generated for compose flavor
+#
+# This file is autogenerated by the configuration management wizard.
+# For a detailed list of configuration variables, see the documentation at
+# https://mailu.io
 
 ###################################
 # Common configuration variables
 ###################################
 
 # Set this to the path where Mailu data and configuration is stored
-ROOT=/mailu
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
 
 # Mailu version to run (1.0, 1.1, etc. or master)
 #VERSION=master
 
 # Set to a randomly generated 16 bytes string
-SECRET_KEY=ChangeMeChangeMe
+SECRET_KEY=1DS36JPBRGPM5JUC
 
 # Address where listening ports should bind
-BIND_ADDRESS4=127.0.0.1
-#BIND_ADDRESS6=::1
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4= 127.0.0.1 (default: 127.0.0.1)
+# PUBLIC_IPV6= ::1 (default: ::1)
+
+# Subnet
+SUBNET=192.168.0.0/24
 
 # Main mail domain
 DOMAIN=mailu.io
 
 # Hostnames for this server, separated with comas
-HOSTNAMES=mail.mailu.io,alternative.mailu.io,yetanother.mailu.io
+HOSTNAMES=mail.mailu.io
 
 # Postmaster local part (will append the main mail domain)
 POSTMASTER=admin
@@ -34,7 +41,7 @@ POSTMASTER=admin
 TLS_FLAVOR=cert
 
 # Authentication rate limit (per source IP address)
-AUTH_RATELIMIT=10/minute;1000/hour
+AUTH_RATELIMIT=10/minute;1000/hour 
 
 # Opt-out of statistics, replace with "True" to opt out
 DISABLE_STATISTICS=False
@@ -44,7 +51,7 @@ DISABLE_STATISTICS=False
 ###################################
 
 # Expose the admin interface (value: true, false)
-ADMIN=false
+ADMIN=true
 
 # Choose which webmail to run if any (values: roundcube, rainloop, none)
 WEBMAIL=none
@@ -53,7 +60,10 @@ WEBMAIL=none
 WEBDAV=none
 
 # Antivirus solution (value: clamav, none)
-ANTIVIRUS=none
+#ANTIVIRUS=none
+
+#Antispam solution
+ANTISPAM=none
 
 ###################################
 # Mail settings
@@ -65,7 +75,7 @@ MESSAGE_SIZE_LIMIT=50000000
 
 # Networks granted relay permissions, make sure that you include your Docker
 # internal network (default to 172.17.0.0/16)
-RELAYNETS=172.16.0.0/12
+RELAYNETS=172.17.0.0/16
 
 # Will relay all outgoing mails if configured
 RELAYHOST=
@@ -74,18 +84,12 @@ RELAYHOST=
 FETCHMAIL_DELAY=600
 
 # Recipient delimiter, character used to delimiter localpart from custom address part
-# e.g. localpart+custom@domain;tld
 RECIPIENT_DELIMITER=+
 
 # DMARC rua and ruf email
 DMARC_RUA=admin
 DMARC_RUF=admin
 
-# Welcome email, enable and set a topic and body if you wish to send welcome
-# emails to all users.
-WELCOME=false
-WELCOME_SUBJECT=Welcome to your new email account
-WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly!
 
 # Maildir Compression
 # choose compression-method, default: none (value: bz2, gz)
@@ -109,12 +113,7 @@ SITENAME=Mailu
 # Linked Website URL
 WEBSITE=https://mailu.io
 
-# Registration reCaptcha settings (warning, this has some privacy impact)
-# RECAPTCHA_PUBLIC_KEY=
-# RECAPTCHA_PRIVATE_KEY=
 
-# Domain registration, uncomment to enable
-# DOMAIN_REGISTRATION=true
 
 ###################################
 # Advanced settings
@@ -127,14 +126,17 @@ WEBSITE=https://mailu.io
 LOG_DRIVER=json-file
 
 # Docker-compose project name, this will prepended to containers names.
-#COMPOSE_PROJECT_NAME=mailu
+COMPOSE_PROJECT_NAME=mailu
 
 # Default password scheme used for newly created accounts and changed passwords
-# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
-PASSWORD_SCHEME=SHA512-CRYPT
+# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME=BLF-CRYPT
 
 # Header to take the real ip from
 REAL_IP_HEADER=
 
 # IPs for nginx set_real_ip_from (CIDR list separated by commas)
 REAL_IP_FROM=
+
+# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
+REJECT_UNLISTED_RECIPIENT=
\ No newline at end of file
diff --git a/tests/compose/rainloop.yml b/tests/compose/rainloop.yml
new file mode 100644
index 00000000..29011eeb
--- /dev/null
+++ b/tests/compose/rainloop.yml
@@ -0,0 +1,84 @@
+# This file is auto-generated by the Mailu configuration wizard.
+# Please read the documentation before attempting any change.
+# Generated for compose flavor
+
+version: '3.6'
+
+services:
+
+  # External dependencies
+  redis:
+    image: redis:alpine
+    volumes:
+      - "/mailu/redis:/data"
+
+  # Core services
+  front:
+    image: mailu/nginx:master
+    env_file: mailu.env
+    ports:
+      - "127.0.0.1:80:80"
+      - "::1:80:80"
+      - "127.0.0.1:443:443"
+      - "::1:443:443"
+      - "127.0.0.1:25:25"
+      - "::1:25:25"
+      - "127.0.0.1:465:465"
+      - "::1:465:465"
+      - "127.0.0.1:587:587"
+      - "::1:587:587"
+      - "127.0.0.1:110:110"
+      - "::1:110:110"
+      - "127.0.0.1:995:995"
+      - "::1:995:995"
+      - "127.0.0.1:143:143"
+      - "::1:143:143"
+      - "127.0.0.1:993:993"
+      - "::1:993:993"
+    volumes:
+      - "/mailu/certs:/certs"
+      
+
+  admin:
+    image: mailu/admin:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/data:/data"
+      - "/mailu/dkim:/dkim"
+    depends_on:
+      - redis
+
+  imap:
+    image: mailu/dovecot:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/mail:/mail"
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  smtp:
+    image: mailu/postfix:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  # Optional services
+
+
+
+
+  # Webmail
+  webmail:
+    image: mailu/rainloop:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/webmail:/data"
+    depends_on:
+      - imap
+      - resolver
+    dns:
+      - 192.168.0.254
+  
diff --git a/tests/compose/roundcube.yml b/tests/compose/roundcube.yml
new file mode 100644
index 00000000..c6e46ed9
--- /dev/null
+++ b/tests/compose/roundcube.yml
@@ -0,0 +1,84 @@
+# This file is auto-generated by the Mailu configuration wizard.
+# Please read the documentation before attempting any change.
+# Generated for compose flavor
+
+version: '3.6'
+
+services:
+
+  # External dependencies
+  redis:
+    image: redis:alpine
+    volumes:
+      - "/mailu/redis:/data"
+
+  # Core services
+  front:
+    image: mailu/nginx:master
+    env_file: mailu.env
+    ports:
+      - "127.0.0.1:80:80"
+      - "::1:80:80"
+      - "127.0.0.1:443:443"
+      - "::1:443:443"
+      - "127.0.0.1:25:25"
+      - "::1:25:25"
+      - "127.0.0.1:465:465"
+      - "::1:465:465"
+      - "127.0.0.1:587:587"
+      - "::1:587:587"
+      - "127.0.0.1:110:110"
+      - "::1:110:110"
+      - "127.0.0.1:995:995"
+      - "::1:995:995"
+      - "127.0.0.1:143:143"
+      - "::1:143:143"
+      - "127.0.0.1:993:993"
+      - "::1:993:993"
+    volumes:
+      - "/mailu/certs:/certs"
+      
+
+  admin:
+    image: mailu/admin:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/data:/data"
+      - "/mailu/dkim:/dkim"
+    depends_on:
+      - redis
+
+  imap:
+    image: mailu/dovecot:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/mail:/mail"
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  smtp:
+    image: mailu/postfix:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  # Optional services
+
+
+
+
+  # Webmail
+  webmail:
+    image: mailu/roundcube:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/webmail:/data"
+    depends_on:
+      - imap
+      - resolver
+    dns:
+      - 192.168.0.254
+  
diff --git a/tests/compose/run.yml b/tests/compose/run.yml
deleted file mode 100644
index eac35b76..00000000
--- a/tests/compose/run.yml
+++ /dev/null
@@ -1,101 +0,0 @@
-version: '2'
-
-services:
-
-  front:
-    image: $DOCKER_ORG/nginx:$VERSION
-    restart: 'no'
-    env_file: $PWD/.env
-    logging:
-      driver: $LOG_DRIVER
-    ports:
-    - "$BIND_ADDRESS4:80:80"
-    - "$BIND_ADDRESS4:443:443"
-    - "$BIND_ADDRESS4:110:110"
-    - "$BIND_ADDRESS4:143:143"
-    - "$BIND_ADDRESS4:993:993"
-    - "$BIND_ADDRESS4:995:995"
-    - "$BIND_ADDRESS4:25:25"
-    - "$BIND_ADDRESS4:465:465"
-    - "$BIND_ADDRESS4:587:587"
-    volumes:
-      - "$ROOT/certs:/certs"
-
-  redis:
-    image: redis:alpine
-    restart: 'no'
-    volumes:
-      - "$ROOT/redis:/data"
-
-  imap:
-    image: $DOCKER_ORG/dovecot:$VERSION
-    restart: 'no'
-    env_file: $PWD/.env
-    volumes:
-      - "$ROOT/data:/data"
-      - "$ROOT/mail:/mail"
-      - "$ROOT/overrides:/overrides"
-    depends_on:
-      - front
-
-  smtp:
-    image: $DOCKER_ORG/postfix:$VERSION
-    restart: 'no'
-    env_file: $PWD/.env
-    volumes:
-      - "$ROOT/data:/data"
-      - "$ROOT/overrides:/overrides"
-    depends_on:
-      - front
-
-  antispam:
-    image: $DOCKER_ORG/rspamd:$VERSION
-    restart: 'no'
-    env_file: $PWD/.env
-    volumes:
-      - "$ROOT/filter:/var/lib/rspamd"
-      - "$ROOT/dkim:/dkim"
-      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
-    depends_on:
-      - front
-
-  antivirus:
-    image: $DOCKER_ORG/$ANTIVIRUS:$VERSION
-    restart: 'no'
-    env_file: $PWD/.env
-    volumes:
-      - "$ROOT/filter:/data"
-
-  webdav:
-    image: $DOCKER_ORG/$WEBDAV:$VERSION
-    restart: 'no'
-    env_file: $PWD/.env
-    volumes:
-      - "$ROOT/dav:/data"
-
-  admin:
-    image: $DOCKER_ORG/admin:$VERSION
-    restart: 'no'
-    env_file: $PWD/.env
-    volumes:
-      - "$ROOT/data:/data"
-      - "$ROOT/dkim:/dkim"
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    depends_on:
-      - redis
-
-  webmail:
-    image: "$DOCKER_ORG/$WEBMAIL:$VERSION"
-    restart: 'no'
-    env_file: $PWD/.env
-    volumes:
-      - "$ROOT/webmail:/data"
-    depends_on:
-      - imap
-
-  fetchmail:
-    image: $DOCKER_ORG/fetchmail:$VERSION
-    restart: 'no'
-    env_file: $PWD/.env
-    volumes:
-      - "$ROOT/data:/data"
diff --git a/tests/compose/webdav.yml b/tests/compose/webdav.yml
new file mode 100644
index 00000000..b4d222ef
--- /dev/null
+++ b/tests/compose/webdav.yml
@@ -0,0 +1,79 @@
+# This file is auto-generated by the Mailu configuration wizard.
+# Please read the documentation before attempting any change.
+# Generated for compose flavor
+
+version: '3.6'
+
+services:
+
+  # External dependencies
+  redis:
+    image: redis:alpine
+    volumes:
+      - "/mailu/redis:/data"
+
+  # Core services
+  front:
+    image: mailu/nginx:master
+    env_file: mailu.env
+    ports:
+      - "127.0.0.1:80:80"
+      - "::1:80:80"
+      - "127.0.0.1:443:443"
+      - "::1:443:443"
+      - "127.0.0.1:25:25"
+      - "::1:25:25"
+      - "127.0.0.1:465:465"
+      - "::1:465:465"
+      - "127.0.0.1:587:587"
+      - "::1:587:587"
+      - "127.0.0.1:110:110"
+      - "::1:110:110"
+      - "127.0.0.1:995:995"
+      - "::1:995:995"
+      - "127.0.0.1:143:143"
+      - "::1:143:143"
+      - "127.0.0.1:993:993"
+      - "::1:993:993"
+    volumes:
+      - "/mailu/certs:/certs"
+      
+
+  admin:
+    image: mailu/admin:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/data:/data"
+      - "/mailu/dkim:/dkim"
+    depends_on:
+      - redis
+
+  imap:
+    image: mailu/dovecot:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/mail:/mail"
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  smtp:
+    image: mailu/postfix:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  # Optional services
+
+
+  webdav:
+    image: mailu/radicale:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/dav:/data"
+
+
+  # Webmail
+  

From 3b1fdc6166a3505e7c516d0f99b210dc5ab066f9 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Thu, 25 Oct 2018 15:27:09 +0300
Subject: [PATCH 15/62] Migrate test script from shell to python

- test.py needs to be called with 2 arguments : test_name and timeout
  - it will cd to test_name dir and use the test_name.yml from there
  - it will sleep for an amount of time equals to timeout in minutes
- it will perform health checks for containers. If healtcheck isn't enabled will check for running state
- it will run hooks inside the test_name dir ( .py and .sh) if there are any
- printing logs in any case
---
 .travis.yml                                   |  7 +-
 tests/compose/{ => core}/core.yml             | 10 +-
 tests/compose/{ => fetchmail}/fetchmail.yml   |  0
 .../compose/{ => filters+dns}/filters+dns.yml |  0
 tests/compose/{ => rainloop}/rainloop.yml     |  0
 tests/compose/{ => roundcube}/roundcube.yml   |  0
 tests/compose/test.py                         | 92 +++++++++++++++++++
 tests/compose/{ => webdav}/webdav.yml         |  0
 tests/requirements.txt                        |  2 +
 9 files changed, 109 insertions(+), 2 deletions(-)
 rename tests/compose/{ => core}/core.yml (85%)
 rename tests/compose/{ => fetchmail}/fetchmail.yml (100%)
 rename tests/compose/{ => filters+dns}/filters+dns.yml (100%)
 rename tests/compose/{ => rainloop}/rainloop.yml (100%)
 rename tests/compose/{ => roundcube}/roundcube.yml (100%)
 create mode 100755 tests/compose/test.py
 rename tests/compose/{ => webdav}/webdav.yml (100%)
 create mode 100644 tests/requirements.txt

diff --git a/.travis.yml b/.travis.yml
index c3a19529..107696aa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,12 +6,17 @@ addons:
       - docker-ce
 env:
   - VERSION=$TRAVIS_BRANCH
+language: python
+python:
+  - "3.6"
+install:
+  - pip install -r tests/requirements.txt
 
 script:
   # Default to mailu for DOCKER_ORG
   - if [ -z "$DOCKER_ORG" ]; then export DOCKER_ORG="mailu"; fi
   - docker-compose -f tests/build.yml build
-  - tests/compose/test-script.sh
+  - python tests/compose/test.py
 
 deploy:
   provider: script
diff --git a/tests/compose/core.yml b/tests/compose/core/core.yml
similarity index 85%
rename from tests/compose/core.yml
rename to tests/compose/core/core.yml
index 460a6908..d85388a4 100644
--- a/tests/compose/core.yml
+++ b/tests/compose/core/core.yml
@@ -66,7 +66,15 @@ services:
       - front
 
   # Optional services
-
+  antispam:
+    image: mailu/rspamd:master
+    env_file: mailu.env
+    volumes:
+      - "/mailu/filter:/var/lib/rspamd"
+      - "/mailu/dkim:/dkim"
+      - "/mailu/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
 
 
 
diff --git a/tests/compose/fetchmail.yml b/tests/compose/fetchmail/fetchmail.yml
similarity index 100%
rename from tests/compose/fetchmail.yml
rename to tests/compose/fetchmail/fetchmail.yml
diff --git a/tests/compose/filters+dns.yml b/tests/compose/filters+dns/filters+dns.yml
similarity index 100%
rename from tests/compose/filters+dns.yml
rename to tests/compose/filters+dns/filters+dns.yml
diff --git a/tests/compose/rainloop.yml b/tests/compose/rainloop/rainloop.yml
similarity index 100%
rename from tests/compose/rainloop.yml
rename to tests/compose/rainloop/rainloop.yml
diff --git a/tests/compose/roundcube.yml b/tests/compose/roundcube/roundcube.yml
similarity index 100%
rename from tests/compose/roundcube.yml
rename to tests/compose/roundcube/roundcube.yml
diff --git a/tests/compose/test.py b/tests/compose/test.py
new file mode 100755
index 00000000..95670e30
--- /dev/null
+++ b/tests/compose/test.py
@@ -0,0 +1,92 @@
+import sys
+import os
+import time
+import docker
+from colorama import Fore, Style
+
+# Declare variables for service name and sleep time
+test_name=sys.argv[1]
+timeout=int(sys.argv[2])
+
+client = docker.APIClient(base_url='unix://var/run/docker.sock')
+
+containers = []
+
+# Start up containers
+def start():
+    os.system("cp mailu.env " + test_name + "/")
+    os.system("docker-compose -f " + test_name + "/" + test_name + ".yml -p $DOCKER_ORG up -d ")
+
+# Stop containers
+def stop(exit_code):
+    print_logs()
+    os.system("docker-compose -f " + test_name + "/" + test_name + ".yml -p $DOCKER_ORG down")
+    os.system("rm " + test_name +"/mailu.env")
+    sys.exit(exit_code)
+  
+# Sleep for a defined amount of time
+def sleep():
+    print(Fore.LIGHTMAGENTA_EX + "Sleeping for " + str(timeout) + "m" + Style.RESET_ALL)
+    time.sleep(timeout*60)
+    
+def health_checks():
+    exit_code = 0
+    #Iterating trough all containers dictionary
+    for container in client.containers(all=True):
+        #Perform "docker container inspect" on container based on container ID and save output to a dictionary
+        container_inspect = client.inspect_container(container['Id']) #Dict
+        
+        if "Health" in container_inspect['State'].keys():
+            if container_inspect['State']['Health']['Status'] == "healthy":
+                print(Fore.GREEN + "Health status for " + container_inspect['Name'].replace("/", "") + " : " + Fore.CYAN + container_inspect['State']['Health']['Status'] + Style.RESET_ALL)
+            if container_inspect['State']['Health']['Status'] != "healthy":
+                print(Fore.RED + "Container " + container_inspect['Name'].replace("/", "") + " is " + Fore.YELLOW + container_inspect['State']['Health']['Status']
+                      + Fore.RED + ", FailingStreak: " + Fore.YELLOW + str(container_inspect['State']['Health']['FailingStreak'])
+                      + Fore.RED + ", Log: " + Fore.YELLOW + str(container_inspect['State']['Health']['Log']) + Style.RESET_ALL)
+                exit_code = 1
+        else:
+            if container_inspect['State']['Status'] == "running":
+                print(Fore.GREEN + "Running status for " + container_inspect['Name'].replace("/", "") + " : " + Fore.BLUE + container_inspect['State']['Status'] + Style.RESET_ALL)
+            if container_inspect['State']['Status'] != "running":
+                print(Fore.RED + "Container " + container_inspect['Name'].replace("/", "") + " state is: " + Fore.YELLOW + container_inspect['State']['Status'] + Style.RESET_ALL)
+                exit_code = 1
+           
+        #Saving Id, Name and state to a new dictionary
+        containers_dict = {}
+        containers_dict['Name'] = container_inspect['Name'].replace("/", "")
+        containers_dict['Id'] = container_inspect['Id']
+        containers_dict['State'] = container_inspect['State']
+        
+        #Adding the generated dictionary to a list
+        containers.append(containers_dict)
+        
+    if exit_code != 0:
+        stop(exit_code)
+        
+def print_logs():
+    print("Printing logs ...")
+    #Iterating through docker container inspect list and print logs
+    for container in containers:
+        print(Fore.LIGHTMAGENTA_EX + "Printing logs for: " + Fore.GREEN + container['Name'] + Style.RESET_ALL)
+        os.system('docker container logs ' + container['Name'])
+      
+#Iterating over hooks in test folder and running them  
+def hooks():
+    print("Running hooks")
+    for test_file in sorted(os.listdir(test_name + "/")):
+        if test_file.endswith(".py"):
+            os.system("python3 " + test_name + "/" + test_file)
+        elif test_file.endswith(".sh"):
+            os.system("./" + test_name + "/" + test_file)
+      
+start()
+print()
+sleep()
+print()
+os.system("docker ps -a")
+print()
+health_checks()
+print()
+hooks()
+print()
+stop(0)
diff --git a/tests/compose/webdav.yml b/tests/compose/webdav/webdav.yml
similarity index 100%
rename from tests/compose/webdav.yml
rename to tests/compose/webdav/webdav.yml
diff --git a/tests/requirements.txt b/tests/requirements.txt
new file mode 100644
index 00000000..36006cfc
--- /dev/null
+++ b/tests/requirements.txt
@@ -0,0 +1,2 @@
+docker
+colorama

From 72e931f4b470b74f2546ffa0483a88208987b7d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Thu, 25 Oct 2018 16:35:08 +0300
Subject: [PATCH 16/62] Modify setup templates to allow for DOCKER_ORG and
 VERSION override. (Needed for Travis)

---
 .travis.yml                              |  7 +++----
 setup/flavors/compose/docker-compose.yml | 18 ++++++++--------
 setup/flavors/stack/docker-compose.yml   | 18 ++++++++--------
 tests/build.yml                          | 26 ++++++++++++------------
 4 files changed, 34 insertions(+), 35 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 107696aa..bae6d04f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ addons:
     packages:
       - docker-ce
 env:
-  - VERSION=$TRAVIS_BRANCH
+  - MAILU_VERSION=$TRAVIS_BRANCH
 language: python
 python:
   - "3.6"
@@ -13,10 +13,9 @@ install:
   - pip install -r tests/requirements.txt
 
 script:
-  # Default to mailu for DOCKER_ORG
-  - if [ -z "$DOCKER_ORG" ]; then export DOCKER_ORG="mailu"; fi
   - docker-compose -f tests/build.yml build
-  - python tests/compose/test.py
+# test.py, test name and timeout between start and tests.
+  - python tests/compose/test.py core 1
 
 deploy:
   provider: script
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index daea844e..14eeba3c 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -16,7 +16,7 @@ services:
 
   # Core services
   front:
-    image: mailu/nginx:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
     logging:
@@ -34,7 +34,7 @@ services:
       - "{{ root }}/certs:/certs"
 
   admin:
-    image: mailu/admin:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
     {% if not admin_enabled %}
@@ -48,7 +48,7 @@ services:
       - redis
 
   imap:
-    image: mailu/dovecot:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
     volumes:
@@ -58,7 +58,7 @@ services:
       - front
 
   smtp:
-    image: mailu/postfix:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
     volumes:
@@ -69,7 +69,7 @@ services:
   # Optional services
   {% if antispam_enabled %}
   antispam:
-    image: mailu/rspamd:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
     volumes:
@@ -82,7 +82,7 @@ services:
 
   {% if antivirus_enabled %}
   antivirus:
-    image: mailu/clamav:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/clamav:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
     volumes:
@@ -91,7 +91,7 @@ services:
 
   {% if webdav_enabled %}
   webdav:
-    image: mailu/radicale:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/radicale:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
     volumes:
@@ -100,7 +100,7 @@ services:
 
   {% if fetchmail_enabled %}
   fetchmail:
-    image: mailu/fetchmail:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/fetchmail:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
   {% endif %}
@@ -108,7 +108,7 @@ services:
   # Webmail
   {% if webmail_type != 'none' %}
   webmail:
-    image: mailu/{{ webmail_type }}:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/{{ webmail_type }}:${MAILU_VERSION:-{{ version }}}
     restart: always
     env_file: {{ env }}
     volumes:
diff --git a/setup/flavors/stack/docker-compose.yml b/setup/flavors/stack/docker-compose.yml
index 728135d5..92a156c7 100644
--- a/setup/flavors/stack/docker-compose.yml
+++ b/setup/flavors/stack/docker-compose.yml
@@ -15,7 +15,7 @@ services:
 
 # Core services
   front:
-    image: mailu/nginx:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     logging:
       driver: {{ log_driver or 'json-file' }}
@@ -31,7 +31,7 @@ services:
       replicas: {{ front_replicas }}
 
   admin:
-    image: mailu/admin:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     {% if not admin_enabled %}
     ports:
@@ -44,7 +44,7 @@ services:
       replicas: {{ admin_replicas }}
 
   imap:
-    image: mailu/dovecot:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     environment:
     # Default to 10.0.1.0/24
@@ -56,7 +56,7 @@ services:
       replicas: {{ imap_replicas }}
 
   smtp:
-    image: mailu/postfix:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     environment:
       - POD_ADDRESS_RANGE={{ subnet }}
@@ -68,7 +68,7 @@ services:
   # Optional services
   {% if antispam_enabled %}
   antispam:
-    image: mailu/rspamd:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     environment:
       - POD_ADDRESS_RANGE={{ subnet }}
@@ -82,7 +82,7 @@ services:
 
   {% if antivirus_enabled %}
   antivirus:
-    image: mailu/clamav:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/clamav:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     volumes:
       - "{{ root }}/filter:/data"
@@ -92,7 +92,7 @@ services:
 
   {% if webdav_enabled %}
   webdav:
-    image: mailu/none:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/none:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     volumes:
       - "{{ root }}/dav:/data"
@@ -102,7 +102,7 @@ services:
 
   {% if fetchmail_enabled %}
   fetchmail:
-    image: mailu/fetchmail:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/fetchmail:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     volumes:
       - "{{ root }}/data:/data"
@@ -112,7 +112,7 @@ services:
 
   {% if webmail_type != 'none' %}
   webmail:
-    image: mailu/roundcube:{{ version }}
+    image: ${DOCKER_ORG:-mailu}/roundcube:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
     volumes:
       - "{{ root }}/webmail:/data"
diff --git a/tests/build.yml b/tests/build.yml
index 5f360ece..dc259608 100644
--- a/tests/build.yml
+++ b/tests/build.yml
@@ -3,54 +3,54 @@ version: '3'
 services:
 
   front:
-    image: ${DOCKER_ORG:-mailu}/nginx:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-local}
     build: ../core/nginx
 
   imap:
-    image: ${DOCKER_ORG:-mailu}/dovecot:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-local}
     build: ../core/dovecot
 
   smtp:
-    image: ${DOCKER_ORG:-mailu}/postfix:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-local}
     build: ../core/postfix
 
   antispam:
-    image: ${DOCKER_ORG:-mailu}/rspamd:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-local}
     build: ../services/rspamd
 
   antivirus:
-    image: ${DOCKER_ORG:-mailu}/clamav:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/clamav:${MAILU_VERSION:-local}
     build: ../optional/clamav
 
   webdav:
-    image: ${DOCKER_ORG:-mailu}/radicale:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/radicale:${MAILU_VERSION:-local}
     build: ../optional/radicale
 
   admin:
-    image: ${DOCKER_ORG:-mailu}/admin:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-local}
     build: ../core/admin
 
   roundcube:
-    image: ${DOCKER_ORG:-mailu}/roundcube:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/roundcube:${MAILU_VERSION:-local}
     build: ../webmails/roundcube
 
   rainloop:
-    image: ${DOCKER_ORG:-mailu}/rainloop:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/rainloop:${MAILU_VERSION:-local}
     build: ../webmails/rainloop
 
   fetchmail:
-    image: ${DOCKER_ORG:-mailu}/fetchmail:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/fetchmail:${MAILU_VERSION:-local}
     build: ../services/fetchmail
 
   none:
-    image: ${DOCKER_ORG:-mailu}/none:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/none:${MAILU_VERSION:-local}
     build: ../core/none
 
   docs:
-    image: ${DOCKER_ORG:-mailu}/docs:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/docs:${MAILU_VERSION:-local}
     build: ../docs
 
   setup:
-    image: ${DOCKER_ORG:-mailu}/setup:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/setup:${MAILU_VERSION:-local}
     build: ../setup
 

From dde7ccca97a72c20143f0cfb76a54077aab675f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Thu, 25 Oct 2018 17:35:53 +0300
Subject: [PATCH 17/62] Created test cases from latest setup revision

---
 .travis.yml                                   |   9 +-
 .../core/{core.yml => docker-compose.yml}     |  23 ++-
 tests/compose/{ => core}/mailu.env            |   9 +-
 .../docker-compose.yml}                       |  43 +++---
 tests/compose/fetchmail/fetchmail.yml         |  77 ----------
 tests/compose/fetchmail/mailu.env             | 139 ++++++++++++++++++
 tests/compose/filters/docker-compose.yml      |  95 ++++++++++++
 tests/compose/filters/mailu.env               | 139 ++++++++++++++++++
 .../docker-compose.yml}                       |  54 ++-----
 tests/compose/rainloop/mailu.env              | 139 ++++++++++++++++++
 .../{roundcube.yml => docker-compose.yml}     |  35 +++--
 tests/compose/roundcube/mailu.env             | 139 ++++++++++++++++++
 tests/compose/test-script.sh                  |  57 -------
 tests/compose/test.py                         |  12 +-
 .../webdav/{webdav.yml => docker-compose.yml} |  32 +++-
 tests/compose/webdav/mailu.env                | 139 ++++++++++++++++++
 16 files changed, 910 insertions(+), 231 deletions(-)
 rename tests/compose/core/{core.yml => docker-compose.yml} (75%)
 rename tests/compose/{ => core}/mailu.env (96%)
 rename tests/compose/{rainloop/rainloop.yml => fetchmail/docker-compose.yml} (63%)
 delete mode 100644 tests/compose/fetchmail/fetchmail.yml
 create mode 100644 tests/compose/fetchmail/mailu.env
 create mode 100644 tests/compose/filters/docker-compose.yml
 create mode 100644 tests/compose/filters/mailu.env
 rename tests/compose/{filters+dns/filters+dns.yml => rainloop/docker-compose.yml} (68%)
 create mode 100644 tests/compose/rainloop/mailu.env
 rename tests/compose/roundcube/{roundcube.yml => docker-compose.yml} (64%)
 create mode 100644 tests/compose/roundcube/mailu.env
 delete mode 100755 tests/compose/test-script.sh
 rename tests/compose/webdav/{webdav.yml => docker-compose.yml} (63%)
 create mode 100644 tests/compose/webdav/mailu.env

diff --git a/.travis.yml b/.travis.yml
index bae6d04f..b1eb2d10 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,10 +12,17 @@ python:
 install:
   - pip install -r tests/requirements.txt
 
-script:
+before_script:
   - docker-compose -f tests/build.yml build
+  
+script:
 # test.py, test name and timeout between start and tests.
   - python tests/compose/test.py core 1
+  - python tests/compose/test.py fetchmail 1
+  - travis_wait python tests/compose/test.py filters 10
+  - python tests/compose/test.py rainloop 1
+  - python tests/compose/test.py roundcube 1
+  - python tests/compose/test.py webdav 1
 
 deploy:
   provider: script
diff --git a/tests/compose/core/core.yml b/tests/compose/core/docker-compose.yml
similarity index 75%
rename from tests/compose/core/core.yml
rename to tests/compose/core/docker-compose.yml
index d85388a4..47c5f179 100644
--- a/tests/compose/core/core.yml
+++ b/tests/compose/core/docker-compose.yml
@@ -9,13 +9,17 @@ services:
   # External dependencies
   redis:
     image: redis:alpine
+    restart: always
     volumes:
       - "/mailu/redis:/data"
 
   # Core services
   front:
-    image: mailu/nginx:master
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
+    logging:
+      driver: json-file
     ports:
       - "127.0.0.1:80:80"
       - "::1:80:80"
@@ -37,10 +41,10 @@ services:
       - "::1:993:993"
     volumes:
       - "/mailu/certs:/certs"
-      
 
   admin:
-    image: mailu/admin:master
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/data:/data"
@@ -49,7 +53,8 @@ services:
       - redis
 
   imap:
-    image: mailu/dovecot:master
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/mail:/mail"
@@ -58,16 +63,17 @@ services:
       - front
 
   smtp:
-    image: mailu/postfix:master
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/overrides:/overrides"
     depends_on:
       - front
 
-  # Optional services
   antispam:
-    image: mailu/rspamd:master
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/filter:/var/lib/rspamd"
@@ -76,7 +82,8 @@ services:
     depends_on:
       - front
 
+  # Optional services
+
 
 
   # Webmail
-  
diff --git a/tests/compose/mailu.env b/tests/compose/core/mailu.env
similarity index 96%
rename from tests/compose/mailu.env
rename to tests/compose/core/mailu.env
index 74c87118..b243adcc 100644
--- a/tests/compose/mailu.env
+++ b/tests/compose/core/mailu.env
@@ -18,16 +18,13 @@
 #VERSION=master
 
 # Set to a randomly generated 16 bytes string
-SECRET_KEY=1DS36JPBRGPM5JUC
+SECRET_KEY=DY8PPWQXCFV84N4K
 
 # Address where listening ports should bind
 # This variables are now set directly in `docker-compose.yml by the setup utility
 # PUBLIC_IPV4= 127.0.0.1 (default: 127.0.0.1)
 # PUBLIC_IPV6= ::1 (default: ::1)
 
-# Subnet
-SUBNET=192.168.0.0/24
-
 # Main mail domain
 DOMAIN=mailu.io
 
@@ -38,7 +35,7 @@ HOSTNAMES=mail.mailu.io
 POSTMASTER=admin
 
 # Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
-TLS_FLAVOR=cert
+TLS_FLAVOR=letsencrypt
 
 # Authentication rate limit (per source IP address)
 AUTH_RATELIMIT=10/minute;1000/hour 
@@ -123,7 +120,7 @@ WEBSITE=https://mailu.io
 # json-file (default)
 # journald (On systemd platforms, useful for Fail2Ban integration)
 # syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
-LOG_DRIVER=json-file
+# LOG_DRIVER=json-file
 
 # Docker-compose project name, this will prepended to containers names.
 COMPOSE_PROJECT_NAME=mailu
diff --git a/tests/compose/rainloop/rainloop.yml b/tests/compose/fetchmail/docker-compose.yml
similarity index 63%
rename from tests/compose/rainloop/rainloop.yml
rename to tests/compose/fetchmail/docker-compose.yml
index 29011eeb..6612dc25 100644
--- a/tests/compose/rainloop/rainloop.yml
+++ b/tests/compose/fetchmail/docker-compose.yml
@@ -9,13 +9,17 @@ services:
   # External dependencies
   redis:
     image: redis:alpine
+    restart: always
     volumes:
       - "/mailu/redis:/data"
 
   # Core services
   front:
-    image: mailu/nginx:master
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
+    logging:
+      driver: json-file
     ports:
       - "127.0.0.1:80:80"
       - "::1:80:80"
@@ -37,10 +41,10 @@ services:
       - "::1:993:993"
     volumes:
       - "/mailu/certs:/certs"
-      
 
   admin:
-    image: mailu/admin:master
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/data:/data"
@@ -49,7 +53,8 @@ services:
       - redis
 
   imap:
-    image: mailu/dovecot:master
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/mail:/mail"
@@ -58,27 +63,31 @@ services:
       - front
 
   smtp:
-    image: mailu/postfix:master
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/overrides:/overrides"
     depends_on:
       - front
 
+  antispam:
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    volumes:
+      - "/mailu/filter:/var/lib/rspamd"
+      - "/mailu/dkim:/dkim"
+      - "/mailu/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
+
   # Optional services
 
 
-
+  fetchmail:
+    image: ${DOCKER_ORG:-mailu}/fetchmail:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
 
   # Webmail
-  webmail:
-    image: mailu/rainloop:master
-    env_file: mailu.env
-    volumes:
-      - "/mailu/webmail:/data"
-    depends_on:
-      - imap
-      - resolver
-    dns:
-      - 192.168.0.254
-  
diff --git a/tests/compose/fetchmail/fetchmail.yml b/tests/compose/fetchmail/fetchmail.yml
deleted file mode 100644
index 8f40ba19..00000000
--- a/tests/compose/fetchmail/fetchmail.yml
+++ /dev/null
@@ -1,77 +0,0 @@
-# This file is auto-generated by the Mailu configuration wizard.
-# Please read the documentation before attempting any change.
-# Generated for compose flavor
-
-version: '3.6'
-
-services:
-
-  # External dependencies
-  redis:
-    image: redis:alpine
-    volumes:
-      - "/mailu/redis:/data"
-
-  # Core services
-  front:
-    image: mailu/nginx:master
-    env_file: mailu.env
-    ports:
-      - "127.0.0.1:80:80"
-      - "::1:80:80"
-      - "127.0.0.1:443:443"
-      - "::1:443:443"
-      - "127.0.0.1:25:25"
-      - "::1:25:25"
-      - "127.0.0.1:465:465"
-      - "::1:465:465"
-      - "127.0.0.1:587:587"
-      - "::1:587:587"
-      - "127.0.0.1:110:110"
-      - "::1:110:110"
-      - "127.0.0.1:995:995"
-      - "::1:995:995"
-      - "127.0.0.1:143:143"
-      - "::1:143:143"
-      - "127.0.0.1:993:993"
-      - "::1:993:993"
-    volumes:
-      - "/mailu/certs:/certs"
-      
-
-  admin:
-    image: mailu/admin:master
-    env_file: mailu.env
-    volumes:
-      - "/mailu/data:/data"
-      - "/mailu/dkim:/dkim"
-    depends_on:
-      - redis
-
-  imap:
-    image: mailu/dovecot:master
-    env_file: mailu.env
-    volumes:
-      - "/mailu/mail:/mail"
-      - "/mailu/overrides:/overrides"
-    depends_on:
-      - front
-
-  smtp:
-    image: mailu/postfix:master
-    env_file: mailu.env
-    volumes:
-      - "/mailu/overrides:/overrides"
-    depends_on:
-      - front
-
-  # Optional services
-
-
-
-  fetchmail:
-    image: mailu/fetchmail:master
-    env_file: mailu.env
-
-  # Webmail
-  
diff --git a/tests/compose/fetchmail/mailu.env b/tests/compose/fetchmail/mailu.env
new file mode 100644
index 00000000..4cea36f7
--- /dev/null
+++ b/tests/compose/fetchmail/mailu.env
@@ -0,0 +1,139 @@
+# Mailu main configuration file
+#
+# Generated for compose flavor
+#
+# This file is autogenerated by the configuration management wizard.
+# For a detailed list of configuration variables, see the documentation at
+# https://mailu.io
+
+###################################
+# Common configuration variables
+###################################
+
+# Set this to the path where Mailu data and configuration is stored
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
+
+# Mailu version to run (1.0, 1.1, etc. or master)
+#VERSION=master
+
+# Set to a randomly generated 16 bytes string
+SECRET_KEY=PDK7N9UC7FDQ2UWE
+
+# Address where listening ports should bind
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4= 127.0.0.1 (default: 127.0.0.1)
+# PUBLIC_IPV6= ::1 (default: ::1)
+
+# Main mail domain
+DOMAIN=mailu.io
+
+# Hostnames for this server, separated with comas
+HOSTNAMES=mail.mailu.io
+
+# Postmaster local part (will append the main mail domain)
+POSTMASTER=admin
+
+# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
+TLS_FLAVOR=letsencrypt
+
+# Authentication rate limit (per source IP address)
+AUTH_RATELIMIT=10/minute;1000/hour 
+
+# Opt-out of statistics, replace with "True" to opt out
+DISABLE_STATISTICS=False
+
+###################################
+# Optional features
+###################################
+
+# Expose the admin interface (value: true, false)
+ADMIN=true
+
+# Choose which webmail to run if any (values: roundcube, rainloop, none)
+WEBMAIL=none
+
+# Dav server implementation (value: radicale, none)
+WEBDAV=none
+
+# Antivirus solution (value: clamav, none)
+#ANTIVIRUS=none
+
+#Antispam solution
+ANTISPAM=none
+
+###################################
+# Mail settings
+###################################
+
+# Message size limit in bytes
+# Default: accept messages up to 50MB
+MESSAGE_SIZE_LIMIT=50000000
+
+# Networks granted relay permissions, make sure that you include your Docker
+# internal network (default to 172.17.0.0/16)
+RELAYNETS=172.17.0.0/16
+
+# Will relay all outgoing mails if configured
+RELAYHOST=
+
+# Fetchmail delay
+FETCHMAIL_DELAY=600
+
+# Recipient delimiter, character used to delimiter localpart from custom address part
+RECIPIENT_DELIMITER=+
+
+# DMARC rua and ruf email
+DMARC_RUA=admin
+DMARC_RUF=admin
+
+
+# Maildir Compression
+# choose compression-method, default: none (value: bz2, gz)
+COMPRESSION=
+# change compression-level, default: 6 (value: 1-9)
+COMPRESSION_LEVEL=
+
+###################################
+# Web settings
+###################################
+
+# Path to the admin interface if enabled
+WEB_ADMIN=/admin
+
+# Path to the webmail if enabled
+WEB_WEBMAIL=/webmail
+
+# Website name
+SITENAME=Mailu
+
+# Linked Website URL
+WEBSITE=https://mailu.io
+
+
+
+###################################
+# Advanced settings
+###################################
+
+# Log driver for front service. Possible values:
+# json-file (default)
+# journald (On systemd platforms, useful for Fail2Ban integration)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+# LOG_DRIVER=json-file
+
+# Docker-compose project name, this will prepended to containers names.
+COMPOSE_PROJECT_NAME=mailu
+
+# Default password scheme used for newly created accounts and changed passwords
+# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME=BLF-CRYPT
+
+# Header to take the real ip from
+REAL_IP_HEADER=
+
+# IPs for nginx set_real_ip_from (CIDR list separated by commas)
+REAL_IP_FROM=
+
+# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
+REJECT_UNLISTED_RECIPIENT=
\ No newline at end of file
diff --git a/tests/compose/filters/docker-compose.yml b/tests/compose/filters/docker-compose.yml
new file mode 100644
index 00000000..31e62e72
--- /dev/null
+++ b/tests/compose/filters/docker-compose.yml
@@ -0,0 +1,95 @@
+# This file is auto-generated by the Mailu configuration wizard.
+# Please read the documentation before attempting any change.
+# Generated for compose flavor
+
+version: '3.6'
+
+services:
+
+  # External dependencies
+  redis:
+    image: redis:alpine
+    restart: always
+    volumes:
+      - "/mailu/redis:/data"
+
+  # Core services
+  front:
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    logging:
+      driver: json-file
+    ports:
+      - "127.0.0.1:80:80"
+      - "::1:80:80"
+      - "127.0.0.1:443:443"
+      - "::1:443:443"
+      - "127.0.0.1:25:25"
+      - "::1:25:25"
+      - "127.0.0.1:465:465"
+      - "::1:465:465"
+      - "127.0.0.1:587:587"
+      - "::1:587:587"
+      - "127.0.0.1:110:110"
+      - "::1:110:110"
+      - "127.0.0.1:995:995"
+      - "::1:995:995"
+      - "127.0.0.1:143:143"
+      - "::1:143:143"
+      - "127.0.0.1:993:993"
+      - "::1:993:993"
+    volumes:
+      - "/mailu/certs:/certs"
+
+  admin:
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    volumes:
+      - "/mailu/data:/data"
+      - "/mailu/dkim:/dkim"
+    depends_on:
+      - redis
+
+  imap:
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    volumes:
+      - "/mailu/mail:/mail"
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  smtp:
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    volumes:
+      - "/mailu/overrides:/overrides"
+    depends_on:
+      - front
+
+  antispam:
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    volumes:
+      - "/mailu/filter:/var/lib/rspamd"
+      - "/mailu/dkim:/dkim"
+      - "/mailu/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
+
+  # Optional services
+  antivirus:
+    image: ${DOCKER_ORG:-mailu}/clamav:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    volumes:
+      - "/mailu/filter:/data"
+
+
+
+  # Webmail
diff --git a/tests/compose/filters/mailu.env b/tests/compose/filters/mailu.env
new file mode 100644
index 00000000..15fae1fc
--- /dev/null
+++ b/tests/compose/filters/mailu.env
@@ -0,0 +1,139 @@
+# Mailu main configuration file
+#
+# Generated for compose flavor
+#
+# This file is autogenerated by the configuration management wizard.
+# For a detailed list of configuration variables, see the documentation at
+# https://mailu.io
+
+###################################
+# Common configuration variables
+###################################
+
+# Set this to the path where Mailu data and configuration is stored
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
+
+# Mailu version to run (1.0, 1.1, etc. or master)
+#VERSION=master
+
+# Set to a randomly generated 16 bytes string
+SECRET_KEY=IUFE7Z57URZH1ZM1
+
+# Address where listening ports should bind
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4= 127.0.0.1 (default: 127.0.0.1)
+# PUBLIC_IPV6= ::1 (default: ::1)
+
+# Main mail domain
+DOMAIN=mailu.io
+
+# Hostnames for this server, separated with comas
+HOSTNAMES=mail.mailu.io
+
+# Postmaster local part (will append the main mail domain)
+POSTMASTER=admin
+
+# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
+TLS_FLAVOR=letsencrypt
+
+# Authentication rate limit (per source IP address)
+AUTH_RATELIMIT=10/minute;1000/hour 
+
+# Opt-out of statistics, replace with "True" to opt out
+DISABLE_STATISTICS=False
+
+###################################
+# Optional features
+###################################
+
+# Expose the admin interface (value: true, false)
+ADMIN=true
+
+# Choose which webmail to run if any (values: roundcube, rainloop, none)
+WEBMAIL=none
+
+# Dav server implementation (value: radicale, none)
+WEBDAV=none
+
+# Antivirus solution (value: clamav, none)
+#ANTIVIRUS=clamav
+
+#Antispam solution
+ANTISPAM=none
+
+###################################
+# Mail settings
+###################################
+
+# Message size limit in bytes
+# Default: accept messages up to 50MB
+MESSAGE_SIZE_LIMIT=50000000
+
+# Networks granted relay permissions, make sure that you include your Docker
+# internal network (default to 172.17.0.0/16)
+RELAYNETS=172.17.0.0/16
+
+# Will relay all outgoing mails if configured
+RELAYHOST=
+
+# Fetchmail delay
+FETCHMAIL_DELAY=600
+
+# Recipient delimiter, character used to delimiter localpart from custom address part
+RECIPIENT_DELIMITER=+
+
+# DMARC rua and ruf email
+DMARC_RUA=admin
+DMARC_RUF=admin
+
+
+# Maildir Compression
+# choose compression-method, default: none (value: bz2, gz)
+COMPRESSION=
+# change compression-level, default: 6 (value: 1-9)
+COMPRESSION_LEVEL=
+
+###################################
+# Web settings
+###################################
+
+# Path to the admin interface if enabled
+WEB_ADMIN=/admin
+
+# Path to the webmail if enabled
+WEB_WEBMAIL=/webmail
+
+# Website name
+SITENAME=Mailu
+
+# Linked Website URL
+WEBSITE=https://mailu.io
+
+
+
+###################################
+# Advanced settings
+###################################
+
+# Log driver for front service. Possible values:
+# json-file (default)
+# journald (On systemd platforms, useful for Fail2Ban integration)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+# LOG_DRIVER=json-file
+
+# Docker-compose project name, this will prepended to containers names.
+COMPOSE_PROJECT_NAME=mailu
+
+# Default password scheme used for newly created accounts and changed passwords
+# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME=BLF-CRYPT
+
+# Header to take the real ip from
+REAL_IP_HEADER=
+
+# IPs for nginx set_real_ip_from (CIDR list separated by commas)
+REAL_IP_FROM=
+
+# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
+REJECT_UNLISTED_RECIPIENT=
\ No newline at end of file
diff --git a/tests/compose/filters+dns/filters+dns.yml b/tests/compose/rainloop/docker-compose.yml
similarity index 68%
rename from tests/compose/filters+dns/filters+dns.yml
rename to tests/compose/rainloop/docker-compose.yml
index 993ed786..730e33d8 100644
--- a/tests/compose/filters+dns/filters+dns.yml
+++ b/tests/compose/rainloop/docker-compose.yml
@@ -9,13 +9,17 @@ services:
   # External dependencies
   redis:
     image: redis:alpine
+    restart: always
     volumes:
       - "/mailu/redis:/data"
 
   # Core services
   front:
-    image: mailu/nginx:master
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
+    logging:
+      driver: json-file
     ports:
       - "127.0.0.1:80:80"
       - "::1:80:80"
@@ -37,17 +41,10 @@ services:
       - "::1:993:993"
     volumes:
       - "/mailu/certs:/certs"
-      
-  resolver:
-    image: mailu/unbound:master
-    env_file: mailu.env
-    restart: always
-    networks:
-      default:
-        ipv4_address: 192.168.0.254
 
   admin:
-    image: mailu/admin:master
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/data:/data"
@@ -56,7 +53,8 @@ services:
       - redis
 
   imap:
-    image: mailu/dovecot:master
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/mail:/mail"
@@ -65,19 +63,17 @@ services:
       - front
 
   smtp:
-    image: mailu/postfix:master
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/overrides:/overrides"
     depends_on:
       - front
-      - resolver
-    dns:
-      - 192.168.0.254
 
-  # Optional services
   antispam:
-    image: mailu/rspamd:master
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/filter:/var/lib/rspamd"
@@ -85,35 +81,17 @@ services:
       - "/mailu/overrides/rspamd:/etc/rspamd/override.d"
     depends_on:
       - front
-      - resolver
-    dns:
-      - 192.168.0.254
 
-  antivirus:
-    image: mailu/clamav:master
-    env_file: mailu.env
-    volumes:
-      - "/mailu/filter:/data"
-    depends_on:
-      - resolver
-    dns:
-      - 192.168.0.254
+  # Optional services
 
 
 
   # Webmail
   webmail:
-    image: mailu/rainloop:master
+    image: ${DOCKER_ORG:-mailu}/rainloop:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/webmail:/data"
     depends_on:
       - imap
-  
-networks:
-  default:
-    driver: bridge
-    ipam:
-      driver: default
-      config:
-        - subnet: 192.168.0.0/24
diff --git a/tests/compose/rainloop/mailu.env b/tests/compose/rainloop/mailu.env
new file mode 100644
index 00000000..7ac75438
--- /dev/null
+++ b/tests/compose/rainloop/mailu.env
@@ -0,0 +1,139 @@
+# Mailu main configuration file
+#
+# Generated for compose flavor
+#
+# This file is autogenerated by the configuration management wizard.
+# For a detailed list of configuration variables, see the documentation at
+# https://mailu.io
+
+###################################
+# Common configuration variables
+###################################
+
+# Set this to the path where Mailu data and configuration is stored
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
+
+# Mailu version to run (1.0, 1.1, etc. or master)
+#VERSION=master
+
+# Set to a randomly generated 16 bytes string
+SECRET_KEY=QWE7CJZOET9BN4BU
+
+# Address where listening ports should bind
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4= 127.0.0.1 (default: 127.0.0.1)
+# PUBLIC_IPV6= ::1 (default: ::1)
+
+# Main mail domain
+DOMAIN=mailu.io
+
+# Hostnames for this server, separated with comas
+HOSTNAMES=mail.mailu.io
+
+# Postmaster local part (will append the main mail domain)
+POSTMASTER=admin
+
+# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
+TLS_FLAVOR=letsencrypt
+
+# Authentication rate limit (per source IP address)
+AUTH_RATELIMIT=10/minute;1000/hour 
+
+# Opt-out of statistics, replace with "True" to opt out
+DISABLE_STATISTICS=False
+
+###################################
+# Optional features
+###################################
+
+# Expose the admin interface (value: true, false)
+ADMIN=true
+
+# Choose which webmail to run if any (values: roundcube, rainloop, none)
+WEBMAIL=rainloop
+
+# Dav server implementation (value: radicale, none)
+WEBDAV=none
+
+# Antivirus solution (value: clamav, none)
+#ANTIVIRUS=none
+
+#Antispam solution
+ANTISPAM=none
+
+###################################
+# Mail settings
+###################################
+
+# Message size limit in bytes
+# Default: accept messages up to 50MB
+MESSAGE_SIZE_LIMIT=50000000
+
+# Networks granted relay permissions, make sure that you include your Docker
+# internal network (default to 172.17.0.0/16)
+RELAYNETS=172.17.0.0/16
+
+# Will relay all outgoing mails if configured
+RELAYHOST=
+
+# Fetchmail delay
+FETCHMAIL_DELAY=600
+
+# Recipient delimiter, character used to delimiter localpart from custom address part
+RECIPIENT_DELIMITER=+
+
+# DMARC rua and ruf email
+DMARC_RUA=admin
+DMARC_RUF=admin
+
+
+# Maildir Compression
+# choose compression-method, default: none (value: bz2, gz)
+COMPRESSION=
+# change compression-level, default: 6 (value: 1-9)
+COMPRESSION_LEVEL=
+
+###################################
+# Web settings
+###################################
+
+# Path to the admin interface if enabled
+WEB_ADMIN=/admin
+
+# Path to the webmail if enabled
+WEB_WEBMAIL=/webmail
+
+# Website name
+SITENAME=Mailu
+
+# Linked Website URL
+WEBSITE=https://mailu.io
+
+
+
+###################################
+# Advanced settings
+###################################
+
+# Log driver for front service. Possible values:
+# json-file (default)
+# journald (On systemd platforms, useful for Fail2Ban integration)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+# LOG_DRIVER=json-file
+
+# Docker-compose project name, this will prepended to containers names.
+COMPOSE_PROJECT_NAME=mailu
+
+# Default password scheme used for newly created accounts and changed passwords
+# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME=BLF-CRYPT
+
+# Header to take the real ip from
+REAL_IP_HEADER=
+
+# IPs for nginx set_real_ip_from (CIDR list separated by commas)
+REAL_IP_FROM=
+
+# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
+REJECT_UNLISTED_RECIPIENT=
\ No newline at end of file
diff --git a/tests/compose/roundcube/roundcube.yml b/tests/compose/roundcube/docker-compose.yml
similarity index 64%
rename from tests/compose/roundcube/roundcube.yml
rename to tests/compose/roundcube/docker-compose.yml
index c6e46ed9..989f6a67 100644
--- a/tests/compose/roundcube/roundcube.yml
+++ b/tests/compose/roundcube/docker-compose.yml
@@ -9,13 +9,17 @@ services:
   # External dependencies
   redis:
     image: redis:alpine
+    restart: always
     volumes:
       - "/mailu/redis:/data"
 
   # Core services
   front:
-    image: mailu/nginx:master
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
+    logging:
+      driver: json-file
     ports:
       - "127.0.0.1:80:80"
       - "::1:80:80"
@@ -37,10 +41,10 @@ services:
       - "::1:993:993"
     volumes:
       - "/mailu/certs:/certs"
-      
 
   admin:
-    image: mailu/admin:master
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/data:/data"
@@ -49,7 +53,8 @@ services:
       - redis
 
   imap:
-    image: mailu/dovecot:master
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/mail:/mail"
@@ -58,27 +63,35 @@ services:
       - front
 
   smtp:
-    image: mailu/postfix:master
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/overrides:/overrides"
     depends_on:
       - front
 
+  antispam:
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    volumes:
+      - "/mailu/filter:/var/lib/rspamd"
+      - "/mailu/dkim:/dkim"
+      - "/mailu/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
+
   # Optional services
 
 
 
-
   # Webmail
   webmail:
-    image: mailu/roundcube:master
+    image: ${DOCKER_ORG:-mailu}/roundcube:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/webmail:/data"
     depends_on:
       - imap
-      - resolver
-    dns:
-      - 192.168.0.254
-  
diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/roundcube/mailu.env
new file mode 100644
index 00000000..3c092f15
--- /dev/null
+++ b/tests/compose/roundcube/mailu.env
@@ -0,0 +1,139 @@
+# Mailu main configuration file
+#
+# Generated for compose flavor
+#
+# This file is autogenerated by the configuration management wizard.
+# For a detailed list of configuration variables, see the documentation at
+# https://mailu.io
+
+###################################
+# Common configuration variables
+###################################
+
+# Set this to the path where Mailu data and configuration is stored
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
+
+# Mailu version to run (1.0, 1.1, etc. or master)
+#VERSION=master
+
+# Set to a randomly generated 16 bytes string
+SECRET_KEY=KXGYDHIHWTS7VRUP
+
+# Address where listening ports should bind
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4= 127.0.0.1 (default: 127.0.0.1)
+# PUBLIC_IPV6= ::1 (default: ::1)
+
+# Main mail domain
+DOMAIN=mailu.io
+
+# Hostnames for this server, separated with comas
+HOSTNAMES=mail.mailu.io
+
+# Postmaster local part (will append the main mail domain)
+POSTMASTER=admin
+
+# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
+TLS_FLAVOR=letsencrypt
+
+# Authentication rate limit (per source IP address)
+AUTH_RATELIMIT=10/minute;1000/hour 
+
+# Opt-out of statistics, replace with "True" to opt out
+DISABLE_STATISTICS=False
+
+###################################
+# Optional features
+###################################
+
+# Expose the admin interface (value: true, false)
+ADMIN=true
+
+# Choose which webmail to run if any (values: roundcube, rainloop, none)
+WEBMAIL=roundcube
+
+# Dav server implementation (value: radicale, none)
+WEBDAV=none
+
+# Antivirus solution (value: clamav, none)
+#ANTIVIRUS=none
+
+#Antispam solution
+ANTISPAM=none
+
+###################################
+# Mail settings
+###################################
+
+# Message size limit in bytes
+# Default: accept messages up to 50MB
+MESSAGE_SIZE_LIMIT=50000000
+
+# Networks granted relay permissions, make sure that you include your Docker
+# internal network (default to 172.17.0.0/16)
+RELAYNETS=172.17.0.0/16
+
+# Will relay all outgoing mails if configured
+RELAYHOST=
+
+# Fetchmail delay
+FETCHMAIL_DELAY=600
+
+# Recipient delimiter, character used to delimiter localpart from custom address part
+RECIPIENT_DELIMITER=+
+
+# DMARC rua and ruf email
+DMARC_RUA=admin
+DMARC_RUF=admin
+
+
+# Maildir Compression
+# choose compression-method, default: none (value: bz2, gz)
+COMPRESSION=
+# change compression-level, default: 6 (value: 1-9)
+COMPRESSION_LEVEL=
+
+###################################
+# Web settings
+###################################
+
+# Path to the admin interface if enabled
+WEB_ADMIN=/admin
+
+# Path to the webmail if enabled
+WEB_WEBMAIL=/webmail
+
+# Website name
+SITENAME=Mailu
+
+# Linked Website URL
+WEBSITE=https://mailu.io
+
+
+
+###################################
+# Advanced settings
+###################################
+
+# Log driver for front service. Possible values:
+# json-file (default)
+# journald (On systemd platforms, useful for Fail2Ban integration)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+# LOG_DRIVER=json-file
+
+# Docker-compose project name, this will prepended to containers names.
+COMPOSE_PROJECT_NAME=mailu
+
+# Default password scheme used for newly created accounts and changed passwords
+# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME=BLF-CRYPT
+
+# Header to take the real ip from
+REAL_IP_HEADER=
+
+# IPs for nginx set_real_ip_from (CIDR list separated by commas)
+REAL_IP_FROM=
+
+# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
+REJECT_UNLISTED_RECIPIENT=
\ No newline at end of file
diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh
deleted file mode 100755
index 0a3c2237..00000000
--- a/tests/compose/test-script.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/bash
-containers=(
-	webmail
-	imap
-	smtp
-	antispam
-	admin
-	redis
-	antivirus
-	webdav
-#	fetchmail
-	front
-)
-
-# Time to sleep in minutes after starting the containers
-WAIT=1
-
-containers_check() {
-	status=0
-	for container in "${containers[@]}"; do
-		name="${DOCKER_ORG}_${container}_1"
-		echo "Checking $name"
-		docker inspect "$name" | grep '"Status": "running"' || status=1
-	done
-	docker ps -a
-	return $status
-}
-
-container_logs() {
-	for container in "${containers[@]}"; do
-		name="${DOCKER_ORG}_${container}_1"
-                echo "Showing logs for $name"
-                docker container logs "$name"
-        done
-}
-
-clean() {
-	docker-compose -f tests/compose/run.yml -p $DOCKER_ORG down || exit 1
-	rm -fv .env
-}
-
-# Cleanup before callig exit
-die() {
-	clean
-	exit $1
-}
-
-for file in tests/compose/*.env ; do
-	cp $file .env
-	docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d
-	echo -e "\nSleeping for ${WAIT} minutes" # Clean terminal distortion from docker-compose in travis
-	travis_wait sleep ${WAIT}m || sleep ${WAIT}m #Fallback sleep for local run
-	container_logs
-	containers_check || die 1
-	clean
-done
-
diff --git a/tests/compose/test.py b/tests/compose/test.py
index 95670e30..dfe07ea8 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -7,21 +7,16 @@ from colorama import Fore, Style
 # Declare variables for service name and sleep time
 test_name=sys.argv[1]
 timeout=int(sys.argv[2])
+compose_file="tests/compose/" + test_name + "/docker-compose.yml"
 
 client = docker.APIClient(base_url='unix://var/run/docker.sock')
 
 containers = []
 
-# Start up containers
-def start():
-    os.system("cp mailu.env " + test_name + "/")
-    os.system("docker-compose -f " + test_name + "/" + test_name + ".yml -p $DOCKER_ORG up -d ")
-
 # Stop containers
 def stop(exit_code):
     print_logs()
-    os.system("docker-compose -f " + test_name + "/" + test_name + ".yml -p $DOCKER_ORG down")
-    os.system("rm " + test_name +"/mailu.env")
+    os.system("docker-compose -f " + compose_file + " -p ${DOCKER_ORG:-mailu} down")
     sys.exit(exit_code)
   
 # Sleep for a defined amount of time
@@ -79,7 +74,8 @@ def hooks():
         elif test_file.endswith(".sh"):
             os.system("./" + test_name + "/" + test_file)
       
-start()
+# Start up containers
+os.system("docker-compose -f " + compose_file + " -p ${DOCKER_ORG:-mailu} up -d ")
 print()
 sleep()
 print()
diff --git a/tests/compose/webdav/webdav.yml b/tests/compose/webdav/docker-compose.yml
similarity index 63%
rename from tests/compose/webdav/webdav.yml
rename to tests/compose/webdav/docker-compose.yml
index b4d222ef..7e28c4cc 100644
--- a/tests/compose/webdav/webdav.yml
+++ b/tests/compose/webdav/docker-compose.yml
@@ -9,13 +9,17 @@ services:
   # External dependencies
   redis:
     image: redis:alpine
+    restart: always
     volumes:
       - "/mailu/redis:/data"
 
   # Core services
   front:
-    image: mailu/nginx:master
+    image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
+    logging:
+      driver: json-file
     ports:
       - "127.0.0.1:80:80"
       - "::1:80:80"
@@ -37,10 +41,10 @@ services:
       - "::1:993:993"
     volumes:
       - "/mailu/certs:/certs"
-      
 
   admin:
-    image: mailu/admin:master
+    image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/data:/data"
@@ -49,7 +53,8 @@ services:
       - redis
 
   imap:
-    image: mailu/dovecot:master
+    image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/mail:/mail"
@@ -58,22 +63,33 @@ services:
       - front
 
   smtp:
-    image: mailu/postfix:master
+    image: ${DOCKER_ORG:-mailu}/postfix:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/overrides:/overrides"
     depends_on:
       - front
 
+  antispam:
+    image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-master}
+    restart: always
+    env_file: mailu.env
+    volumes:
+      - "/mailu/filter:/var/lib/rspamd"
+      - "/mailu/dkim:/dkim"
+      - "/mailu/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
+
   # Optional services
 
-
   webdav:
-    image: mailu/radicale:master
+    image: ${DOCKER_ORG:-mailu}/radicale:${MAILU_VERSION:-master}
+    restart: always
     env_file: mailu.env
     volumes:
       - "/mailu/dav:/data"
 
 
   # Webmail
-  
diff --git a/tests/compose/webdav/mailu.env b/tests/compose/webdav/mailu.env
new file mode 100644
index 00000000..acb48abb
--- /dev/null
+++ b/tests/compose/webdav/mailu.env
@@ -0,0 +1,139 @@
+# Mailu main configuration file
+#
+# Generated for compose flavor
+#
+# This file is autogenerated by the configuration management wizard.
+# For a detailed list of configuration variables, see the documentation at
+# https://mailu.io
+
+###################################
+# Common configuration variables
+###################################
+
+# Set this to the path where Mailu data and configuration is stored
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
+
+# Mailu version to run (1.0, 1.1, etc. or master)
+#VERSION=master
+
+# Set to a randomly generated 16 bytes string
+SECRET_KEY=QDHY5C0EME3YBI2W
+
+# Address where listening ports should bind
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4= 127.0.0.1 (default: 127.0.0.1)
+# PUBLIC_IPV6= ::1 (default: ::1)
+
+# Main mail domain
+DOMAIN=mailu.io
+
+# Hostnames for this server, separated with comas
+HOSTNAMES=mail.mailu.io
+
+# Postmaster local part (will append the main mail domain)
+POSTMASTER=admin
+
+# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
+TLS_FLAVOR=letsencrypt
+
+# Authentication rate limit (per source IP address)
+AUTH_RATELIMIT=10/minute;1000/hour 
+
+# Opt-out of statistics, replace with "True" to opt out
+DISABLE_STATISTICS=False
+
+###################################
+# Optional features
+###################################
+
+# Expose the admin interface (value: true, false)
+ADMIN=true
+
+# Choose which webmail to run if any (values: roundcube, rainloop, none)
+WEBMAIL=none
+
+# Dav server implementation (value: radicale, none)
+WEBDAV=radicale
+
+# Antivirus solution (value: clamav, none)
+#ANTIVIRUS=none
+
+#Antispam solution
+ANTISPAM=none
+
+###################################
+# Mail settings
+###################################
+
+# Message size limit in bytes
+# Default: accept messages up to 50MB
+MESSAGE_SIZE_LIMIT=50000000
+
+# Networks granted relay permissions, make sure that you include your Docker
+# internal network (default to 172.17.0.0/16)
+RELAYNETS=172.17.0.0/16
+
+# Will relay all outgoing mails if configured
+RELAYHOST=
+
+# Fetchmail delay
+FETCHMAIL_DELAY=600
+
+# Recipient delimiter, character used to delimiter localpart from custom address part
+RECIPIENT_DELIMITER=+
+
+# DMARC rua and ruf email
+DMARC_RUA=admin
+DMARC_RUF=admin
+
+
+# Maildir Compression
+# choose compression-method, default: none (value: bz2, gz)
+COMPRESSION=
+# change compression-level, default: 6 (value: 1-9)
+COMPRESSION_LEVEL=
+
+###################################
+# Web settings
+###################################
+
+# Path to the admin interface if enabled
+WEB_ADMIN=/admin
+
+# Path to the webmail if enabled
+WEB_WEBMAIL=/webmail
+
+# Website name
+SITENAME=Mailu
+
+# Linked Website URL
+WEBSITE=https://mailu.io
+
+
+
+###################################
+# Advanced settings
+###################################
+
+# Log driver for front service. Possible values:
+# json-file (default)
+# journald (On systemd platforms, useful for Fail2Ban integration)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+# LOG_DRIVER=json-file
+
+# Docker-compose project name, this will prepended to containers names.
+COMPOSE_PROJECT_NAME=mailu
+
+# Default password scheme used for newly created accounts and changed passwords
+# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME=BLF-CRYPT
+
+# Header to take the real ip from
+REAL_IP_HEADER=
+
+# IPs for nginx set_real_ip_from (CIDR list separated by commas)
+REAL_IP_FROM=
+
+# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
+REJECT_UNLISTED_RECIPIENT=
\ No newline at end of file

From 1e3392e4175ff41d0dfc61c8fba40914e7d179ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Thu, 25 Oct 2018 17:37:22 +0300
Subject: [PATCH 18/62] Antispam not an optional service, postfix fails without
 it

---
 setup/flavors/compose/docker-compose.yml       | 4 +---
 setup/flavors/stack/docker-compose.yml         | 4 +---
 setup/templates/steps/compose/02_services.html | 6 ------
 setup/templates/steps/stack/02_services.html   | 6 ------
 4 files changed, 2 insertions(+), 18 deletions(-)

diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index 14eeba3c..f2cbad23 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -66,8 +66,6 @@ services:
     depends_on:
       - front
 
-  # Optional services
-  {% if antispam_enabled %}
   antispam:
     image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-{{ version }}}
     restart: always
@@ -78,8 +76,8 @@ services:
       - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d"
     depends_on:
       - front
-  {% endif %}
 
+  # Optional services
   {% if antivirus_enabled %}
   antivirus:
     image: ${DOCKER_ORG:-mailu}/clamav:${MAILU_VERSION:-{{ version }}}
diff --git a/setup/flavors/stack/docker-compose.yml b/setup/flavors/stack/docker-compose.yml
index 92a156c7..dd9ed7dc 100644
--- a/setup/flavors/stack/docker-compose.yml
+++ b/setup/flavors/stack/docker-compose.yml
@@ -65,8 +65,6 @@ services:
     deploy:
       replicas: {{ smtp_replicas }}
 
-  # Optional services
-  {% if antispam_enabled %}
   antispam:
     image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-{{ version }}}
     env_file: {{ env }}
@@ -78,8 +76,8 @@ services:
       - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d"
     deploy:
       replicas: 1
-  {% endif %}
 
+  # Optional services
   {% if antivirus_enabled %}
   antivirus:
     image: ${DOCKER_ORG:-mailu}/clamav:${MAILU_VERSION:-{{ version }}}
diff --git a/setup/templates/steps/compose/02_services.html b/setup/templates/steps/compose/02_services.html
index 7117c490..424b22dc 100644
--- a/setup/templates/steps/compose/02_services.html
+++ b/setup/templates/steps/compose/02_services.html
@@ -32,12 +32,6 @@ will prevent Mailu from doing spam filtering, virus filtering, and from applying
 white and blacklists that you may configure in the admin interface. You may
 also disable the antivirus if required (it does use aroung 1GB of ram).

 
-

-  
-

 

   
 

 
+
+
+
+
 {% endcall %}
diff --git a/setup/templates/steps/config.html b/setup/templates/steps/config.html
index 88c17597..a3352b7e 100644
--- a/setup/templates/steps/config.html
+++ b/setup/templates/steps/config.html
@@ -68,11 +68,29 @@ Or in plain english: if receivers start to classify your mail as spam, this post
 manage your email domains, users, etc.

 
 

-  
-  

-    

-    
-  

+    
+  	
+    
 

 
+
+
+
+
 {% endcall %}
diff --git a/setup/templates/steps/stack/02_services.html b/setup/templates/steps/stack/02_services.html
index 424b22dc..53763658 100644
--- a/setup/templates/steps/stack/02_services.html
+++ b/setup/templates/steps/stack/02_services.html
@@ -15,15 +15,14 @@ accessing messages for beginner users.

 
 
   

-  
   	{% for webmailtype in ["none", "roundcube", "rainloop"] %}
   	
   	{% endfor %}
   
   

   

-
-    
+    
   

 
 
@@ -53,4 +52,27 @@ also disable the antivirus if required (it does use aroung 1GB of ram).

   
 
 
+
+
+
+
 {% endcall %}

From 003c36c98a68a137a95bf105341bae4c0a301683 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Wed, 31 Oct 2018 17:47:05 +0200
Subject: [PATCH 23/62] Fix imap login when no webmail selected

---
 core/admin/mailu/internal/views/dovecot.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py
index 036140f0..e6e4c10f 100644
--- a/core/admin/mailu/internal/views/dovecot.py
+++ b/core/admin/mailu/internal/views/dovecot.py
@@ -3,7 +3,7 @@ from mailu.internal import internal
 
 import flask
 import socket
-
+import os
 
 @internal.route("/dovecot/passdb/")
 def dovecot_passdb_dict(user_email):
@@ -13,7 +13,8 @@ def dovecot_passdb_dict(user_email):
         app.config.get("POD_ADDRESS_RANGE") or
         socket.gethostbyname(app.config["HOST_FRONT"])
     )
-    allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"]))
+    if os.environ["WEBMAIL"] != "none":
+        allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"]))
     print(allow_nets)
     return flask.jsonify({
         "password": None,

From 4e9dc0c3c93fbc65b2704b50b9f92dab46ceac86 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Thu, 1 Nov 2018 11:40:54 +0200
Subject: [PATCH 24/62] Implemented sending/reading email(local tests)

---
 tests/certs/cert.pem                  | 29 ++++++++++++++
 tests/certs/key.pem                   | 52 +++++++++++++++++++++++++
 tests/compose/core/email_test.py      | 55 +++++++++++++++++++++++++++
 tests/compose/core/mailu.env          |  2 +-
 tests/compose/fetchmail/email_test.py | 55 +++++++++++++++++++++++++++
 tests/compose/fetchmail/mailu.env     |  2 +-
 tests/compose/filters/email_test.py   | 55 +++++++++++++++++++++++++++
 tests/compose/filters/mailu.env       |  2 +-
 tests/compose/rainloop/email_test.py  | 55 +++++++++++++++++++++++++++
 tests/compose/rainloop/mailu.env      |  2 +-
 tests/compose/roundcube/email_test.py | 55 +++++++++++++++++++++++++++
 tests/compose/roundcube/mailu.env     |  2 +-
 tests/compose/test.py                 | 14 ++++++-
 tests/compose/webdav/email_test.py    | 55 +++++++++++++++++++++++++++
 tests/compose/webdav/mailu.env        |  2 +-
 15 files changed, 430 insertions(+), 7 deletions(-)
 create mode 100644 tests/certs/cert.pem
 create mode 100644 tests/certs/key.pem
 create mode 100755 tests/compose/core/email_test.py
 create mode 100755 tests/compose/fetchmail/email_test.py
 create mode 100755 tests/compose/filters/email_test.py
 create mode 100755 tests/compose/rainloop/email_test.py
 create mode 100755 tests/compose/roundcube/email_test.py
 create mode 100755 tests/compose/webdav/email_test.py

diff --git a/tests/certs/cert.pem b/tests/certs/cert.pem
new file mode 100644
index 00000000..d6dc928f
--- /dev/null
+++ b/tests/certs/cert.pem
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE/jCCAuagAwIBAgIJAKVnyadXS7SuMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCWxvY2FsaG9zdDAeFw0xODEwMzExMDE1MzFaFw0yODEwMjgxMDE1MzFaMBQx
+EjAQBgNVBAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC
+ggIBAOQ2ZDqR+YvW5FKykBXz/Ec+jSb0Lv7GYQkT5t+TB1NXuR+QH1LfNWFmXOo7
+YXcPVXlmcuLDuUldrctdS59fx8dnFu5gRRUqJwZuEQICypsX0rTDtsV6xqZB8c8y
+2+BztP9OHfPpZdnU1IBx2fDbjpdKUaoAMFMFvyTaEcIyp6aGAhejvJCwc3D8fIJI
+NhWA2O11sZQHUs7/MHzpu/IHpgutgk8EsNOUNLwB3+9p3IlOlTT6GilIXOYeTzoD
+hiI6B5BQqXHsRrkao3v0YL6Ekun4hOx3MYx09AZtmuyrlq1mkNueKS5JwKDrXXbq
+Ta0oyJ18UTZFRwVqApcuR4CA8vuhI9PsoDCvBQH1rW6FyiM4bhybatFJAYjQAODe
+gwh2p6JWux5C1gaBUubOrKO7o5ePI6s0MmK8ZxrL4PpBYt3B33ztFfjWmVbCTSvP
+GuQ2Ux73OY2NNxx2aNt4Th0IxrvMdsGLrZsdma2rWa5eTJTAuqbSjI/Wb1zjO0pi
+pwoxk6f1COFLopo2xgJj6+KKG1nKLfOzQFexcpdq/mpuulcVcLDPJzJTLX3qsgtD
+iBpm1ozNRT+M7XUavg8aHNfn6S+TcDb5hp+1yZ6obZq/VlA6atk0fuPzf+ndQ0fq
+YN1jlAIzZXt/Dpc+ObjS09WGDVQXobGesdwA6BH14OV+TxOHAgMBAAGjUzBRMB0G
+A1UdDgQWBBQy7kA8FbdcFpVU1AoFgzE7Fw1QqDAfBgNVHSMEGDAWgBQy7kA8Fbdc
+FpVU1AoFgzE7Fw1QqDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IC
+AQBLFlQKxztxm7MtsHs01Pl8/FpKzWekWK1ksf15d8mHBT30OTs+NXaJDuHTGL4r
+rPeFf3NZ1PZkGRnJCEWur+8e8Y5KwuMAaagneSYXU0gcZfvTidvf865Jiml8xO5x
+PAo8qTZQCHmYcvJQwBXMkq/2sFJCYeMOLoJdXXbTTe2ZQ/N3MSQbpgWJ8pF7srKU
+biw2RkNH39QPq9GpWRQGx2gwvZDy2oFG8cM1hJYmz0Y9clpBE0mSqypvA1E8ufKC
+uaUc0tpPI5H4efeWv/ObnFAJ3DMEmzUnQ8hdM/7cpf6AL8VRm4Wrw112gK7SbSdd
+mMsUfFIDfyE9vsZ3OC8C8LqXKLwMcm7Fdq0ym0NINtoVW0ukmVJzB78CdWaJ7ux1
+WqitcnewgiMWuuwuepBmNurZtgDrg+zgMhNpuK0NzYyE+ZReoJIOJCub3SSEsWdl
+x5aJEYuFYJR5EvmxWeYv5p1GVOTL1TJqW7iRodzRoMc9u2vb0+tCbM5XSZVPul6P
+QimDui2Ogq0zYNbSkHaUGBpjGDvHYG0zXO2sWrdrAJQMHo8dGEe7FuSuAlWbQdb/
+xgN4uwejxV6B2e6rjT6YMni+r5Qw0EhNka+Xohw5E68bEcQSrCP8j64qQLAeipuz
+ImqBTNyyR4WTcL+1HIVM7ZIw3igHH55zo5qTvyjKyZX9Uw==
+-----END CERTIFICATE-----
diff --git a/tests/certs/key.pem b/tests/certs/key.pem
new file mode 100644
index 00000000..938633fd
--- /dev/null
+++ b/tests/certs/key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDkNmQ6kfmL1uRS
+spAV8/xHPo0m9C7+xmEJE+bfkwdTV7kfkB9S3zVhZlzqO2F3D1V5ZnLiw7lJXa3L
+XUufX8fHZxbuYEUVKicGbhECAsqbF9K0w7bFesamQfHPMtvgc7T/Th3z6WXZ1NSA
+cdnw246XSlGqADBTBb8k2hHCMqemhgIXo7yQsHNw/HyCSDYVgNjtdbGUB1LO/zB8
+6bvyB6YLrYJPBLDTlDS8Ad/vadyJTpU0+hopSFzmHk86A4YiOgeQUKlx7Ea5GqN7
+9GC+hJLp+ITsdzGMdPQGbZrsq5atZpDbnikuScCg61126k2tKMidfFE2RUcFagKX
+LkeAgPL7oSPT7KAwrwUB9a1uhcojOG4cm2rRSQGI0ADg3oMIdqeiVrseQtYGgVLm
+zqyju6OXjyOrNDJivGcay+D6QWLdwd987RX41plWwk0rzxrkNlMe9zmNjTccdmjb
+eE4dCMa7zHbBi62bHZmtq1muXkyUwLqm0oyP1m9c4ztKYqcKMZOn9QjhS6KaNsYC
+Y+viihtZyi3zs0BXsXKXav5qbrpXFXCwzycyUy196rILQ4gaZtaMzUU/jO11Gr4P
+GhzX5+kvk3A2+YaftcmeqG2av1ZQOmrZNH7j83/p3UNH6mDdY5QCM2V7fw6XPjm4
+0tPVhg1UF6GxnrHcAOgR9eDlfk8ThwIDAQABAoICACoHsnHvDIyqqSZp6IuCggYF
+CS4Rbs5RbvGjDrRCeejpkRi1DG/Q2B32IkqpYQvycQWIzsPg1DEk5as8pX7Wvw6E
+d/6zEEYTm1hd0RgTt4jU3GOaYAEC2a8pGgXVEhXGeaFDm9SeObnirrhxP3hSl3JZ
+p6ytmDjSKB/7YaXoemP67ku4RjRHqxs2BSBheESBlHI3aNsgdinVafK3gXvT2Mrx
+y7wN2xs8gnHVzo5jatCG/ofhQAw2XZWsI19F4uBO27HCiVKH94aD13Quz9qGxB//
+O0vpr+B0cbT1XsET4Q5Sg39PI7p4rtd0QaRzBpdLmZcXnEVogOoIWi3JwjVyik1g
+lcg+4A8wj4pDGsCmANt90YqedktQGiYsYozZHO3YCrnjO6lqYJLOBocRG9NJqldY
+kzs6UfJ+96FoYQVGNXyeQZizC26rQHll/rwsJnsB7GvM38f3q3cr3Borpwx3HosN
+mmM+WRcvV3WWjjx1870Jm+tIDu0clWvT7hdHSf4938/Xr9cUTyuX2LrqTfp6JThl
++NbYgbuvd5leP94wPwRxfJL+PR5B4kbLPwDNCbpM8QTBm+9Y4kU+6ePmgcuRemMQ
+8J41ocUjC4wR2j9Zgy0f0Rz4KiKM6IiVgKyqPUMaY+aJQ+yB5J+tlBkPJeZzft/e
+XAoxt0STTassHC+p9COxAoIBAQD2Vd2Q1rbxWGnWl0m1LcH5q4hsuUAhIYmuTMSO
+RkDLD/8yfPR4uUbTgrtdL2FaeOsCK7nrQAPxcfdD//+SoNVsAkMuNw6QvJn4ZXLf
+5C45tN4pfoz/EwIRBvyJnI+HZuNaCUCfsQB9ggeEHgM2n36GBiOX82inQey3eREz
+wZjQqmCp+b1QiYoWrVCgOPOvB86kbNgHGacIS7cDe94OeP4dH+FAfWaIBab8sDnG
+K6+N6dWdj+b7veUWpXBs8beVCTO4GPnW5hnYOfuWkdpNCej/QbMeivMA4U7g+CeF
+Y5QB07EE5f35Epp8WoNtwVZoFgP72xMT1taz1Rx7dohdYvLVAoIBAQDtKoDiwi2V
+07rOgsjgW972HdA0nOnja/lky6CKkY5BqNGMj63h0ysy8Fe8mEWdPXyY9f7TgWP9
+sDMZMq+d8ZwAjfdYjYTKpxA3pA9oj66OCxtR6usElmeyultPjZ8FXJNXzOLv4dju
+FnELSFSSx8o6WHGq9l2eWNMFf46g70Bt+aiHV/VGLLSFTUcvd51H7jP+PFxrBn1k
+kz1u0n/RRuPMIru68lKJxrpDsr917Spw16O+uzjR99IqNPskVJxUnXV8qvMxeWVl
+wTOP9soqYv/KvqjsBO+nLNkLSH402Fp78e2Oe6KPKlF21kl5oA7Yn/w4MtyFpj65
+fg6uDaPhgoLrAoIBAQCb9uWfzLJrwETSn1sFoYENKPPpkqjt0SQw/V39jrF7YBd9
+yeune/dB96XVbChBdgmliDXgotlcR4H8xdr05Wv7RLtwSV+peCAsS18eLoSt+Lwo
+nX18CnbmfPvrzPp7CkOsP+twsErVLDzCA5aZQQaEqOJkVLLQI0dTKw4fLNYqV5V4
+SSz6DvslPHqt1yFCkrjdFiT46d79u6KWTBjeJPEPU530jPEb8ig2GQWbWRF/0qtz
+ZSckAKlJW1oBQFGxxO/AAeA9ldaLNrr6LEKBQGMLKnfUQLl2tzCP885iABg3x+Zu
+aYgR6Rty3IQWO7EPmdDP53b+uqmZlra/3N6d8gY5AoIBADxkBk23hEQSlg7f3qbC
+vhONo+bBzgzLAcZY05h1V/QAONvB+lT2oJln+e9cFt3jOkb43NqeqAeBRoG0FmPx
+kffSLpmt75Jq2AZTEFlfvOMOkPZbC10vr1gje/zV4xhKanqBAYhzyflWXZKx6Fc3
+6JbSzp7p/QzFMXbE9Fymj5FxcSiFjT9BQvZupyG/I52dWj/yvtXB4Uwq8gm2MDXq
+BzeD4KnJ6pqKsANtELPGoHf7cQawRdexcyKsOwcVRHmHXtNP9H00nE081RRjkzcX
+3mqSAhGXcC7xjJMC8qAiN2g4QnV1pf8ul2/bQPpnd2BR3Leyu9SMcIxrPPG1J3XU
+9eECggEBAMMhMURUfLSXIkreMfxH4rSqk0r2xQ1rE1ChAIBQPfyx4KWUkBTdpoiv
+uKcPzAgN+bm3Y5wRGwoE22Ac0lWobnzaIYyYN9N7HU+86q92ozWW1lCUEE0kBt2r
+FnWCD/3B0LOX2Cn8HHYzroRmzMlRvBa7/GO1dqURz/OzjTWN0+k9mgE7oS5M8fQV
+AS3mxXZMPKSB0xTfJoXW8ui9MQZHcNSkNORNP/2doCkR2qDUkazbhi/3ghLmDGVJ
+p5OrIPQUwcp1bFOciX22fAaZwoa63ng3K+WZjSqqma05AiOc59MhDLAu6a0rKKO1
+W3079UVfBB4hkfN2721fqyj+r/0z+R0=
+-----END PRIVATE KEY-----
diff --git a/tests/compose/core/email_test.py b/tests/compose/core/email_test.py
new file mode 100755
index 00000000..c16ce9d8
--- /dev/null
+++ b/tests/compose/core/email_test.py
@@ -0,0 +1,55 @@
+import string
+import random
+import smtplib
+import imaplib
+import time
+
+def secret(length=16):
+    charset = string.ascii_uppercase + string.digits
+    return ''.join(
+        random.SystemRandom().choice(charset)
+        for _ in range(length)
+    )
+  
+#Generating secret message    
+secret_message = secret(16)
+
+#Login to smt server and sending email with secret message    
+def send_email(msg):
+    print("Sending email ...")
+    server = smtplib.SMTP('localhost')
+    server.set_debuglevel(1)
+    server.connect('localhost', 587)
+    server.ehlo()
+    server.starttls()
+    server.ehlo()
+    server.login("admin@mailu.io", "password")
+    
+    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
+    server.quit()
+
+    print("email sent with message " + msg)
+
+#Login to imap server, read latest email and check for secret message    
+def read_email():
+    print("Receiving email ...")
+    server = imaplib.IMAP4_SSL('localhost')
+    server.login('user@mailu.io', 'password')
+    
+    stat, count = server.select('inbox')
+    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
+    
+    print("email received with message " + str(data[0][1])) 
+    
+    if secret_message in str(data[0][1]):
+        print("Success!")
+    else:
+        print("Failed! Something went wrong")    
+    server.close()
+    server.logout()
+
+    
+send_email(secret_message)
+print("Sleeping for 1m")
+time.sleep(60)
+read_email()
diff --git a/tests/compose/core/mailu.env b/tests/compose/core/mailu.env
index dabd523f..9a744e35 100644
--- a/tests/compose/core/mailu.env
+++ b/tests/compose/core/mailu.env
@@ -29,7 +29,7 @@ SECRET_KEY=HGZCYGVI6FVG31HS
 DOMAIN=mailu.io
 
 # Hostnames for this server, separated with comas
-HOSTNAMES=mail.mailu.io
+HOSTNAMES=localhost
 
 # Postmaster local part (will append the main mail domain)
 POSTMASTER=admin
diff --git a/tests/compose/fetchmail/email_test.py b/tests/compose/fetchmail/email_test.py
new file mode 100755
index 00000000..c16ce9d8
--- /dev/null
+++ b/tests/compose/fetchmail/email_test.py
@@ -0,0 +1,55 @@
+import string
+import random
+import smtplib
+import imaplib
+import time
+
+def secret(length=16):
+    charset = string.ascii_uppercase + string.digits
+    return ''.join(
+        random.SystemRandom().choice(charset)
+        for _ in range(length)
+    )
+  
+#Generating secret message    
+secret_message = secret(16)
+
+#Login to smt server and sending email with secret message    
+def send_email(msg):
+    print("Sending email ...")
+    server = smtplib.SMTP('localhost')
+    server.set_debuglevel(1)
+    server.connect('localhost', 587)
+    server.ehlo()
+    server.starttls()
+    server.ehlo()
+    server.login("admin@mailu.io", "password")
+    
+    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
+    server.quit()
+
+    print("email sent with message " + msg)
+
+#Login to imap server, read latest email and check for secret message    
+def read_email():
+    print("Receiving email ...")
+    server = imaplib.IMAP4_SSL('localhost')
+    server.login('user@mailu.io', 'password')
+    
+    stat, count = server.select('inbox')
+    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
+    
+    print("email received with message " + str(data[0][1])) 
+    
+    if secret_message in str(data[0][1]):
+        print("Success!")
+    else:
+        print("Failed! Something went wrong")    
+    server.close()
+    server.logout()
+
+    
+send_email(secret_message)
+print("Sleeping for 1m")
+time.sleep(60)
+read_email()
diff --git a/tests/compose/fetchmail/mailu.env b/tests/compose/fetchmail/mailu.env
index ef78508b..a987c853 100644
--- a/tests/compose/fetchmail/mailu.env
+++ b/tests/compose/fetchmail/mailu.env
@@ -29,7 +29,7 @@ SECRET_KEY=JS48Q9KE3B6T97E6
 DOMAIN=mailu.io
 
 # Hostnames for this server, separated with comas
-HOSTNAMES=mail.mailu.io
+HOSTNAMES=localhost
 
 # Postmaster local part (will append the main mail domain)
 POSTMASTER=admin
diff --git a/tests/compose/filters/email_test.py b/tests/compose/filters/email_test.py
new file mode 100755
index 00000000..c16ce9d8
--- /dev/null
+++ b/tests/compose/filters/email_test.py
@@ -0,0 +1,55 @@
+import string
+import random
+import smtplib
+import imaplib
+import time
+
+def secret(length=16):
+    charset = string.ascii_uppercase + string.digits
+    return ''.join(
+        random.SystemRandom().choice(charset)
+        for _ in range(length)
+    )
+  
+#Generating secret message    
+secret_message = secret(16)
+
+#Login to smt server and sending email with secret message    
+def send_email(msg):
+    print("Sending email ...")
+    server = smtplib.SMTP('localhost')
+    server.set_debuglevel(1)
+    server.connect('localhost', 587)
+    server.ehlo()
+    server.starttls()
+    server.ehlo()
+    server.login("admin@mailu.io", "password")
+    
+    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
+    server.quit()
+
+    print("email sent with message " + msg)
+
+#Login to imap server, read latest email and check for secret message    
+def read_email():
+    print("Receiving email ...")
+    server = imaplib.IMAP4_SSL('localhost')
+    server.login('user@mailu.io', 'password')
+    
+    stat, count = server.select('inbox')
+    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
+    
+    print("email received with message " + str(data[0][1])) 
+    
+    if secret_message in str(data[0][1]):
+        print("Success!")
+    else:
+        print("Failed! Something went wrong")    
+    server.close()
+    server.logout()
+
+    
+send_email(secret_message)
+print("Sleeping for 1m")
+time.sleep(60)
+read_email()
diff --git a/tests/compose/filters/mailu.env b/tests/compose/filters/mailu.env
index ab97e191..8609a287 100644
--- a/tests/compose/filters/mailu.env
+++ b/tests/compose/filters/mailu.env
@@ -29,7 +29,7 @@ SECRET_KEY=11H6XURLGE7GW3U1
 DOMAIN=mailu.io
 
 # Hostnames for this server, separated with comas
-HOSTNAMES=mail.mailu.io
+HOSTNAMES=localhost
 
 # Postmaster local part (will append the main mail domain)
 POSTMASTER=admin
diff --git a/tests/compose/rainloop/email_test.py b/tests/compose/rainloop/email_test.py
new file mode 100755
index 00000000..c16ce9d8
--- /dev/null
+++ b/tests/compose/rainloop/email_test.py
@@ -0,0 +1,55 @@
+import string
+import random
+import smtplib
+import imaplib
+import time
+
+def secret(length=16):
+    charset = string.ascii_uppercase + string.digits
+    return ''.join(
+        random.SystemRandom().choice(charset)
+        for _ in range(length)
+    )
+  
+#Generating secret message    
+secret_message = secret(16)
+
+#Login to smt server and sending email with secret message    
+def send_email(msg):
+    print("Sending email ...")
+    server = smtplib.SMTP('localhost')
+    server.set_debuglevel(1)
+    server.connect('localhost', 587)
+    server.ehlo()
+    server.starttls()
+    server.ehlo()
+    server.login("admin@mailu.io", "password")
+    
+    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
+    server.quit()
+
+    print("email sent with message " + msg)
+
+#Login to imap server, read latest email and check for secret message    
+def read_email():
+    print("Receiving email ...")
+    server = imaplib.IMAP4_SSL('localhost')
+    server.login('user@mailu.io', 'password')
+    
+    stat, count = server.select('inbox')
+    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
+    
+    print("email received with message " + str(data[0][1])) 
+    
+    if secret_message in str(data[0][1]):
+        print("Success!")
+    else:
+        print("Failed! Something went wrong")    
+    server.close()
+    server.logout()
+
+    
+send_email(secret_message)
+print("Sleeping for 1m")
+time.sleep(60)
+read_email()
diff --git a/tests/compose/rainloop/mailu.env b/tests/compose/rainloop/mailu.env
index 747dac58..678ea048 100644
--- a/tests/compose/rainloop/mailu.env
+++ b/tests/compose/rainloop/mailu.env
@@ -29,7 +29,7 @@ SECRET_KEY=V5J4SHRYVW9PZIQU
 DOMAIN=mailu.io
 
 # Hostnames for this server, separated with comas
-HOSTNAMES=mail.mailu.io
+HOSTNAMES=localhost
 
 # Postmaster local part (will append the main mail domain)
 POSTMASTER=admin
diff --git a/tests/compose/roundcube/email_test.py b/tests/compose/roundcube/email_test.py
new file mode 100755
index 00000000..c16ce9d8
--- /dev/null
+++ b/tests/compose/roundcube/email_test.py
@@ -0,0 +1,55 @@
+import string
+import random
+import smtplib
+import imaplib
+import time
+
+def secret(length=16):
+    charset = string.ascii_uppercase + string.digits
+    return ''.join(
+        random.SystemRandom().choice(charset)
+        for _ in range(length)
+    )
+  
+#Generating secret message    
+secret_message = secret(16)
+
+#Login to smt server and sending email with secret message    
+def send_email(msg):
+    print("Sending email ...")
+    server = smtplib.SMTP('localhost')
+    server.set_debuglevel(1)
+    server.connect('localhost', 587)
+    server.ehlo()
+    server.starttls()
+    server.ehlo()
+    server.login("admin@mailu.io", "password")
+    
+    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
+    server.quit()
+
+    print("email sent with message " + msg)
+
+#Login to imap server, read latest email and check for secret message    
+def read_email():
+    print("Receiving email ...")
+    server = imaplib.IMAP4_SSL('localhost')
+    server.login('user@mailu.io', 'password')
+    
+    stat, count = server.select('inbox')
+    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
+    
+    print("email received with message " + str(data[0][1])) 
+    
+    if secret_message in str(data[0][1]):
+        print("Success!")
+    else:
+        print("Failed! Something went wrong")    
+    server.close()
+    server.logout()
+
+    
+send_email(secret_message)
+print("Sleeping for 1m")
+time.sleep(60)
+read_email()
diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/roundcube/mailu.env
index e87d1f61..b8a8b266 100644
--- a/tests/compose/roundcube/mailu.env
+++ b/tests/compose/roundcube/mailu.env
@@ -29,7 +29,7 @@ SECRET_KEY=PGGO2JRQ59QV3DW7
 DOMAIN=mailu.io
 
 # Hostnames for this server, separated with comas
-HOSTNAMES=mail.mailu.io
+HOSTNAMES=localhost
 
 # Postmaster local part (will append the main mail domain)
 POSTMASTER=admin
diff --git a/tests/compose/test.py b/tests/compose/test.py
index 819c7222..d3b2d109 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -74,8 +74,19 @@ def hooks():
             os.system("python3 " + test_path + test_file)
         elif test_file.endswith(".sh"):
             os.system("./" + test_path + test_file)
-      
+
+#Create admin and user
+def create_users():
+    print("Creating admin account...")
+    os.system("docker-compose -p $DOCKER_ORG exec admin python manage.py admin admin mailu.io password")
+    print("Admin account created")
+    print("Creating user account...")
+    os.system("docker-compose -p $DOCKER_ORG exec admin python manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password'")
+    print("User account created")
+
 # Start up containers
+os.system("mkdir -p /mailu && cp -r tests/certs /mailu")
+os.system("chmod 600 /mailu/certs/* ")
 os.system("docker-compose -f " + compose_file + " -p ${DOCKER_ORG:-mailu} up -d ")
 print()
 sleep()
@@ -84,6 +95,7 @@ os.system("docker ps -a")
 print()
 health_checks()
 print()
+create_users()
 hooks()
 print()
 stop(0)
diff --git a/tests/compose/webdav/email_test.py b/tests/compose/webdav/email_test.py
new file mode 100755
index 00000000..c16ce9d8
--- /dev/null
+++ b/tests/compose/webdav/email_test.py
@@ -0,0 +1,55 @@
+import string
+import random
+import smtplib
+import imaplib
+import time
+
+def secret(length=16):
+    charset = string.ascii_uppercase + string.digits
+    return ''.join(
+        random.SystemRandom().choice(charset)
+        for _ in range(length)
+    )
+  
+#Generating secret message    
+secret_message = secret(16)
+
+#Login to smt server and sending email with secret message    
+def send_email(msg):
+    print("Sending email ...")
+    server = smtplib.SMTP('localhost')
+    server.set_debuglevel(1)
+    server.connect('localhost', 587)
+    server.ehlo()
+    server.starttls()
+    server.ehlo()
+    server.login("admin@mailu.io", "password")
+    
+    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
+    server.quit()
+
+    print("email sent with message " + msg)
+
+#Login to imap server, read latest email and check for secret message    
+def read_email():
+    print("Receiving email ...")
+    server = imaplib.IMAP4_SSL('localhost')
+    server.login('user@mailu.io', 'password')
+    
+    stat, count = server.select('inbox')
+    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
+    
+    print("email received with message " + str(data[0][1])) 
+    
+    if secret_message in str(data[0][1]):
+        print("Success!")
+    else:
+        print("Failed! Something went wrong")    
+    server.close()
+    server.logout()
+
+    
+send_email(secret_message)
+print("Sleeping for 1m")
+time.sleep(60)
+read_email()
diff --git a/tests/compose/webdav/mailu.env b/tests/compose/webdav/mailu.env
index ac7b434c..21dd3981 100644
--- a/tests/compose/webdav/mailu.env
+++ b/tests/compose/webdav/mailu.env
@@ -29,7 +29,7 @@ SECRET_KEY=XVDDSWOAGVF5J9QJ
 DOMAIN=mailu.io
 
 # Hostnames for this server, separated with comas
-HOSTNAMES=mail.mailu.io
+HOSTNAMES=localhost
 
 # Postmaster local part (will append the main mail domain)
 POSTMASTER=admin

From c3bc7988c90d25d65d1eb8d2142a58686bacab40 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 2 Nov 2018 14:27:46 +0200
Subject: [PATCH 25/62] Implemented email test for travis

---
 .travis.yml                            |  4 ++
 tests/compose/{core => }/email_test.py |  0
 tests/compose/fetchmail/email_test.py  | 55 --------------------------
 tests/compose/filters/email_test.py    | 55 --------------------------
 tests/compose/rainloop/email_test.py   | 55 --------------------------
 tests/compose/roundcube/email_test.py  | 55 --------------------------
 tests/compose/test.py                  | 15 ++-----
 tests/compose/webdav/email_test.py     | 55 --------------------------
 8 files changed, 7 insertions(+), 287 deletions(-)
 rename tests/compose/{core => }/email_test.py (100%)
 delete mode 100755 tests/compose/fetchmail/email_test.py
 delete mode 100755 tests/compose/filters/email_test.py
 delete mode 100755 tests/compose/rainloop/email_test.py
 delete mode 100755 tests/compose/roundcube/email_test.py
 delete mode 100755 tests/compose/webdav/email_test.py

diff --git a/.travis.yml b/.travis.yml
index 54b3b3c7..ffb6af43 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,6 +18,10 @@ install:
 before_script:
   - docker-compose -v
   - docker-compose -f tests/build.yml build
+  - docker-compose up -d admin
+  - docker-compose exec admin python manage.py admin admin mailu.io password
+  - docker-compose exec admin python manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password'
+  - docker-compose down
   
 script:
 # test.py, test name and timeout between start and tests.
diff --git a/tests/compose/core/email_test.py b/tests/compose/email_test.py
similarity index 100%
rename from tests/compose/core/email_test.py
rename to tests/compose/email_test.py
diff --git a/tests/compose/fetchmail/email_test.py b/tests/compose/fetchmail/email_test.py
deleted file mode 100755
index c16ce9d8..00000000
--- a/tests/compose/fetchmail/email_test.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import string
-import random
-import smtplib
-import imaplib
-import time
-
-def secret(length=16):
-    charset = string.ascii_uppercase + string.digits
-    return ''.join(
-        random.SystemRandom().choice(charset)
-        for _ in range(length)
-    )
-  
-#Generating secret message    
-secret_message = secret(16)
-
-#Login to smt server and sending email with secret message    
-def send_email(msg):
-    print("Sending email ...")
-    server = smtplib.SMTP('localhost')
-    server.set_debuglevel(1)
-    server.connect('localhost', 587)
-    server.ehlo()
-    server.starttls()
-    server.ehlo()
-    server.login("admin@mailu.io", "password")
-    
-    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
-    server.quit()
-
-    print("email sent with message " + msg)
-
-#Login to imap server, read latest email and check for secret message    
-def read_email():
-    print("Receiving email ...")
-    server = imaplib.IMAP4_SSL('localhost')
-    server.login('user@mailu.io', 'password')
-    
-    stat, count = server.select('inbox')
-    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
-    
-    print("email received with message " + str(data[0][1])) 
-    
-    if secret_message in str(data[0][1]):
-        print("Success!")
-    else:
-        print("Failed! Something went wrong")    
-    server.close()
-    server.logout()
-
-    
-send_email(secret_message)
-print("Sleeping for 1m")
-time.sleep(60)
-read_email()
diff --git a/tests/compose/filters/email_test.py b/tests/compose/filters/email_test.py
deleted file mode 100755
index c16ce9d8..00000000
--- a/tests/compose/filters/email_test.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import string
-import random
-import smtplib
-import imaplib
-import time
-
-def secret(length=16):
-    charset = string.ascii_uppercase + string.digits
-    return ''.join(
-        random.SystemRandom().choice(charset)
-        for _ in range(length)
-    )
-  
-#Generating secret message    
-secret_message = secret(16)
-
-#Login to smt server and sending email with secret message    
-def send_email(msg):
-    print("Sending email ...")
-    server = smtplib.SMTP('localhost')
-    server.set_debuglevel(1)
-    server.connect('localhost', 587)
-    server.ehlo()
-    server.starttls()
-    server.ehlo()
-    server.login("admin@mailu.io", "password")
-    
-    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
-    server.quit()
-
-    print("email sent with message " + msg)
-
-#Login to imap server, read latest email and check for secret message    
-def read_email():
-    print("Receiving email ...")
-    server = imaplib.IMAP4_SSL('localhost')
-    server.login('user@mailu.io', 'password')
-    
-    stat, count = server.select('inbox')
-    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
-    
-    print("email received with message " + str(data[0][1])) 
-    
-    if secret_message in str(data[0][1]):
-        print("Success!")
-    else:
-        print("Failed! Something went wrong")    
-    server.close()
-    server.logout()
-
-    
-send_email(secret_message)
-print("Sleeping for 1m")
-time.sleep(60)
-read_email()
diff --git a/tests/compose/rainloop/email_test.py b/tests/compose/rainloop/email_test.py
deleted file mode 100755
index c16ce9d8..00000000
--- a/tests/compose/rainloop/email_test.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import string
-import random
-import smtplib
-import imaplib
-import time
-
-def secret(length=16):
-    charset = string.ascii_uppercase + string.digits
-    return ''.join(
-        random.SystemRandom().choice(charset)
-        for _ in range(length)
-    )
-  
-#Generating secret message    
-secret_message = secret(16)
-
-#Login to smt server and sending email with secret message    
-def send_email(msg):
-    print("Sending email ...")
-    server = smtplib.SMTP('localhost')
-    server.set_debuglevel(1)
-    server.connect('localhost', 587)
-    server.ehlo()
-    server.starttls()
-    server.ehlo()
-    server.login("admin@mailu.io", "password")
-    
-    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
-    server.quit()
-
-    print("email sent with message " + msg)
-
-#Login to imap server, read latest email and check for secret message    
-def read_email():
-    print("Receiving email ...")
-    server = imaplib.IMAP4_SSL('localhost')
-    server.login('user@mailu.io', 'password')
-    
-    stat, count = server.select('inbox')
-    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
-    
-    print("email received with message " + str(data[0][1])) 
-    
-    if secret_message in str(data[0][1]):
-        print("Success!")
-    else:
-        print("Failed! Something went wrong")    
-    server.close()
-    server.logout()
-
-    
-send_email(secret_message)
-print("Sleeping for 1m")
-time.sleep(60)
-read_email()
diff --git a/tests/compose/roundcube/email_test.py b/tests/compose/roundcube/email_test.py
deleted file mode 100755
index c16ce9d8..00000000
--- a/tests/compose/roundcube/email_test.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import string
-import random
-import smtplib
-import imaplib
-import time
-
-def secret(length=16):
-    charset = string.ascii_uppercase + string.digits
-    return ''.join(
-        random.SystemRandom().choice(charset)
-        for _ in range(length)
-    )
-  
-#Generating secret message    
-secret_message = secret(16)
-
-#Login to smt server and sending email with secret message    
-def send_email(msg):
-    print("Sending email ...")
-    server = smtplib.SMTP('localhost')
-    server.set_debuglevel(1)
-    server.connect('localhost', 587)
-    server.ehlo()
-    server.starttls()
-    server.ehlo()
-    server.login("admin@mailu.io", "password")
-    
-    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
-    server.quit()
-
-    print("email sent with message " + msg)
-
-#Login to imap server, read latest email and check for secret message    
-def read_email():
-    print("Receiving email ...")
-    server = imaplib.IMAP4_SSL('localhost')
-    server.login('user@mailu.io', 'password')
-    
-    stat, count = server.select('inbox')
-    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
-    
-    print("email received with message " + str(data[0][1])) 
-    
-    if secret_message in str(data[0][1]):
-        print("Success!")
-    else:
-        print("Failed! Something went wrong")    
-    server.close()
-    server.logout()
-
-    
-send_email(secret_message)
-print("Sleeping for 1m")
-time.sleep(60)
-read_email()
diff --git a/tests/compose/test.py b/tests/compose/test.py
index d3b2d109..84281123 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -17,7 +17,7 @@ containers = []
 # Stop containers
 def stop(exit_code):
     print_logs()
-    os.system("docker-compose -f " + compose_file + " -p ${DOCKER_ORG:-mailu} down")
+    os.system("docker-compose -f " + compose_file + " down")
     sys.exit(exit_code)
   
 # Sleep for a defined amount of time
@@ -69,25 +69,17 @@ def print_logs():
 #Iterating over hooks in test folder and running them  
 def hooks():
     print("Running hooks")
+    os.system("python3 tests/compose/email_test.py")
     for test_file in sorted(os.listdir(test_path)):
         if test_file.endswith(".py"):
             os.system("python3 " + test_path + test_file)
         elif test_file.endswith(".sh"):
             os.system("./" + test_path + test_file)
 
-#Create admin and user
-def create_users():
-    print("Creating admin account...")
-    os.system("docker-compose -p $DOCKER_ORG exec admin python manage.py admin admin mailu.io password")
-    print("Admin account created")
-    print("Creating user account...")
-    os.system("docker-compose -p $DOCKER_ORG exec admin python manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password'")
-    print("User account created")
-
 # Start up containers
 os.system("mkdir -p /mailu && cp -r tests/certs /mailu")
 os.system("chmod 600 /mailu/certs/* ")
-os.system("docker-compose -f " + compose_file + " -p ${DOCKER_ORG:-mailu} up -d ")
+os.system("docker-compose -f " + compose_file + " up -d ")
 print()
 sleep()
 print()
@@ -95,7 +87,6 @@ os.system("docker ps -a")
 print()
 health_checks()
 print()
-create_users()
 hooks()
 print()
 stop(0)
diff --git a/tests/compose/webdav/email_test.py b/tests/compose/webdav/email_test.py
deleted file mode 100755
index c16ce9d8..00000000
--- a/tests/compose/webdav/email_test.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import string
-import random
-import smtplib
-import imaplib
-import time
-
-def secret(length=16):
-    charset = string.ascii_uppercase + string.digits
-    return ''.join(
-        random.SystemRandom().choice(charset)
-        for _ in range(length)
-    )
-  
-#Generating secret message    
-secret_message = secret(16)
-
-#Login to smt server and sending email with secret message    
-def send_email(msg):
-    print("Sending email ...")
-    server = smtplib.SMTP('localhost')
-    server.set_debuglevel(1)
-    server.connect('localhost', 587)
-    server.ehlo()
-    server.starttls()
-    server.ehlo()
-    server.login("admin@mailu.io", "password")
-    
-    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
-    server.quit()
-
-    print("email sent with message " + msg)
-
-#Login to imap server, read latest email and check for secret message    
-def read_email():
-    print("Receiving email ...")
-    server = imaplib.IMAP4_SSL('localhost')
-    server.login('user@mailu.io', 'password')
-    
-    stat, count = server.select('inbox')
-    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
-    
-    print("email received with message " + str(data[0][1])) 
-    
-    if secret_message in str(data[0][1]):
-        print("Success!")
-    else:
-        print("Failed! Something went wrong")    
-    server.close()
-    server.logout()
-
-    
-send_email(secret_message)
-print("Sleeping for 1m")
-time.sleep(60)
-read_email()

From 88f5e6e4cff54757bbbd3bd8653d8f8cad7bee51 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 2 Nov 2018 15:05:16 +0200
Subject: [PATCH 26/62] Moved users creation in core

---
 .travis.yml                           |  4 ----
 tests/compose/core/00_create_users.sh |  3 +++
 tests/compose/test.py                 | 21 +++++++++++----------
 3 files changed, 14 insertions(+), 14 deletions(-)
 create mode 100755 tests/compose/core/00_create_users.sh

diff --git a/.travis.yml b/.travis.yml
index ffb6af43..54b3b3c7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,10 +18,6 @@ install:
 before_script:
   - docker-compose -v
   - docker-compose -f tests/build.yml build
-  - docker-compose up -d admin
-  - docker-compose exec admin python manage.py admin admin mailu.io password
-  - docker-compose exec admin python manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password'
-  - docker-compose down
   
 script:
 # test.py, test name and timeout between start and tests.
diff --git a/tests/compose/core/00_create_users.sh b/tests/compose/core/00_create_users.sh
new file mode 100755
index 00000000..babe307d
--- /dev/null
+++ b/tests/compose/core/00_create_users.sh
@@ -0,0 +1,3 @@
+echo "Creating users ..."
+docker-compose -f tests/compose/core/docker-compose.yml exec admin python manage.py admin admin mailu.io password || exit 1
+docker-compose -f tests/compose/core/docker-compose.yml exec admin python manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password' || exit 1
diff --git a/tests/compose/test.py b/tests/compose/test.py
index 84281123..c90f481b 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -19,19 +19,19 @@ def stop(exit_code):
     print_logs()
     os.system("docker-compose -f " + compose_file + " down")
     sys.exit(exit_code)
-  
+
 # Sleep for a defined amount of time
 def sleep():
     print(Fore.LIGHTMAGENTA_EX + "Sleeping for " + str(timeout) + "m" + Style.RESET_ALL)
     time.sleep(timeout*60)
-    
+
 def health_checks():
     exit_code = 0
     #Iterating trough all containers dictionary
     for container in client.containers(all=True):
         #Perform "docker container inspect" on container based on container ID and save output to a dictionary
         container_inspect = client.inspect_container(container['Id']) #Dict
-        
+
         if "Health" in container_inspect['State'].keys():
             if container_inspect['State']['Health']['Status'] == "healthy":
                 print(Fore.GREEN + "Health status for " + container_inspect['Name'].replace("/", "") + " : " + Fore.CYAN + container_inspect['State']['Health']['Status'] + Style.RESET_ALL)
@@ -46,36 +46,37 @@ def health_checks():
             if container_inspect['State']['Status'] != "running":
                 print(Fore.RED + "Container " + container_inspect['Name'].replace("/", "") + " state is: " + Fore.YELLOW + container_inspect['State']['Status'] + Style.RESET_ALL)
                 exit_code = 1
-           
+
         #Saving Id, Name and state to a new dictionary
         containers_dict = {}
         containers_dict['Name'] = container_inspect['Name'].replace("/", "")
         containers_dict['Id'] = container_inspect['Id']
         containers_dict['State'] = container_inspect['State']
-        
+
         #Adding the generated dictionary to a list
         containers.append(containers_dict)
-        
+
     if exit_code != 0:
         stop(exit_code)
-        
+
 def print_logs():
     print("Printing logs ...")
     #Iterating through docker container inspect list and print logs
     for container in containers:
         print(Fore.LIGHTMAGENTA_EX + "Printing logs for: " + Fore.GREEN + container['Name'] + Style.RESET_ALL)
         os.system('docker container logs ' + container['Name'])
-      
-#Iterating over hooks in test folder and running them  
+
+#Iterating over hooks in test folder and running them
 def hooks():
     print("Running hooks")
-    os.system("python3 tests/compose/email_test.py")
     for test_file in sorted(os.listdir(test_path)):
         if test_file.endswith(".py"):
             os.system("python3 " + test_path + test_file)
         elif test_file.endswith(".sh"):
             os.system("./" + test_path + test_file)
 
+    os.system("python3 tests/compose/email_test.py")
+
 # Start up containers
 os.system("mkdir -p /mailu && cp -r tests/certs /mailu")
 os.system("chmod 600 /mailu/certs/* ")

From fd8ed3dfa6c901bd2a493d64042ac5a6e3c4bd39 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 2 Nov 2018 15:17:59 +0200
Subject: [PATCH 27/62] Moved certs copy in travis

---
 .travis.yml           | 3 ++-
 tests/compose/test.py | 2 --
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 54b3b3c7..a0762280 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,7 +18,8 @@ install:
 before_script:
   - docker-compose -v
   - docker-compose -f tests/build.yml build
-  
+  - mkdir -p /mailu && sudo cp -r tests/certs /mailu && chmod 600 /mailu/certs/*
+
 script:
 # test.py, test name and timeout between start and tests.
   - python tests/compose/test.py core 1
diff --git a/tests/compose/test.py b/tests/compose/test.py
index c90f481b..a44382a1 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -78,8 +78,6 @@ def hooks():
     os.system("python3 tests/compose/email_test.py")
 
 # Start up containers
-os.system("mkdir -p /mailu && cp -r tests/certs /mailu")
-os.system("chmod 600 /mailu/certs/* ")
 os.system("docker-compose -f " + compose_file + " up -d ")
 print()
 sleep()

From 0d6a203a9d8009e8c7348f9f7692c748bf6fdba3 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 2 Nov 2018 15:32:36 +0200
Subject: [PATCH 28/62] Use sudo for running sh commands

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index a0762280..22e024ec 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,7 +18,7 @@ install:
 before_script:
   - docker-compose -v
   - docker-compose -f tests/build.yml build
-  - mkdir -p /mailu && sudo cp -r tests/certs /mailu && chmod 600 /mailu/certs/*
+  - sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*'
 
 script:
 # test.py, test name and timeout between start and tests.

From 982e586e499a6bc30e1961882ba2c14291e71396 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 2 Nov 2018 16:25:55 +0200
Subject: [PATCH 29/62] Replaced os.system calls with native python calls

---
 tests/compose/core/00_create_users.sh |  1 +
 tests/compose/test.py                 | 14 +++++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/tests/compose/core/00_create_users.sh b/tests/compose/core/00_create_users.sh
index babe307d..fd998a15 100755
--- a/tests/compose/core/00_create_users.sh
+++ b/tests/compose/core/00_create_users.sh
@@ -1,3 +1,4 @@
 echo "Creating users ..."
 docker-compose -f tests/compose/core/docker-compose.yml exec admin python manage.py admin admin mailu.io password || exit 1
 docker-compose -f tests/compose/core/docker-compose.yml exec admin python manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password' || exit 1
+echo "Admin and user successfully created!"
\ No newline at end of file
diff --git a/tests/compose/test.py b/tests/compose/test.py
index a44382a1..7172aab9 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -17,7 +17,7 @@ containers = []
 # Stop containers
 def stop(exit_code):
     print_logs()
-    os.system("docker-compose -f " + compose_file + " down")
+    print(os.popen("docker-compose -f " + compose_file + " down").read())
     sys.exit(exit_code)
 
 # Sleep for a defined amount of time
@@ -64,25 +64,25 @@ def print_logs():
     #Iterating through docker container inspect list and print logs
     for container in containers:
         print(Fore.LIGHTMAGENTA_EX + "Printing logs for: " + Fore.GREEN + container['Name'] + Style.RESET_ALL)
-        os.system('docker container logs ' + container['Name'])
+        print(os.popen('docker container logs ' + container['Name']).read())
 
 #Iterating over hooks in test folder and running them
 def hooks():
     print("Running hooks")
     for test_file in sorted(os.listdir(test_path)):
         if test_file.endswith(".py"):
-            os.system("python3 " + test_path + test_file)
+            print(os.popen("python3 " + test_path + test_file).read())
         elif test_file.endswith(".sh"):
-            os.system("./" + test_path + test_file)
+            print(os.popen("./" + test_path + test_file).read())
 
-    os.system("python3 tests/compose/email_test.py")
+    print(os.popen("python3 tests/compose/email_test.py").read())
 
 # Start up containers
-os.system("docker-compose -f " + compose_file + " up -d ")
+print(os.popen("docker-compose -f " + compose_file + " up -d ").read())
 print()
 sleep()
 print()
-os.system("docker ps -a")
+print(os.popen("docker ps -a").read())
 print()
 health_checks()
 print()

From b3ac4465f29d59755ed694aaa2db31331f2a8c55 Mon Sep 17 00:00:00 2001
From: Daniel Huber 
Date: Sat, 3 Nov 2018 14:54:17 +0100
Subject: [PATCH 30/62] Add docker image prefix

---
 tests/build.yml | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tests/build.yml b/tests/build.yml
index ed5b75fe..3a5d8c5f 100644
--- a/tests/build.yml
+++ b/tests/build.yml
@@ -3,58 +3,58 @@ version: '3'
 services:
 
   front:
-    image: ${DOCKER_ORG:-mailu}/nginx:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}nginx:${VERSION:-local}
     build: ../core/nginx
 
   resolver:
-    image: ${DOCKER_ORG:-mailu}/unbound:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}unbound:${VERSION:-local}
     build: ../services/unbound
 
   imap:
-    image: ${DOCKER_ORG:-mailu}/dovecot:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}dovecot:${VERSION:-local}
     build: ../core/dovecot
 
   smtp:
-    image: ${DOCKER_ORG:-mailu}/postfix:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}postfix:${VERSION:-local}
     build: ../core/postfix
 
   antispam:
-    image: ${DOCKER_ORG:-mailu}/rspamd:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}rspamd:${VERSION:-local}
     build: ../services/rspamd
 
   antivirus:
-    image: ${DOCKER_ORG:-mailu}/clamav:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}clamav:${VERSION:-local}
     build: ../optional/clamav
 
   webdav:
-    image: ${DOCKER_ORG:-mailu}/radicale:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}radicale:${VERSION:-local}
     build: ../optional/radicale
 
   admin:
-    image: ${DOCKER_ORG:-mailu}/admin:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}admin:${VERSION:-local}
     build: ../core/admin
 
   roundcube:
-    image: ${DOCKER_ORG:-mailu}/roundcube:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}roundcube:${VERSION:-local}
     build: ../webmails/roundcube
 
   rainloop:
-    image: ${DOCKER_ORG:-mailu}/rainloop:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}rainloop:${VERSION:-local}
     build: ../webmails/rainloop
 
   fetchmail:
-    image: ${DOCKER_ORG:-mailu}/fetchmail:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}fetchmail:${VERSION:-local}
     build: ../services/fetchmail
 
   none:
-    image: ${DOCKER_ORG:-mailu}/none:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}none:${VERSION:-local}
     build: ../core/none
 
   docs:
-    image: ${DOCKER_ORG:-mailu}/docs:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}docs:${VERSION:-local}
     build: ../docs
 
   setup:
-    image: ${DOCKER_ORG:-mailu}/setup:${VERSION:-local}
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}setup:${VERSION:-local}
     build: ../setup
 

From 9a7fc1416a29d81faac7710aaa7c796cbe33bb05 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Sat, 3 Nov 2018 19:15:05 +0200
Subject: [PATCH 31/62] Replaces os.popen with subprocess.check_output

---
 tests/compose/test.py             | 15 ++++++++-------
 tests/{compose => }/email_test.py |  4 +++-
 2 files changed, 11 insertions(+), 8 deletions(-)
 rename tests/{compose => }/email_test.py (94%)

diff --git a/tests/compose/test.py b/tests/compose/test.py
index 7172aab9..b04bad00 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -3,6 +3,7 @@ import os
 import time
 import docker
 from colorama import Fore, Style
+import subprocess
 
 # Declare variables for service name and sleep time
 test_name=sys.argv[1]
@@ -17,7 +18,7 @@ containers = []
 # Stop containers
 def stop(exit_code):
     print_logs()
-    print(os.popen("docker-compose -f " + compose_file + " down").read())
+    print(subprocess.check_output("docker-compose -f " + compose_file + " down", shell=True))
     sys.exit(exit_code)
 
 # Sleep for a defined amount of time
@@ -64,25 +65,25 @@ def print_logs():
     #Iterating through docker container inspect list and print logs
     for container in containers:
         print(Fore.LIGHTMAGENTA_EX + "Printing logs for: " + Fore.GREEN + container['Name'] + Style.RESET_ALL)
-        print(os.popen('docker container logs ' + container['Name']).read())
+        print(subprocess.check_output('docker container logs ' + container['Name'], shell=True))
 
 #Iterating over hooks in test folder and running them
 def hooks():
     print("Running hooks")
     for test_file in sorted(os.listdir(test_path)):
         if test_file.endswith(".py"):
-            print(os.popen("python3 " + test_path + test_file).read())
+            print(subprocess.check_output("python3 " + test_path + test_file, shell=True))
         elif test_file.endswith(".sh"):
-            print(os.popen("./" + test_path + test_file).read())
+            print(subprocess.check_output("./" + test_path + test_file, shell=True))
 
-    print(os.popen("python3 tests/compose/email_test.py").read())
+    print(subprocess.check_output("python3 tests/email_test.py", shell=True))
 
 # Start up containers
-print(os.popen("docker-compose -f " + compose_file + " up -d ").read())
+print(subprocess.check_output("docker-compose -f " + compose_file + " up -d", shell=True))
 print()
 sleep()
 print()
-print(os.popen("docker ps -a").read())
+print(subprocess.check_output("docker ps -a", shell=True))
 print()
 health_checks()
 print()
diff --git a/tests/compose/email_test.py b/tests/email_test.py
similarity index 94%
rename from tests/compose/email_test.py
rename to tests/email_test.py
index c16ce9d8..c56c1f57 100755
--- a/tests/compose/email_test.py
+++ b/tests/email_test.py
@@ -3,6 +3,7 @@ import random
 import smtplib
 import imaplib
 import time
+import sys
 
 def secret(length=16):
     charset = string.ascii_uppercase + string.digits
@@ -44,7 +45,8 @@ def read_email():
     if secret_message in str(data[0][1]):
         print("Success!")
     else:
-        print("Failed! Something went wrong")    
+        print("Failed! Something went wrong")
+        sys.exit(1)
     server.close()
     server.logout()
 

From fca3dc4e7005250a3a49028e2131b439663e0829 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Sat, 3 Nov 2018 19:34:47 +0200
Subject: [PATCH 32/62] Flushing stream before calling subprocess

---
 tests/compose/test.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/compose/test.py b/tests/compose/test.py
index b04bad00..576e128d 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -18,6 +18,7 @@ containers = []
 # Stop containers
 def stop(exit_code):
     print_logs()
+    sys.stdout.flush()
     print(subprocess.check_output("docker-compose -f " + compose_file + " down", shell=True))
     sys.exit(exit_code)
 
@@ -65,6 +66,7 @@ def print_logs():
     #Iterating through docker container inspect list and print logs
     for container in containers:
         print(Fore.LIGHTMAGENTA_EX + "Printing logs for: " + Fore.GREEN + container['Name'] + Style.RESET_ALL)
+        sys.stdout.flush()
         print(subprocess.check_output('docker container logs ' + container['Name'], shell=True))
 
 #Iterating over hooks in test folder and running them
@@ -72,17 +74,22 @@ def hooks():
     print("Running hooks")
     for test_file in sorted(os.listdir(test_path)):
         if test_file.endswith(".py"):
+            sys.stdout.flush()
             print(subprocess.check_output("python3 " + test_path + test_file, shell=True))
         elif test_file.endswith(".sh"):
+            sys.stdout.flush()
             print(subprocess.check_output("./" + test_path + test_file, shell=True))
-
+    
+    sys.stdout.flush()
     print(subprocess.check_output("python3 tests/email_test.py", shell=True))
 
 # Start up containers
+sys.stdout.flush()
 print(subprocess.check_output("docker-compose -f " + compose_file + " up -d", shell=True))
 print()
 sleep()
 print()
+sys.stdout.flush()
 print(subprocess.check_output("docker ps -a", shell=True))
 print()
 health_checks()

From 4032e7128d2093cf12a0557cca25503ed1b19d9b Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Mon, 5 Nov 2018 12:34:52 +0200
Subject: [PATCH 33/62] Calling email test as hook with msg as arg

---
 tests/compose/core/01_email_test.sh    |  1 +
 tests/compose/filters/01_email_test.sh |  1 +
 tests/compose/test.py                  | 29 +++++++++++++-------------
 tests/email_test.py                    | 18 ++++------------
 4 files changed, 21 insertions(+), 28 deletions(-)
 create mode 100755 tests/compose/core/01_email_test.sh
 create mode 100755 tests/compose/filters/01_email_test.sh

diff --git a/tests/compose/core/01_email_test.sh b/tests/compose/core/01_email_test.sh
new file mode 100755
index 00000000..97dd6e4b
--- /dev/null
+++ b/tests/compose/core/01_email_test.sh
@@ -0,0 +1 @@
+python3 tests/email_test.py message-core
\ No newline at end of file
diff --git a/tests/compose/filters/01_email_test.sh b/tests/compose/filters/01_email_test.sh
new file mode 100755
index 00000000..64fa3fba
--- /dev/null
+++ b/tests/compose/filters/01_email_test.sh
@@ -0,0 +1 @@
+python3 tests/email_test.py message-filters
\ No newline at end of file
diff --git a/tests/compose/test.py b/tests/compose/test.py
index 576e128d..690855d4 100755
--- a/tests/compose/test.py
+++ b/tests/compose/test.py
@@ -19,7 +19,7 @@ containers = []
 def stop(exit_code):
     print_logs()
     sys.stdout.flush()
-    print(subprocess.check_output("docker-compose -f " + compose_file + " down", shell=True))
+    print(subprocess.check_output("docker-compose -f " + compose_file + " down", shell=True).decode())
     sys.exit(exit_code)
 
 # Sleep for a defined amount of time
@@ -67,30 +67,31 @@ def print_logs():
     for container in containers:
         print(Fore.LIGHTMAGENTA_EX + "Printing logs for: " + Fore.GREEN + container['Name'] + Style.RESET_ALL)
         sys.stdout.flush()
-        print(subprocess.check_output('docker container logs ' + container['Name'], shell=True))
+        print(subprocess.check_output('docker container logs ' + container['Name'], shell=True).decode())
 
 #Iterating over hooks in test folder and running them
 def hooks():
-    print("Running hooks")
+    print(Fore.LIGHTMAGENTA_EX + "Running hooks" + Style.RESET_ALL)
     for test_file in sorted(os.listdir(test_path)):
-        if test_file.endswith(".py"):
-            sys.stdout.flush()
-            print(subprocess.check_output("python3 " + test_path + test_file, shell=True))
-        elif test_file.endswith(".sh"):
-            sys.stdout.flush()
-            print(subprocess.check_output("./" + test_path + test_file, shell=True))
-    
-    sys.stdout.flush()
-    print(subprocess.check_output("python3 tests/email_test.py", shell=True))
+        try:
+            if test_file.endswith(".py"):
+                sys.stdout.flush()
+                print(subprocess.check_output("python3 " + test_path + test_file, shell=True).decode())
+            elif test_file.endswith(".sh"):
+                sys.stdout.flush()
+                print(subprocess.check_output("./" + test_path + test_file, shell=True).decode())
+        except subprocess.CalledProcessError as e:
+            sys.stderr.write("[ERROR]: output = %s, error code = %s\n" % (e.output.decode(), e.returncode))
+            stop(1)
 
 # Start up containers
 sys.stdout.flush()
-print(subprocess.check_output("docker-compose -f " + compose_file + " up -d", shell=True))
+print(subprocess.check_output("docker-compose -f " + compose_file + " up -d", shell=True).decode())
 print()
 sleep()
 print()
 sys.stdout.flush()
-print(subprocess.check_output("docker ps -a", shell=True))
+print(subprocess.check_output("docker ps -a", shell=True).decode())
 print()
 health_checks()
 print()
diff --git a/tests/email_test.py b/tests/email_test.py
index c56c1f57..7148365e 100755
--- a/tests/email_test.py
+++ b/tests/email_test.py
@@ -1,19 +1,9 @@
-import string
-import random
 import smtplib
 import imaplib
 import time
 import sys
 
-def secret(length=16):
-    charset = string.ascii_uppercase + string.digits
-    return ''.join(
-        random.SystemRandom().choice(charset)
-        for _ in range(length)
-    )
-  
-#Generating secret message    
-secret_message = secret(16)
+email_msg = sys.argv[1]
 
 #Login to smt server and sending email with secret message    
 def send_email(msg):
@@ -42,16 +32,16 @@ def read_email():
     
     print("email received with message " + str(data[0][1])) 
     
-    if secret_message in str(data[0][1]):
+    if email_msg in str(data[0][1]):
         print("Success!")
     else:
-        print("Failed! Something went wrong")
+        print("Failed receiving email with message %s" % email_msg)
         sys.exit(1)
     server.close()
     server.logout()
 
     
-send_email(secret_message)
+send_email(email_msg)
 print("Sleeping for 1m")
 time.sleep(60)
 read_email()

From 76d9fc3865efb010c0dfc33f796df4af981bed8f Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Mon, 5 Nov 2018 17:28:40 +0200
Subject: [PATCH 34/62] Rewrite of email test script and added eicar virus file

---
 tests/compose/filters/01_email_test.sh |  7 ++-
 tests/compose/filters/eicar.com        |  1 +
 tests/email_test.py                    | 86 +++++++++++++++-----------
 3 files changed, 56 insertions(+), 38 deletions(-)
 create mode 100644 tests/compose/filters/eicar.com

diff --git a/tests/compose/filters/01_email_test.sh b/tests/compose/filters/01_email_test.sh
index 64fa3fba..5af395c4 100755
--- a/tests/compose/filters/01_email_test.sh
+++ b/tests/compose/filters/01_email_test.sh
@@ -1 +1,6 @@
-python3 tests/email_test.py message-filters
\ No newline at end of file
+python3 tests/email_test.py message-virus "tests/compose/filters/eicar.com"
+if [ $? -eq 99 ]; then
+	exit 0
+else
+	exit 1
+fi
\ No newline at end of file
diff --git a/tests/compose/filters/eicar.com b/tests/compose/filters/eicar.com
new file mode 100644
index 00000000..704cac85
--- /dev/null
+++ b/tests/compose/filters/eicar.com
@@ -0,0 +1 @@
+X5O!P%@AP[4\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*
diff --git a/tests/email_test.py b/tests/email_test.py
index 7148365e..853b76b5 100755
--- a/tests/email_test.py
+++ b/tests/email_test.py
@@ -2,46 +2,58 @@ import smtplib
 import imaplib
 import time
 import sys
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+import ntpath
+from email.mime.base import MIMEBase
+from email import encoders
 
-email_msg = sys.argv[1]
+msg = MIMEMultipart()
+msg['From'] = "admin@mailu.io"
+msg['To'] = "user@mailu.io"
+msg['Subject'] = "File Test"
+msg.attach(MIMEText(sys.argv[1], 'plain'))
 
-#Login to smt server and sending email with secret message    
-def send_email(msg):
-    print("Sending email ...")
-    server = smtplib.SMTP('localhost')
-    server.set_debuglevel(1)
-    server.connect('localhost', 587)
-    server.ehlo()
-    server.starttls()
-    server.ehlo()
-    server.login("admin@mailu.io", "password")
-    
-    server.sendmail("admin@mailu.io", "user@mailu.io", msg)
-    server.quit()
+if len(sys.argv) == 3:
+    part = MIMEBase('application', 'octet-stream')
+    part.set_payload((open(sys.argv[2], "rb")).read())
+    encoders.encode_base64(part)
+    part.add_header('Content-Disposition', "attachment; filename=%s" % ntpath.basename(sys.argv[2]))
+    msg.attach(part)
 
-    print("email sent with message " + msg)
+try:
+    smtp_server = smtplib.SMTP('localhost')
+    smtp_server.set_debuglevel(1)
+    smtp_server.connect('localhost', 587)
+    smtp_server.ehlo()
+    smtp_server.starttls()
+    smtp_server.ehlo()
+    smtp_server.login("admin@mailu.io", "password")
+    
+    smtp_server.sendmail("admin@mailu.io", "user@mailu.io", msg.as_string())
+    smtp_server.quit()
+except:
+    sys.exit(25)
 
-#Login to imap server, read latest email and check for secret message    
-def read_email():
-    print("Receiving email ...")
-    server = imaplib.IMAP4_SSL('localhost')
-    server.login('user@mailu.io', 'password')
-    
-    stat, count = server.select('inbox')
-    stat, data = server.fetch(count[0], '(UID BODY[TEXT])')
-    
-    print("email received with message " + str(data[0][1])) 
-    
-    if email_msg in str(data[0][1]):
-        print("Success!")
-    else:
-        print("Failed receiving email with message %s" % email_msg)
-        sys.exit(1)
-    server.close()
-    server.logout()
+time.sleep(30)
 
+try:
+    imap_server = imaplib.IMAP4_SSL('localhost')
+    imap_server.login('user@mailu.io', 'password')
+except:
+    sys.exit(110)
+
+stat, count = imap_server.select('inbox')
+try:
+    stat, data = imap_server.fetch(count[0], '(UID BODY[TEXT])')
+except :
+    sys.exit(99)
+        
+if sys.argv[1] in str(data[0][1]):
+    print("Success sending and receiving email!")
+else:
+    print("Failed receiving email with message %s" % sys.argv[1])
+    sys.exit(99)
     
-send_email(email_msg)
-print("Sleeping for 1m")
-time.sleep(60)
-read_email()
+imap_server.close()
+imap_server.logout()

From 1bbf3f235d41208b75939c486745226f47d9e508 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Wed, 7 Nov 2018 09:58:49 +0200
Subject: [PATCH 35/62] Using a new class when captcha is enabled

---
 core/admin/mailu/ui/forms.py       | 3 ++-
 core/admin/mailu/ui/views/users.py | 6 +++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py
index 4f7a30ae..57c106c3 100644
--- a/core/admin/mailu/ui/forms.py
+++ b/core/admin/mailu/ui/forms.py
@@ -90,9 +90,10 @@ class UserSignupForm(flask_wtf.FlaskForm):
     localpart = fields.StringField(_('Email address'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)])
     pw = fields.PasswordField(_('Password'), [validators.DataRequired()])
     pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')])
-    captcha = flask_wtf.RecaptchaField()
     submit = fields.SubmitField(_('Sign up'))
 
+class UserSignupFormCaptcha(UserSignupForm):
+    captcha = flask_wtf.RecaptchaField()
 
 class UserSettingsForm(flask_wtf.FlaskForm):
     displayed_name = fields.StringField(_('Displayed name'))
diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py
index c54d22ec..f7f75c32 100644
--- a/core/admin/mailu/ui/views/users.py
+++ b/core/admin/mailu/ui/views/users.py
@@ -170,7 +170,11 @@ def user_signup(domain_name=None):
             available_domains=available_domains)
     domain = available_domains.get(domain_name) or flask.abort(404)
     quota_bytes = domain.max_quota_bytes or app.config['DEFAULT_QUOTA']
-    form = forms.UserSignupForm()
+    if app.config['RECAPTCHA_PUBLIC_KEY'] == "" or app.config['RECAPTCHA_PRIVATE_KEY'] == "":
+        form = forms.UserSignupForm()
+    else:
+        form = forms.UserSignupFormCaptcha()
+
     if form.validate_on_submit():
         if domain.has_email(form.localpart.data):
             flask.flash('Email is already used', 'error')

From 2a76451a98edbf6d5f8feb02cc13f1577e75fcf9 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Wed, 7 Nov 2018 16:41:11 +0200
Subject: [PATCH 36/62] Moved scripts to an external js file

---
 setup/Dockerfile                              |  1 +
 setup/static/render.js                        | 34 +++++++++++++++++++
 .../templates/steps/compose/02_services.html  | 22 ++----------
 setup/templates/steps/config.html             | 18 +---------
 setup/templates/steps/stack/02_services.html  | 21 +-----------
 5 files changed, 39 insertions(+), 57 deletions(-)
 create mode 100644 setup/static/render.js

diff --git a/setup/Dockerfile b/setup/Dockerfile
index 83711af5..e39d7c3b 100644
--- a/setup/Dockerfile
+++ b/setup/Dockerfile
@@ -12,6 +12,7 @@ COPY setup.py ./setup.py
 COPY main.py ./main.py
 COPY flavors /data/master/flavors
 COPY templates /data/master/templates
+COPY static ./static
 
 #RUN python setup.py https://github.com/mailu/mailu /data
 
diff --git a/setup/static/render.js b/setup/static/render.js
new file mode 100644
index 00000000..a1c3fb0d
--- /dev/null
+++ b/setup/static/render.js
@@ -0,0 +1,34 @@
+$(document).ready(function() {
+	if ($("#webmail").val() == 'none') {
+		$("#webmail_path").hide();
+		$("#webmail_path").attr("value", "");
+	} else {
+		$("#webmail_path").show();
+		$("#webmail_path").attr("value", "/webmail");
+	}
+	$("#webmail").click(function() {
+		if (this.value == 'none') {
+			$("#webmail_path").hide();
+			$("#webmail_path").attr("value", "");
+		} else {
+			$("#webmail_path").show();
+			$("#webmail_path").attr("value", "/webmail");
+		}
+	});
+});
+
+$(document).ready(function() {
+	if ($('#admin').prop('checked')) {
+		$("#admin_path").show();
+		$("#admin_path").attr("value", "/admin");
+	}
+	$("#admin").change(function() {
+		if ($(this).is(":checked")) {
+			$("#admin_path").show();
+			$("#admin_path").attr("value", "/admin");
+		} else {
+			$("#admin_path").hide();
+			$("#admin_path").attr("value", "");
+		}
+	});
+});
\ No newline at end of file
diff --git a/setup/templates/steps/compose/02_services.html b/setup/templates/steps/compose/02_services.html
index 53763658..11e7a14e 100644
--- a/setup/templates/steps/compose/02_services.html
+++ b/setup/templates/steps/compose/02_services.html
@@ -54,25 +54,7 @@ also disable the antivirus if required (it does use aroung 1GB of ram).

 
 
 
-
+
+
 
 {% endcall %}
diff --git a/setup/templates/steps/config.html b/setup/templates/steps/config.html
index a3352b7e..d843d684 100644
--- a/setup/templates/steps/config.html
+++ b/setup/templates/steps/config.html
@@ -75,22 +75,6 @@ manage your email domains, users, etc.

 
 
 
-
+
 
 {% endcall %}
diff --git a/setup/templates/steps/stack/02_services.html b/setup/templates/steps/stack/02_services.html
index 53763658..36493e05 100644
--- a/setup/templates/steps/stack/02_services.html
+++ b/setup/templates/steps/stack/02_services.html
@@ -54,25 +54,6 @@ also disable the antivirus if required (it does use aroung 1GB of ram).

 
 
 
-
+
 
 {% endcall %}

From 206cce0b47dec42d42442e2dd45ca83edc43990a Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 20:29:52 +0100
Subject: [PATCH 37/62] Finish the configuration bits

---
 core/admin/mailu/__init__.py      | 3 ++-
 core/admin/mailu/configuration.py | 7 +++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py
index 8996fa20..be216e59 100644
--- a/core/admin/mailu/__init__.py
+++ b/core/admin/mailu/__init__.py
@@ -20,8 +20,9 @@ def create_app_from_config(config):
     utils.limiter.init_app(app)
     utils.babel.init_app(app)
     utils.login.init_app(app)
-    utils.login.user_loader(models.User.query.get)
+    utils.login.user_loader(models.User.get)
     utils.proxy.init_app(app)
+    utils.migrate.init_app(app, models.db)
 
     # Initialize debugging tools
     if app.config.get("DEBUG"):
diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py
index 05a3d570..48599d5e 100644
--- a/core/admin/mailu/configuration.py
+++ b/core/admin/mailu/configuration.py
@@ -4,7 +4,7 @@ import os
 DEFAULT_CONFIG = {
     # Specific to the admin UI
     'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db',
-    'SQLALCHEMY_TRACK_MODIFICATIONS': True,
+    'SQLALCHEMY_TRACK_MODIFICATIONS': False,
     'DOCKER_SOCKET': 'unix:///var/run/docker.sock',
     'BABEL_DEFAULT_LOCALE': 'en',
     'BABEL_DEFAULT_TIMEZONE': 'UTC',
@@ -54,7 +54,7 @@ DEFAULT_CONFIG = {
 }
 
 
-class ConfigManager(object):
+class ConfigManager(dict):
     """ Naive configuration manager that uses environment only
     """
 
@@ -77,6 +77,9 @@ class ConfigManager(object):
     def get(self, *args):
         return self.config.get(*args)
 
+    def keys(self):
+        return self.config.keys()
+
     def __getitem__(self, key):
         return self.config.get(key)
 

From f6013aa29fddf9883f8f0bea4b7733718b9d8846 Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 20:30:20 +0100
Subject: [PATCH 38/62] Fix an old migration that was reading configuration
 before migrating

---
 core/admin/migrations/versions/3f6994568962_.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/core/admin/migrations/versions/3f6994568962_.py b/core/admin/migrations/versions/3f6994568962_.py
index ffe8569c..65620c00 100644
--- a/core/admin/migrations/versions/3f6994568962_.py
+++ b/core/admin/migrations/versions/3f6994568962_.py
@@ -13,8 +13,6 @@ down_revision = '2335c80a6bc3'
 from alembic import op
 import sqlalchemy as sa
 
-from mailu import app
-
 
 fetch_table = sa.Table(
     'fetch',
@@ -24,13 +22,7 @@ fetch_table = sa.Table(
 
 
 def upgrade():
-    connection = op.get_bind()
     op.add_column('fetch', sa.Column('keep', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
-    # also apply the current config value if set
-    if app.config.get("FETCHMAIL_KEEP", "False") == "True":
-        connection.execute(
-            fetch_table.update().values(keep=True)
-        )
 
 
 def downgrade():

From f57d4859f36069e967d8ee6a5530db6917f722c0 Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 20:30:41 +0100
Subject: [PATCH 39/62] Provide an in-context wrapper for getting users

---
 core/admin/mailu/models.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index bc1044db..22839e3b 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -349,6 +349,10 @@ class User(Base, Email):
             self.sendmail(app.config["WELCOME_SUBJECT"],
                 app.config["WELCOME_BODY"])
 
+    @classmethod
+    def get(cls, email):
+        return cls.query.get(email)
+
     @classmethod
     def login(cls, email, password):
         user = cls.query.get(email)

From 2a8808bdec519912f0badb0c5a14d3205f1e2892 Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 20:32:06 +0100
Subject: [PATCH 40/62] Add the configuration table migration

---
 core/admin/mailu/dockercli.py                 | 26 -------------------
 .../migrations/versions/cd79ed46d9c2_.py      | 25 ++++++++++++++++++
 core/admin/run.py                             |  7 -----
 3 files changed, 25 insertions(+), 33 deletions(-)
 delete mode 100644 core/admin/mailu/dockercli.py
 create mode 100644 core/admin/migrations/versions/cd79ed46d9c2_.py
 delete mode 100644 core/admin/run.py

diff --git a/core/admin/mailu/dockercli.py b/core/admin/mailu/dockercli.py
deleted file mode 100644
index 94f71144..00000000
--- a/core/admin/mailu/dockercli.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from mailu import app
-
-import docker
-import signal
-
-
-# Connect to the Docker socket
-cli = docker.Client(base_url=app.config['DOCKER_SOCKET'])
-
-
-def get(*names):
-    result = {}
-    all_containers = cli.containers(all=True)
-    for brief in all_containers:
-        if brief['Image'].startswith('mailu/'):
-            container = cli.inspect_container(brief['Id'])
-            container['Image'] = cli.inspect_image(container['Image'])
-            name = container['Config']['Labels']['com.docker.compose.service']
-            if not names or name in names:
-                result[name] = container
-    return result
-
-
-def reload(*names):
-    for name, container in get(*names).items():
-        cli.kill(container["Id"], signal.SIGHUP.value)
diff --git a/core/admin/migrations/versions/cd79ed46d9c2_.py b/core/admin/migrations/versions/cd79ed46d9c2_.py
new file mode 100644
index 00000000..0e9da5d3
--- /dev/null
+++ b/core/admin/migrations/versions/cd79ed46d9c2_.py
@@ -0,0 +1,25 @@
+""" Add a configuration table
+
+Revision ID: cd79ed46d9c2
+Revises: 25fd6c7bcb4a
+Create Date: 2018-10-17 21:44:48.924921
+
+"""
+
+revision = 'cd79ed46d9c2'
+down_revision = '25fd6c7bcb4a'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('config',
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.Column('value', sa.String(length=255), nullable=True),
+    sa.PrimaryKeyConstraint('name')
+    )
+
+
+def downgrade():
+    op.drop_table('config')
diff --git a/core/admin/run.py b/core/admin/run.py
deleted file mode 100644
index a55ac1e2..00000000
--- a/core/admin/run.py
+++ /dev/null
@@ -1,7 +0,0 @@
-import os
-
-
-if __name__ == "__main__":
-    os.environ["DEBUG"] = "True"
-    from mailu import app
-    app.run()

From 4a7eb1eb6cd59e93edece14f17aeb05e8688f3e4 Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 20:32:23 +0100
Subject: [PATCH 41/62] Explicitely declare flask migrate

---
 core/admin/mailu/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py
index 18292041..b11b1689 100644
--- a/core/admin/mailu/utils.py
+++ b/core/admin/mailu/utils.py
@@ -47,3 +47,7 @@ class PrefixMiddleware(object):
         app.wsgi_app = self
 
 proxy = PrefixMiddleware()
+
+
+# Data migrate
+migrate = flask_migrate.Migrate()

From f9e30bd87c3bd08307ada55d303598b01f54d90e Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 21:29:11 +0100
Subject: [PATCH 42/62] Update the dockerfile and upgrade dependencies

---
 core/admin/Dockerfile            |  2 +-
 core/admin/requirements-prod.txt | 47 ++++++++++++++------------------
 core/admin/requirements.txt      |  1 -
 core/admin/start.sh              |  7 +++--
 4 files changed, 25 insertions(+), 32 deletions(-)

diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile
index 2e637206..29eb69f6 100644
--- a/core/admin/Dockerfile
+++ b/core/admin/Dockerfile
@@ -11,12 +11,12 @@ RUN apk add --no-cache openssl \
 
 COPY mailu ./mailu
 COPY migrations ./migrations
-COPY manage.py .
 COPY start.sh /start.sh
 
 RUN pybabel compile -d mailu/translations
 
 EXPOSE 80/tcp
 VOLUME ["/data"]
+ENV FLASK_APP mailu
 
 CMD ["/start.sh"]
diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt
index e321a4d6..2ca59edc 100644
--- a/core/admin/requirements-prod.txt
+++ b/core/admin/requirements-prod.txt
@@ -1,53 +1,46 @@
-alembic==0.9.9
+alembic==1.0.2
 asn1crypto==0.24.0
-Babel==2.5.3
+Babel==2.6.0
 bcrypt==3.1.4
 blinker==1.4
-certifi==2018.4.16
 cffi==1.11.5
-chardet==3.0.4
-click==6.7
-cryptography==2.2.2
+Click==7.0
+cryptography==2.3.1
 decorator==4.3.0
 dnspython==1.15.0
-docker-py==1.10.6
-docker-pycreds==0.2.2
-dominate==2.3.1
-Flask==0.12.2
-Flask-Babel==0.11.2
+dominate==2.3.4
+Flask==1.0.2
+Flask-Babel==0.12.2
 Flask-Bootstrap==3.3.7.1
 Flask-DebugToolbar==0.10.1
 Flask-Limiter==1.0.1
 Flask-Login==0.4.1
-Flask-Migrate==2.1.1
+Flask-Migrate==2.3.0
 Flask-Script==2.0.6
 Flask-SQLAlchemy==2.3.2
 Flask-WTF==0.14.2
-gunicorn==19.7.1
-idna==2.6
+gunicorn==19.9.0
+idna==2.7
 infinity==1.4
 intervals==0.8.1
-itsdangerous==0.24
+itsdangerous==1.1.0
 Jinja2==2.10
 limits==1.3
 Mako==1.0.7
-MarkupSafe==1.0
+MarkupSafe==1.1.0
 passlib==1.7.1
-pycparser==2.18
-pyOpenSSL==17.5.0
-python-dateutil==2.7.2
+pycparser==2.19
+pyOpenSSL==18.0.0
+python-dateutil==2.7.5
 python-editor==1.0.3
-pytz==2018.4
-PyYAML==3.12
+pytz==2018.7
+PyYAML==3.13
 redis==2.10.6
-requests==2.18.4
 six==1.11.0
-SQLAlchemy==1.2.6
+SQLAlchemy==1.2.13
 tabulate==0.8.2
-urllib3==1.22
-validators==0.12.1
+validators==0.12.2
 visitor==0.1.3
-websocket-client==0.47.0
 Werkzeug==0.14.1
-WTForms==2.1
+WTForms==2.2.1
 WTForms-Components==0.10.3
diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt
index d6e7adb1..95a65bbe 100644
--- a/core/admin/requirements.txt
+++ b/core/admin/requirements.txt
@@ -12,7 +12,6 @@ redis
 WTForms-Components
 passlib
 gunicorn
-docker-py
 tabulate
 PyYAML
 PyOpenSSL
diff --git a/core/admin/start.sh b/core/admin/start.sh
index 8208e4a1..b3538204 100755
--- a/core/admin/start.sh
+++ b/core/admin/start.sh
@@ -1,5 +1,6 @@
 #!/bin/sh
 
-python manage.py advertise
-python manage.py db upgrade
-gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload mailu:app
+flask mailu advertise
+flask db upgrade
+
+gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload "$FLASK_APP:create_app()"

From 02995f0a15d46272b219c01acec8b9f3ec938b8c Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 21:29:30 +0100
Subject: [PATCH 43/62] Add a mailu command line to flask

---
 core/admin/mailu/__init__.py |  3 ++-
 core/admin/mailu/manage.py   | 28 ++++++++++++++--------------
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py
index be216e59..3914e9f6 100644
--- a/core/admin/mailu/__init__.py
+++ b/core/admin/mailu/__init__.py
@@ -1,7 +1,7 @@
 import flask
 import flask_bootstrap
 
-from mailu import utils, debug, models, configuration
+from mailu import utils, debug, models, manage, configuration
 
 
 def create_app_from_config(config):
@@ -9,6 +9,7 @@ def create_app_from_config(config):
     """
     app = flask.Flask(__name__)
     app.app_context().push()
+    app.cli.add_command(manage.mailu)
 
     # Bootstrap is used for basic JS and CSS loading
     # TODO: remove this and use statically generated assets instead
diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py
index e0aa6286..7f32f6c8 100644
--- a/core/admin/mailu/manage.py
+++ b/core/admin/mailu/manage.py
@@ -1,4 +1,4 @@
-from mailu import models, create_app
+from mailu import models
 
 from flask import current_app as app
 from flask import cli as flask_cli
@@ -14,12 +14,12 @@ db = models.db
 
 
 @click.group()
-def cli(cls=flask_cli.FlaskGroup, create_app=create_app):
-    """ Main command group
+def mailu(cls=flask_cli.FlaskGroup):
+    """ Mailu command line
     """
 
 
-@cli.command()
+@mailu.command()
 @flask_cli.with_appcontext
 def advertise():
     """ Advertise this server against statistic services.
@@ -38,7 +38,7 @@ def advertise():
             pass
 
 
-@cli.command()
+@mailu.command()
 @click.argument('localpart')
 @click.argument('domain_name')
 @click.argument('password')
@@ -60,7 +60,7 @@ def admin(localpart, domain_name, password):
     db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.argument('localpart')
 @click.argument('domain_name')
 @click.argument('password')
@@ -85,7 +85,7 @@ def user(localpart, domain_name, password, hash_scheme=None):
     db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.option('-n', '--domain_name')
 @click.option('-u', '--max_users')
 @click.option('-a', '--max_aliases')
@@ -99,7 +99,7 @@ def domain(domain_name, max_users=0, max_aliases=0, max_quota_bytes=0):
         db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.argument('localpart')
 @click.argument('domain_name')
 @click.argument('password_hash')
@@ -124,7 +124,7 @@ def user_import(localpart, domain_name, password_hash, hash_scheme = None):
     db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.option('-v', '--verbose')
 @click.option('-d', '--delete_objects')
 @flask_cli.with_appcontext
@@ -266,7 +266,7 @@ def config_update(verbose=False, delete_objects=False):
     db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.argument('email')
 @flask_cli.with_appcontext
 def user_delete(email):
@@ -277,7 +277,7 @@ def user_delete(email):
     db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.argument('email')
 @flask_cli.with_appcontext
 def alias_delete(email):
@@ -288,7 +288,7 @@ def alias_delete(email):
     db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.argument('localpart')
 @click.argument('domain_name')
 @click.argument('destination')
@@ -310,7 +310,7 @@ def alias(localpart, domain_name, destination):
     db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.argument('domain_name')
 @click.argument('max_users')
 @click.argument('max_aliases')
@@ -327,7 +327,7 @@ def setlimits(domain_name, max_users, max_aliases, max_quota_bytes):
     db.session.commit()
 
 
-@cli.command()
+@mailu.command()
 @click.argument('domain_name')
 @click.argument('user_name')
 @flask_cli.with_appcontext

From 30716b8bdf0694ebdd7f06abb9a928c1a29a3513 Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 21:35:41 +0100
Subject: [PATCH 44/62] Update docs with the new flask command

---
 docs/cli.rst                   | 12 ++++++------
 docs/compose/setup.rst         |  4 ++--
 docs/contributors/database.rst |  4 ++--
 docs/kubernetes/1.6/README.md  |  4 ++--
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/cli.rst b/docs/cli.rst
index 4a5250a2..8cfb440b 100644
--- a/docs/cli.rst
+++ b/docs/cli.rst
@@ -15,7 +15,7 @@ alias
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py alias foo example.net "mail1@example.com,mail2@example.com"
+  docker-compose exec admin flask mailu alias foo example.net "mail1@example.com,mail2@example.com"
 
 
 alias_delete
@@ -23,14 +23,14 @@ alias_delete
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py alias_delete foo@example.net
+  docker-compose exec admin flask mailu alias_delete foo@example.net
 
 user
 ----
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py user --hash_scheme='SHA512-CRYPT' myuser example.net 'password123'
+  docker-compose exec admin flask mailu user --hash_scheme='SHA512-CRYPT' myuser example.net 'password123'
 
 user_import
 -----------
@@ -39,14 +39,14 @@ primary difference with simple `user` command is that password is being imported
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py user --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce'
+  docker-compose exec admin flask mailu user --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce'
 
 user_delete
 ------------
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py user_delete foo@example.net
+  docker-compose exec admin flask mailu user_delete foo@example.net
 
 config_update
 -------------
@@ -55,7 +55,7 @@ The sole purpose of this command is for importing users/aliases in bulk and sync
 
 .. code-block:: bash
 
-  cat mail-config.yml | docker-compose run --rm admin python manage.py config_update --delete_objects
+  cat mail-config.yml | docker-compose exec admin flask mailu config_update --delete_objects
 
 where mail-config.yml looks like:
 
diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst
index 64ad7b25..942a368e 100644
--- a/docs/compose/setup.rst
+++ b/docs/compose/setup.rst
@@ -151,6 +151,6 @@ Finally, you must create the initial admin user account:
 
 .. code-block:: bash
 
-  docker-compose run --rm admin python manage.py admin root example.net password
+  docker-compose exec admin flask mailu admin me example.net password
 
-This will create a user named ``root@example.net`` with password ``password`` and administration privileges. Connect to the Web admin interface and change the password to a strong one.
+This will create a user named ``me@example.net`` with password ``password`` and administration privileges. Connect to the Web admin interface and change the password to a strong one.
diff --git a/docs/contributors/database.rst b/docs/contributors/database.rst
index ca97f604..a987c980 100644
--- a/docs/contributors/database.rst
+++ b/docs/contributors/database.rst
@@ -17,7 +17,7 @@ migration script:
 
 .. code-block:: bash
 
-  python manage.py db migrate
+  flask db migrate
 
 This will generate a new script in ``migrations/versions`` that you must review
 before adding it for commit.
@@ -54,7 +54,7 @@ At that point, to start working on the changed database structure, you will need
 
 .. code-block:: bash
 
-  python manage.py db upgrade
+  flask db upgrade
 
 If any error arises, restore the backup, fix the migration script and try again.
 
diff --git a/docs/kubernetes/1.6/README.md b/docs/kubernetes/1.6/README.md
index c0dd935b..21780a0c 100644
--- a/docs/kubernetes/1.6/README.md
+++ b/docs/kubernetes/1.6/README.md
@@ -82,7 +82,7 @@ And in the pod run the following command. The command uses following entries:
 - `password` the chosen password for the user
 
 ```bash
-python manage.py admin root example.com password
+flask mailu admin root example.com password
 ```
 
 Now you should be able to login on the mail account: `https://mail.example.com/admin`
@@ -154,4 +154,4 @@ Wait for the pod to recreate and you're online!
 Happy mailing!
 
 Wait for the pod to recreate and you're online!
-Happy mailing!
\ No newline at end of file
+Happy mailing!

From 72e1b444caaa5dd07700bd8f993a8ffa40f83a21 Mon Sep 17 00:00:00 2001
From: kaiyou 
Date: Thu, 8 Nov 2018 21:55:39 +0100
Subject: [PATCH 45/62] Merge alembic migrations

---
 core/admin/migrations/versions/cd79ed46d9c2_.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/migrations/versions/cd79ed46d9c2_.py b/core/admin/migrations/versions/cd79ed46d9c2_.py
index 0e9da5d3..ccf210fe 100644
--- a/core/admin/migrations/versions/cd79ed46d9c2_.py
+++ b/core/admin/migrations/versions/cd79ed46d9c2_.py
@@ -7,7 +7,7 @@ Create Date: 2018-10-17 21:44:48.924921
 """
 
 revision = 'cd79ed46d9c2'
-down_revision = '25fd6c7bcb4a'
+down_revision = '3b281286c7bd'
 
 from alembic import op
 import sqlalchemy as sa

From 4783e6169371e6016260fb768942abf71bd724c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Fri, 9 Nov 2018 11:45:08 +0200
Subject: [PATCH 46/62] Fix password context

Fixes the following error:
```
admin_1      | [2018-11-09 09:44:10,533] ERROR in app: Exception on /internal/auth/email [GET]
admin_1      | Traceback (most recent call last):
admin_1      |   File "/usr/lib/python3.6/site-packages/flask/app.py", line 2292, in wsgi_app
admin_1      |     response = self.full_dispatch_request()
admin_1      |   File "/usr/lib/python3.6/site-packages/flask/app.py", line 1815, in full_dispatch_request
admin_1      |     rv = self.handle_user_exception(e)
admin_1      |   File "/usr/lib/python3.6/site-packages/flask/app.py", line 1718, in handle_user_exception
admin_1      |     reraise(exc_type, exc_value, tb)
admin_1      |   File "/usr/lib/python3.6/site-packages/flask/_compat.py", line 35, in reraise
admin_1      |     raise value
admin_1      |   File "/usr/lib/python3.6/site-packages/flask/app.py", line 1813, in full_dispatch_request
admin_1      |     rv = self.dispatch_request()
admin_1      |   File "/usr/lib/python3.6/site-packages/flask/app.py", line 1799, in dispatch_request
admin_1      |     return self.view_functions[rule.endpoint](**req.view_args)
admin_1      |   File "/usr/lib/python3.6/site-packages/flask_limiter/extension.py", line 544, in __inner
admin_1      |     return obj(*a, **k)
admin_1      |   File "/app/mailu/internal/views/auth.py", line 18, in nginx_authentication
admin_1      |     headers = nginx.handle_authentication(flask.request.headers)
admin_1      |   File "/app/mailu/internal/nginx.py", line 48, in handle_authentication
admin_1      |     if user.check_password(password):
admin_1      |   File "/app/mailu/models.py", line 333, in check_password
admin_1      |     context = User.pw_context
admin_1      | AttributeError: type object 'User' has no attribute 'pw_context'
```
---
 core/admin/mailu/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index d1c6b370..c25d30cb 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -330,7 +330,7 @@ class User(Base, Email):
         )
 
     def check_password(self, password):
-        context = User.pw_context
+        context = self.get_password_context()
         reference = re.match('({[^}]+})?(.*)', self.password).group(2)
         result = context.verify(password, reference)
         if result and context.identify(reference) != context.default_scheme():

From fed7146873c5de0f3fab288451242850c7c9c67d Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 9 Nov 2018 12:30:49 +0200
Subject: [PATCH 47/62] Captcha check on signup form

---
 core/admin/mailu/ui/templates/user/signup.html | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/admin/mailu/ui/templates/user/signup.html b/core/admin/mailu/ui/templates/user/signup.html
index 59c1a988..f540d660 100644
--- a/core/admin/mailu/ui/templates/user/signup.html
+++ b/core/admin/mailu/ui/templates/user/signup.html
@@ -14,7 +14,9 @@
   {% call macros.box() %}
   {{ macros.form_field(form.localpart, append='@'+domain.name+'') }}
   {{ macros.form_fields((form.pw, form.pw2)) }}
-  {{ macros.form_field(form.captcha) }}
+  {% if form.captcha %}
+    {{ macros.form_field(form.captcha) }}
+  {% endif %}
   {{ macros.form_field(form.submit) }}
   {% endcall %}
 

From 2b6f5ea3d079440dacbcb1b5758babee17064e8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Sat, 1 Dec 2018 02:37:34 +0200
Subject: [PATCH 48/62] Update setup's own docker-compose.yml file

---
 setup/docker-compose.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup/docker-compose.yml b/setup/docker-compose.yml
index e91332e1..42e7ee18 100644
--- a/setup/docker-compose.yml
+++ b/setup/docker-compose.yml
@@ -1,13 +1,13 @@
 # This file is used to run the mailu/setup utility
 
-version: '2'
+version: '3.6'
 
 services:
   redis:
     image: redis:alpine
 
   setup:
-    image: mailu/setup
+    image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX}setup:${MAILU_VERSION:-master}
     ports:
       - "8000:80"
     build: .

From 88c174fb7aef88e8796256158509840060b6dfa5 Mon Sep 17 00:00:00 2001
From: David Rothera 
Date: Sun, 2 Dec 2018 10:46:47 +0000
Subject: [PATCH 49/62] Query alternative table for domain matches

At present postfix checks this view for matches in the domain table and is used to accept/deny messages sent into it however it never checks for matches in the alternative table.

Fixes #718
---
 core/admin/mailu/internal/views/postfix.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py
index 79fbdb8a..74e49864 100644
--- a/core/admin/mailu/internal/views/postfix.py
+++ b/core/admin/mailu/internal/views/postfix.py
@@ -6,7 +6,9 @@ import flask
 
 @internal.route("/postfix/domain/")
 def postfix_mailbox_domain(domain_name):
-    domain = models.Domain.query.get(domain_name) or flask.abort(404)
+    domain = models.Domain.query.get(domain_name) or \
+             models.Alternative.query.get(domain_name) or \
+             flask.abort(404)
     return flask.jsonify(domain.name)
 
 

From eff6c34632c8c4097c94162b38d5ca97b06ba15a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Tue, 4 Dec 2018 15:40:07 +0200
Subject: [PATCH 50/62] Catch asterisk  before resolve_domain

Asterisk results in IDNA error and a 500 return code.
---
 core/admin/mailu/internal/views/postfix.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py
index 894532a3..7343459a 100644
--- a/core/admin/mailu/internal/views/postfix.py
+++ b/core/admin/mailu/internal/views/postfix.py
@@ -27,6 +27,8 @@ def postfix_alias_map(alias):
 
 @internal.route("/postfix/transport/")
 def postfix_transport(email):
+    if email == '*':
+        return flask.abort(404)
     localpart, domain_name = models.Email.resolve_domain(email)
     relay = models.Relay.query.get(domain_name) or flask.abort(404)
     return flask.jsonify("smtp:[{}]".format(relay.smtp))

From c9df311a0de4f379a7a9f9f2845b2d7fa01b1828 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Tue, 4 Dec 2018 16:22:18 +0200
Subject: [PATCH 51/62] Set forward_destination to an empty list

The value of `None` resulted in an error, since a list was expected.
---
 core/admin/mailu/models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index ffe1ad08..62577dd5 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -267,7 +267,7 @@ class User(Base, Email):
 
     # Filters
     forward_enabled = db.Column(db.Boolean(), nullable=False, default=False)
-    forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=None)
+    forward_destination = db.Column(CommaSeparatedList(), nullable=True, default=[])
     forward_keep = db.Column(db.Boolean(), nullable=False, default=True)
     reply_enabled = db.Column(db.Boolean(), nullable=False, default=False)
     reply_subject = db.Column(db.String(255), nullable=True, default=None)

From b564b879aabc27858915ad5476d0c361d39a2007 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Wed, 5 Dec 2018 12:38:06 +0200
Subject: [PATCH 52/62] Update .mergify.yml - New syntax for mergify engine v.
 2 - Relax review rules for trusted users

---
 .mergify.yml | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/.mergify.yml b/.mergify.yml
index 7195e58e..023bf59b 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -1,10 +1,16 @@
-rules:
-  default: null
-  branches:
-    master:
-      protection:
-        required_status_checks:
-          contexts:
-            - continuous-integration/travis-ci 
-        required_pull_request_reviews:
-          required_approving_review_count: 2
+pull_request_rules:
+  - name: Successful travis and 2 approved reviews
+    conditions:
+      - status-success=continuous-integration/travis-ci/pr
+      - "#approved-reviews-by>=2"
+    actions:
+      merge:
+        method: merge
+  - name: Trusted author, successful travis and 1 approved review
+    conditions:
+      - author~=(kaiyou|muhlemmer|mildred|HorayNarea|adi90x|hoellen|ofthesun9)
+      - status-success=continuous-integration/travis-ci/pr
+      - "#approved-reviews-by>=1"
+    actions:
+      merge:
+        method: merge

From d84254ccd83f8359b6cf3f78d568df668b4a2e11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Wed, 5 Dec 2018 16:23:16 +0200
Subject: [PATCH 53/62] Update user creation to python3

---
 tests/compose/core/00_create_users.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/compose/core/00_create_users.sh b/tests/compose/core/00_create_users.sh
index fd998a15..967e13b7 100755
--- a/tests/compose/core/00_create_users.sh
+++ b/tests/compose/core/00_create_users.sh
@@ -1,4 +1,4 @@
 echo "Creating users ..."
-docker-compose -f tests/compose/core/docker-compose.yml exec admin python manage.py admin admin mailu.io password || exit 1
-docker-compose -f tests/compose/core/docker-compose.yml exec admin python manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password' || exit 1
-echo "Admin and user successfully created!"
\ No newline at end of file
+docker-compose -f tests/compose/core/docker-compose.yml exec admin python3 manage.py admin admin mailu.io password || exit 1
+docker-compose -f tests/compose/core/docker-compose.yml exec admin python3 manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password' || exit 1
+echo "Admin and user successfully created!"

From d18cf7cb25723f60abb7e57279673eb2fcaa051d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Wed, 5 Dec 2018 17:43:42 +0200
Subject: [PATCH 54/62] Prevent redirects during health checking

---
 core/admin/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile
index 315b2e39..3f132a88 100644
--- a/core/admin/Dockerfile
+++ b/core/admin/Dockerfile
@@ -25,4 +25,4 @@ VOLUME ["/data"]
 
 CMD /start.py
 
-HEALTHCHECK CMD curl -f -L http://localhost/ui || exit 1
+HEALTHCHECK CMD curl -f -L http://localhost/ui/login?next=ui.index || exit 1

From 9dd447e23b7d0b8fc34c7cda6b71f2b6c644b1d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Thu, 6 Dec 2018 01:00:16 +0200
Subject: [PATCH 55/62] Add login method to smtp_auth under ssl Fixes #704

---
 core/nginx/conf/nginx.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf
index 17d67526..eced2c46 100644
--- a/core/nginx/conf/nginx.conf
+++ b/core/nginx/conf/nginx.conf
@@ -245,7 +245,7 @@ mail {
       listen 465 ssl;
       listen [::]:465 ssl;
       protocol smtp;
-      smtp_auth plain;
+      smtp_auth plain login;
     }
 
     server {

From fee52e87eddbf445b15359eea95cde576bc13666 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Thu, 6 Dec 2018 11:34:28 +0200
Subject: [PATCH 56/62] Don't allow for 1 review when review/need2 label is set

---
 .mergify.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.mergify.yml b/.mergify.yml
index 023bf59b..f18eec13 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -10,6 +10,7 @@ pull_request_rules:
     conditions:
       - author~=(kaiyou|muhlemmer|mildred|HorayNarea|adi90x|hoellen|ofthesun9)
       - status-success=continuous-integration/travis-ci/pr
+      - label!=["review/need2"]
       - "#approved-reviews-by>=1"
     actions:
       merge:

From 8acf9451fac69bccb158cbd24ecf767adff588b4 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 23 Nov 2018 14:12:22 +0200
Subject: [PATCH 57/62] Changed admin and user creation command

---
 tests/compose/core/00_create_users.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/compose/core/00_create_users.sh b/tests/compose/core/00_create_users.sh
index 967e13b7..43e82f3f 100755
--- a/tests/compose/core/00_create_users.sh
+++ b/tests/compose/core/00_create_users.sh
@@ -1,4 +1,4 @@
 echo "Creating users ..."
-docker-compose -f tests/compose/core/docker-compose.yml exec admin python3 manage.py admin admin mailu.io password || exit 1
-docker-compose -f tests/compose/core/docker-compose.yml exec admin python3 manage.py user --hash_scheme='SHA512-CRYPT' user mailu.io 'password' || exit 1
+docker-compose -f tests/compose/core/docker-compose.yml exec admin flask mailu admin admin mailu.io password || exit 1
+docker-compose -f tests/compose/core/docker-compose.yml exec admin flask mailu user --hash_scheme='SHA512-CRYPT' user mailu.io 'password' || exit 1
 echo "Admin and user successfully created!"

From 35be1710a674e5f432ba7df397724d9292322ae3 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 23 Nov 2018 14:37:55 +0200
Subject: [PATCH 58/62] Changed user creation command

---
 tests/compose/core/00_create_users.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/compose/core/00_create_users.sh b/tests/compose/core/00_create_users.sh
index 43e82f3f..40d0bd6e 100755
--- a/tests/compose/core/00_create_users.sh
+++ b/tests/compose/core/00_create_users.sh
@@ -1,4 +1,4 @@
 echo "Creating users ..."
 docker-compose -f tests/compose/core/docker-compose.yml exec admin flask mailu admin admin mailu.io password || exit 1
-docker-compose -f tests/compose/core/docker-compose.yml exec admin flask mailu user --hash_scheme='SHA512-CRYPT' user mailu.io 'password' || exit 1
+docker-compose -f tests/compose/core/docker-compose.yml exec admin flask mailu user user mailu.io 'password' 'SHA512-CRYPT' || exit 1
 echo "Admin and user successfully created!"

From a2a9512afaf3cc550820dc0f42c77c3b0fcd7929 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Fri, 7 Dec 2018 11:53:43 +0200
Subject: [PATCH 59/62] Enable mergify strict mode

In the past we had strict mode in branch protection. This didn't really work as it broke mergify. Now mergify supports this options and takes care of the merging automatically. Let's see how it goes ;)

Reason is the recent build failures we had on master, during a busy merge day. This could have been prevented if sequential PR's where re-merging with master.

More info: https://doc.mergify.io/strict-workflow.html
---
 .mergify.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.mergify.yml b/.mergify.yml
index 023bf59b..1cbb0999 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -6,6 +6,7 @@ pull_request_rules:
     actions:
       merge:
         method: merge
+        strict: true
   - name: Trusted author, successful travis and 1 approved review
     conditions:
       - author~=(kaiyou|muhlemmer|mildred|HorayNarea|adi90x|hoellen|ofthesun9)
@@ -14,3 +15,4 @@ pull_request_rules:
     actions:
       merge:
         method: merge
+        strict: true

From 94edb48f08653957db54a8ff6ae3a85ccb7438d1 Mon Sep 17 00:00:00 2001
From: Ionut Filip 
Date: Fri, 7 Dec 2018 13:37:40 +0200
Subject: [PATCH 60/62] Dynamic attachment size

---
 docs/compose/.env               | 1 +
 setup/flavors/compose/mailu.env | 1 +
 webmails/rainloop/Dockerfile    | 2 +-
 webmails/rainloop/config.ini    | 2 +-
 webmails/rainloop/php.ini       | 5 +++--
 webmails/rainloop/start.py      | 3 +++
 webmails/roundcube/Dockerfile   | 4 ++--
 webmails/roundcube/php.ini      | 5 +++--
 webmails/roundcube/start.py     | 7 +++++++
 9 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/docs/compose/.env b/docs/compose/.env
index 2100e27a..7b1e6d01 100644
--- a/docs/compose/.env
+++ b/docs/compose/.env
@@ -61,6 +61,7 @@ ANTIVIRUS=none
 
 # Message size limit in bytes
 # Default: accept messages up to 50MB
+# Max attachment size will be 33% smaller
 MESSAGE_SIZE_LIMIT=50000000
 
 # Networks granted relay permissions, make sure that you include your Docker
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 3f67b0dd..b4505c62 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -73,6 +73,7 @@ ANTISPAM={{ antispam_enabled or 'none'}}
 
 # Message size limit in bytes
 # Default: accept messages up to 50MB
+# Max attachment size will be 33% smaller
 MESSAGE_SIZE_LIMIT={{ message_size_limit or '50000000' }}
 
 # Networks granted relay permissions, make sure that you include your Docker
diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile
index db7403f5..92479489 100644
--- a/webmails/rainloop/Dockerfile
+++ b/webmails/rainloop/Dockerfile
@@ -22,7 +22,7 @@ RUN apt-get update && apt-get install -y \
  && rm -rf /var/lib/apt/lists
 
 COPY include.php /var/www/html/include.php
-COPY php.ini /usr/local/etc/php/conf.d/rainloop.ini
+COPY php.ini /php.ini
 
 COPY config.ini /config.ini
 COPY default.ini /default.ini
diff --git a/webmails/rainloop/config.ini b/webmails/rainloop/config.ini
index 7fb13889..6ae5fff7 100644
--- a/webmails/rainloop/config.ini
+++ b/webmails/rainloop/config.ini
@@ -1,7 +1,7 @@
 ; RainLoop Webmail configuration file
 
 [webmail]
-attachment_size_limit = 25
+attachment_size_limit = {{ MAX_FILESIZE }}
 
 [security]
 allow_admin_panel = Off
diff --git a/webmails/rainloop/php.ini b/webmails/rainloop/php.ini
index 9b241b46..39abbdd5 100644
--- a/webmails/rainloop/php.ini
+++ b/webmails/rainloop/php.ini
@@ -1,3 +1,4 @@
 date.timezone=UTC
-upload_max_filesize = 25M
-post_max_size = 25M
+upload_max_filesize = {{ MAX_FILESIZE }}M
+post_max_size = {{ MAX_FILESIZE }}M
+
diff --git a/webmails/rainloop/start.py b/webmails/rainloop/start.py
index 9e8465a2..4c116e09 100755
--- a/webmails/rainloop/start.py
+++ b/webmails/rainloop/start.py
@@ -10,6 +10,8 @@ convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()
 os.environ["FRONT_ADDRESS"] = os.environ.get("FRONT_ADDRESS", "front")
 os.environ["IMAP_ADDRESS"] = os.environ.get("IMAP_ADDRESS", "imap")
 
+os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576))
+
 base = "/data/_data_/_default_/"
 shutil.rmtree(base + "domains/", ignore_errors=True)
 os.makedirs(base + "domains", exist_ok=True)
@@ -17,6 +19,7 @@ os.makedirs(base + "configs", exist_ok=True)
 
 convert("/default.ini", "/data/_data_/_default_/domains/default.ini")
 convert("/config.ini", "/data/_data_/_default_/configs/config.ini")
+convert("/php.ini", "/usr/local/etc/php/conf.d/rainloop.ini")
 
 os.system("chown -R www-data:www-data /data")
 
diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile
index 14bee56e..00b843b2 100644
--- a/webmails/roundcube/Dockerfile
+++ b/webmails/roundcube/Dockerfile
@@ -7,7 +7,7 @@ RUN apt-get update && apt-get install -y \
 ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.8/roundcubemail-1.3.8-complete.tar.gz
 
 RUN apt-get update && apt-get install -y \
-      zlib1g-dev \
+      zlib1g-dev python3-jinja2 \
  && docker-php-ext-install zip \
  && echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini \
  && rm -rf /var/www/html/ \
@@ -22,7 +22,7 @@ RUN apt-get update && apt-get install -y \
  && chown -R www-data: logs temp \
  && rm -rf /var/lib/apt/lists
 
-COPY php.ini /usr/local/etc/php/conf.d/roundcube.ini
+COPY php.ini /php.ini
 COPY config.inc.php /var/www/html/config/
 COPY start.py /start.py
 
diff --git a/webmails/roundcube/php.ini b/webmails/roundcube/php.ini
index 9b241b46..39abbdd5 100644
--- a/webmails/roundcube/php.ini
+++ b/webmails/roundcube/php.ini
@@ -1,3 +1,4 @@
 date.timezone=UTC
-upload_max_filesize = 25M
-post_max_size = 25M
+upload_max_filesize = {{ MAX_FILESIZE }}M
+post_max_size = {{ MAX_FILESIZE }}M
+
diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py
index 07b3a567..3a0bd0bc 100755
--- a/webmails/roundcube/start.py
+++ b/webmails/roundcube/start.py
@@ -1,6 +1,13 @@
 #!/usr/bin/python3
 
 import os
+import jinja2
+
+convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+
+os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576))
+
+convert("/php.ini", "/usr/local/etc/php/conf.d/roundcube.ini")
 
 # Fix some permissions
 os.system("mkdir -p /data/gpg")

From 626559f99bee40723ffb2e9b5e6faf88446d0db5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Fri, 7 Dec 2018 13:43:53 +0200
Subject: [PATCH 61/62] Mergify dismiss reviews

---
 .mergify.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.mergify.yml b/.mergify.yml
index 1cbb0999..f00c4086 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -7,6 +7,9 @@ pull_request_rules:
       merge:
         method: merge
         strict: true
+      dismiss_reviews:
+        approved: true
+
   - name: Trusted author, successful travis and 1 approved review
     conditions:
       - author~=(kaiyou|muhlemmer|mildred|HorayNarea|adi90x|hoellen|ofthesun9)
@@ -16,3 +19,5 @@ pull_request_rules:
       merge:
         method: merge
         strict: true
+      dismiss_reviews:
+        approved: true

From 8e5ccf27541e9bda25fbf989db915dfe58dfd991 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= 
Date: Fri, 7 Dec 2018 13:47:22 +0200
Subject: [PATCH 62/62] Don't merge when WIP or Blocked

---
 .mergify.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.mergify.yml b/.mergify.yml
index f18eec13..d7299d36 100644
--- a/.mergify.yml
+++ b/.mergify.yml
@@ -2,6 +2,7 @@ pull_request_rules:
   - name: Successful travis and 2 approved reviews
     conditions:
       - status-success=continuous-integration/travis-ci/pr
+      - label!=["status"/wip","status/blocked"]
       - "#approved-reviews-by>=2"
     actions:
       merge:
@@ -10,7 +11,7 @@ pull_request_rules:
     conditions:
       - author~=(kaiyou|muhlemmer|mildred|HorayNarea|adi90x|hoellen|ofthesun9)
       - status-success=continuous-integration/travis-ci/pr
-      - label!=["review/need2"]
+      - label!=["status"/wip","status/blocked","review/need2"]
       - "#approved-reviews-by>=1"
     actions:
       merge: