Merge remote-tracking branch 'upstream/master' into feat-unbound-dns

core/admin/Dockerfile

@@ -4,10 +4,10 @@ RUN mkdir -p /app
 WORKDIR /app
 
 COPY requirements-prod.txt requirements.txt
-RUN apk add --no-cache openssl \
+RUN apk add --no-cache openssl curl \
  && apk add --no-cache --virtual build-dep openssl-dev libffi-dev python-dev build-base \
  && pip install -r requirements.txt \
- && apk del build-dep
+ && apk del --no-cache build-dep
 
 COPY mailu ./mailu
 COPY migrations ./migrations
@@ -20,3 +20,5 @@ EXPOSE 80/tcp
 VOLUME ["/data"]
 
 CMD ["/start.sh"]
+
+HEALTHCHECK CMD curl -f -L http://localhost/ui || exit 1

core/admin/mailu/__init__.py

@@ -11,9 +11,8 @@ import os
 import docker
 import socket
 import uuid
-import redis
 
-from werkzeug.contrib import fixers
+from werkzeug.contrib import fixers, profiler
 
 # Create application
 app = flask.Flask(__name__)
@@ -58,12 +57,15 @@ default_config = {
     'RECAPTCHA_PUBLIC_KEY': '',
     'RECAPTCHA_PRIVATE_KEY': '',
     # Advanced settings
-    'PASSWORD_SCHEME': 'SHA512-CRYPT',
+    'PASSWORD_SCHEME': 'BLF-CRYPT',
     # Host settings
     'HOST_IMAP': 'imap',
     'HOST_POP3': 'imap',
     'HOST_SMTP': 'smtp',
+    'HOST_WEBMAIL': 'webmail',
+    'HOST_FRONT': 'front',
     'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'),
+    'POD_ADDRESS_RANGE': None
 }
 
 # Load configuration from the environment if available
@@ -81,6 +83,10 @@ if app.config.get("DEBUG"):
     import flask_debugtoolbar
     toolbar = flask_debugtoolbar.DebugToolbarExtension(app)
 
+# Profiler
+if app.config.get("DEBUG"):
+    app.wsgi_app = profiler.ProfilerMiddleware(app.wsgi_app, restrictions=[30])
+
 # Manager commnad
 manager = flask_script.Manager(app)
 manager.add_command('db', flask_migrate.MigrateCommand)
@@ -89,9 +95,6 @@ manager.add_command('db', flask_migrate.MigrateCommand)
 babel = flask_babel.Babel(app)
 translations = list(map(str, babel.list_translations()))
 
-# Quota manager
-quota = redis.Redis.from_url(app.config.get("QUOTA_STORAGE_URL"))
-
 @babel.localeselector
 def get_locale():
     return flask.request.accept_languages.best_match(translations)
@@ -133,4 +136,5 @@ class PrefixMiddleware(object):
             environ['SCRIPT_NAME'] = prefix
         return self.app(environ, start_response)
 
+
 app.wsgi_app = PrefixMiddleware(fixers.ProxyFix(app.wsgi_app))

core/admin/mailu/internal/__init__.py

@@ -6,7 +6,8 @@ import socket
 import flask
 
 
-internal = flask.Blueprint('internal', __name__)
+internal = flask.Blueprint('internal', __name__, template_folder='templates')
+
 
 @internal.app_errorhandler(RateLimitExceeded)
 def rate_limit_handler(e):
@@ -17,6 +18,7 @@ def rate_limit_handler(e):
         response.headers['Auth-Wait'] = '3'
     return response
 
+
 @limiter.request_filter
 def whitelist_webmail():
     try:
@@ -26,4 +28,4 @@ def whitelist_webmail():
         return False
 
 
-from mailu.internal import views
+from mailu.internal.views import *

core/admin/mailu/internal/templates/default.sieve

@@ -8,8 +8,6 @@ require "regex";
 require "relational";
 require "date";
 require "comparator-i;ascii-numeric";
-require "vnd.dovecot.extdata";
-require "vnd.dovecot.execute";
 require "spamtestplus";
 require "editheader";
 require "index";
@@ -20,21 +18,20 @@ if header :index 2 :matches "Received" "from * by * for <*>; *"
   addheader "Delivered-To" "<${3}>";
 }
 
-if allof (string :is "${extdata.spam_enabled}" "1",
+{% if user.spam_enabled %}
-          spamtest :percent :value "gt" :comparator "i;ascii-numeric"  "${extdata.spam_threshold}")
+if spamtest :percent :value "gt" :comparator "i;ascii-numeric"  "{{ user.spam_threshold }}"
 {
   setflag "\\seen";
   fileinto :create "Junk";
   stop;
 }
+{% endif %}
 
 if exists "X-Virus" {
   discard;
   stop;
 }
 
-if allof (string :is "${extdata.reply_enabled}" "1",
+{% if user.reply_active  %}
-          currentdate :value "le" "date" "${extdata.reply_enddate}")
+vacation :days 1 :subject "{{ user.reply_subject }}" "{{ user.reply_body }}";
-{
+{% endif %}
-  vacation :days 1 :subject "${extdata.reply_subject}" "${extdata.reply_body}";
-}

core/admin/mailu/internal/views/__init__.py

@@ -0,0 +1,3 @@
+__all__ = [
+    'auth', 'postfix', 'dovecot', 'fetch'
+]

core/admin/mailu/internal/views/auth.py

@@ -4,7 +4,6 @@ from mailu.internal import internal, nginx
 import flask
 import flask_login
 import base64
-import urllib
 
 
 @internal.route("/auth/email")

core/admin/mailu/internal/views/dovecot.py

@@ -0,0 +1,50 @@
+from mailu import db, models, app
+from mailu.internal import internal
+
+import flask
+import socket
+
+
+@internal.route("/dovecot/passdb/<user_email>")
+def dovecot_passdb_dict(user_email):
+    user = models.User.query.get(user_email) or flask.abort(404)
+    allow_nets = []
+    allow_nets.append(
+        app.config.get("POD_ADDRESS_RANGE") or
+        socket.gethostbyname(app.config["HOST_FRONT"])
+    )
+    allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"]))
+    print(allow_nets)
+    return flask.jsonify({
+        "password": None,
+        "nopassword": "Y",
+        "allow_nets": ",".join(allow_nets)
+    })
+
+
+@internal.route("/dovecot/userdb/<user_email>")
+def dovecot_userdb_dict(user_email):
+    user = models.User.query.get(user_email) or flask.abort(404)
+    return flask.jsonify({
+        "quota_rule": "*:bytes={}".format(user.quota_bytes)
+    })
+
+
+@internal.route("/dovecot/quota/<ns>/<user_email>", methods=["POST"])
+def dovecot_quota(ns, user_email):
+    user = models.User.query.get(user_email) or flask.abort(404)
+    if ns == "storage":
+        user.quota_bytes_used = flask.request.get_json()
+        db.session.commit()
+    return flask.jsonify(None)
+
+
+@internal.route("/dovecot/sieve/name/<script>/<user_email>")
+def dovecot_sieve_name(script, user_email):
+    return flask.jsonify(script)
+
+
+@internal.route("/dovecot/sieve/data/default/<user_email>")
+def dovecot_sieve_data(user_email):
+    user = models.User.query.get(user_email) or flask.abort(404)
+    return flask.jsonify(flask.render_template("default.sieve", user=user))

core/admin/mailu/internal/views/fetch.py

@@ -0,0 +1,32 @@
+from mailu import db, models
+from mailu.internal import internal
+
+import flask
+import datetime
+
+
+@internal.route("/fetch")
+def fetch_list():
+    return flask.jsonify([
+        {
+            "id": fetch.id,
+            "tls": fetch.tls,
+            "keep": fetch.keep,
+            "user_email": fetch.user_email,
+            "protocol": fetch.protocol,
+            "host": fetch.host,
+            "port": fetch.port,
+            "username": fetch.username,
+            "password": fetch.password
+        } for fetch in models.Fetch.query.all()
+    ])
+
+
+@internal.route("/fetch/<fetch_id>", methods=["POST"])
+def fetch_done(fetch_id):
+    fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
+    fetch.last_check = datetime.datetime.now()
+    fetch.error_message = str(flask.request.get_json())
+    db.session.add(fetch)
+    db.session.commit()
+    return ""

core/admin/mailu/internal/views/postfix.py

@@ -0,0 +1,54 @@
+from mailu import db, models
+from mailu.internal import internal
+
+import flask
+
+
+@internal.route("/postfix/domain/<domain_name>")
+def postfix_mailbox_domain(domain_name):
+    domain = models.Domain.query.get(domain_name) or flask.abort(404)
+    return flask.jsonify(domain.name)
+
+
+@internal.route("/postfix/mailbox/<email>")
+def postfix_mailbox_map(email):
+    user = models.User.query.get(email) or flask.abort(404)
+    return flask.jsonify(user.email)
+
+
+@internal.route("/postfix/alias/<alias>")
+def postfix_alias_map(alias):
+    localpart, domain = alias.split('@', 1) if '@' in alias else (None, alias)
+    alternative = models.Alternative.query.get(domain)
+    if alternative:
+        domain = alternative.domain_name
+    email = '{}@{}'.format(localpart, domain)
+    if localpart is None:
+        return flask.jsonify(domain)
+    else:
+        alias_obj = models.Alias.resolve(localpart, domain)
+        if alias_obj:
+            return flask.jsonify(",".join(alias_obj.destination))
+        user_obj = models.User.query.get(email)
+        if user_obj:
+            return flask.jsonify(user_obj.destination)
+        return flask.abort(404)
+
+
+@internal.route("/postfix/transport/<email>")
+def postfix_transport(email):
+    localpart, domain = email.split('@', 1) if '@' in email else (None, email)
+    relay = models.Relay.query.get(domain) or flask.abort(404)
+    return flask.jsonify("smtp:[{}]".format(relay.smtp))
+
+
+@internal.route("/postfix/sender/<sender>")
+def postfix_sender(sender):
+    """ Simply reject any sender that pretends to be from a local domain
+    """
+    localpart, domain_name = sender.split('@', 1) if '@' in sender else (None, sender)
+    domain = models.Domain.query.get(domain_name)
+    alternative = models.Alternative.query.get(domain_name)
+    if domain or alternative:
+        return flask.jsonify("REJECT")
+    return flask.abort(404)

core/admin/mailu/models.py

@@ -1,11 +1,11 @@
-from mailu import app, db, dkim, login_manager, quota
+from mailu import app, db, dkim, login_manager
 
 from sqlalchemy.ext import declarative
 from passlib import context, hash
 from datetime import datetime, date
 from email.mime import text
 
-
+import sqlalchemy
 import re
 import time
 import os
@@ -235,6 +235,7 @@ class User(Base, Email):
         backref=db.backref('users', cascade='all, delete-orphan'))
     password = db.Column(db.String(255), nullable=False)
     quota_bytes = db.Column(db.Integer(), nullable=False, default=10**9)
+    quota_bytes_used = db.Column(db.Integer(), nullable=False, default=0)
     global_admin = db.Column(db.Boolean(), nullable=False, default=False)
     enabled = db.Column(db.Boolean(), nullable=False, default=True)
 
@@ -249,6 +250,8 @@ class User(Base, Email):
     reply_enabled = db.Column(db.Boolean(), nullable=False, default=False)
     reply_subject = db.Column(db.String(255), nullable=True, default=None)
     reply_body = db.Column(db.Text(), nullable=True, default=None)
+    reply_startdate = db.Column(db.Date, nullable=False,
+        default=date(1900, 1, 1))
     reply_enddate = db.Column(db.Date, nullable=False,
         default=date(2999, 12, 31))
 
@@ -266,10 +269,27 @@ class User(Base, Email):
         return self.email
 
     @property
-    def quota_bytes_used(self):
+    def destination(self):
-        return quota.get(self.email + "/quota/storage") or 0
+        if self.forward_enabled:
+            result = self.self.forward_destination
+            if self.forward_keep:
+                result += ',' + self.email
+            return result
+        else:
+            return self.email
 
-    scheme_dict = {'SHA512-CRYPT': "sha512_crypt",
+    @property
+    def reply_active(self):
+        now = date.today()
+        return (
+            self.reply_enabled and
+            self.reply_startdate < now and
+            self.reply_enddate > now
+        )
+
+    scheme_dict = {'PBKDF2': "pbkdf2_sha512",
+                   'BLF-CRYPT': "bcrypt",
+                   'SHA512-CRYPT': "sha512_crypt",
                    'SHA256-CRYPT': "sha256_crypt",
                    'MD5-CRYPT': "md5_crypt",
                    'CRYPT': "des_crypt"}
@@ -279,8 +299,14 @@ class User(Base, Email):
     )
 
     def check_password(self, password):
+        context = User.pw_context
         reference = re.match('({[^}]+})?(.*)', self.password).group(2)
-        return User.pw_context.verify(password, reference)
+        result = context.verify(password, reference)
+        if result and context.identify(reference) != context.default_scheme():
+            self.set_password(password)
+            db.session.add(self)
+            db.session.commit()
+        return result
 
     def set_password(self, password, hash_scheme=app.config['PASSWORD_SCHEME'], raw=False):
         """Set password for user with specified encryption scheme
@@ -329,6 +355,22 @@ class Alias(Base, Email):
     wildcard = db.Column(db.Boolean(), nullable=False, default=False)
     destination = db.Column(CommaSeparatedList, nullable=False, default=[])
 
+    @classmethod
+    def resolve(cls, localpart, domain_name):
+        return cls.query.filter(
+            sqlalchemy.and_(cls.domain_name == domain_name,
+                sqlalchemy.or_(
+                    sqlalchemy.and_(
+                        cls.wildcard == False,
+                        cls.localpart == localpart
+                    ), sqlalchemy.and_(
+                        cls.wildcard == True,
+                        sqlalchemy.bindparam("l", localpart).like(cls.localpart)
+                    )
+                )
+            )
+        ).first()
+
 
 class Token(Base):
     """ A token is an application password for a given user.

core/admin/mailu/ui/forms.py

@@ -117,6 +117,7 @@ class UserReplyForm(flask_wtf.FlaskForm):
     reply_subject = fields.StringField(_('Reply subject'))
     reply_body = fields.StringField(_('Reply body'),
         widget=widgets.TextArea())
+    reply_startdate = fields.html5.DateField(_('Start of vacation'))
     reply_enddate = fields.html5.DateField(_('End of vacation'))
     submit = fields.SubmitField(_('Update'))
 

core/admin/mailu/ui/templates/user/reply.html

@@ -13,7 +13,7 @@
 <form class="form" method="post" role="form">
   {{ form.hidden_tag() }}
   {{ macros.form_field(form.reply_enabled,
-      onchange="if(this.checked){$('#reply_subject,#reply_body,#reply_enddate').removeAttr('readonly')}
+      onchange="if(this.checked){$('#reply_subject,#reply_body,#reply_enddate,#reply_startdate').removeAttr('readonly')}
                 else{$('#reply_subject,#reply_body,#reply_enddate').attr('readonly', '')}") }}
   {{ macros.form_field(form.reply_subject,
       **{("rw" if user.reply_enabled else "readonly"): ""}) }}
@@ -21,6 +21,9 @@
         **{("rw" if user.reply_enabled else "readonly"): ""}) }}
   {{ macros.form_field(form.reply_enddate,
         **{("rw" if user.reply_enabled else "readonly"): ""}) }}
+  {{ macros.form_field(form.reply_startdate,
+        **{("rw" if user.reply_enabled else "readonly"): ""}) }}
+
   {{ macros.form_field(form.submit) }}
 </form>
 {% endcall %}

core/admin/migrations/versions/25fd6c7bcb4a_.py

@@ -0,0 +1,28 @@
+""" Add a column for used quota
+
+Revision ID: 25fd6c7bcb4a
+Revises: 049fed905da7
+Create Date: 2018-07-25 21:56:09.729153
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '25fd6c7bcb4a'
+down_revision = '049fed905da7'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    with op.batch_alter_table('user') as batch:
+        batch.add_column(sa.Column('quota_bytes_used', sa.Integer(), nullable=False, server_default='0'))
+
+
+def downgrade():
+    with op.batch_alter_table('user') as batch:
+        batch.drop_column('user', 'quota_bytes_used')

core/admin/migrations/versions/3b281286c7bd_.py

@@ -0,0 +1,24 @@
+""" Add a start day for vacations
+
+Revision ID: 3b281286c7bd
+Revises: 25fd6c7bcb4a
+Create Date: 2018-09-27 22:20:08.158553
+
+"""
+
+revision = '3b281286c7bd'
+down_revision = '25fd6c7bcb4a'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    with op.batch_alter_table('user') as batch:
+        batch.add_column(sa.Column('reply_startdate', sa.Date(), nullable=False,
+            server_default="1900-01-01"))
+
+
+def downgrade():
+    with op.batch_alter_table('user') as batch:
+        batch.drop_column('reply_startdate')

core/admin/requirements-prod.txt

@@ -1,6 +1,7 @@
 alembic==0.9.9
 asn1crypto==0.24.0
 Babel==2.5.3
+bcrypt==3.1.4
 blinker==1.4
 certifi==2018.4.16
 cffi==1.11.5

core/admin/requirements.txt

@@ -17,3 +17,4 @@ tabulate
 PyYAML
 PyOpenSSL
 dnspython
+bcrypt

core/dovecot/Dockerfile

@@ -1,17 +1,17 @@
-FROM alpine:3.7
+FROM alpine:3.8
 
-RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
+RUN apk add --no-cache \
- && apk add --no-cache \
+     dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \
-     dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \
+     bash python3 py3-pip \
-     dovecot-fts-lucene rspamd-client@testing python py-jinja2 py-pip \
+ && pip3 install --upgrade pip \
- && pip install --upgrade pip \
+ && pip3 install jinja2 podop tenacity
- && pip install tenacity
 
 COPY conf /conf
-COPY sieve /var/lib/dovecot
 COPY start.py /start.py
 
 EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp
 VOLUME ["/data", "/mail"]
 
 CMD /start.py
+
+HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready."

core/dovecot/conf/auth.conf

@@ -0,0 +1,5 @@
+uri = proxy:/tmp/podop.socket:auth
+iterate_disable = yes
+default_pass_scheme = plain
+password_key = passdb/%u
+user_key = userdb/%u

core/dovecot/conf/bin/ham

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+tee >(rspamc -h antispam:11334 -P mailu learn_ham /dev/stdin) \
+    | rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /dev/stdin

core/dovecot/conf/bin/spam

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+tee >(rspamc -h antispam:11334 -P mailu learn_spam /dev/stdin) \
+    >(rspamc -h antispam:11334 -P mailu -f 11 fuzzy_add /dev/stdin)

core/dovecot/conf/dovecot-sql.conf.ext

@@ -1,18 +0,0 @@
-driver = sqlite
-connect = /data/main.db
-
-# Return the user hashed password
-password_query = \
- SELECT NULL as password, 'Y' as nopassword, '{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% if WEBMAIL_ADDRESS %},{{ WEBMAIL_ADDRESS }}{% endif %}{% endif %}' as allow_nets \
-   FROM user \
-  WHERE user.email = '%u'
-
-# Mostly get the user quota
-user_query = \
- SELECT '*:bytes=' || user.quota_bytes AS quota_rule \
-   FROM user \
-  WHERE user.email = '%u'
-
-# For using doveadm -A:
-iterate_query = \
- SELECT user.email AS user FROM user

core/dovecot/conf/dovecot.conf

@@ -7,17 +7,6 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
 hostname = {{ HOSTNAMES.split(",")[0] }}
 submission_host = {{ FRONT_ADDRESS }}
 
-service dict {
-  unix_listener dict {
-    group = mail
-    mode = 0660
-  }
-}
-
-dict {
-  sieve = sqlite:/etc/dovecot/pigeonhole-sieve.dict
-}
-
 ###############
 # Full-text search
 ###############
@@ -50,28 +39,18 @@ mail_plugins = $mail_plugins quota quota_clone zlib
 
 namespace inbox {
   inbox = yes
-  mailbox Trash {
+  {% for mailbox in ("Trash", "Drafts", "Sent", "Junk") %}
+  mailbox {{ mailbox }} {
     auto = subscribe
-    special_use = \Trash
+    special_use = \{{ mailbox }}
-  }
-  mailbox Drafts {
-    auto = subscribe
-    special_use = \Drafts
-  }
-  mailbox Sent {
-    auto = subscribe
-    special_use = \Sent
-  }
-  mailbox Junk {
-    auto = subscribe
-    special_use = \Junk
   }
+  {% endfor %}
 }
 
 plugin {
   quota = count:User quota
   quota_vsizes = yes
-  quota_clone_dict = redis:host={{ REDIS_ADDRESS }}:port=6379:db=1
+  quota_clone_dict = proxy:/tmp/podop.socket:quota
 
   {% if COMPRESSION in [ 'gz', 'bz2' ] %}
   zlib_save = {{ COMPRESSION }}
@@ -87,16 +66,15 @@ plugin {
 ###############
 auth_mechanisms = plain login
 disable_plaintext_auth = no
-ssl_protocols = !SSLv3
 
 passdb {
-  driver = sql
+  driver = dict
-  args = /etc/dovecot/dovecot-sql.conf.ext
+  args = /etc/dovecot/auth.conf
 }
 
 userdb {
-  driver = sql
+  driver = dict
-  args = /etc/dovecot/dovecot-sql.conf.ext
+  args = /etc/dovecot/auth.conf
 }
 
 service auth {
@@ -117,7 +95,6 @@ service auth-worker {
 ###############
 # IMAP & POP
 ###############
-
 protocol imap {
   mail_plugins = $mail_plugins imap_quota imap_sieve
 }
@@ -135,7 +112,6 @@ service imap-login {
 ###############
 # Delivery
 ###############
-
 protocol lmtp {
   mail_plugins = $mail_plugins sieve
   recipient_delimiter = {{ RECIPIENT_DELIMITER }}
@@ -147,11 +123,9 @@ service lmtp {
   }
 }
 
-
 ###############
 # Filtering
 ###############
-
 service managesieve-login {
   inet_listener sieve {
     port = 4190
@@ -163,15 +137,13 @@ service managesieve {
 
 plugin {
   sieve = file:~/sieve;active=~/.dovecot.sieve
-  sieve_plugins = sieve_extdata sieve_imapsieve sieve_extprograms
+  sieve_before = dict:proxy:/tmp/podop.socket:sieve
-  sieve_global_extensions = +vnd.dovecot.extdata +spamtest +spamtestplus +vnd.dovecot.execute +editheader
+  sieve_plugins = sieve_imapsieve sieve_extprograms
-  sieve_before = /var/lib/dovecot/before.sieve
+  sieve_extensions = +spamtest +spamtestplus +editheader
-  sieve_default = /var/lib/dovecot/default.sieve
+  sieve_global_extensions = +vnd.dovecot.execute
-  sieve_after = /var/lib/dovecot/after.sieve
-  sieve_extdata_dict_uri = proxy::sieve
 
   # Sieve execute
-  sieve_execute_bin_dir = /var/lib/dovecot/bin
+  sieve_execute_bin_dir = /conf/bin
 
   # Send vacation replies even for aliases
   # See the Pigeonhole documentation about warnings: http://wiki2.dovecot.org/Pigeonhole/Sieve/Extensions/Vacation
@@ -190,11 +162,11 @@ plugin {
   # Learn from spam
   imapsieve_mailbox1_name = Junk
   imapsieve_mailbox1_causes = COPY
-  imapsieve_mailbox1_before = file:/var/lib/dovecot/report-spam.sieve
+  imapsieve_mailbox1_before = file:/conf/report-spam.sieve
   imapsieve_mailbox2_name = *
   imapsieve_mailbox2_from = Junk
   imapsieve_mailbox2_causes = COPY
-  imapsieve_mailbox2_before = file:/var/lib/dovecot/report-ham.sieve
+  imapsieve_mailbox2_before = file:/conf/report-ham.sieve
 }
 
 ###############

core/dovecot/conf/pigeonhole-sieve.dict

@@ -1,43 +0,0 @@
-connect = /data/main.db
-
-map {
-  pattern = priv/spam_enabled
-  table = user
-  username_field = email
-  value_field = spam_enabled
-}
-
-map {
-  pattern = priv/spam_threshold
-  table = user
-  username_field = email
-  value_field = spam_threshold
-}
-
-map {
-  pattern = priv/reply_enabled
-  table = user
-  username_field = email
-  value_field = reply_enabled
-}
-
-map {
-  pattern = priv/reply_subject
-  table = user
-  username_field = email
-  value_field = reply_subject
-}
-
-map {
-  pattern = priv/reply_body
-  table = user
-  username_field = email
-  value_field = reply_body
-}
-
-map {
-  pattern = priv/reply_enddate
-  table = user
-  username_field = email
-  value_field = reply_enddate
-}

core/dovecot/conf/report-ham.sieve

@@ -8,4 +8,4 @@ if string "${mailbox}" "Trash" {
   stop;
 }
 
-execute :pipe "mailtrain" "ham";
+execute :pipe "ham";

core/dovecot/conf/report-spam.sieve

@@ -0,0 +1,3 @@
+require "vnd.dovecot.execute";
+
+execute :pipe "spam";

core/dovecot/sieve/bin/mailtrain

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-rspamc -h antispam:11334 -P mailu "learn_$1" /dev/stdin <&0

core/dovecot/sieve/report-spam.sieve

@@ -1,3 +0,0 @@
-require "vnd.dovecot.execute";
-
-execute :pipe "mailtrain" "spam";

core/dovecot/start.py

@@ -1,11 +1,23 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import jinja2
 import os
 import socket
 import glob
+import multiprocessing
 import tenacity
+
 from tenacity import retry
+from podop import run_server
+
+
+def start_podop():
+    os.setuid(8)
+    run_server(3 if "DEBUG" in os.environ else 0, "dovecot", "/tmp/podop.socket", [
+		("quota", "url", "http://admin/internal/dovecot/§"),
+		("auth", "url", "http://admin/internal/dovecot/§"),
+		("sieve", "url", "http://admin/internal/dovecot/§"),
+    ])
 
 convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
@@ -18,9 +30,11 @@ def resolve():
 
 # Actual startup script
 resolve()
-for dovecot_file in glob.glob("/conf/*"):
+
+for dovecot_file in glob.glob("/conf/*.conf"):
     convert(dovecot_file, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
 
-# Run postfix
+# Run Podop, then postfix
-os.system("chown -R mail:mail /mail /var/lib/dovecot")
+multiprocessing.Process(target=start_podop).start()
+os.system("chown -R mail:mail /mail /var/lib/dovecot /conf")
 os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

core/nginx/Dockerfile

@@ -1,6 +1,9 @@
-FROM alpine:3.7
+FROM alpine:3.8
 
-RUN apk add --no-cache nginx nginx-mod-mail python py-jinja2 certbot openssl
+RUN apk add --no-cache certbot nginx nginx-mod-mail openssl curl \
+ python py-jinja2 py-requests-toolbelt py-pip \
+ && pip install --upgrade pip \
+ && pip install idna
 
 COPY conf /conf
 COPY *.py /
@@ -9,3 +12,5 @@ EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 100
 VOLUME ["/certs"]
 
 CMD /start.py
+
+HEALTHCHECK CMD curl -k -f -L http://localhost/health || exit 1

core/nginx/conf/nginx.conf

@@ -34,6 +34,8 @@ http {
       ''      $scheme;
     }
 
+    # Disable the main http server when on kubernetes (port 80 and 443)
+    {% if KUBERNETES_INGRESS != 'true' %}
     # Main HTTP server
     server {
       # Variables for proxifying
@@ -48,8 +50,8 @@ http {
 
       # Only enable HTTPS if TLS is enabled with no error
       {% if TLS and not TLS_ERROR %}
-      listen 443 ssl;
+      listen 443 ssl http2;
-      listen [::]:443 ssl;
+      listen [::]:443 ssl http2;
 
       include /etc/nginx/tls.conf;
       ssl_session_cache shared:SSLHTTP:50m;
@@ -91,8 +93,10 @@ http {
       {% endif %}
 
       location {{ WEB_WEBMAIL }} {
+        {% if WEB_WEBMAIL != '/' %}
         rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent;
         rewrite ^{{ WEB_WEBMAIL }}/(.*) /$1 break;
+        {% endif %}
         include /etc/nginx/proxy.conf;
         client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }};
         proxy_pass http://$webmail;
@@ -146,7 +150,12 @@ http {
         proxy_pass_request_body off;
         proxy_set_header Content-Length "";
       }
+
+      location /health {
+        return 204;
       }
+    }
+    {% endif %}
 
     # Forwarding authentication server
     server {

core/none/Dockerfile

@@ -1,5 +1,5 @@
 # This is an idle image to dynamically replace any component if disabled.
 
-FROM alpine
+FROM alpine:3.8
 
 CMD sleep 1000000d

core/postfix/Dockerfile

@@ -1,8 +1,9 @@
-FROM alpine:3.7
+FROM alpine:3.8
 
-RUN apk add --no-cache postfix postfix-sqlite postfix-pcre rsyslog python py-jinja2 py-pip \
+RUN apk add --no-cache postfix postfix-pcre rsyslog \
- && pip install --upgrade pip \
+    python3 py3-pip \
- && pip install tenacity
+ && pip3 install --upgrade pip \
+ && pip3 install jinja2 podop tenacity
 
 COPY conf /conf
 COPY start.py /start.py
@@ -11,3 +12,5 @@ EXPOSE 25/tcp 10025/tcp
 VOLUME ["/data"]
 
 CMD /start.py
+
+HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix"

core/postfix/conf/main.cf

@@ -19,8 +19,8 @@ mynetworks = 127.0.0.1/32 [::1]/128 {{ RELAYNETS }}
 # Empty alias list to override the configuration variable and disable NIS
 alias_maps =
 
-# SQLite configuration
+# Podop configuration
-sql = sqlite:${config_directory}/
+podop = socketmap:unix:/tmp/podop.socket:
 
 # Only accept virtual emails
 mydestination =
@@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }}
 recipient_delimiter = {{ RECIPIENT_DELIMITER }}
 
 # Only the front server is allowed to perform xclient
-smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }}
+smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
 
 ###############
 # TLS
@@ -56,13 +56,14 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
 
 # The alias map actually returns both aliases and local mailboxes, which is
 # required for reject_unlisted_sender to work properly
-virtual_alias_maps = ${sql}sqlite-virtual_alias_maps.cf
+virtual_alias_domains =
-virtual_mailbox_domains = ${sql}sqlite-virtual_mailbox_domains.cf
+virtual_alias_maps = ${podop}alias
-virtual_mailbox_maps = $virtual_alias_maps
+virtual_mailbox_domains = ${podop}domain
+virtual_mailbox_maps = ${podop}mailbox
 
 # Mails are transported if required, then forwarded to Dovecot for delivery
-relay_domains = ${sql}sqlite-transport.cf
+relay_domains = ${podop}transport
-transport_maps = ${sql}sqlite-transport.cf
+transport_maps = ${podop}transport
 virtual_transport = lmtp:inet:{{ HOST_LMTP }}
 
 # In order to prevent Postfix from running DNS query, enforce the use of the
@@ -82,15 +83,20 @@ smtpd_sender_login_maps = $virtual_alias_maps
 # Restrictions for incoming SMTP, other restrictions are applied in master.cf
 smtpd_helo_required = yes
 
-smtpd_recipient_restrictions =
+smtpd_client_restrictions =
   permit_mynetworks,
-  check_sender_access ${sql}sqlite-reject-spoofed.cf,
+  check_sender_access ${podop}sender,
   reject_non_fqdn_sender,
   reject_unknown_sender_domain,
   reject_unknown_recipient_domain,
   reject_unverified_recipient,
   permit
 
+smtpd_relay_restrictions =
+  permit_mynetworks,
+  permit_sasl_authenticated,
+  reject_unauth_destination
+
 unverified_recipient_reject_reason = Address lookup failure
 
 ###############

core/postfix/conf/master.cf

@@ -7,7 +7,7 @@ smtp      inet  n       -       n       -       -       smtpd
 # Internal SMTP service
 10025     inet  n       -       n       -       -       smtpd
   -o smtpd_sasl_auth_enable=yes
-  -o smtpd_recipient_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
+  -o smtpd_client_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
   -o smtpd_reject_unlisted_recipient={% if REJECT_UNLISTED_RECIPIENT %}{{ REJECT_UNLISTED_RECIPIENT }}{% else %}no{% endif %}
   -o cleanup_service_name=outclean
 outclean  unix n       -       n       -       0       cleanup

core/postfix/conf/sqlite-reject-spoofed.cf

@@ -1,5 +0,0 @@
-dbpath = /data/main.db
-query =
-  SELECT 'REJECT' FROM domain WHERE name='%s'
-  UNION
-  SELECT 'REJECT' FROM alternative WHERE name='%s'

core/postfix/conf/sqlite-transport.cf

@@ -1,3 +0,0 @@
-dbpath = /data/main.db
-query =
-  SELECT 'smtp:['||smtp||']' FROM relay WHERE name='%s'

core/postfix/conf/sqlite-virtual_alias_maps.cf

@@ -1,23 +0,0 @@
-dbpath = /data/main.db
-query =
-  SELECT destination
-   FROM
-     (SELECT destination, email, wildcard, localpart, localpart||'@'||alternative.name AS alt_email FROM alias LEFT JOIN alternative ON alias.domain_name = alternative.domain_name
-      UNION
-      SELECT (CASE WHEN forward_enabled=1 THEN (CASE WHEN forward_keep=1 THEN email||',' ELSE '' END)||forward_destination ELSE email END) AS destination, email, 0 as wildcard, localpart, localpart||'@'||alternative.name as alt_email FROM user LEFT JOIN alternative ON user.domain_name = alternative.domain_name
-      UNION
-      SELECT '@'||domain_name as destination, '@'||name as email, 0 as wildcard, '' as localpart, NULL AS alt_email FROM alternative)
-   WHERE
-     (
-      wildcard = 0
-      AND
-      (email = '%s' OR alt_email = '%s')
-     ) OR (
-      wildcard = 1
-      AND
-      '%s' LIKE email
-     )
-   ORDER BY
-     wildcard ASC,
-     length(localpart) DESC
-   LIMIT 1

core/postfix/conf/sqlite-virtual_mailbox_domains.cf

@@ -1,5 +0,0 @@
-dbpath = /data/main.db
-query =
-  SELECT name FROM domain WHERE name='%s'
-  UNION
-  SELECT name FROM alternative WHERE name='%s'

core/postfix/start.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import jinja2
 import os
@@ -6,7 +6,21 @@ import socket
 import glob
 import shutil
 import tenacity
+import multiprocessing
+
 from tenacity import retry
+from podop import run_server
+
+
+def start_podop():
+    os.setuid(100)
+    run_server(3 if "DEBUG" in os.environ else 0, "postfix", "/tmp/podop.socket", [
+		("transport", "url", "http://admin/internal/postfix/transport/§"),
+		("alias", "url", "http://admin/internal/postfix/alias/§"),
+		("domain", "url", "http://admin/internal/postfix/domain/§"),
+        ("mailbox", "url", "http://admin/internal/postfix/mailbox/§"),
+        ("sender", "url", "http://admin/internal/postfix/sender/§")
+    ])
 
 convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
@@ -38,7 +52,8 @@ for map_file in glob.glob("/overrides/*.map"):
 
 convert("/conf/rsyslog.conf", "/etc/rsyslog.conf")
 
-# Run postfix
+# Run Podop and Postfix
+multiprocessing.Process(target=start_podop).start()
 if os.path.exists("/var/run/rsyslogd.pid"):
     os.remove("/var/run/rsyslogd.pid")
 os.system("/usr/lib/postfix/post-install meta_directory=/etc/postfix create-missing")

docs/Dockerfile

@@ -2,13 +2,21 @@ FROM python:3-alpine
 
 COPY requirements.txt /requirements.txt
 
+ARG version=master
+ENV VERSION=$version
+
 RUN pip install -r /requirements.txt \
- && apk add --no-cache nginx \
+ && apk add --no-cache nginx curl \
  && mkdir /run/nginx
 
 COPY ./nginx.conf /etc/nginx/conf.d/default.conf
 COPY . /docs
 
-RUN sphinx-build /docs /build
+RUN mkdir -p /build/$VERSION \
+ && sphinx-build /docs /build/$VERSION
+
+EXPOSE 80/tcp
 
 CMD nginx -g "daemon off;"
+
+HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1

docs/_templates/layout.html

@@ -1,2 +1,9 @@
-{% set version=github_version %}
 {% extends "!layout.html" %}
+{% block document %}
+{% if version != stable_version %}
+<div class="wy-alert info">
+    <p>You are currently browsing documentation for the <b>{{ version }}</b> branch. Documentation for the stable <b>{{ stable_version }}</b> branch can be found <a href="/{{ stable_version }}/">here</a>.</p>
+</div>
+{% endif %}
+{{ super() }}
+{% endblock %}

docs/_templates/page.html

@@ -1,4 +0,0 @@
-{%- extends "layout.html" %}
-{% block body %}
-  {{ body|replace("VERSION_TAG", version) }}
-{% endblock %}

docs/_templates/versions.html

@@ -0,0 +1,16 @@
+  <div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions">
+    <span class="rst-current-version" data-toggle="rst-current-version">
+      <span class="fa fa-book"> Versions</span>
+      v: {{ version }}
+      <span class="fa fa-caret-down"></span>
+    </span>
+    <div class="rst-other-versions">
+      <dl>
+        <dt>{{ _('Versions') }}</dt>
+        {% for slug, url in versions %}
+          <dd><a href="{{ url }}">{{ slug }}</a></dd>
+        {% endfor %}
+      </dl>
+    </div>
+  </div>
+

docs/compose/.env

@@ -120,12 +120,18 @@ WEBSITE=https://mailu.io
 # Advanced settings
 ###################################
 
+# Log driver for front service. Possible values:
+# json-file (default)
+# journald (On systemd platforms, useful for Fail2Ban integration)
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+LOG_DRIVER=json-file
+
 # Docker-compose project name, this will prepended to containers names.
 COMPOSE_PROJECT_NAME=mailu
 
 # Default password scheme used for newly created accounts and changed passwords
-# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT)
-PASSWORD_SCHEME=SHA512-CRYPT
+PASSWORD_SCHEME=PBKDF2
 
 # Header to take the real ip from
 REAL_IP_HEADER=

docs/compose/docker-compose.yml

@@ -6,6 +6,8 @@ services:
     image: mailu/nginx:$VERSION
     restart: always
     env_file: .env
+    logging:
+      driver: $LOG_DRIVER
     ports:
     - "$BIND_ADDRESS4:80:80"
     - "$BIND_ADDRESS4:443:443"
@@ -54,7 +56,6 @@ services:
     restart: always
     env_file: .env
     volumes:
-      - "$ROOT/data:/data"
       - "$ROOT/mail:/mail"
       - "$ROOT/overrides:/overrides"
     depends_on:
@@ -68,7 +69,6 @@ services:
     restart: always
     env_file: .env
     volumes:
-      - "$ROOT/data:/data"
       - "$ROOT/overrides:/overrides"
     depends_on:
       - front
@@ -142,8 +142,6 @@ services:
     image: mailu/fetchmail:$VERSION
     restart: always
     env_file: .env
-    volumes:
-      - "$ROOT/data:/data"
     depends_on:
       - unbound
     dns:
@@ -156,3 +154,4 @@ networks:
       driver: default
       config:
         - subnet: 10.177.20.0/24
+

docs/compose/setup.rst

@@ -26,36 +26,61 @@ for the ``VERSION_TAG`` branch, use:
   wget https://mailu.io/VERSION_TAG/_downloads/docker-compose.yml
   wget https://mailu.io/VERSION_TAG/_downloads/.env
 
-Then open the ``.env`` file to setup the mail server. Modify the ``ROOT`` setting
+Important configuration variables
-to match your setup directory if different from ``/mailu``.
+---------------------------------
 
-Modify the ``VERSION`` configuration in the ``.env`` file to reflect the version you picked.
+Open the ``.env`` file and review the following variable settings:
 
-Set the common configuration values
+- Change ``ROOT`` if you have your setup directory in a different location then ``/mailu``.
------------------------------------
+- Check ``VERSION`` to reflect the version you picked. (``master`` or ``1.5``).
 
-Open the ``.env`` file and set configuration settings after reading the configuration
+Make sure to read the comments in the file and instructions from the :ref:`common_cfg` section.
-documentation. Some settings are specific to the Docker Compose setup.
 
-Modify ``BIND_ADDRESS4`` to match the public IP address assigned to your server.
+TLS certificates
-This address should be configured on one of the network interfaces of the server.
+````````````````
-If the address is not configured directly (NAT) on any of the network interfaces or if
-you would simply like the server to listen on all interfaces, use ``0.0.0.0``.
-
-Modify ``BIND_ADDRESS6`` to match the public IPv6 address assigned to your server.
-The behavior is identical to ``BIND_ADDRESS4``.
 
 Set the ``TLS_FLAVOR`` to one of the following
 values:
 
 - ``cert`` is the default and requires certificates to be setup manually;
-- ``letsencrypt`` will use the Letsencrypt! CA to generate automatic ceriticates;
+- ``letsencrypt`` will use the *Letsencrypt!* CA to generate automatic ceriticates;
 - ``mail`` is similar to ``cert`` except that TLS will only be served for
   emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
 - ``mail-letsencrypt`` is similar to ``letsencrypt`` except that TLS will only be served for
   emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
 - ``notls`` will disable TLS, this is not recommended except for testing.
 
+.. note::
+
+  When using *Letsencrypt!* you have to make sure that the DNS ``A`` and ``AAAA`` records for the
+  all hostnames mentioned in the ``HOSTNAMES`` variable match with the ip adresses of you server.
+  Or else certificate generation will fail! See also: :ref:`dns_setup`.
+
+Bind address
+````````````
+
+Modify ``BIND_ADDRESS4`` and ``BIND_ADDRESS6`` to match the public IP addresses assigned to your server. For IPv6 you will need the ``<global>`` scope address. 
+
+You can find those addresses by running the following:
+
+.. code-block:: bash
+
+  [root@mailu ~]$ ifconfig eth0
+  eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
+          inet 125.189.138.127  netmask 255.255.255.0  broadcast 5.189.138.255
+          inet6 fd21:aab2:717c:cc5a::1  prefixlen 64  scopeid 0x0<global>
+          inet6 fe2f:2a73:43a8:7a1b::1  prefixlen 64  scopeid 0x20<link>
+          ether 00:50:56:3c:b2:23  txqueuelen 1000  (Ethernet)
+          RX packets 174866612  bytes 127773819607 (118.9 GiB)
+          RX errors 0  dropped 0  overruns 0  frame 0
+          TX packets 19905110  bytes 2191519656 (2.0 GiB)
+          TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
+
+If the address is not configured directly (NAT) on any of the network interfaces or if
+you would simply like the server to listen on all interfaces, use ``0.0.0.0`` and ``::``. Note that running is this mode is not supported and can lead to `issues`_.
+
+.. _issues: https://github.com/Mailu/Mailu/issues/641
+
 Enable optional features
 ------------------------
 

docs/conf.py

@@ -2,6 +2,8 @@
 # -*- coding: utf-8 -*-
 #
 
+import os
+
 extensions = ['sphinx.ext.imgmath', 'sphinx.ext.viewcode']
 templates_path = ['_templates']
 source_suffix = '.rst'
@@ -9,9 +11,9 @@ master_doc = 'index'
 project = 'Mailu'
 copyright = '2018, Mailu authors'
 author = 'Mailu authors'
-version = release = 'latest'
+version = release = os.environ.get('VERSION', 'master')
 language = None
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'Dockerfile', 'docker-compose.yml']
 pygments_style = 'sphinx'
 todo_include_todos = False
 html_theme = 'sphinx_rtd_theme'
@@ -33,6 +35,11 @@ html_context = {
     'display_github': True,
     'github_user': 'mailu',
     'github_repo': 'mailu',
-    'github_version': 'master',
+    'github_version': version,
+    'stable_version': '1.5',
+    'versions': [
+        ('1.5', '/1.5/'),
+        ('master', '/master/')
+    ],
     'conf_py_path': '/docs/'
 }

docs/configuration.rst

@@ -1,12 +1,20 @@
 Mailu configuration settings
 ============================
 
+.. _common_cfg:
+
 Common configuration
 --------------------
 
 The ``SECRET_KEY`` **must** be changed for every setup and set to a 16 bytes
 randomly generated value. It is intended to secure authentication cookies
-among other critical uses.
+among other critical uses. This can be generated with a utility such as *pwgen*,
+which can be installed on most Linux systems:
+
+.. code-block:: bash
+
+  apt-get install pwgen
+  pwgen 16 1
 
 The ``DOMAIN`` holds the main e-mail domain for the server. This email domain
 is used for bounce emails, for generating the postmaster email and other

docs/contributors/environment.rst

@@ -5,39 +5,51 @@ Docker containers
 -----------------
 
 The development environment is quite similar to the production one. You should always use
-the ``master`` version when developing. Simply add a build directive to the images
+the ``master`` version when developing.
-you are working on in the ``docker-compose.yml``:
 
-.. code-block:: yaml
+Building images
+```````````````
 
-  webdav:
+We supply a separate ``test/build.yml`` file for
-    build: ./optional/radicale
+convenience. To build all Mailu containers:
-    image: mailu/$WEBDAV:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-      - "$ROOT/dav:/data"
-
-  admin:
-    build: ./core/admin
-    image: mailu/admin:$VERSION
-    restart: always
-    env_file: .env
-    volumes:
-      - "$ROOT/data:/data"
-      - "$ROOT/dkim:/dkim"
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    depends_on:
-      - redis
-
-
-The build these containers.
 
 .. code-block:: bash
 
-  docker-compose build admin webdav
+  docker-compose -f tests/build.yml build
 
-Then you can simply start the stack as normal, newly-built images will be used.
+The ``build.yml`` file has two variables:
+
+#. ``$DOCKER_ORG``: First part of the image tag. Defaults to *mailu* and needs to be changed
+   only  when pushing to your own Docker hub account.
+#. ``$VERSION``: Last part of the image tag. Defaults to *local* to differentiate from pulled
+   images.
+
+To re-build only specific containers at a later time.
+
+.. code-block:: bash
+
+  docker-compose -f tests/build.yml build admin webdav
+
+If you have to push the images to Docker Hub for testing in Docker Swarm or a remote
+host, you have to define ``DOCKER_ORG`` (usually your Docker user-name) and login to
+the hub.
+
+.. code-block:: bash
+
+  docker login
+  Username: Foo
+  Password: Bar
+  export DOCKER_ORG="Foo"
+  export VERSION="feat-extra-app"
+  docker-compose -f tests/build.yml build
+  docker-compose -f tests/build.yml push
+
+Running containers
+``````````````````
+
+To run the newly created images: ``cd`` to your project directory. Edit ``.env`` to set
+``VERSION`` to the same value as used during the build, which defaults to ``local``.
+After that you can run:
 
 .. code-block:: bash
 
@@ -101,7 +113,8 @@ Documentation is maintained in the ``docs`` directory and are maintained as `reS
   docker build -t docs docs
   docker run -p 127.0.0.1:8080:80 docs
 
-You can now read the local documentation by navigating to http://localhost:8080.
+In a local build Docker always assumes the version to be master.
+You can read the local documentation by navigating to http://localhost:8080/master.
 
 .. note:: After modifying the documentation, the image needs to be rebuild and the container restarted for the changes to become visible.
 

docs/dns.rst

@@ -1,3 +1,5 @@
+.. _dns_setup:
+
 Setting up your DNS
 ===================
 

docs/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+
+
+services:
+  docs_master:
+    image: mailu/docs:master
+    labels:
+      - traefik.enable=true
+      - traefik.port=80
+      - traefik.main.frontend.rule=Host:${hostname};PathPrefix:/master/
+
+  docs_15:
+    image: mailu/docs:1.5
+    labels:
+      - traefik.enable=true
+      - traefik.port=80
+      - traefik.root.frontend.redirect.regex=.*
+      - traefik.root.frontend.redirect.replacement=/1.5/
+      - traefik.root.frontend.rule=Host:${hostname};PathPrefix:/
+      - traefik.main.frontend.rule=Host:${hostname};PathPrefix:/1.5/
+

docs/index.rst

@@ -55,7 +55,7 @@ the version of Mailu that you are running.
     configuration
     compose/requirements
     compose/setup
-    kubernetes/stable/index
+    kubernetes/mailu/index
     dns
     reverse
 

docs/kubernetes/1.6/README.md

@@ -1,157 +0,0 @@
-# Install Mailu master on kubernetes
-
-## Prequisites
-
-### Structure
-
-There's chosen to have a double NGINX stack for Mailu, this way the main ingress can still be used to access other websites/domains on your cluster. This is the current structure:
-
-- `NGINX Ingress controller`: Listens to the nodes ports 80 & 443 and directly forwards all TCP traffic on the E-amail ports (993,143,25,587,...). This is because this `DaemonSet` already consumes ports 80 & 443 and uses `hostNetwork: true`
-- `Cert manager`: Creates automatic Lets Encrypt certificates based on an `Ingress`-objects domain name.
-- `Mailu NGINX Front container`: This container receives all the mail traffic forwarded from the ingress controller. The web traffic is also forwarded based on an ingress
-- `Mailu components`: All Mailu components are split into separate files to make them more 
-
-### What you need
-- A working Kubernetes cluster (tested with 1.10.5)
-- A working [cert-manager](https://github.com/jetstack/cert-manager) installation
-- A working nginx-ingress controller needed for the lets-encrypt certificates. You can find those files in the `nginx` subfolder
-
-#### Cert manager
-
-The `Cert-manager` is quite easy to deploy using Helm when reading the [docs](https://cert-manager.readthedocs.io/en/latest/getting-started/2-installing.html). 
-After booting the `Cert-manager` you'll need a `ClusterIssuer` which takes care of all required certificates through `Ingress` items. An example:
-
-```yaml
-apiVersion: certmanager.k8s.io/v1alpha1
-kind: ClusterIssuer
-metadata:
-  name: letsencrypt-prod
-spec:
-  acme:
-    email: something@example.com
-    http01: {}
-    privateKeySecretRef:
-      key: ""
-      name: letsencrypt-stage
-    server: https://acme-v02.api.letsencrypt.org/directory
-```
-
-## Deploying Mailu
-
-All manifests can be found in the `mailu` subdirectory. All commands below need to be run from this subdirectory
-
-### Personalization
-- All services run in the same namespace, currently `mailu-mailserver`. So if you want to use a different one, change the `namespace` value in **every** file
-- Check the `storage-class` field in the `pvc.yaml` file, you can also change the sizes to your liking. Note that you need `RWX` (read-write-many) and `RWO` (read-write-once) storageclasses.
-- Check the `configmap.yaml` and adapt it to your needs. Be sure to check the kubernetes DNS values at the end (if you use a different namespace)
-- Check the `ingress-ssl.yaml` and change it to the domain you want (this is for the kubernetes ingress controller, it will forward to `mailu/nginx` a.k.a. the `front` pod)
-
-## Installation
-First run the command to start Mailu:
-
-```bash
-kubectl create -f rbac.yaml
-kubectl create -f configmap.yaml
-kubectl create -f pvc.yaml
-kubectl create -f ingress-ssl.yaml
-kubectl create -f redis.yaml
-kubectl create -f front.yaml
-kubectl create -f webmail.yaml
-kubectl create -f imap.yaml
-kubectl create -f security.yaml
-kubectl create -f smtp.yaml
-kubectl create -f fetchmail.yaml
-kubectl create -f admin.yaml
-kubectl create -f webdav.yaml
-```
-
-## Create the first admin account
-
-When the cluster is online you need to create you master user to access `https://mail.example.com/admin`.
-Enter the main `admin` pod to create the root account:
-
-```bash
-kubectl -n mailu-mailserver get po
-kubectl -n mailu-mailserver exec -it mailu-admin-.... /bin/sh
-```
-
-And in the pod run the following command. The command uses following entries:
-- `admin` Make it an admin user
-- `root` The first part of the e-mail adres (ROOT@example.com)
-- `example.com` the domain appendix
-- `password` the chosen password for the user
-
-```bash
-python manage.py admin root example.com password
-```
-
-Now you should be able to login on the mail account: `https://mail.example.com/admin`
-
-## Adaptations
-
-### Postfix
-I noticed you need an override for the `postfix` server in order to be able to send mail. I noticed Google wasn't able to deliver mail to my account and it had to do with the `smtpd_authorized_xclient_hosts` value in the config file. The config can be read [here](https://github.com/hacor/Mailu/blob/master/core/postfix/conf/main.cf#L35) and is pointing to a single IP of the service. But the requests come from the host IPs (the NGINX Ingress proxy) and they don't use the service specific IP.
-
-Enter the `postfix` pod:
-
-```bash
-kubectl -n mailu-mailserver get po
-kubectl -n mailu-mailserver exec -it mailu-smtp-.... /bin/sh
-```
-
-Now you're in the pod, create an override file like so:
-
-```bash
-vi /overrides/postfix.cf
-```
-
-And give it the following contents, off course replacing `10.2.0.0/16` with the CIDR of your pod range. This way the NGINX pods can also restart and your mail server will still operate
-
-```bash
-not_needed = true
-smtpd_authorized_xclient_hosts = 10.2.0.0/16
-```
-
-The first line seems stupid, but is needed because its pasted after a #, so from the second line we're really in action.
-Save and close the file and exit. Now you need to delete the pod in order to recreate the config file.
-
-```bash
-kubectl -n mailu-mailserver delete po/mailu-smtp-....
-```
-
-### Dovecot
-- If you are using Dovecot on a shared file system (Glusterfs, NFS,...), you need to create a special override otherwise a lot of indexing errors will occur on your Dovecot pod.
-- I also higher the number of max connections per IP. Now it's limited to 10.
-Enter the dovecot pod:
-
-```bash
-kubectl -n mailu-mailserver get po
-kubectl -n mailu-mailserver exec -it mailu-imap-.... /bin/sh
-```
-
-Create the file `/overrides/dovecot.conf`
-
-```bash
-vi /overrides/dovecot.conf
-```
-
-And enter following contents:
-```bash
-mail_nfs_index = yes
-mail_nfs_storage = yes
-mail_fsync = always
-mmap_disable = yes
-mail_max_userip_connections=100
-```
-
-Save and close the file and delete the imap pod to get it recreated.
-
-```bash
-kubectl -n mailu-mailserver delete po/mailu-imap-....
-```
-
-Wait for the pod to recreate and you're online!
-Happy mailing!
-
-Wait for the pod to recreate and you're online!
-Happy mailing!

docs/kubernetes/1.6/mailu/ingress-ssl.yaml

@@ -1,32 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  name: mailu-ssl-ingress
-  namespace: mailu-mailserver
-  annotations:
-    kubernetes.io/ingress.class: tectonic
-    kubernetes.io/tls-acme: "true"
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"
-    ingress.kubernetes.io/ssl-redirect: "true"
-    # Replace letsencrypt-prod with the name of the certificate issuer
-    certmanager.k8s.io/cluster-issuer: letsencrypt-prod
-    #ingress.kubernetes.io/rewrite-target: "/"
-    #ingress.kubernetes.io/app-root: "/ui"
-    #ingress.kubernetes.io/follow-redirects: "true"
-  labels:
-    app: mailu
-    role: mail
-    tier: backend
-spec:
-  tls:
-  - hosts:
-    - "mail.example.com"
-    secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt
-  rules:
-  - host: "mail.example.com"
-    http:
-      paths:
-      - path: "/"
-        backend:
-          serviceName: front
-          servicePort: 80

docs/kubernetes/mailu/admin-ingress.yaml

@@ -0,0 +1,86 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: mailu-admin-ingress
+  namespace: mailu-mailserver
+  annotations:
+    kubernetes.io/tls-acme: "true"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    certmanager.k8s.io/cluster-issuer: letsencrypt-stage
+    ingress.kubernetes.io/permanent-redirect: "https://mail.example.com/admin/ui/"
+    ingress.kubernetes.io/follow-redirects: "true"
+  labels:
+    app: mailu
+    role: mail
+    tier: backend
+spec:
+  tls:
+  - hosts:
+    - "mail.example.com"
+    secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt
+  rules:
+  - host: "mail.example.com"
+    http:
+      paths:
+      - path: "/admin"
+        backend:
+          serviceName: admin
+          servicePort: 80
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: mailu-admin-ui-ingress
+  namespace: mailu-mailserver
+  annotations:
+    kubernetes.io/tls-acme: "true"
+    certmanager.k8s.io/cluster-issuer: letsencrypt-stage
+    ingress.kubernetes.io/rewrite-target: "/ui"
+    ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header X-Forwarded-Prefix /admin;
+  labels:
+    app: mailu
+    role: mail
+    tier: backend
+spec:
+  tls:
+  - hosts:
+    - "mail.example.com"
+    secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt
+  rules:
+  - host: "mail.example.com"
+    http:
+      paths:
+      - path: "/admin/ui"
+        backend:
+          serviceName: admin
+          servicePort: 80
+---
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: mailu-admin-static-ingress
+  namespace: mailu-mailserver
+  annotations:
+    kubernetes.io/tls-acme: "true"
+    certmanager.k8s.io/cluster-issuer: letsencrypt-stage
+    ingress.kubernetes.io/rewrite-target: "/static"
+    ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header X-Forwarded-Prefix /admin;
+  labels:
+    app: mailu
+    role: mail
+    tier: backend
+spec:
+  tls:
+  - hosts:
+    - "mail.example.com"
+    secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt
+  rules:
+  - host: "mail.example.com"
+    http:
+      paths:
+      - path: "/admin/static"
+        backend:
+          serviceName: admin
+          servicePort: 80

docs/kubernetes/mailu/admin.yaml

@@ -1,4 +1,3 @@
-
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:

docs/kubernetes/mailu/configmap.yaml

@@ -21,7 +21,7 @@
     VERSION: "master"
 
     # Set to a randomly generated 16 bytes string
-    SECRET_KEY: "YourKeyHere"
+    SECRET_KEY: "MySup3rS3cr3tPas"
 
     # Address where listening ports should bind
     BIND_ADDRESS4: "127.0.0.1"
@@ -45,6 +45,14 @@
     # Opt-out of statistics, replace with "True" to opt out
     DISABLE_STATISTICS: "False"
 
+    ###################################
+    # Kubernetes configuration
+    ###################################
+
+    # Use Kubernetes Ingress Controller to handle all actions on port 80 and 443
+    # This way we can make use of the advantages of the cert-manager deployment
+    KUBERNETES_INGRESS: "true"
+
     ###################################
     # Optional features
     ###################################
@@ -71,19 +79,18 @@
     # Default: accept messages up to 50MB
     MESSAGE_SIZE_LIMIT: "50000000"
 
-    # Networks granted relay permissions, make sure that you include your Docker
-    # internal network (default to 172.17.0.0/16)
-    # For kubernetes this is the CIDR of the pod network
-    RELAYNETS: "10.2.0.0/16"
-    POD_ADDRESS_RANGE: "10.2.0.0/16"
-
-
     # Will relay all outgoing mails if configured
     #RELAYHOST=
 
     # This part is needed for the XCLIENT login for postfix. This should be the POD ADDRESS range
     FRONT_ADDRESS: "front.mailu-mailserver.svc.cluster.local"
 
+    # This value  is needed by the webmail to find the correct imap backend
+    IMAP_ADDRESS: "imap.mailu-mailserver.svc.cluster.local"
+
+    # This value is used by Dovecot to find the Redis server in the cluster
+    REDIS_ADDRESS: "redis.mailu-mailserver.svc.cluster.local"
+
     # Fetchmail delay
     FETCHMAIL_DELAY: "600"
 
@@ -106,13 +113,16 @@
     ###################################
 
     # Path to the admin interface if enabled
+    # Kubernetes addition: You need to change ALL the ingresses, when you want this URL to be different!!!
     WEB_ADMIN: "/admin"
 
     # Path to the webmail if enabled
+    # Currently, this is not used, because we intended to use a different subdomain: webmail.example.com
+    # This option can be added in a feature release
     WEB_WEBMAIL: "/webmail"
 
     # Website name
-    SITENAME: "AppSynth"
+    SITENAME: "Mailu"
 
     # Linked Website URL
     WEBSITE: "https://example.com"

docs/kubernetes/mailu/front.yaml

@@ -1,23 +1,41 @@
-
+apiVersion: apps/v1beta2
-apiVersion: extensions/v1beta1
+kind: DaemonSet
-kind: Deployment
 metadata:
   name: mailu-front
   namespace: mailu-mailserver
+  labels:
+    k8s-app: mail-loadbalancer
+    component: ingress-controller
+    type: nginx
 spec:
-  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: mail-loadbalancer
+      component: ingress-controller
+      type: nginx
   template:
     metadata:
       labels:
-        app: mailu-front
+        k8s-app: mail-loadbalancer
-        role: mail
+        component: ingress-controller
-        tier: backend
+        type: nginx
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/node
+                operator: Exists
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/node: ""
+      dnsPolicy: ClusterFirstWithHostNet
       restartPolicy: Always
       terminationGracePeriodSeconds: 60
       containers:
       - name: front
-        image: mailu/nginx:latest
+        image: mailu/nginx:master
         imagePullPolicy: Always
         envFrom:
           - configMapRef:
@@ -26,12 +44,6 @@ spec:
           - name: certs
             mountPath: /certs
         ports:
-          - name: http
-            containerPort: 80
-            protocol: TCP
-          - name: https
-            containerPort: 443
-            protocol: TCP
           - name: pop3
             containerPort: 110
             protocol: TCP
@@ -85,21 +97,15 @@ metadata:
   name: front
   namespace: mailu-mailserver
   labels:
-    app: mailu-admin
+    k8s-app: mail-loadbalancer
-    role: mail
+    component: ingress-controller
-    tier: backend
+    type: nginx
 spec:
   selector:
-    app: mailu-front
+    k8s-app: mail-loadbalancer
-    role: mail
+    component: ingress-controller
-    tier: backend
+    type: nginx
   ports:
-  - name: http
-    port: 80
-    protocol: TCP
-  - name: https
-    port: 443
-    protocol: TCP
   - name: pop3
     port: 110
     protocol: TCP

docs/kubernetes/mailu/imap.yaml

@@ -37,8 +37,8 @@ spec:
           - containerPort: 4190
         resources:
           requests:
-            memory: 500Mi
+            memory: 1Gi
-            cpu: 500m
+            cpu: 1000m
           limits:
             memory: 1Gi
             cpu: 1000m

docs/kubernetes/mailu/index.rst

@@ -0,0 +1,193 @@
+Install Mailu master on kubernetes
+==================================
+
+Prequisites
+-----------
+
+Structure
+~~~~~~~~~
+
+There’s chosen to have a double NGINX stack for Mailu, this way the main
+ingress can still be used to access other websites/domains on your
+cluster. This is the current structure:
+
+-  ``NGINX Ingress controller``: Listens to the nodes ports 80 & 443. We have chosen to have a double NGINX stack for Mailu.
+-  ``Cert manager``: Creates automatic Lets Encrypt certificates based on an ``Ingress``-objects domain name.
+-  ``Mailu NGINX Front daemonset``: This daemonset runs in parallel with the Nginx Ingress Controller and only listens on all E-mail specific ports (25, 110, 143, 587,...)
+-  ``Mailu components``: All Mailu components (imap, smtp, security, webmail,...) are split into separate files to make them more handy to use, you can find the ``YAML`` files in this directory
+
+What you need
+~~~~~~~~~~~~~
+
+-  A working Kubernetes cluster (tested with 1.10.5)
+-  A working `cert-manager`_ installation
+-  A working nginx-ingress controller needed for the lets-encrypt
+   certificates. You can find those files in the ``nginx`` subfolder
+
+Cert manager
+^^^^^^^^^^^^
+
+The ``Cert-manager`` is quite easy to deploy using Helm when reading the
+`docs`_. After booting the ``Cert-manager`` you’ll need a
+``ClusterIssuer`` which takes care of all required certificates through
+``Ingress`` items. We chose to provide a ``clusterIssuer`` so you can provide SSL certificates
+for other namespaces (different websites/services), if you don't need this option, you can easily change this by
+changing ``clusterIssuer`` to ``Issuer`` and adding the ``namespace: mailu-mailserver`` to the metadata.
+An example of a production and a staging ``clusterIssuer``:
+
+.. code:: yaml
+
+   # This clusterIssuer example uses the staging environment for testing first
+   apiVersion: certmanager.k8s.io/v1alpha1
+   kind: ClusterIssuer
+   metadata:
+     name: letsencrypt-stage
+   spec:
+     acme:
+       email: something@example.com
+       http01: {}
+       privateKeySecretRef:
+         name: letsencrypt-stage
+       server: https://acme-staging-v02.api.letsencrypt.org/directory
+
+.. code:: yaml
+
+   # This clusterIssuer example uses the production environment
+   apiVersion: certmanager.k8s.io/v1alpha1
+   kind: ClusterIssuer
+   metadata:
+     name: letsencrypt-prod
+   spec:
+     acme:
+       email: something@example.com
+       http01: {}
+       privateKeySecretRef:
+         name: letsencrypt-prod
+       server: https://acme-v02.api.letsencrypt.org/directory
+
+**IMPORTANT**: All ``*-ingress.yaml`` files use the ``letsencrypt-stage`` ``clusterIssuer``. If you are ready for production,
+change this field in all ``*-ingress.yaml`` files to ``letsencrypt-prod`` or whatever name you chose for the production.
+If you choose for ``Issuer`` instead of ``clusterIssuer`` you also need to change the annotation to ``certmanager.k8s.io/issuer`` instead of ``certmanager.k8s.io/cluster-issuer``
+
+Deploying Mailu
+---------------
+
+All manifests can be found in the ``mailu`` subdirectory. All commands
+below need to be run from this subdirectory
+
+Personalization
+~~~~~~~~~~~~~~~
+
+-  All services run in the same namespace, currently ``mailu-mailserver``. So if you want to use a different one, change the ``namespace`` value in **every** file
+-  Check the ``storage-class`` field in the ``pvc.yaml`` file, you can also change the sizes to your liking. Note that you need ``RWX`` (read-write-many) and ``RWO`` (read-write-once) storageclasses.
+-  Check the ``configmap.yaml`` and adapt it to your needs. Be sure to check the kubernetes DNS values at the end (if you use a different namespace)
+-  Check the ``*-ingress.yaml`` files and change it to the domain you want (this is for the kubernetes ingress controller to handle the admin, webmail, webdav and auth connections)
+
+Installation
+------------
+
+Boot the Mailu components
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To start Mailu, run the following commands from the ``docs/kubernetes/mailu`` directory
+
+.. code-block:: bash
+
+    kubectl create -f rbac.yaml
+    kubectl create -f configmap.yaml
+    kubectl create -f pvc.yaml
+    kubectl create -f redis.yaml
+    kubectl create -f front.yaml
+    kubectl create -f webmail.yaml
+    kubectl create -f imap.yaml
+    kubectl create -f security.yaml
+    kubectl create -f smtp.yaml
+    kubectl create -f fetchmail.yaml
+    kubectl create -f admin.yaml
+    kubectl create -f webdav.yaml
+    kubectl create -f admin-ingress.yaml
+    kubectl create -f webdav-ingress.yaml
+    kubectl create -f security-ingress.yaml
+    kubectl create -f webmail-ingress.yaml
+
+
+Create the first admin account
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When the cluster is online you need to create you master user to access https://mail.example.com/admin
+Enter the main ``admin`` pod to create the root account:
+
+.. code-block:: bash
+
+    kubectl -n mailu-mailserver get po
+    kubectl -n mailu-mailserver exec -it mailu-admin-.... /bin/sh
+
+And in the pod run the following command. The command uses following entries:
+
+.. code-block:: bash
+
+    python manage.py admin root example.com password
+
+- ``admin`` Make it an admin user
+- ``root`` The first part of the e-mail adres (ROOT@example.com)
+- ``example.com`` the domain appendix
+- ``password`` the chosen password for the user
+
+
+Now you should be able to login on the mail account: https://mail.example.com/admin
+
+Adaptations
+-----------
+
+Dovecot
+~~~~~~~
+
+- If you are using Dovecot on a shared file system (Glusterfs, NFS,...), you need to create a special override otherwise a lot of indexing errors will occur on your Dovecot pod.
+- I also higher the number of max connections per IP. Now it's limited to 10.
+
+Enter the dovecot pod:
+
+.. code:: bash
+
+    kubectl -n mailu-mailserver get po
+    kubectl -n mailu-mailserver exec -it mailu-imap-.... /bin/sh
+
+Create the file ``overrides/dovecot.conf``
+
+.. code:: bash
+
+    vi /overrides/dovecot.conf
+
+And enter following contents:
+
+.. code:: bash
+
+    mail_nfs_index = yes
+    mail_nfs_storage = yes
+    mail_fsync = always
+    mmap_disable = yes
+    mail_max_userip_connections=100
+
+Save and close the file and delete the imap pod to get it recreated.
+
+.. code:: bash
+
+    kubectl -n mailu-mailserver delete po/mailu-imap-....
+
+Wait for the pod to recreate and you're online!
+Happy mailing!
+
+.. _here: https://github.com/hacor/Mailu/blob/master/core/postfix/conf/main.cf#L35
+.. _cert-manager: https://github.com/jetstack/cert-manager
+.. _docs: https://cert-manager.readthedocs.io/en/latest/getting-started/2-installing.html
+
+Imap login fix
+~~~~~~~~~~~~~~
+
+If it seems you're not able to login using IMAP on your Mailu accounts, check the logs of the imap container to see whether it's a permissions problem on the database.
+This problem can be easily fixed by running following commands:
+
+.. code:: bash
+
+    kubectl -n mailu-mailserver exec -it maolu-imap-... /bin/sh
+    chmod 777 /data/main.db

docs/kubernetes/mailu/security-ingress.yaml

@@ -0,0 +1,30 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: mailu-antispam-ingress
+  namespace: mailu-mailserver
+  annotations:
+    kubernetes.io/tls-acme: "true"
+    certmanager.k8s.io/cluster-issuer: letsencrypt-stage
+    ingress.kubernetes.io/configuration-snippet: |
+      rewrite ^/admin/antispam/(.*) /$1 break;
+      auth_request /internal/auth/admin;
+      proxy_set_header X-Real-IP "";
+      proxy_set_header X-Forwarded-For "";
+  labels:
+    app: mailu
+    role: mail
+    tier: frontend
+spec:
+  tls:
+  - hosts:
+    - "mail.example.com"
+    secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt
+  rules:
+  - host: "mail.example.com"
+    http:
+      paths:
+      - path: "/admin/antispam"
+        backend:
+          serviceName: antispam
+          servicePort: 11334

docs/kubernetes/mailu/security.yaml

@@ -31,6 +31,9 @@ spec:
           - name: antispam
             containerPort: 11332
             protocol: TCP
+          - name: antispam-http
+            containerPort: 11334
+            protocol: TCP
         volumeMounts:
           - name: filter
             subPath: filter
@@ -87,6 +90,9 @@ spec:
   - name: antispam
     port: 11332
     protocol: TCP
+  - name: antispam-http
+    protocol: TCP
+    port: 11334
 
 ---
 

docs/kubernetes/mailu/smtp.yaml

@@ -21,10 +21,10 @@ spec:
               name: mailu-config
         resources:
           requests:
-            memory: 500Mi
+            memory: 2Gi
-            cpu: 200m
+            cpu: 500m
           limits:
-            memory: 1Gi
+            memory: 2Gi
             cpu: 500m
         volumeMounts:
           - mountPath: /data

docs/kubernetes/mailu/webdav-ingress.yaml

@@ -0,0 +1,46 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: mailu-webdav-ingress
+  namespace: mailu-mailserver
+  annotations:
+    kubernetes.io/tls-acme: "true"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    certmanager.k8s.io/cluster-issuer: letsencrypt-stage
+    #ingress.kubernetes.io/auth-url: http://admin.mailu-mailserver.svc.cluster.local/internal/auth/basic
+    ingress.kubernetes.io/configuration-snippet: |
+      rewrite ^/webdav/(.*) /$1 break;
+      auth_request /internal/auth/basic;
+      proxy_set_header Host $http_host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      auth_request_set $user $upstream_http_x_user;
+      proxy_set_header X-Remote-User $user;
+      proxy_set_header X-Script-Name /webdav;
+    ingress.kubernetes.io/server-snippet: |
+      location /internal {
+        internal;
+
+        proxy_set_header Authorization $http_authorization;
+        proxy_pass_header Authorization;
+        proxy_pass http://admin.mailu-mailserver.svc.cluster.local;
+        proxy_pass_request_body off;
+        proxy_set_header Content-Length "";
+      }
+  labels:
+    app: mailu
+    role: mail
+    tier: frontend
+spec:
+  tls:
+  - hosts:
+    - "mail.example.com"
+    secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt
+  rules:
+  - host: "mail.example.com"
+    http:
+      paths:
+      - path: "/webdav"
+        backend:
+          serviceName: webdav
+          servicePort: 5232

docs/kubernetes/mailu/webmail-ingress.yaml

@@ -0,0 +1,31 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: mailu-webmail-ingress
+  namespace: mailu-mailserver
+  annotations:
+    kubernetes.io/tls-acme: "true"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+    certmanager.k8s.io/cluster-issuer: letsencrypt-stage
+    nginx.ingress.kubernetes.io/configuration-snippet: |
+      proxy_set_header Host $http_host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
+  labels:
+    app: mailu
+    role: mail
+    tier: backend
+spec:
+  tls:
+  - hosts:
+    - "webmail.example.com"
+    secretName: letsencrypt-webmail # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt
+  rules:
+    - host: "webmail.example.com"
+      http:
+        paths:
+        - path: "/"
+          backend:
+            serviceName: webmail
+            servicePort: 80

docs/kubernetes/mailu/webmail.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: roundcube
-        image: mailu/roundcube:1.5
+        image: mailu/roundcube:master
         imagePullPolicy: Always
         envFrom:
           - configMapRef:

docs/kubernetes/nginx/nginx-ingress.yaml

@@ -2,15 +2,15 @@ apiVersion: v1
 kind: Service
 metadata:
   # keep it under 24 chars
-  name: appsynth-lb
+  name: ingress-lb
   namespace: kube-ingress
   labels:
-    k8s-app: appsynth-lb
+    k8s-app: ingress-lb
     component: ingress-controller
 spec:
   type: ClusterIP
   selector:
-    k8s-app: appsynth-lb
+    k8s-app: ingress-lb
     component: ingress-controller
   ports:
     - name: http
@@ -35,13 +35,6 @@ metadata:
   name: tcp-services
   namespace: kube-ingress
 data:
-  25: "mailu-mailserver/front:25"
-  110: "mailu-mailserver/front:110"
-  465: "mailu-mailserver/front:465"
-  587: "mailu-mailserver/front:587"
-  143: "mailu-mailserver/front:143"
-  993: "mailu-mailserver/front:993"
-  995: "mailu-mailserver/front:995"
 
 ---
 apiVersion: v1
@@ -61,7 +54,7 @@ metadata:
     prometheus.io/port: "10254"
     prometheus.io/scrape: "true"
   labels:
-    k8s-app: appsynth-lb
+    k8s-app: ingress-lb
     component: ingress-controller
     type: nginx
 spec:
@@ -71,13 +64,13 @@ spec:
     type: RollingUpdate
   selector:
     matchLabels:
-      k8s-app: appsynth-lb
+      k8s-app: ingress-lb
       component: ingress-controller
       type: nginx
   template:
     metadata:
       labels:
-        k8s-app: appsynth-lb
+        k8s-app: ingress-lb
         component: ingress-controller
         type: nginx
     spec:
@@ -94,14 +87,11 @@ spec:
           image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.2
           args:
           - /nginx-ingress-controller
-          - --configmap=$(POD_NAMESPACE)/tectonic-custom-error
           - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
-          #- --default-ssl-certificate=tectonic-system/tectonic-ingress-tls-secret
           - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
           - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
           - --annotations-prefix=ingress.kubernetes.io
           - --enable-ssl-passthrough
-          - --ingress-class=tectonic
           # use downward API
           env:
             - name: POD_NAME
@@ -115,10 +105,8 @@ spec:
           ports:
             - name: http
               containerPort: 80
-              hostPort: 80
             - name: https
               containerPort: 443
-              hostPort: 443
           readinessProbe:
             httpGet:
               path: /healthz
@@ -134,6 +122,6 @@ spec:
       hostNetwork: true
       nodeSelector:
         node-role.kubernetes.io/node: ""
-      dnsPolicy: ClusterFirst
+      dnsPolicy: ClusterFirstWithHostNet
       restartPolicy: Always
       terminationGracePeriodSeconds: 60

docs/kubernetes/stable/index.rst

@@ -1,26 +0,0 @@
-Kubernetes setup
-================
-
-Please note that Kubernetes setup is not yet well supported or documented, all
-tests currently run on Docker Compose. The configuration has not yet been updated
-to work properly with ngin authentication proxy.
-
-Prepare the environment
------------------------
-
-The resource configurations in this folder assume that you have `Kubernetes Ingress`_
-set up for your cluster. If you are not using the `NGINX Ingress Controller for Kubernetes`_,
-please ensure that the configuration specified in the file matches your set up.
-
-.. _`Kubernetes Ingress`: https://kubernetes.io/docs/concepts/services-networking/ingress/
-.. _`NGINX Ingress Controller for Kubernetes`: https://github.com/kubernetes/ingress/tree/master/controllers/nginx
-
-Setup the Kubernetes service
-----------------------------
-
-Using the resource configurations is simple:
-
-1. ``kubectl apply -f kubernetes-nginx-ingress-controller.yaml`` to configure an ingress controller with the proper settings. (If you have one set up already you may need to port the configuration to your own ingress).
-2. ``kubectl apply -f kubernetes-mailu.yaml`` to create the resources required to run Mailu.
-
-Based on the configuration, your Mailu instance should be available at ``mail.<hostname>.tld/admin`` (note that visiting just ``mail.<hostname>.tld`` will likely result in a 404 error).

docs/kubernetes/stable/kubernetes-mailu.yaml

@@ -1,419 +0,0 @@
----
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  name: mailu-admin-ing
-  labels:
-    app: mailu
-    role: mail
-    tier: backend
-spec:
-  tls:
-  - hosts:
-    - "mail.example.com"
-    secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt
-  rules:
-  - host: "mail.example.com"
-    http:
-      paths:
-      - path: "/admin"
-        backend:
-          serviceName: mailu-admin
-          servicePort: 80
-
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: mailu-redis
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: mailu-redis
-        role: mail
-        tier: backend
-    spec:
-      containers:
-      - name: redis
-        image: redis:4.0-alpine
-        imagePullPolicy: Always
-        volumeMounts:
-          - mountPath: /data
-            name: redisdata
-        ports:
-          - containerPort: 6379
-            name: redis
-            protocol: TCP
-      volumes:
-        - name: redisdata
-          hostPath:
-            path: /var/data/mailu/redisdata
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis
-  labels:
-    app: mailu-redis
-    role: mail
-    tier: backend
-spec:
-  selector:
-    app: mailu
-    role: mail
-    tier: backend
-  ports:
-  - name: redis
-    port: 6379
-    protocol: TCP
-
----
-
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: mailu-imap
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: mailu-imap
-        role: mail
-        tier: backend
-    spec:
-      containers:
-      - name: imap
-        image: mailu/dovecot:stable
-        imagePullPolicy: Always
-        env:
-          - name  : DOMAIN
-            value : example.com
-          - name  : HOSTNAME
-            value : mail.example.com
-          - name  : POSTMASTER
-            value : admin
-        volumeMounts:
-          - mountPath: /data
-            name: maildata
-          - mountPath: /mail
-            name: mailstate
-          - mountPath: /overrides
-            name: overrides
-          - mountPath: /certs
-            name: certs
-            readOnly: true
-        ports:
-          - containerPort: 2102
-          - containerPort: 2525
-          - containerPort: 143
-          - containerPort: 993
-          - containerPort: 4190
-      volumes:
-        - name: maildata
-          hostPath:
-            path: /var/data/mailu/maildata
-        - name: mailstate
-          hostPath:
-            path: /var/data/mailu/mailstate
-        - name: overrides
-          hostPath:
-            path: /var/data/mailu/overrides
-        - name: certs
-          secret:
-            items:
-              - key: tls.crt
-                path: cert.pem
-              - key: tls.key
-                path: key.pem
-            secretName: letsencrypt-certs-all
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: imap
-  labels:
-    app: mailu
-    role: mail
-    tier: backend
-spec:
-  selector:
-    app: mailu-imap
-    role: mail
-    tier: backend
-  ports:
-  ports:
-  - name: imap-auth
-    port: 2102
-    protocol: TCP
-  - name: imap-transport
-    port: 2525
-    protocol: TCP
-  - name: imap-default
-    port: 143
-    protocol: TCP
-  - name: imap-ssl
-    port: 993
-    protocol: TCP
-  - name: sieve
-    port: 4190
-    protocol: TCP
-
----
-
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: mailu-smtp
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: mailu-smtp
-        role: mail
-        tier: backend
-    spec:
-      containers:
-      - name: smtp
-        image: mailu/postfix:stable
-        imagePullPolicy: Always
-        env:
-          - name  : DOMAIN
-            value : example.com
-          - name  : HOSTNAME
-            value : mail.example.com
-          - name  : MESSAGE_SIZE_LIMIT
-            value : "50000000"
-          - name  : RELAYHOST
-            value : ""
-        volumeMounts:
-          - mountPath: /data
-            name: maildata
-          - mountPath: /overrides
-            name: overrides
-          - mountPath: /certs
-            name: certs
-            readOnly: true
-        ports:
-          - name: smtp
-            containerPort: 25
-            protocol: TCP
-          - name: smtp-ssl
-            containerPort: 465
-            protocol: TCP
-          - name: smtp-starttls
-            containerPort: 587
-            protocol: TCP
-      volumes:
-        - name: maildata
-          hostPath:
-            path: /var/data/mailu/maildata
-        - name: overrides
-          hostPath:
-            path: /var/data/mailu/overrides
-        - name: certs
-          secret:
-            items:
-              - key: tls.crt
-                path: cert.pem
-              - key: tls.key
-                path: key.pem
-            secretName: letsencrypt-certs-all
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: smtp
-  labels:
-    app: mailu
-    role: mail
-    tier: backend
-spec:
-  selector:
-    app: mailu-smtp
-    role: mail
-    tier: backend
-  ports:
-  - name: smtp
-    port: 25
-    protocol: TCP
-  - name: smtp-ssl
-    port: 465
-    protocol: TCP
-  - name: smtp-starttls
-    port: 587
-    protocol: TCP
-
----
-
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: mailu-security
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: mailu-security
-        role: mail
-        tier: backend
-    spec:
-      containers:
-      - name: antispam
-        image: mailu/rspamd:stable
-        imagePullPolicy: Always
-        ports:
-          - name: antispam
-            containerPort: 11333
-            protocol: TCP
-        volumeMounts:
-          - name: filter
-            mountPath: /var/lib/rspamd
-      - name: antivirus
-        image: mailu/clamav:stable
-        imagePullPolicy: Always
-        ports:
-          - name: antivirus
-            containerPort: 3310
-            protocol: TCP
-        volumeMounts:
-          - name: filter
-            mountPath: /data
-      volumes:
-        - name: filter
-          hostPath:
-            path: /var/data/mailu/filter
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: antispam
-  labels:
-    app: mailu-antispam
-    role: mail
-    tier: backend
-spec:
-  selector:
-    app: mailu-security
-    role: mail
-    tier: backend
-  ports:
-  - name: antispam
-    port: 11333
-    protocol: TCP
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: antivirus
-  labels:
-    app: mailu-antivirus
-    role: mail
-    tier: backend
-spec:
-  selector:
-    app: mailu-security
-    role: mail
-    tier: backend
-  ports:
-  - name: antivirus
-    port: 3310
-    protocol: TCP
-
----
-
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: mailu-admin
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: mailu-admin
-        role: mail
-        tier: backend
-    spec:
-      containers:
-      - name: admin
-        image: mailu/admin:stable
-        imagePullPolicy: Always
-        env:
-          - name  : DOMAIN
-            value : example.com
-          - name  : HOSTNAME
-            value : mail.example.com
-          - name  : POSTMASTER
-            value : core
-          - name  : SECRET_KEY
-            value : pleasereplacethiswithabetterkey
-          - name  : DEBUG
-            value : "True"
-        volumeMounts:
-          - name: maildata
-            mountPath: /data
-          - name: dkim
-            mountPath: /dkim
-          - name: certs
-            mountPath: /certs
-            readOnly: true
-          # - name: docker
-          #   mountPath: /var/run/docker.sock
-          #   readOnly: true
-        ports:
-          - name: http
-            containerPort: 80
-            protocol: TCP
-      volumes:
-        - name: maildata
-          hostPath:
-            path: /var/data/mailu/maildata
-        - name: dkim
-          hostPath:
-            path: /var/data/mailu/dkim
-        - name: certs
-          secret:
-            items:
-              - key: tls.crt
-                path: cert.pem
-              - key: tls.key
-                path: key.pem
-            secretName: letsencrypt-certs-all
-        # - name: docker
-        #   hostPath:
-        #     path: /var/run/docker.sock
-
----
-
-apiVersion: v1
-kind: Service
-metadata:
-  name: mailu-admin
-  labels:
-    app: mailu-admin
-    role: mail
-    tier: backend
-spec:
-  selector:
-    app: mailu-admin
-    role: mail
-    tier: backend
-  ports:
-  - name: http
-    port: 80
-    protocol: TCP

docs/kubernetes/stable/kubernetes-nginx-ingress-controller.yaml

@@ -1,84 +0,0 @@
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: nginx-configuration
-  namespace: ingress-nginx
-  labels:
-    app: ingress-nginx
-
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: udp-services
-  namespace: ingress-nginx
-
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: tcp-services
-  namespace: ingress-nginx
-data:
-  25: "mailu/smtp:25"
-  465: "mailu/smtp:465"
-  587: "mailu/smtp:587"
-  143: "mailu/imap:143"
-  993: "mailu/imap:993"
-
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: nginx-ingress-controller
-  namespace: kube-system
-  labels:
-    k8s-app: nginx-ingress-controller
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        k8s-app: nginx-ingress-controller
-      annotations:
-        prometheus.io/port: '10254'
-        prometheus.io/scrape: 'true'
-    spec:
-      # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration
-      # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host
-      # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used
-      # like with kubeadm
-      # hostNetwork: true
-      terminationGracePeriodSeconds: 60
-      containers:
-      - image: gcr.io/google_containers/nginx-ingress-controller:0.11.0
-        name: nginx-ingress-controller
-        args:
-          - /nginx-ingress-controller
-          - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
-          - --configmap=$(POD_NAMESPACE)/nginx-configuration
-          - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
-          - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
-          - --annotations-prefix=nginx.ingress.kubernetes.io
-        readinessProbe:
-          httpGet:
-            path: /healthz
-            port: 10254
-            scheme: HTTP
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 10254
-            scheme: HTTP
-          initialDelaySeconds: 10
-          timeoutSeconds: 1
-        env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace

docs/reverse.rst

@@ -6,6 +6,7 @@ One of Mailu use cases is as part of a larger services platform, where maybe oth
 In such a configuration, one would usually run a frontend reverse proxy to serve all Web contents based on criteria like the requested hostname (virtual hosts) and/or the requested path. Mailu Web frontend is disabled in the default setup for security reasons, it is however expected that most users will enable it at some point. Also, due to Docker Compose configuration structure, it is impossible for us to make disabling the Web frontend completely available through a configuration variable. This guide was written to help users setup such an architecture.
 
 There are basically three options, from the most to the least recommended one:
+
 - have Mailu Web frontend listen locally and use your own Web frontend on top of it
 - override Mailu Web frontend configuration
 - disable Mailu Web frontend completely and use your own

docs/setup.rst

@@ -32,7 +32,7 @@ user. Make sure you complete the requirements for the flavor you chose.
 
 You should also have at least a DNS hostname and a DNS name for receiving
 emails. Some instructions are provided on the matter in the article
-[Setup your DNS](dns).
+:ref:`dns_setup`.
 
 .. _`MFAshby's fork`: https://github.com/MFAshby/Mailu
 
@@ -68,10 +68,9 @@ Make sure that you test properly before going live!
 - Try to receive an email from an external service
 - Check the logs (``docker-compose logs -f servicenamehere``) to look for
   warnings or errors
-- Use an open relay checker like `mailradar`_
+- Use an open relay checker like `mxtoolbox`_
   to ensure you're not contributing to the spam problem on the internet.
-  All tests there should result in "Relay denied".
 - If using DMARC, be sure to check the reports you get to verify that legitimate
   email is getting through and forgeries are being properly blocked.
 
-  .. _mailradar: http://www.mailradar.com/openrelay/
+  .. _mxtoolbox: https://mxtoolbox.com/diagnostic.aspx

docs/swarm/master/README.md

@@ -0,0 +1,252 @@
+# Install Mailu on a docker swarm
+
+## Prequisites
+
+### Swarm
+
+In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
+
+The main command will be:
+```bash
+docker swarm init --advertise-addr <IP_ADDR>
+```
+See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
+
+If you want to add other managers or workers, please use:
+```bash
+docker swarm join --token xxxxx 
+```
+See https://docs.docker.com/engine/swarm/join-nodes/
+
+You have now a working swarm, and you can check its status with:
+```bash
+core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
+ID                            HOSTNAME            STATUS              AVAILABILITY        MANAGER STATUS      ENGINE VERSION
+xhgeekkrlttpmtgmapt5hyxrb     black-pearl         Ready               Active                                  18.06.0-ce
+sczlqjgfhehsfdjhfhhph1nvb *   coreos-01           Ready               Active              Leader              18.03.1-ce
+mzrm9nbdggsfz4sgq6dhs5i6n     flying-dutchman     Ready               Active                                  18.06.0-ce
+```
+
+### Volume definition
+For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
+
+Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs" (GlusterFS and nfs shares have been successfully used).
+
+On this example, we are using:
+- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. 
+- the default ingress mode.
+
+### Allow authentification with the mesh routing
+In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
+
+Let's create the mailu_default network:
+```bash
+core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
+core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
+                    "Subnet": "10.0.1.0/24",
+```
+In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 
+In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of  "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
+
+### Limitation with the ingress mode
+With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
+
+This issue is known and discussed here:
+
+https://github.com/moby/moby/issues/25526
+
+A workaround (using network host mode and global deployment) is discussed here:
+
+https://github.com/moby/moby/issues/25526#issuecomment-336363408 
+
+### Don't create an open relay !
+As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
+
+
+## Scalability
+- smtp and imap are scalable
+- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) 
+- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
+
+## Variable substitution and docker-compose.yml
+The docker stack deploy command doesn't support variable substitution in the .yml file itself. 
+As a consequence, we cannot simply use ``` docker stack deploy -c docker.compose.yml mailu ```
+Instead, we will use the following work-around:
+``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ```
+
+We need also to:
+- add a deploy section for every service
+- modify the way the ports are defined for the front service
+- add the POD_ADDRESS_RANGE definition for imap, smtp and antispam services
+
+## Docker compose 
+An example of docker-compose-stack.yml file is available here:
+
+```yaml
+
+version: '3.2'
+
+services:
+
+  front:
+    image: mailu/nginx:$VERSION
+    restart: always
+    env_file: .env
+    ports:
+      - target: 80
+        published: 80
+      - target: 443
+        published: 443
+      - target: 110
+        published: 110
+      - target: 143
+        published: 143
+      - target: 993
+        published: 993
+      - target: 995
+        published: 995
+      - target: 25
+        published: 25
+      - target: 465
+        published: 465
+      - target: 587
+        published: 587
+    volumes:
+      - "$ROOT/certs:/certs"
+    deploy:
+      replicas: 2
+
+  redis:
+    image: redis:alpine
+    restart: always
+    volumes:
+      - "$ROOT/redis:/data"
+    deploy:
+      replicas: 1
+
+  imap:
+    image: mailu/dovecot:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+      - "$ROOT/mail:/mail"
+      - "$ROOT/overrides:/overrides"
+    depends_on:
+      - front
+    deploy:
+      replicas: 2
+
+  smtp:
+    image: mailu/postfix:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+      - "$ROOT/overrides:/overrides"
+    depends_on:
+      - front
+    deploy:
+      replicas: 2
+
+  antispam:
+    image: mailu/rspamd:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+      - "$ROOT/filter:/var/lib/rspamd"
+      - "$ROOT/dkim:/dkim"
+      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
+    deploy:
+      replicas: 1
+
+  antivirus:
+    image: mailu/none:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+      - "$ROOT/filter:/data"
+    deploy:
+      replicas: 1
+
+  webdav:
+    image: mailu/none:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+      - "$ROOT/dav:/data"
+    deploy:
+      replicas: 1
+
+  admin:
+    image: mailu/admin:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+      - "$ROOT/data:/data"
+      - "$ROOT/dkim:/dkim"
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      - redis
+    deploy:
+      replicas: 1
+
+  webmail:
+    image: mailu/roundcube:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+      - "$ROOT/webmail:/data"
+    depends_on:
+      - imap
+    deploy:
+      replicas: 2
+
+  fetchmail:
+    image: mailu/fetchmail:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+    deploy:
+      replicas: 1
+
+networks:
+  default:
+    external:
+      name: mailu_default
+```
+
+## Deploy Mailu on the docker swarm
+Run the following command:
+```bash
+echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu
+```
+See how the services are being deployed:
+```bash
+core@coreos-01 ~ $ docker service ls
+ID                  NAME                                 MODE                REPLICAS            IMAGE                                     PORTS
+ywnsetmtkb1l        mailu_antivirus                      replicated          1/1                 mailu/none:master
+pqokiaz0q128        mailu_fetchmail                      replicated          1/1                 mailu/fetchmail:master
+```
+check a specific service:
+```bash
+core@coreos-01 ~ $ docker service ps mailu_fetchmail
+ID                  NAME                IMAGE                 NODE                DESIRED STATE       CURRENT STATE         ERROR               PORTS
+tbu8ppgsdffj        mailu_fetchmail.1   mailu/fetchmail:master   coreos-01           Running             Running 11 days ago
+```
+You might also have a look on the logs:
+```bash
+core@coreos-01 ~ $ docker service logs -f mailu_fetchmail
+```
+
+## Remove the stack
+Run the follwoing command:
+```bash
+core@coreos-01 ~ $ docker stack rm mailu
+```

docs/swarm/master/README_nfs_example.md

@@ -0,0 +1,357 @@
+# Install Mailu on a docker swarm
+
+## Prequisites
+
+### Swarm
+
+In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
+
+The main command will be:
+```bash
+docker swarm init --advertise-addr <IP_ADDR>
+```
+See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
+
+If you want to add other managers or workers, please use:
+```bash
+docker swarm join --token xxxxx 
+```
+See https://docs.docker.com/engine/swarm/join-nodes/
+
+You have now a working swarm, and you can check its status with:
+```bash
+core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
+ID                            HOSTNAME            STATUS              AVAILABILITY        MANAGER STATUS      ENGINE VERSION
+xhgeekkrlttpmtgmapt5hyxrb     black-pearl         Ready               Active                                  18.06.0-ce
+sczlqjgfhehsfdjhfhhph1nvb *   coreos-01           Ready               Active              Leader              18.03.1-ce
+mzrm9nbdggsfz4sgq6dhs5i6n     flying-dutchman     Ready               Active                                  18.06.0-ce
+```
+
+### Volume definition
+For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
+Hereafter we will use a NFS share:
+```bash
+core@coreos-01 ~ $ showmount -e 192.168.0.30
+Export list for 192.168.0.30:
+/mnt/Pool1/pv            192.168.0.0
+```
+
+on the nfs server, I am using the following /etc/exports
+```bash
+$more /etc/exports
+/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 
+```
+on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up)
+```bash
+$mkdir /mnt/Pool1/pv/mailu
+```
+
+On your manager node, mount the nfs share to check that the share is available:
+```bash
+core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/
+```
+If this is ok, you can umount it:
+```bash
+core@coreos-01 ~ $ sudo umount /mnt/local/
+```
+
+
+## Networking mode
+On this example, we are using:
+- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. 
+- the default ingress mode.
+
+### Allow authentification with the mesh routing
+In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
+
+Let's create the mailu_default network:
+```bash
+core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
+core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
+                    "Subnet": "10.0.1.0/24",
+```
+In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 
+In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of  "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
+
+### Limitation with the ingress mode
+With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
+
+This issue is known and discussed here:
+
+https://github.com/moby/moby/issues/25526
+
+A workaround (using network host mode and global deployment) is discussed here:
+
+https://github.com/moby/moby/issues/25526#issuecomment-336363408 
+
+### Don't create an open relay !
+As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
+
+
+## Scalability
+- smtp and imap are scalable
+- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) 
+- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
+
+## Variable substitution and docker-compose.yml
+The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around:
+``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ```
+
+We need also to:
+- change the way we define the volumes (nfs share in our case)
+- add a deploy section for every service
+- the way the ports are defined for the front service
+
+## Docker compose 
+An example of docker-compose-stack.yml file is available here:
+
+```yaml
+
+version: '3.2'
+
+services:
+
+  front:
+    image: mailu/nginx:$VERSION
+    restart: always
+    env_file: .env
+    ports:
+      - target: 80
+        published: 80
+      - target: 443
+        published: 443
+      - target: 110
+        published: 110
+      - target: 143
+        published: 143
+      - target: 993
+        published: 993
+      - target: 995
+        published: 995
+      - target: 25
+        published: 25
+      - target: 465
+        published: 465
+      - target: 587
+        published: 587
+    volumes:
+#      - "$ROOT/certs:/certs"
+      - type: volume
+        source: mailu_certs
+        target: /certs
+    deploy:
+      replicas: 2
+
+  redis:
+    image: redis:alpine
+    restart: always
+    volumes:
+#      - "$ROOT/redis:/data"
+      - type: volume
+        source: mailu_redis
+        target: /data
+    deploy:
+      replicas: 1
+
+  imap:
+    image: mailu/dovecot:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+#      - "$ROOT/mail:/mail"
+      - type: volume
+        source: mailu_mail
+        target: /mail
+#      - "$ROOT/overrides:/overrides"
+      - type: volume
+        source: mailu_overrides
+        target: /overrides
+    depends_on:
+      - front
+    deploy:
+      replicas: 2
+
+  smtp:
+    image: mailu/postfix:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    volumes:
+#      - "$ROOT/overrides:/overrides"
+      - type: volume
+        source: mailu_overrides
+        target: /overrides
+    depends_on:
+      - front
+    deploy:
+      replicas: 2
+
+  antispam:
+    image: mailu/rspamd:$VERSION
+    restart: always
+    env_file: .env
+    environment:
+      - POD_ADDRESS_RANGE=10.0.1.0/24
+    depends_on:
+      - front
+    volumes:
+#      - "$ROOT/filter:/var/lib/rspamd"
+      - type: volume
+        source: mailu_filter
+        target: /var/lib/rspamd
+#      - "$ROOT/dkim:/dkim"
+      - type: volume
+        source: mailu_dkim
+        target: /dkim
+#      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+      - type: volume
+        source: mailu_overrides_rspamd
+        target: /etc/rspamd/override.d
+    deploy:
+      replicas: 1
+
+  antivirus:
+    image: mailu/none:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+#      - "$ROOT/filter:/data"
+      - type: volume
+        source: mailu_filter
+        target: /data
+    deploy:
+      replicas: 1
+
+  webdav:
+    image: mailu/none:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+#      - "$ROOT/dav:/data"
+      - type: volume
+        source: mailu_dav
+        target: /data
+    deploy:
+      replicas: 1
+
+  admin:
+    image: mailu/admin:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+#      - "$ROOT/data:/data"
+      - type: volume
+        source: mailu_data
+        target: /data
+#      - "$ROOT/dkim:/dkim"
+      - type: volume
+        source: mailu_dkim
+        target: /dkim
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      - redis
+    deploy:
+      replicas: 1
+
+  webmail:
+    image: mailu/roundcube:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+#      - "$ROOT/webmail:/data"
+      - type: volume
+        source: mailu_data
+        target: /data
+    depends_on:
+      - imap
+    deploy:
+      replicas: 2
+
+  fetchmail:
+    image: mailu/fetchmail:$VERSION
+    restart: always
+    env_file: .env
+    volumes:
+    deploy:
+      replicas: 1
+
+networks:
+  default:
+    external:
+      name: mailu_default
+
+volumes:
+  mailu_filter:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/filter"
+  mailu_dkim:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/dkim"
+  mailu_overrides_rspamd:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/overrides/rspamd"
+  mailu_data:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/data"
+  mailu_mail:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/mail"
+  mailu_overrides:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/overrides"
+  mailu_dav:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/dav"
+  mailu_certs:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/certs"
+  mailu_redis:
+    driver_opts:
+      type: "nfs"
+      o: "addr=192.168.0.30,soft,rw"
+      device: ":/mnt/Pool1/pv/mailu/redis"
+```
+
+## Deploy Mailu on the docker swarm
+Run the following command:
+```bash
+echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu
+```
+See how the services are being deployed:
+```bash
+core@coreos-01 ~ $ docker service ls
+ID                  NAME                                 MODE                REPLICAS            IMAGE                                     PORTS
+ywnsetmtkb1l        mailu_antivirus                      replicated          1/1                 mailu/none:master
+pqokiaz0q128        mailu_fetchmail                      replicated          1/1                 mailu/fetchmail:master
+```
+check a specific service:
+```bash
+core@coreos-01 ~ $ docker service ps mailu_fetchmail
+ID                  NAME                IMAGE                 NODE                DESIRED STATE       CURRENT STATE         ERROR               PORTS
+tbu8ppgsdffj        mailu_fetchmail.1   mailu/fetchmail:master   coreos-01           Running             Running 11 days ago
+```
+
+## Remove the stack
+Run the follwoing command:
+```bash
+core@coreos-01 ~ $ docker stack rm mailu
+```

optional/clamav/Dockerfile

@@ -1,11 +1,14 @@
-FROM alpine:edge
+FROM alpine:3.8
 
 RUN apk add --no-cache clamav rsyslog wget clamav-libunrar
 
 COPY conf /etc/clamav
 COPY start.sh /start.sh
+COPY health.sh /health.sh
 
 EXPOSE 3310/tcp
 VOLUME ["/data"]
 
 CMD ["/start.sh"]
+
+HEALTHCHECK CMD /health.sh

optional/clamav/health.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [ "$(echo PING | nc localhost 3310)" = "PONG" ]; then
+	echo "ping successful"
+else
+	echo "ping failed"
+	exit 1
+fi

optional/radicale/Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine:edge
 
 RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
- && apk add --no-cache radicale@testing py-dulwich@testing
+ && apk add --no-cache radicale@testing py-dulwich@testing curl
 
 COPY radicale.conf /radicale.conf
 
@@ -9,3 +9,5 @@ EXPOSE 5232/tcp
 VOLUME ["/data"]
 
 CMD radicale -f -S -C /radicale.conf
+
+HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1

services/fetchmail/Dockerfile

@@ -1,7 +1,9 @@
-FROM python:alpine
+FROM python:3-alpine
 
-RUN apk add --no-cache fetchmail ca-certificates
+RUN apk add --no-cache fetchmail ca-certificates \
+ && pip install requests
 
 COPY fetchmail.py /fetchmail.py
+USER fetchmail
 
 CMD ["/fetchmail.py"]

services/fetchmail/fetchmail.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 
-import sqlite3
 import time
 import os
 import tempfile
 import shlex
 import subprocess
 import re
+import requests
 
 
 FETCHMAIL = """
@@ -15,6 +15,7 @@ fetchmail -N \
     -f {}
 """
 
+
 RC_LINE = """
 poll "{host}" proto {protocol}  port {port}
     user "{username}" password "{password}"
@@ -24,10 +25,12 @@ poll "{host}" proto {protocol}  port {port}
     sslproto 'AUTO'
 """
 
+
 def extract_host_port(host_and_port, default_port):
     host, _, port = re.match('^(.*)(:([0-9]*))?$', host_and_port).groups()
     return host, int(port) if port else default_port
 
+
 def escape_rc_string(arg):
     return arg.replace("\\", "\\\\").replace('"', '\\"')
 
@@ -41,30 +44,26 @@ def fetchmail(fetchmailrc):
         return output
 
 
-def run(connection, cursor, debug):
+def run(debug):
-    cursor.execute("""
+    fetches = requests.get("http://admin/internal/fetch").json()
-        SELECT user_email, protocol, host, port, tls, username, password, keep
-        FROM fetch
-    """)
     smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
     if smtpport is None:
         smtphostport = smtphost
     else:
         smtphostport = "%s/%d" % (smtphost, smtpport)
-    for line in cursor.fetchall():
+    for fetch in fetches:
         fetchmailrc = ""
-        user_email, protocol, host, port, tls, username, password, keep = line
         options = "options antispam 501, 504, 550, 553, 554"
-        options += " ssl" if tls else ""
+        options += " ssl" if fetch["tls"] else ""
-        options += " keep" if keep else " fetchall"
+        options += " keep" if fetch["keep"] else " fetchall"
         fetchmailrc += RC_LINE.format(
-            user_email=escape_rc_string(user_email),
+            user_email=escape_rc_string(fetch["user_email"]),
-            protocol=protocol,
+            protocol=fetch["protocol"],
-            host=escape_rc_string(host),
+            host=escape_rc_string(fetch["host"]),
-            port=port,
+            port=fetch["port"],
             smtphost=smtphostport,
-            username=escape_rc_string(username),
+            username=escape_rc_string(fetch["username"]),
-            password=escape_rc_string(password),
+            password=escape_rc_string(fetch["password"]),
             options=options
         )
         if debug:
@@ -77,26 +76,20 @@ def run(connection, cursor, debug):
             # No mail is not an error
             if not error_message.startswith("fetchmail: No mail"):
                 print(error_message)
-            user_info = "for %s at %s" % (user_email, host)
+            user_info = "for %s at %s" % (fetch["user_email"], fetch["host"])
             # Number of messages seen is not a error as well
             if ("messages" in error_message and
                     "(seen " in error_message and
                     user_info in error_message):
                 print(error_message)
         finally:
-            cursor.execute("""
+            requests.post("http://admin/internal/fetch/{}".format(fetch["id"]),
-                UPDATE fetch SET error=?, last_check=datetime('now')
+                json=error_message.split("\n")[0]
-                WHERE user_email=?
+            )
-            """, (error_message.split("\n")[0], user_email))
-            connection.commit()
 
 
 if __name__ == "__main__":
-    debug = os.environ.get("DEBUG", None) == "True"
-    db_path = os.environ.get("DB_PATH", "/data/main.db")
-    connection = sqlite3.connect(db_path)
     while True:
-        cursor = connection.cursor()
-        run(connection, cursor, debug)
-        cursor.close()
         time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60)))
+        run(os.environ.get("DEBUG", None) == "True")
+

services/rspamd/Dockerfile

@@ -1,6 +1,6 @@
-FROM alpine:edge
+FROM alpine:3.8
 
-RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \
+RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates py-pip curl\
  && pip install --upgrade pip \
  && pip install tenacity
 
@@ -9,11 +9,10 @@ RUN mkdir /run/rspamd
 COPY conf/ /conf
 COPY start.py /start.py
 
-# Temporary fix to remove references to rspamd-fuzzy for now
+EXPOSE 11332/tcp 11334/tcp 11335/tcp
-RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
-
-EXPOSE 11332/tcp 11334/tcp
 
 VOLUME ["/var/lib/rspamd"]
 
 CMD /start.py
+
+HEALTHCHECK --start-period=350s CMD curl -f -L http://localhost:11334/ || exit 1

services/rspamd/conf/fuzzy_check.conf

@@ -0,0 +1,34 @@
+rule "local" {
+    # Fuzzy storage server list
+    servers = "localhost:11335";
+    # Default symbol for unknown flags
+    symbol = "LOCAL_FUZZY_UNKNOWN";
+    # Additional mime types to store/check
+    mime_types = ["application/*"];
+    # Hash weight threshold for all maps
+    max_score = 20.0;
+    # Whether we can learn this storage
+    read_only = no;
+    # Ignore unknown flags
+    skip_unknown = yes;
+    # Hash generation algorithm
+    algorithm = "mumhash";
+
+    # Map flags to symbols
+    fuzzy_map = {
+        LOCAL_FUZZY_DENIED {
+            # Local threshold
+            max_score = 20.0;
+            # Flag to match
+            flag = 11;
+        }
+        LOCAL_FUZZY_PROB {
+            max_score = 10.0;
+            flag = 12;
+        }
+        LOCAL_FUZZY_WHITE {
+            max_score = 2.0;
+            flag = 13;
+        }
+    }
+}

services/rspamd/conf/metrics.conf

@@ -0,0 +1,19 @@
+group "fuzzy" {
+    max_score = 12.0;
+    symbol "LOCAL_FUZZY_UNKNOWN" {
+        weight = 5.0;
+        description = "Generic fuzzy hash match";
+    }
+    symbol "LOCAL_FUZZY_DENIED" {
+        weight = 12.0;
+        description = "Denied fuzzy hash";
+    }
+    symbol "LOCAL_FUZZY_PROB" {
+        weight = 5.0;
+        description = "Probable fuzzy hash";
+    }
+    symbol "LOCAL_FUZZY_WHITE" {
+        weight = -2.1;
+        description = "Whitelisted fuzzy hash";
+    }
+}

services/rspamd/conf/worker-controller.inc

@@ -1,3 +1,4 @@
+type = "controller";
 bind_socket = "*:11334";
 password = "mailu";
-secure_ip = "{{ FRONT_ADDRESS }}";
+secure_ip = "{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% endif %}";

services/rspamd/conf/worker-fuzzy.inc

@@ -0,0 +1,6 @@
+type = "fuzzy";
+bind_socket = "*:11335";
+count = 1;
+backend = "redis";
+expire = 90d;
+allow_update = ["127.0.0.1"];

services/rspamd/conf/worker-normal.inc

@@ -1 +1,2 @@
+type = "normal";
 enabled = false;

setup/Dockerfile

@@ -4,15 +4,19 @@ RUN mkdir -p /app
 WORKDIR /app
 
 COPY requirements.txt requirements.txt
-RUN apk add --no-cache git \
+RUN apk add --no-cache git curl \
  && pip install -r requirements.txt
 
 COPY server.py ./server.py
 COPY setup.py ./setup.py
 COPY main.py ./main.py
+COPY flavors /data/master/flavors
+COPY templates /data/master/templates
 
-RUN python setup.py https://github.com/mailu/mailu /data
+#RUN python setup.py https://github.com/mailu/mailu /data
 
 EXPOSE 80/tcp
 
 CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app
+
+HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1

setup/docker-compose.yml

@@ -9,5 +9,6 @@ services:
   setup:
     image: mailu/setup
     ports:
-      - "80:80"
+      - "8000:80"
+    build: .
 

setup/flavors/compose/docker-compose.yml

@@ -1,124 +1,106 @@
 {% set env='mailu.env' %}
 # This file is auto-generated by the Mailu configuration wizard.
 # Please read the documentation before attempting any change.
+# Generated for {{ flavor }} flavor
 
-version: '2'
+version: '3.6'
 
 services:
 
   # External dependencies
   redis:
     image: redis:alpine
-    restart: always
     volumes:
-      - "$ROOT/redis:/data"
+      - "{{ root }}/redis:/data"
 
   # Core services
   front:
     image: mailu/nginx:{{ version }}
-    restart: always
     env_file: {{ env }}
-    env:
-      - TLS_FLAVOR={{ tls_flavor or 'letsencrypt' }}
-      - ADMIN={{ expose_admin or 'no' }}
     ports:
     {% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
     {% if bind4 %}
-      - "$PUBLIC_IPV4:{{ port }}:{{ port }}"
+      - "{{ bind4 }}:{{ port }}:{{ port }}"
     {% endif %}
     {% if bind6 %}
-      - "$PUBLIC_IPV6:{{ port }}:{{ port }}"
+      - "{{ bind6 }}:{{ port }}:{{ port }}"
     {% endif %}
     {% endfor %}
-    {% if flavor in ('cert', 'mail') %}
     volumes:
-      - "$ROOT/certs:/certs"
+      - "{{ root }}/certs:/certs"
-    {% endif %}
 
   admin:
     image: mailu/admin:{{ version }}
-    restart: always
     env_file: {{ env }}
-    {% if not expose_admin %}
+    {% if not admin_enabled %}
     ports:
       - 127.0.0.1:8080:80
     {% endif %}
     volumes:
-      - "$ROOT/data:/data"
+      - "{{ root }}/data:/data"
-      - "$ROOT/dkim:/dkim"
+      - "{{ root }}/dkim:/dkim"
     depends_on:
       - redis
 
   imap:
     image: mailu/dovecot:{{ version }}
-    restart: always
     env_file: {{ env }}
     volumes:
-      - "$ROOT/data:/data"
+      - "{{ root }}/mail:/mail"
-      - "$ROOT/mail:/mail"
+      - "{{ root }}/overrides:/overrides"
-      - "$ROOT/overrides:/overrides"
     depends_on:
       - front
 
   smtp:
     image: mailu/postfix:{{ version }}
-    restart: always
     env_file: {{ env }}
     volumes:
-      - "$ROOT/data:/data"
+      - "{{ root }}/overrides:/overrides"
-      - "$ROOT/overrides:/overrides"
     depends_on:
       - front
 
   # Optional services
-  {% if enable_antispam %}
+  {% if antispam_enabled %}
   antispam:
     image: mailu/rspamd:{{ version }}
-    restart: always
     env_file: {{ env }}
     volumes:
-      - "$ROOT/filter:/var/lib/rspamd"
+      - "{{ root }}/filter:/var/lib/rspamd"
-      - "$ROOT/dkim:/dkim"
+      - "{{ root }}/dkim:/dkim"
-      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+      - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d"
     depends_on:
       - front
   {% endif %}
 
-  {% if enable_antivirus %}
+  {% if antivirus_enabled %}
   antivirus:
     image: mailu/clamav:{{ version }}
-    restart: always
     env_file: {{ env }}
     volumes:
-      - "$ROOT/filter:/data"
+      - "{{ root }}/filter:/data"
   {% endif %}
 
-  {% if enable_webdav %}
+  {% if webdav_enabled %}
   webdav:
-    image: mailu/radivale:{{ version }}
+    image: mailu/radicale:{{ version }}
-    restart: always
     env_file: {{ env }}
     volumes:
-      - "$ROOT/dav:/data"
+      - "{{ root }}/dav:/data"
   {% endif %}
 
-  {% if enable_fetchmail %}
+  {% if fetchmail_enabled %}
   fetchmail:
     image: mailu/fetchmail:{{ version }}
-    restart: always
     env_file: {{ env }}
-    volumes:
-      - "$ROOT/data:/data"
   {% endif %}
 
   # Webmail
-  {% if enable_webmail %}
+  {% if webmail_type != 'none' %}
   webmail:
-    image: mailu/{{ webmail }}:{{ version }}
+    image: mailu/{{ webmail_type }}:{{ version }}
-    restart: always
     env_file: {{ env }}
     volumes:
-      - "$ROOT/webmail:/data"
+      - "{{ root }}/webmail:/data"
     depends_on:
       - imap
   {% endif %}

setup/flavors/compose/mailu.env

@@ -1,5 +1,7 @@
 # Mailu main configuration file
 #
+# Generated for {{ flavor }} flavor
+#
 # This file is autogenerated by the configuration management wizard.
 # For a detailed list of configuration variables, see the documentation at
 # https://mailu.io
@@ -9,60 +11,118 @@
 ###################################
 
 # Set this to the path where Mailu data and configuration is stored
-ROOT=/mailu
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT={{ root }}
+
+# Mailu version to run (1.0, 1.1, etc. or master)
+#VERSION={{ version }}
 
 # Set to a randomly generated 16 bytes string
 SECRET_KEY={{ secret(16) }}
 
 # Address where listening ports should bind
-{% if bind4 %}PUBLIC_IPV4={{ bind4 }}{% endif %}
+# This variables are now set directly in `docker-compose.yml by the setup utility
-{% if bind6 %}PUBLIC_IPV6={{ bind6 }}{% endif %}
+# PUBLIC_IPV4= {{ bind4 }} (default: 127.0.0.1)
+# PUBLIC_IPV6= {{ bind6 }} (default: ::1)
 
-# Mail address of the postmaster
+# Main mail domain
-POSTMASTER={{ postmaster }}
+DOMAIN={{ domain }}
 
 # Hostnames for this server, separated with comas
 HOSTNAMES={{ hostnames }}
 
+# Postmaster local part (will append the main mail domain)
+POSTMASTER={{ postmaster }}
+
+# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
+TLS_FLAVOR={{ tls_flavor }}
+
 # Authentication rate limit (per source IP address)
-AUTH_RATELIMIT={{ auth_ratelimit }}
+{% if auth_ratelimit_pm > '0' and auth_ratelimit_ph > '0' %}
+AUTH_RATELIMIT={{ auth_ratelimit_pm }}/minute;{{ auth_ratelimit_ph }}/hour 
+{% endif %}
 
 # Opt-out of statistics, replace with "True" to opt out
-DISABLE_STATISTICS={{ disable_statistics }}
+DISABLE_STATISTICS={{ disable_statistics or 'False' }}
 
 ###################################
-# Server behavior
+# Optional features
+###################################
+
+# Expose the admin interface (value: true, false)
+ADMIN={{ admin_enabled or 'false' }}
+
+# Choose which webmail to run if any (values: roundcube, rainloop, none)
+WEBMAIL={{ webmail_type }}
+
+# Dav server implementation (value: radicale, none)
+WEBDAV={{ webdav_enabled or 'none' }}
+
+# Antivirus solution (value: clamav, none)
+#ANTIVIRUS={{ antivirus_enabled or 'none' }}
+
+#Antispam solution
+ANTISPAM={{ antispam_enabled or 'none'}}
+
+###################################
+# Mail settings
 ###################################
 
 # Message size limit in bytes
 # Default: accept messages up to 50MB
-MESSAGE_SIZE_LIMIT={{ message_size_limit }}
+MESSAGE_SIZE_LIMIT={{ message_size_limit or '50000000' }}
 
 # Networks granted relay permissions, make sure that you include your Docker
 # internal network (default to 172.17.0.0/16)
-RELAYNETS={{ relaynets }}
+RELAYNETS={{ relaynets or '172.17.0.0/16' }}
 
 # Will relay all outgoing mails if configured
 RELAYHOST={{ relayhost }}
 
 # Fetchmail delay
-FETCHMAIL_DELAY={{ fetchmail_delay }}
+FETCHMAIL_DELAY={{ fetchmail_delay or '600' }}
 
 # Recipient delimiter, character used to delimiter localpart from custom address part
-RECIPIENT_DELIMITER={{ recipient_delimiter }}
+RECIPIENT_DELIMITER={{ recipient_delimiter or '+' }}
 
-{% if dmarc_rua or dmarc_ruf %}
 # DMARC rua and ruf email
-{% if dmarc_rua %}DMARC_RUA={{ dmarc_rua }}{% endif %}
+DMARC_RUA={{ dmarc_rua or 'admin' }}
-{% if dmarc_ruf %}DMARC_RUF={{ dmarc_ruf }}{% endif %}
+DMARC_RUF={{ dmarc_ruf or 'admin' }}
-{% endif %}
 
 {% if welcome_enabled %}
 # Welcome email, enable and set a topic and body if you wish to send welcome
 # emails to all users.
-WELCOME={{ welcome_enable }}
+WELCOME={{ welcome_enable or 'false' }}
-WELCOME_SUBJECT={{ welcome_subject }}
+WELCOME_SUBJECT={{ welcome_subject or 'Welcome to your new email account' }}
-WELCOME_BODY={{ welcome_body }}
+WELCOME_BODY={{ welcome_body or 'Welcome to your new email account, if you can read this, then it is configured properly!' }}
+{% endif %}
+
+# Maildir Compression
+# choose compression-method, default: none (value: bz2, gz)
+COMPRESSION={{ compression }}
+# change compression-level, default: 6 (value: 1-9)
+COMPRESSION_LEVEL={{ compression_level }}
+
+###################################
+# Web settings
+###################################
+
+# Path to the admin interface if enabled
+WEB_ADMIN={{ admin_path }}
+
+# Path to the webmail if enabled
+WEB_WEBMAIL={{ webmail_path }}
+
+# Website name
+SITENAME={{ site_name }}
+
+# Linked Website URL
+WEBSITE={{ website }}
+
+{% if recaptcha_public_key and recaptcha_private_key %}
+# Registration reCaptcha settings (warning, this has some privacy impact)
+# RECAPTCHA_PUBLIC_KEY={{ recaptcha_public_key }}
+# RECAPTCHA_PRIVATE_KEY={{ recaptcha_private_key }}
 {% endif %}
 
 {% if domain_registration %}
@@ -70,39 +130,28 @@ WELCOME_BODY={{ welcome_body }}
 DOMAIN_REGISTRATION=true
 {% endif %}
 
-###################################
-# Web settings
-###################################
-
-# Path to the admin interface if enabled
-WEB_ADMIN=/admin
-
-# Path to the webmail if enabled
-WEB_WEBMAIL=/webmail
-
-# Website name
-SITENAME=Mailu
-
-# Linked Website URL
-WEBSITE=https://mailu.io
-
-{% if recaptcha_public_key and recaptcha_private_key %}
-# Registration reCaptcha settings (warning, this has some privacy impact)
-# RECAPTCHA_PUBLIC_KEY={{ recaptcha_public_key }}
-# RECAPTCHA_PRIVATE_KEY={{ recaptcha_private_key }}
-{% endif %}
-
 ###################################
 # Advanced settings
 ###################################
 
-{% if password_scheme %}
+# Log driver for front service. Possible values:
-# Specific password storage scheme
+# json-file (default)
-PASSWORD_SCHEME={{ password_scheme }}
+# journald (On systemd platforms, useful for Fail2Ban integration)
-{% endif %}
+# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!)
+LOG_DRIVER={{ log_driver or 'json-file' }}
+
+# Docker-compose project name, this will prepended to containers names.
+COMPOSE_PROJECT_NAME={{ compose_project_name or 'mailu' }}
+
+# Default password scheme used for newly created accounts and changed passwords
+# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME={{ password_scheme or 'BLF-CRYPT' }}
 
 # Header to take the real ip from
 REAL_IP_HEADER={{ real_ip_header }}
 
 # IPs for nginx set_real_ip_from (CIDR list separated by commas)
 REAL_IP_FROM={{ real_ip_from }}
+
+# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
+REJECT_UNLISTED_RECIPIENT={{ reject_unlisted_recipient }}

setup/flavors/compose/setup.html

@@ -4,15 +4,15 @@
 <p>Docker Compose expects a project file, named <code>docker-compose.yml</code>
 in a project directory. First create your project directory.</p>
 
-<pre><code>mkdir /mailu
+<pre><code>mkdir {{ root }}
 </pre></code>
 
 <p>Then download the project file. A side configuration file makes it easier
 to read and check the configuration variables generated by the wizard.</p>
 
-<pre><code>cd /mailu
+<pre><code>cd {{ root }}
-wget {{ url_for('.file', uid=uid, filepath='docker-compose.yml', _external=True) }}
+curl {{ url_for('.file', uid=uid, filepath='docker-compose.yml', _external=True) }} > docker-compose.yml
-wget {{ url_for('.file', uid=uid, filepath='mailu.env', _external=True) }}
+curl {{ url_for('.file', uid=uid, filepath='mailu.env', _external=True) }} > mailu.env
 </pre></code>
 {% endcall %}
 
@@ -30,7 +30,22 @@ files before going any further.</p>
 <p>To start your compose project, simply run the Docker Compose <code>up</code>
 command.</p>
 
-<pre><code>cd /mailu
+<pre><code>cd {{ root }}
 docker-compose up -d
 </pre></code>
+
+Before you can use Mailu, you must create the primary administrator user account. This should be {{ postmaster }}@{{ domain }}. Use the following command, changing PASSWORD to your liking:
+
+<pre><code>docker-compose exec admin python manage.py admin {{ postmaster }} {{ domain }} PASSWORD
+</pre></code>
+
+<p>Login to the admin interface to change the password for a safe one, at
+{% if admin_enabled %}
+one of the hostnames 
+<a href="https://{{ hostnames.split(',')[0] }}{{ admin_path }}">{{ hostnames.split(',')[0] }}{{ admin_path }}</a>.
+{% else %}
+<a href="http://127.0.0.1:8080">http://127.0.0.1:8080</a> (only directly from the host running docker).
+{% endif %}
+And choose the "Update password" option in the left menu.
+</p>
 {% endcall %}