diff --git a/.mergify.yml b/.mergify.yml deleted file mode 100644 index 927bfc3e..00000000 --- a/.mergify.yml +++ /dev/null @@ -1,23 +0,0 @@ -pull_request_rules: - - name: Successful travis and 2 approved reviews - conditions: - - status-success=continuous-integration/travis-ci/pr - - label!=["status"/wip","status/blocked"] - - "#approved-reviews-by>=2" - actions: - merge: - method: merge - dismiss_reviews: - approved: true - - - name: Trusted author, successful travis and 1 approved review - conditions: - - author~=(kaiyou|muhlemmer|mildred|HorayNarea|adi90x|hoellen|ofthesun9) - - status-success=continuous-integration/travis-ci/pr - - label!=["status"/wip","status/blocked","review/need2"] - - "#approved-reviews-by>=1" - actions: - merge: - method: merge - dismiss_reviews: - approved: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bc867cd..e2749591 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,11 @@ Notable changes to this project are documented in the current file. For more details about individual changes, see the Git log. You should read this before upgrading Freposte.io as some changes will include useful notes. -v1.6.0 - unreleased +v1.6.1 - unreleased +------------------- +- Enhancement: Make Unbound drop privileges after binding to port + +v1.6.0 - 2019-01-18 ------------------- - Global: Architecture of the central container ([#56](https://github.com/Mailu/Mailu/issues/56), [#108](https://github.com/Mailu/Mailu/issues/108)) @@ -117,6 +121,8 @@ v1.6.0 - unreleased - Bug: Don't recursivly chown on mailboxes ([#776](https://github.com/Mailu/Mailu/issues/776)) - Bug: Fix forced password input for user edit ([#745](https://github.com/Mailu/Mailu/issues/745)) - Bug: Fetched accounts: Password field is of type "text" ([#789](https://github.com/Mailu/Mailu/issues/789)) +- Bug: Auto-forward destination not accepting top level domains ([#818](https://github.com/Mailu/Mailu/issues/818)) +- Bug: DOMAIN_REGISTRATION=False in .env was not treated correctly ([#830](https://github.com/Mailu/Mailu/issues/830)) v1.5.1 - 2017-11-21 ------------------- diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 95004017..8ccafce2 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -18,6 +18,7 @@ DEFAULT_CONFIG = { 'DB_PW': None, 'DB_HOST': 'database', 'DB_NAME': 'mailu', + 'SQLITE_DATABASE_FILE':'data/main.db', 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', 'SQLALCHEMY_TRACK_MODIFICATIONS': False, # Statistics management @@ -30,11 +31,11 @@ DEFAULT_CONFIG = { 'POSTMASTER': 'postmaster', 'TLS_FLAVOR': 'cert', 'AUTH_RATELIMIT': '10/minute;1000/hour', - 'DISABLE_STATISTICS': 'False', + 'DISABLE_STATISTICS': False, # Mail settings 'DMARC_RUA': None, 'DMARC_RUF': None, - 'WELCOME': 'False', + 'WELCOME': False, 'WELCOME_SUBJECT': 'Dummy welcome topic', 'WELCOME_BODY': 'Dummy welcome body', 'DKIM_SELECTOR': 'dkim', @@ -66,7 +67,7 @@ class ConfigManager(dict): """ DB_TEMPLATES = { - 'sqlite': 'sqlite:////{DB_HOST}', + 'sqlite': 'sqlite:////{SQLITE_DATABASE_FILE}', 'postgresql': 'postgresql://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}', 'mysql': 'mysql://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}' } @@ -74,13 +75,21 @@ class ConfigManager(dict): def __init__(self): self.config = dict() + def __coerce_value(self, value): + if isinstance(value, str) and value.lower() in ('true','yes'): + return True + elif isinstance(value, str) and value.lower() in ('false', 'no'): + return False + return value + def init_app(self, app): self.config.update(app.config) # get environment variables self.config.update({ - key: os.environ.get(key, value) + key: self.__coerce_value(os.environ.get(key, value)) for key, value in DEFAULT_CONFIG.items() }) + # automatically set the sqlalchemy string if self.config['DB_FLAVOR']: template = self.DB_TEMPLATES[self.config['DB_FLAVOR']] diff --git a/core/admin/mailu/manage.py b/core/admin/mailu/manage.py index 4846c2d6..e11644e7 100644 --- a/core/admin/mailu/manage.py +++ b/core/admin/mailu/manage.py @@ -31,7 +31,7 @@ def advertise(): instance_id = str(uuid.uuid4()) with open(app.config["INSTANCE_ID_PATH"], "w") as handle: handle.write(instance_id) - if app.config["DISABLE_STATISTICS"].lower() != "true": + if not app.config["DISABLE_STATISTICS"]: try: socket.gethostbyname(app.config["STATS_ENDPOINT"].format(instance_id)) except: diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 37823f02..8bfb12fb 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -101,8 +101,8 @@ class Base(db.Model): } ) - created_at = db.Column(db.Date, nullable=False, default=datetime.now) - updated_at = db.Column(db.Date, nullable=True, onupdate=datetime.now) + created_at = db.Column(db.Date, nullable=False, default=date.today) + updated_at = db.Column(db.Date, nullable=True, onupdate=date.today) comment = db.Column(db.String(255), nullable=True) @@ -131,7 +131,7 @@ class Domain(Base): backref=db.backref('manager_of'), lazy='dynamic') max_users = db.Column(db.Integer, nullable=False, default=-1) max_aliases = db.Column(db.Integer, nullable=False, default=-1) - max_quota_bytes = db.Column(db.Integer(), nullable=False, default=0) + max_quota_bytes = db.Column(db.BigInteger(), nullable=False, default=0) signup_enabled = db.Column(db.Boolean(), nullable=False, default=False) @property @@ -307,8 +307,8 @@ class User(Base, Email): domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) password = db.Column(db.String(255), nullable=False) - quota_bytes = db.Column(db.Integer(), nullable=False, default=10**9) - quota_bytes_used = db.Column(db.Integer(), nullable=False, default=0) + quota_bytes = db.Column(db.BigInteger(), nullable=False, default=10**9) + quota_bytes_used = db.Column(db.BigInteger(), nullable=False, default=0) global_admin = db.Column(db.Boolean(), nullable=False, default=False) enabled = db.Column(db.Boolean(), nullable=False, default=True) @@ -410,7 +410,7 @@ class User(Base, Email): return emails def send_welcome(self): - if app.config["WELCOME"].lower() == "true": + if app.config["WELCOME"]: self.sendmail(app.config["WELCOME_SUBJECT"], app.config["WELCOME_BODY"]) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index ed0a0a2c..356137e8 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -37,7 +37,7 @@ class MultipleEmailAddressesVerify(object): self.message = message def __call__(self, form, field): - pattern = re.compile(r'^([_a-z0-9\-]+)(\.[_a-z0-9\-]+)*@([a-z0-9\-]{2,}\.)*([a-z]{2,4})(,([_a-z0-9\-]+)(\.[_a-z0-9\-]+)*@([a-z0-9\-]{2,}\.)*([a-z]{2,4}))*$') + pattern = re.compile(r'^([_a-z0-9\-]+)(\.[_a-z0-9\-]+)*@([a-z0-9\-]{2,}\.)*([a-z]{2,})(,([_a-z0-9\-]+)(\.[_a-z0-9\-]+)*@([a-z0-9\-]{2,}\.)*([a-z]{2,}))*$') if not pattern.match(field.data.replace(" ", "")): raise validators.ValidationError(self.message) diff --git a/core/admin/migrations/versions/3b7eee912b41_.py b/core/admin/migrations/versions/3b7eee912b41_.py new file mode 100644 index 00000000..17f1e0ef --- /dev/null +++ b/core/admin/migrations/versions/3b7eee912b41_.py @@ -0,0 +1,30 @@ +"""change quota type to bigint + +Revision ID: 3b7eee912b41 +Revises: fc099bd15cbe +Create Date: 2019-01-15 08:51:05.346257 + +""" + +# revision identifiers, used by Alembic. +revision = '3b7eee912b41' +down_revision = 'fc099bd15cbe' + +from alembic import op +import sqlalchemy as sa + +def upgrade(): + with op.batch_alter_table('domain') as batch: + batch.alter_column('max_quota_bytes', type_=sa.BigInteger(), nullable=False, server_default='0') + + with op.batch_alter_table('user') as batch: + batch.alter_column('quota_bytes', type_=sa.BigInteger(), nullable=False) + batch.alter_column('quota_bytes_used', type_=sa.BigInteger(), nullable=False, server_default='0') + +def downgrade(): + with op.batch_alter_table('domain') as batch: + batch.alter_column('max_quota_bytes', type_=sa.Integer(), nullable=False, server_default='0') + + with op.batch_alter_table('user') as batch: + batch.alter_column('quota_bytes', type_=sa.Integer(), nullable=False) + batch.alter_column('quota_bytes_used', type_=sa.Integer(), nullable=False, server_default='0') diff --git a/docs/compose/.env b/docs/compose/.env index cf906b58..218b94d2 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -1,3 +1,5 @@ +# WARNING: this file is being deprecated over the new setup utility, found at https://setup.mailu.io + # Mailu main configuration file ## Most configuration variables can be modified through the Web interface, # these few settings must however be configured before starting the mail diff --git a/docs/compose/docker-compose.yml b/docs/compose/docker-compose.yml index 2cff9608..2686ee27 100644 --- a/docs/compose/docker-compose.yml +++ b/docs/compose/docker-compose.yml @@ -1,3 +1,5 @@ +# WARNING: this file is being deprecated over the new setup utility, found at https://setup.mailu.io + version: '2' services: diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst index 942a368e..c1a620e6 100644 --- a/docs/compose/setup.rst +++ b/docs/compose/setup.rst @@ -12,34 +12,22 @@ Mailu will store all of its persistent data in a path of your choice mkdir /mailu cd /mailu -Download the initial configuration file ---------------------------------------- +Create the configuration files +------------------------------ -Docker Compose configuration is stored in a file named -:download:`docker-compose.yml`. Additionally, Mailu -relies on a :download:`.env` file for various settings. Download -the proper template files from the git repository. To download the configuration -for the ``VERSION_TAG`` branch, use: +Docker Compose configuration is stored in a file named ``docker-compose.yml``. +Additionally, Mailu relies on a ``mailu.env`` file for various settings. +Both files can be generated by the `mailu setup utility`_. The setup utility +is mostly self-explanatory, with some more additional information in this section. -.. code-block:: bash +.. _`mailu setup utility`: https://setup.mailu.io - wget https://mailu.io/VERSION_TAG/_downloads/docker-compose.yml - wget https://mailu.io/VERSION_TAG/_downloads/.env - -Important configuration variables ---------------------------------- - -Open the ``.env`` file and review the following variable settings: - -- Change ``ROOT`` if you have your setup directory in a different location then ``/mailu``. -- Check ``VERSION`` to reflect the version you picked. (``master`` or ``1.5``). - -Make sure to read the comments in the file and instructions from the :ref:`common_cfg` section. +.. _tls_flavor: TLS certificates ```````````````` -Set the ``TLS_FLAVOR`` to one of the following +Sets the ``TLS_FLAVOR`` to one of the following values: - ``cert`` is the default and requires certificates to be setup manually; @@ -59,7 +47,7 @@ values: Bind address ```````````` -Modify ``BIND_ADDRESS4`` and ``BIND_ADDRESS6`` to match the public IP addresses assigned to your server. For IPv6 you will need the ```` scope address. +The bind addresses need to match the public IP addresses assigned to your server. For IPv6 you will need the ```` scope address. You can find those addresses by running the following: @@ -81,56 +69,17 @@ you would simply like the server to listen on all interfaces, use ``0.0.0.0`` an .. _issues: https://github.com/Mailu/Mailu/issues/641 -Enable optional features ------------------------- +Review configuration variables +------------------------------ -Some of Mailu features are not used by every user and are thus not enabled in a -default configuration. - -A Webmail is a Web interface exposing an email client. Mailu webmails are -bound to the internal IMAP and SMTP server for users to access their mailbox through -the Web. By exposing a complex application such as a Webmail, you should be aware of -the security implications caused by such an increase of attack surface. The ``WEBMAIL`` -configuration option must be one of the following: - -- ``none`` is the default value, no Webmail service will be exposed; -- ``roundcube`` will run the popular Roundcube Webmail; -- ``rainloop`` will run the popular Rainloop Webmail. - -The administration interface is not exposed on the public address by default, -you will need to set the ``ADMIN`` variable accordingly: - -- ``true`` will expose the admin interface in ``/admin``; -- ``false`` (or any other value) will disable this behaviour. - -A Webdav server exposes a Dav interface over HTTP so that clients can store -contacts or calendars using the mail account. This can be enabled using the `WEBDAV` -setting. The configuration option must be one of the following: - -- ``none`` is the default value, no webdav service will be exposed; -- ``radicale`` exposes the radicale Webdav service. - -An antivirus server helps fighting large scale virus spreading campaigns -that leverage e-mail for initial infection. This can be setup using the ``ANTIVIRUS`` -setting. The configuration option must be one of the following: - -- ``none`` disables antivirus checks; -- ``clamav`` is the default values, the popular ClamAV antivirus is enabled. - -Make sure that you have at least 1GB of memory for ClamAV to load its signature -database. - -If you run Mailu behind a reverse proxy you can use ``REAL_IP_HEADER`` and -``REAL_IP_FROM`` to set the values of respective the Nginx directives -``real_ip_header`` and ``set_real_ip_from``. The ``REAL_IP_FROM`` configuration -option is a comma-separated list of IPs (or CIDRs) of which for each a -``set_real_ip_from`` directive is added in the Nginx configuration file. +After downloading the files, open ``mailu.env`` and review the variable settings. +Make sure to read the comments in the file and instructions from the :ref:`common_cfg` page. Finish setting up TLS --------------------- Mailu relies heavily on TLS and must have a key pair and a certificate -available, at least for the hostname configured in the ``.env`` file. +available, at least for the hostname configured in the ``mailu.env`` file. If you set ``TLS_FLAVOR`` to ``cert`` or ``mail`` then you must create a ``certs`` directory in your root path and setup a key-certificate pair there: @@ -154,3 +103,5 @@ Finally, you must create the initial admin user account: docker-compose exec admin flask mailu admin me example.net password This will create a user named ``me@example.net`` with password ``password`` and administration privileges. Connect to the Web admin interface and change the password to a strong one. + + .. note:: It is vitally important that either a user with the same email as ``POSTMASTER`` in your ``mailu.env`` exists, or you remember to create an alias with this name after you log in. All kinds of strange errors will occur as a result of not doing so! diff --git a/docs/conf.py b/docs/conf.py index 64997eb1..7a0cbeb4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,9 +36,10 @@ html_context = { 'github_user': 'mailu', 'github_repo': 'mailu', 'github_version': version, - 'stable_version': '1.5', + 'stable_version': '1.6', 'versions': [ ('1.5', '/1.5/'), + ('1.6', '/1.6/'), ('master', '/master/') ], 'conf_py_path': '/docs/' diff --git a/docs/configuration.rst b/docs/configuration.rst index ec114c97..e7dfa2af 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,5 +1,9 @@ -Mailu configuration settings -============================ +Configuration reference +======================= + +This page explains the variables found in ``mailu.env``. +In most cases ``mailu.env`` is setup correctly by the setup utility and can be left as-is. +However, some advanced settings or modifications can be done by modifying this file. .. _common_cfg: @@ -37,6 +41,9 @@ The ``AUTH_RATELIMIT`` holds a security setting for fighting attackers that try to guess user passwords. The value is the limit of requests that a single IP address can perform against IMAP, POP and SMTP authentication endpoints. +The ``TLS_FLAVOR`` sets how Mailu handles TLS connections. Setting this value to +``notls`` will cause Mailu not to server any web content! More on :ref:`tls_flavor`. + Mail settings ------------- diff --git a/docs/docker-compose.yml b/docs/docker-compose.yml index b7026564..9c5d2473 100644 --- a/docs/docker-compose.yml +++ b/docs/docker-compose.yml @@ -1,28 +1,10 @@ +# This file is used to test the mailu/docs website +# Deployment files can be found on github.com/mailu/infra + version: '3' - services: - docs_master: - image: mailu/docs:master - networks: - - web - labels: - - traefik.enable=true - - traefik.port=80 - - traefik.main.frontend.rule=Host:${ADDRESS};PathPrefix:/master/ - - docs_15: - image: mailu/docs:1.5 - networks: - - web - labels: - - traefik.enable=true - - traefik.port=80 - - traefik.root.frontend.redirect.regex=.* - - traefik.root.frontend.redirect.replacement=/1.5/ - - traefik.root.frontend.rule=Host:${ADDRESS};PathPrefix:/ - - traefik.main.frontend.rule=Host:${ADDRESS};PathPrefix:/1.5/ - -networks: - web: - external: true + docs: + image: ${DOCKER_ORG:-mailu}/docs:${MAILU_VERSION:-master} + ports: + - 127.0.0.1:8000:80 diff --git a/docs/faq.rst b/docs/faq.rst index 2669d9d1..4abd8874 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -1,3 +1,5 @@ +.. _faq: + Frequently asked questions ========================== diff --git a/docs/index.rst b/docs/index.rst index 98825ab6..0808010c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -53,10 +53,10 @@ the version of Mailu that you are running. :caption: Setup setup - configuration compose/requirements compose/setup kubernetes/mailu/index + configuration dns reverse database diff --git a/docs/releases.rst b/docs/releases.rst index 275c0ea1..b3bef61c 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -1,6 +1,95 @@ Release notes ============= +Mailu 1.6 - 2019-01-18 +---------------------- + +Its been more than a year since the release of 1.5! And what a year it has been... +More then 800 commits are done since 1.5, containing thousands of additions. +We had the honor of welcoming more and more contributors and we actually established +a dedicated team of trusted contributors. + +With new review guidelines we now allow the project to grow without dependence +on any single person. And thus merging pull requests at much shorter time. +On top of that we finally got around to creating a simple test suite on TravisCI, +which is doing some e-mail sending and receiving. This greatly helps the reviewing process. + +For a complete overview of changes, see our `changelog`_. +Here we'll try to give you the highlights. + +.. _`changelog`: https://github.com/Mailu/Mailu/blob/master/CHANGELOG.md + +New functionality +````````````````` + +We offer a lot new functions in the user experience. Some of the highlights would be quota +support from the admin interface, optional user sign up with recaptcha, auto-reply start date, +and a client setup page. + +Mailu now also offers a `setup utility`_. +This utility helps users to generate a `docker-compose.yml` and `mailu.env` through guided steps. + +Documentation +````````````` + +Quite some efforts were done in expanding the documentation of Mailu. +We've added support for :ref:`kubernetes`, `Docker Swarm`_ and a :ref:`faq` section. +There is now also a section on running the Mailu web interfaces behind :ref:`traefik_proxy`. + +We now also Dockerized the documentation, allowing for easy local running and versions +management on our web server. + +.. _`Docker Swarm`: https://github.com/Mailu/Mailu/blob/master/docs/swarm/master/README.md + +Back-end +```````` + +Lots and lots of hours went in to the back-end. Work on numerous bugs, +increased the general performance and allowing for better maintainability. + +We've reworked the complete interface with the database. All queries are now done +through the Admin container, with that being the single point of contact with the +database. Now we also support the usage of MySQL and PostgreSQL databases and Mailu +comes with its own PostgreSQL image! This allows for Mailu to be used in larger scaled +operations. + +Main software versions +`````````````````````` +- Alpine 3.8.2 +- Python 3.6.6 +- SQLite 3.25.3 +- Postfix 3.3.0 +- Dovecot 2.3.2.1 +- Radicale 2.1.10 +- Rspamd 1.7.6 +- ClamAV 0.100.2 +- Nginx 1.14.2 +- Rainloop 1.12.1 +- Roundcube 1.3.8 +- Fetchmail 6.3.26 +- Unbound 1.7.3 +- Postgresql 10.5 + +Upgrading +````````` + +We've done some pretty intrusive works on the DB migrations scripts. Although thoroughly +tested, we would recommend users to create a backup copy of ``/mailu/data/main.db`` somewhere. + +Use the `setup utility`_ to obtain new ``docker-compose.yml`` and ``mailu.env`` files. +For this upgrade it is necessary to bring the project down and up, due to network definition changes: + +.. code-block:: bash + + docker-compose pull + docker-compose down --remove-orphans + docker-compose up -d + +After everything runs successfully, ``/mailu/certs/dhparam.pem`` is no longer needed and can be deleted. +It's included in the Mailu distribution by default now. Also the old ``.env`` can be deleted. + +.. _`setup utility`: https://setup.mailu.io + Mailu 1.5 - 2017-11-05 ---------------------- diff --git a/docs/reverse.rst b/docs/reverse.rst index 5f64b8f3..02f90fbb 100644 --- a/docs/reverse.rst +++ b/docs/reverse.rst @@ -115,8 +115,10 @@ Depending on how you access the front server, you might want to add a ``proxy_re This will stop redirects (301 and 302) sent by the Webmail, nginx front and admin interface from sending you to ``localhost``. -Use Traefik in another container as central system-reverse-proxy --------------------------------------------------------------------- +.. _traefik_proxy: + +Traefik as reverse proxy +------------------------ `Traefik`_ is a popular reverse-proxy aimed at containerized systems. As such, many may wish to integrate Mailu into a system which already uses Traefik as its sole ingress/reverse-proxy. diff --git a/docs/setup.rst b/docs/setup.rst index 9771f886..eb79a697 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -41,7 +41,7 @@ Pick a Mailu version Mailu is shipped in multiple versions. -- ``1.5`` features the most recent stable version for Mailu. This is the +- ``1.6`` features the most recent stable version for Mailu. This is the recommended build for new setups, old setups should migrate when possible. - ``1.0``, ``1.1``, and other version branches feature old versions of Mailu diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index 61319cfd..58723c33 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -1,5 +1,43 @@ # Install Mailu on a docker swarm +## Some warnings + +### How Docker swarm works + +Docker swarm enables replication and fail-over scenarios. As a feature, if a node dies or goes away, Docker will re-schedule it's containers on the remaining nodes. +In order to take this decisions, docker swarm works on a consensus between managers regarding the state of nodes. Therefore it recommends to always have an uneven amount of manager nodes. This will always give a majority on either halve of a potential network split. + +### Storage + +On top of this some of Mailu's containers heavily rely on disk storage. As noted below, every host will need the same dataset on every host where related containers are run. So Dovecot IMAP needs `/mailu/mail` replicated to every node it *may* be scheduled to run. There are various solutions for this like NFS and GlusterFS. + +### When disaster strikes + +So imagine 3 swarm nodes and 3 GlusterFS endpoints: + +``` +node-A -> gluster-A --| +node-B -> gluster-B --|--> Single file system +node-C -> gluster-C --| +``` + +Each node has a connection to the shared file system and maintains connections between the other nodes. Let's say Dovecot is running on `node-A`. Now a network error / outage occurs on the route between `node-A` and the remaining nodes, but stays connected to the `gluster-A` endpoint. `node-B` and `node-C` conclude that `node-A` is down. They reschedule Dovecot to start on either one of them. Dovecot starts reading and writing its indexes to the **shared** filesystem. However, it is possible the Dovecot on `node-A` is still up and handling some client requests. I've seen cases where this situations resulted in: + +- Retained locks +- Corrupted indexes +- Users no longer able to read any of mail +- Lost mail + +### It gets funkier + +Our original deployment also included `main.db` on the GlusterFS. Due to the above we corrupted it once and we decided to move it to local storage and restirct the `admin` container to that host only. This inspired us to put some legwork is supporting different database back-ends like MySQL and PostgreSQL. We highly recommend to use either of them, in favor of sqlite. + +### Conclusion + +Although the above situation is less-likely to occur on a stable (local) network, it does indicate a failure case where there is a probability of data-loss or downtime. It may help to create redundant networks, but the effort might be too much for the actual results. We will need to look into better and safer methods of replicating mail data. For now, we regret to have to inform you that Docker swarm deployment is **unstable** and should be avoided in production environments. + +-- @muhlemmer, 17th of January 2019. + ## Prequisites ### Swarm @@ -250,3 +288,44 @@ Run the follwoing command: ```bash core@coreos-01 ~ $ docker stack rm mailu ``` + +## Notes on unbound resolver + +In Docker compose flavor we currently have the option to include the unbound DNS resolver. This does not work in Docker Swarm, as it in not possible to configure any static IP addresses. There is an [open issue](https://github.com/moby/moby/issues/24170) for this at Docker. However, this doesn't seem to move anywhere since some time now. For that reasons we've chosen not to include the unbound resolver in the stack flavor. + +If you still want to benefit from Unbound as a system resolver, you can install it system-wide. The following procedure was done on a Fedora 28 system and might needs some adjustments for your system. Note that this will need to be done on every swarm node. In this example we will make use of `dnssec-trigger`, which is used to configure unbound. When installing this and running the service, unbound is pulled in as dependency and does not need to be installed, configured or run separately. + +Install required packages(unbound will be installed as dependency): + +``` +sudo dnf install dnssec-trigger +``` + +Enable and start the *dnssec-trigger* daemon: + +``` +sudo systemctl enable --now dnssec-triggerd.service +``` + +Configure NetworkManager to use unbound, create the file `/etc/NetworkManager/conf.d/unbound.conf` with contents: + +``` +[main] +dns=unbound +``` + +You might need to restart NetworkManager for the changes to take effect: + +``` +sudo systemctl restart NetworkManager +``` + +Verify `resolv.conf`: + +``` +$ cat /etc/resolv.conf +# Generated by dnssec-trigger-script +nameserver 127.0.0.1 +``` + +Most of this info was take from this [Fedora Project page](https://fedoraproject.org/wiki/Changes/Default_Local_DNS_Resolver#How_To_Test). diff --git a/services/unbound/unbound.conf b/services/unbound/unbound.conf index d54cbfbc..8abd4325 100644 --- a/services/unbound/unbound.conf +++ b/services/unbound/unbound.conf @@ -10,7 +10,7 @@ server: do-daemonize: no access-control: {{ SUBNET }} allow directory: "/etc/unbound" - username: root + username: unbound auto-trust-anchor-file: trusted-key.key root-hints: "/etc/unbound/root.hints" hide-identity: yes diff --git a/setup/docker-compose.yml b/setup/docker-compose.yml index 6d14153a..9c93fd6f 100644 --- a/setup/docker-compose.yml +++ b/setup/docker-compose.yml @@ -1,50 +1,16 @@ -# This file is used to run the mailu/setup utility +# This file is used to test the mailu/setup utility +# Deployment files can be found on github.com/mailu/infra version: '3.6' services: redis: image: redis:alpine - networks: - - default - setup_master: - image: mailu/setup:master - networks: - - web - - default + setup: + image: ${DOCKER_ORG:-mailu}/setup:${MAILU_VERSION:-master} env_file: .env - environment: - this_version: "master" - labels: - - traefik.enable=true - - traefik.port=80 - - traefik.docker.network=web - - traefik.main.frontend.rule=Host:${ADDRESS};PathPrefix:/master/ depends_on: - redis - - setup_release: - image: mailu/setup:${RELEASE} - networks: - - web - - default - env_file: .env - environment: - this_version: ${RELEASE} - labels: - - traefik.enable=true - - traefik.port=80 - - traefik.docker.network=web - - traefik.root.frontend.redirect.regex=.* - - traefik.root.frontend.redirect.replacement=/${RELEASE}/ - - traefik.root.frontend.rule=Host:${ADDRESS};PathPrefix:/ - - traefik.main.frontend.rule=Host:${ADDRESS};PathPrefix:/${RELEASE}/ - depends_on: - - redis - -networks: - web: - external: true - default: - external: false + ports: + - 127.0.0.1:8001:80 diff --git a/setup/flavors/stack/docker-compose.yml b/setup/flavors/stack/docker-compose.yml index 480aafeb..e1382d20 100644 --- a/setup/flavors/stack/docker-compose.yml +++ b/setup/flavors/stack/docker-compose.yml @@ -31,15 +31,6 @@ services: deploy: replicas: {{ front_replicas }} - {% if resolver_enabled %} - resolver: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}unbound:${MAILU_VERSION:-{{ version }}} - env_file: {{ env }} - networks: - default: - ipv4_address: {{ dns }} - {% endif %} - admin: image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-{{ version }}} env_file: {{ env }} @@ -69,10 +60,6 @@ services: - "{{ root }}/overrides:/overrides" deploy: replicas: {{ smtp_replicas }} - {% if resolver_enabled %} - dns: - - {{ dns }} - {% endif %} antispam: image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-{{ version }}} @@ -83,10 +70,6 @@ services: - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d" deploy: replicas: 1 - {% if resolver_enabled %} - dns: - - {{ dns }} - {% endif %} # Optional services {% if antivirus_enabled %} @@ -97,10 +80,6 @@ services: - "{{ root }}/filter:/data" deploy: replicas: 1 - {% if resolver_enabled %} - dns: - - {{ dns }} - {% endif %} {% endif %} {% if webdav_enabled %} @@ -121,10 +100,6 @@ services: - "{{ root }}/data:/data" deploy: replicas: 1 - {% if resolver_enabled %} - dns: - - {{ dns }} - {% endif %} {% endif %} {% if webmail_type != 'none' %} diff --git a/setup/server.py b/setup/server.py index fea27ead..556d4b3a 100644 --- a/setup/server.py +++ b/setup/server.py @@ -11,7 +11,7 @@ import ipaddress import hashlib -version = os.getenv("this_version") +version = os.getenv("this_version", "master") static_url_path = "/" + version + "/static" app = flask.Flask(__name__, static_url_path=static_url_path) flask_bootstrap.Bootstrap(app) diff --git a/setup/templates/steps/compose/02_services.html b/setup/templates/steps/compose/02_services.html index 11e7a14e..a78a3f62 100644 --- a/setup/templates/steps/compose/02_services.html +++ b/setup/templates/steps/compose/02_services.html @@ -1,13 +1,13 @@ {% call macros.panel("info", "Step 3 - pick some features") %}

Mailu comes with multiple base features, including a specific admin -interface, Web email clients (webmails), antispam, antivirus, etc. If you -wish to disable some of these features, you are free to do so.

- -

Emails will be available through IMAP and POP3. You may also enable a Web -email client. These do add some complexity but provide an easier way of -accessing messages for beginner users.

+interface, Web email clients, antispam, antivirus, etc. +In this section you can enable the services to you liking.

+

A Webmail is a Web interface exposing an email client. Mailu webmails are +bound to the internal IMAP and SMTP server for users to access their mailbox through +the Web. By exposing a complex application such as a Webmail, you should be aware of +the security implications caused by such an increase of attack surface.

@@ -26,10 +26,9 @@ accessing messages for beginner users.

-

Email filtering is a really important features. You can still disable it, which -will prevent Mailu from doing spam filtering, virus filtering, and from applying -white and blacklists that you may configure in the admin interface. You may -also disable the antivirus if required (it does use aroung 1GB of ram).

+

An antivirus server helps fighting large scale virus spreading campaigns that leverage +e-mail for initial infection. Make sure that you have at least 1GB of memory for ClamAV to +load its signature database.

+

A Webdav server exposes a Dav interface over HTTP so that clients can store +contacts or calendars using the mail account.

+
+

Fetchmail allows to download mails over IMAP/POP3 and uploads it your Mailu mailbox.

+