diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 036140f0..e6e4c10f 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -3,7 +3,7 @@ from mailu.internal import internal import flask import socket - +import os @internal.route("/dovecot/passdb/") def dovecot_passdb_dict(user_email): @@ -13,7 +13,8 @@ def dovecot_passdb_dict(user_email): app.config.get("POD_ADDRESS_RANGE") or socket.gethostbyname(app.config["HOST_FRONT"]) ) - allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"])) + if os.environ["WEBMAIL"] != "none": + allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"])) print(allow_nets) return flask.jsonify({ "password": None, diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 3653011f..62c25c81 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -271,7 +271,7 @@ class User(Base, Email): @property def destination(self): if self.forward_enabled: - result = self.self.forward_destination + result = self.forward_destination if self.forward_keep: result += ',' + self.email return result diff --git a/docs/cli.rst b/docs/cli.rst index 4a5250a2..038f1247 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -39,7 +39,7 @@ primary difference with simple `user` command is that password is being imported .. code-block:: bash - docker-compose run --rm admin python manage.py user --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' + docker-compose run --rm admin python manage.py user_import --hash_scheme='SHA512-CRYPT' myuser example.net '$6$51ebe0cb9f1dab48effa2a0ad8660cb489b445936b9ffd812a0b8f46bca66dd549fea530ce' user_delete ------------ diff --git a/docs/faq.rst b/docs/faq.rst new file mode 100644 index 00000000..395b739c --- /dev/null +++ b/docs/faq.rst @@ -0,0 +1,276 @@ +Frequently asked questions +========================== + +Informational +------------- + +Where to ask questions? +``````````````````````` + +First, please read this FAQ to check if your question is listed here. +Simple questions best fit in our `Matrix`_ room. +For more complex questions, you can always open a `new issue`_ on GitHub. +We actively monitor the issues list. + + +My installation is broken! +`````````````````````````` + +We're sorry to hear that. Please check for common mistakes and troubleshooting +advice in the `Technical issues`_ section of this page. + +I think I found a bug! +`````````````````````` + +If you did not manage to solve the issue using this FAQ and there is not any +`open issues`_ describing the same problem, you can continue to open a +`new issue`_ on GitHub. + +I want a new feature or enhancement! +```````````````````````````````````` + +Great! We are always open for suggestions. We currently maintain two tags: + +- `Enhancement issues`_: Typically used for optimization of features in the project. +- `Feature request issues`_: For implementing new functionality, + plugins and applications. + +Please check if your idea (or something similar) is already mentioned there. +If there is one open, you can choose to vote with a thumbs up, so we can +estimate the popular demand. Please refrain from writing comments like +*"me too"* as it clobbers the actual discussion. + +If you can't find anything similar, you can open a `new issue`_. +Please also share (where applicable): + +- Use case: how does this improve the project? +- Any research done on the subject. Perhaps some links to upstream website, + reference implementations etc. + +Why does my feature/bug take so long to solve? +`````````````````````````````````````````````` + +You should be aware that creating, maintaining and expanding a mail server +distribution requires a lot of effort. Mail servers are highly exposed to hacking attempts, +open relay scanners, spam and malware distributors etc. We need to work in a safe way and +have to prevent pushing out something quickly. + +**TODO: Move the next section into the contributors part of docs** +We currently maintain a strict work flow: + +#. Someone writes a solution and sends a pull request; +#. We use Travis-CI for some very basic building and testing; +#. The pull request needs to be code-reviewed and tested by at least two members + from the contributors team. + +Please consider that this project is mostly developed in people their free time. +We thank you for your understanding and patience. + +I would like to donate (for a feature) +`````````````````````````````````````` + +Donations are welcome at the `patreon`_ account of the project lead. It will be used to pay +for infra structure and project related costs. If there are leftovers, it will be distributed +among the developers. + +It is not yet possible to pay for a specific feature. We don't have +any bounty system implemented. Feel free to come with suggestions in +our ongoing `project management`_ discussion issue. + + +.. _`Matrix`: https://matrix.to/#/#mailu:tedomum.net +.. _`open issues`: https://github.com/Mailu/Mailu/issues +.. _`new issue`: https://github.com/Mailu/Mailu/issues/new +.. _`Enhancement issues`: https://github.com/Mailu/Mailu/issues?q=is%3Aissue+is%3Aopen+label%3Atype%2Fenhancement +.. _`Feature request issues`: https://github.com/Mailu/Mailu/issues?q=is%3Aopen+is%3Aissue+label%3Atype%2Ffeature +.. _`patreon`: https://patreon.com/kaiyou +.. _`project management`: https://github.com/Mailu/Mailu/issues/508 + +Deployment related +------------------ + +How does Mailu scale up? +```````````````````````` + +Recent works allow Mailu to be deployed in Docker Swarm and Kubernetes. +This means it can be scaled horizontally. For more information, refer to :ref:`kubernetes` +or the `Docker swarm howto`_. + +*Issue reference:* `165`_, `520`_. + +How to achieve HA / failover? +````````````````````````````` + +The mailboxes and databases for Mailu are kept on the host filesystem under ``$ROOT/``. +For making the **storage** highly available, all sorts of techniques can be used: + +- Local raid-1 +- btrfs in raid configuration +- Distributed network filesystems such as GlusterFS or CEPH + +Note that no storage HA solution can protect against incidental deletes or file corruptions. +Therefore it is advised to create backups on a regular base! + +A backup MX can be configured as **failover**. For this you need a separate server running +Mailu. On that server, your domains will need to be setup as "Relayed domains", pointing +to you main server. MX records for the mail domains with a higher priority number will have +to point to this server. Please be aware that a backup MX can act as a `spam magnet`_. + +For **service** HA, please see: `How does Mailu scale up?`_ + + +*Issue reference:* `177`_, `591`_. + +.. _`spam magnet`: https://blog.zensoftware.co.uk/2012/07/02/why-we-tend-to-recommend-not-having-a-secondary-mx-these-days/ + + +Can I run Mailu without host iptables? +`````````````````````````````````````` + +When disabling iptables in docker, its forwarding proxy process takes over. +This creates the situation that every incoming connection on port 25 seems to come from the +local network (docker's 172.17.x.x) and is accepted. This causes an open relay! + +For that reason we do **not** support deployment on Docker hosts without iptables. + +*Issue reference:* `332`_. + +How can I override settings? +```````````````````````````` + +Postfix, dovecot and Rspamd support overriding configuration files. Override files belong in +``$ROOT/overrides``. Please refer to the official documentation of those programs for the +correct syntax. The following file names will be taken as override configuration: + +- `Postfix`_ - ``postfix.cf``; +- `Dovecot`_ - ``dovecot.conf``; +- `Rspamd`_ - All files in the ``rspamd`` sub-directory. + +.. _`Postfix`: http://www.postfix.org/postconf.5.html +.. _`Dovecot`: https://wiki.dovecot.org/ConfigFile +.. _`Rspamd`: https://www.rspamd.com/doc/configuration/index.html + +.. _`Docker swarm howto`: https://github.com/Mailu/Mailu/tree/master/docs/swarm/master +.. _`165`: https://github.com/Mailu/Mailu/issues/165 +.. _`177`: https://github.com/Mailu/Mailu/issues/177 +.. _`332`: https://github.com/Mailu/Mailu/issues/332 +.. _`520`: https://github.com/Mailu/Mailu/issues/520 +.. _`591`: https://github.com/Mailu/Mailu/issues/591 + +Technical issues +---------------- + +In this section we are trying to cover the most common problems our users are having. +If your issue is not listed here, please consult issues with the `troubleshooting tag`_. + +Changes in .env don't propagate +``````````````````````````````` + +Variables are sent to the containers at creation time. This means you need to take the project +down and up again. A container restart is not sufficient. + +.. code-block:: bash + + docker-compose down && \ + docker-compose up -d + +*Issue reference:* `615`_. + +TLS certificate issues +`````````````````````` + +When there are issues with the TLS/SSL certificates, Mailu denies service on secure ports. +This is a security precaution. Symptoms are: + +- 403 browser errors; + +These issues are typically caused by four scenarios: + +#. ``TLS_FLAVOR=notls`` in ``.env``; +#. Certificates expired; +#. When ``TLS_FLAVOR=letsencrypt``, it might be that the *certbot* script is not capable of + obtaining the certificates for your domain. See `letsencrypt issues`_ +#. When ``TLS_FLAVOR=certs``, certificates are supposed to be copied to ``/mailu/certs``. + Using an external ``letsencrypt`` program, it tends to happen people copy the whole + ``letsencrypt/live`` directory containing symlinks. Symlinks do not resolve inside the + container and therefore it breaks the TLS implementation. + +letsencrypt issues +.................. + +In order to determine the exact problem on TLS / Let's encrypt issues, it might be helpful +to check the logs. + +.. code-block:: bash + + docker-compose logs front | less -R + docker-compose exec front less /var/log/letsencrypt/letsencrypt.log + +Common problems: + +- Port 80 not reachable from outside. +- Faulty DNS records: make sure that all ``HOSTNAMES`` have **A** (IPv4) and **AAAA** (IPv6) + records, pointing the the ``BIND_ADDRESS4`` and ``BIND_ADDRESS6``. +- DNS cache not yet expired. It might be that old / faulty DNS records are stuck in a cache + en-route to letsencrypt's server. The time this takes is set by the ``TTL`` field in the + records. You'll have to wait at least this time after changing the DNS entries. + Don't keep trying, as you might hit `rate-limits`_. + +.. _`rate-limits`: https://letsencrypt.org/docs/rate-limits/ + +Copying certificates +.................... + +As mentioned above, care must be taken not to copy symlinks to the ``/mailu/certs`` location. + +**The wrong way!:** + +.. code-block:: bash + + cp -r /etc/letsencrypt/live/domain.com /mailu/certs + +**The right way!:** + +.. code-block:: bash + + mkdir -p /mailu/certs + cp /etc/letsencrypt/live/domain.com/privkey.pem /mailu/certs/key.pem + cp /etc/letsencrypt/live/domain.com/fullchain.pem /mailu/certs/cert.pem + +See also :ref:`external_certs`. + +*Issue reference:* `426`_, `615`_. + +Do you support Fail2Ban? +```````````````````````` +Fail2Ban is not included in Mailu. Fail2Ban needs to modify the host's IP tables in order to +ban the addresses. We consider such a program should be run on the host system and not +inside a container. The ``front`` container does use authentication rate limiting to slow +down brute force attacks. + +We *do* provide a possibility to export the logs from the ``front`` service to the host. +For this you need to set ``LOG_DRIVER=journald`` or ``syslog``, depending on the log +manager of the host. You will need to setup the proper Regex in the Fail2Ban configuration. +Be aware that webmail authentication appears to come from the Docker network, +so don't ban those addresses! + +*Issue reference:* `85`_, `116`_, `171`_, `584`_, `592`_. + +Users can't change their password from webmail +`````````````````````````````````````````````` + +All users have the abilty to login to the admin interface. Non-admin users +have only restricted funtionality such as changing their password and the +spam filter weight settings. + +*Issue reference:* `503`_. + +.. _`troubleshooting tag`: https://github.com/Mailu/Mailu/issues?utf8=%E2%9C%93&q=label%3Afaq%2Ftroubleshooting +.. _`85`: https://github.com/Mailu/Mailu/issues/85 +.. _`116`: https://github.com/Mailu/Mailu/issues/116 +.. _`171`: https://github.com/Mailu/Mailu/issues/171 +.. _`426`: https://github.com/Mailu/Mailu/issues/426 +.. _`503`: https://github.com/Mailu/Mailu/issues/503 +.. _`584`: https://github.com/Mailu/Mailu/issues/584 +.. _`592`: https://github.com/Mailu/Mailu/issues/592 +.. _`615`: https://github.com/Mailu/Mailu/issues/615 diff --git a/docs/index.rst b/docs/index.rst index ebfbd7a7..e1f924d2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,7 @@ the version of Mailu that you are running. general features + faq releases demo diff --git a/docs/kubernetes/mailu/index.rst b/docs/kubernetes/mailu/index.rst index ef12eb58..4b6ba8f7 100644 --- a/docs/kubernetes/mailu/index.rst +++ b/docs/kubernetes/mailu/index.rst @@ -1,5 +1,7 @@ -Install Mailu master on kubernetes -================================== +.. _kubernetes: + +Kubernetes setup +================ Prequisites ----------- @@ -189,5 +191,5 @@ This problem can be easily fixed by running following commands: .. code:: bash - kubectl -n mailu-mailserver exec -it maolu-imap-... /bin/sh + kubectl -n mailu-mailserver exec -it mailu-imap-... /bin/sh chmod 777 /data/main.db diff --git a/docs/maintain.rst b/docs/maintain.rst index d570690e..ffb51a50 100644 --- a/docs/maintain.rst +++ b/docs/maintain.rst @@ -28,6 +28,33 @@ Logs are managed by Docker directly. You can easily read your logs using: Docker is able to forward logs to multiple log engines. Read the following documentation for details: https://docs.docker.com/engine/admin/logging/overview/. +.. _external_certs: + +Managing of external Let's encrypt certificates +----------------------------------------------- + +When you are not using the embedded ``letsencrypt`` option from Mailu, +you cannot make use of it's symlink functionality in the ``letsencrypt/live`` directory. +You should take care that after every renewal new certificates are copied to ``/mailu/certs`` and +the *nginx* process in the ``front`` container is reloaded. + +In the case of *certbot* you could write a script to be executed as `deploy hook`_. Example: + +.. code-block:: bash + + #!/bin/sh + cp /etc/letsencrypt/live/domain.com/privkey.pem /mailu/certs/key.pem || exit 1 + cp /etc/letsencrypt/live/domain.com/fullchain.pem /mailu/certs/cert.pem || exit 1 + docker exec mailu_front_1 nginx -s reload + +And the certbot command you will use in crontab would look something like: + +.. code-block:: bash + + 52 0,12 * * * root /usr/bin/certbot renew --deploy-hook /path/to/script.sh + +.. _`deploy hook`: https://certbot.eff.org/docs/using.html#renewing-certificates + Migrating an instance --------------------- diff --git a/services/rspamd/conf/history_redis.conf b/services/rspamd/conf/history_redis.conf new file mode 100644 index 00000000..fee07024 --- /dev/null +++ b/services/rspamd/conf/history_redis.conf @@ -0,0 +1 @@ +servers = "{{ HOST_REDIS }}"; diff --git a/services/unbound/Dockerfile b/services/unbound/Dockerfile new file mode 100644 index 00000000..1b84855c --- /dev/null +++ b/services/unbound/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3-alpine + +RUN apk add --no-cache unbound curl bind-tools \ + && pip3 install jinja2 \ + && curl -o /etc/unbound/root.hints https://www.internic.net/domain/named.cache \ + && chown root:unbound /etc/unbound \ + && chmod 775 /etc/unbound \ + && apk del --no-cache curl \ + && /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key | true + +COPY start.py /start.py +COPY unbound.conf /unbound.conf + +EXPOSE 53/udp 53/tcp + +CMD /start.py + +HEALTHCHECK CMD dig @127.0.0.1 || exit 1 diff --git a/services/unbound/start.py b/services/unbound/start.py new file mode 100755 index 00000000..82e017f7 --- /dev/null +++ b/services/unbound/start.py @@ -0,0 +1,9 @@ +#!/usr/local/bin/python3 + +import jinja2 +import os + +convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +convert("/unbound.conf", "/etc/unbound/unbound.conf") + +os.execv("/usr/sbin/unbound", ["-c /etc/unbound/unbound.conf"]) diff --git a/services/unbound/unbound.conf b/services/unbound/unbound.conf new file mode 100644 index 00000000..d54cbfbc --- /dev/null +++ b/services/unbound/unbound.conf @@ -0,0 +1,19 @@ +server: + verbosity: 1 + interface: 0.0.0.0 + interface: ::0 + logfile: /dev/stdout + do-ip4: yes + do-ip6: yes + do-udp: yes + do-tcp: yes + do-daemonize: no + access-control: {{ SUBNET }} allow + directory: "/etc/unbound" + username: root + auto-trust-anchor-file: trusted-key.key + root-hints: "/etc/unbound/root.hints" + hide-identity: yes + hide-version: yes + max-udp-size: 4096 + msg-buffer-size: 65552 diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index f2cbad23..cc45ca99 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -32,6 +32,16 @@ services: {% endfor %} volumes: - "{{ root }}/certs:/certs" + + {% if resolver_enabled %} + resolver: + image: mailu/unbound:{{ version }} + env_file: {{ env }} + restart: always + networks: + default: + ipv4_address: {{ dns }} + {% endif %} admin: image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-{{ version }}} @@ -65,6 +75,11 @@ services: - "{{ root }}/overrides:/overrides" depends_on: - front + {% if resolver_enabled %} + - resolver + dns: + - {{ dns }} + {% endif %} antispam: image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-{{ version }}} @@ -76,6 +91,11 @@ services: - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d" depends_on: - front + {% if resolver_enabled %} + - resolver + dns: + - {{ dns }} + {% endif %} # Optional services {% if antivirus_enabled %} @@ -85,6 +105,12 @@ services: env_file: {{ env }} volumes: - "{{ root }}/filter:/data" + {% if resolver_enabled %} + depends_on: + - resolver + dns: + - {{ dns }} + {% endif %} {% endif %} {% if webdav_enabled %} @@ -101,6 +127,12 @@ services: image: ${DOCKER_ORG:-mailu}/fetchmail:${MAILU_VERSION:-{{ version }}} restart: always env_file: {{ env }} + {% if resolver_enabled %} + depends_on: + - resolver + dns: + - {{ dns }} + {% endif %} {% endif %} # Webmail @@ -114,3 +146,13 @@ services: depends_on: - imap {% endif %} + +{% if resolver_enabled %} +networks: + default: + driver: bridge + ipam: + driver: default + config: + - subnet: {{ subnet }} +{% endif %} diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 9433ecce..3f67b0dd 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -25,6 +25,9 @@ SECRET_KEY={{ secret(16) }} # PUBLIC_IPV4= {{ bind4 }} (default: 127.0.0.1) # PUBLIC_IPV6= {{ bind6 }} (default: ::1) +# Subnet +SUBNET={{ subnet }} + # Main mail domain DOMAIN={{ domain }} diff --git a/setup/flavors/stack/docker-compose.yml b/setup/flavors/stack/docker-compose.yml index dd9ed7dc..98ba61b1 100644 --- a/setup/flavors/stack/docker-compose.yml +++ b/setup/flavors/stack/docker-compose.yml @@ -29,6 +29,15 @@ services: - "{{ root }}/certs:/certs" deploy: replicas: {{ front_replicas }} + + {% if resolver_enabled %} + resolver: + image: mailu/unbound:{{ version }} + env_file: {{ env }} + networks: + default: + ipv4_address: {{ dns }} + {% endif %} admin: image: ${DOCKER_ORG:-mailu}/admin:${MAILU_VERSION:-{{ version }}} @@ -64,6 +73,10 @@ services: - "{{ root }}/overrides:/overrides" deploy: replicas: {{ smtp_replicas }} + {% if resolver_enabled %} + dns: + - {{ dns }} + {% endif %} antispam: image: ${DOCKER_ORG:-mailu}/rspamd:${MAILU_VERSION:-{{ version }}} @@ -76,6 +89,10 @@ services: - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d" deploy: replicas: 1 + {% if resolver_enabled %} + dns: + - {{ dns }} + {% endif %} # Optional services {% if antivirus_enabled %} @@ -86,6 +103,10 @@ services: - "{{ root }}/filter:/data" deploy: replicas: 1 + {% if resolver_enabled %} + dns: + - {{ dns }} + {% endif %} {% endif %} {% if webdav_enabled %} @@ -106,6 +127,10 @@ services: - "{{ root }}/data:/data" deploy: replicas: 1 + {% if resolver_enabled %} + dns: + - {{ dns }} + {% endif %} {% endif %} {% if webmail_type != 'none' %} diff --git a/setup/server.py b/setup/server.py index bfe5ef15..6f60c3c0 100644 --- a/setup/server.py +++ b/setup/server.py @@ -7,6 +7,7 @@ import jinja2 import uuid import string import random +import ipaddress app = flask.Flask(__name__) @@ -75,6 +76,7 @@ def build_app(path): def submit(): data = flask.request.form.copy() data['uid'] = str(uuid.uuid4()) + data['dns'] = str(ipaddress.IPv4Network(data['subnet'])[-2]) db.set(data['uid'], json.dumps(data)) return flask.redirect(flask.url_for('.setup', uid=data['uid'])) diff --git a/setup/templates/steps/compose/03_expose.html b/setup/templates/steps/compose/03_expose.html index df121c7d..783c2037 100644 --- a/setup/templates/steps/compose/03_expose.html +++ b/setup/templates/steps/compose/03_expose.html @@ -26,6 +26,19 @@ avoid generic all-interfaces addresses like 0.0.0.0 or :: +
+ +
+ +
+ + +
+

You server will be available under a main hostname but may expose multiple public hostnames. Every e-mail domain that points to this server must have one of the hostnames in its MX record. Hostnames must be coma-separated.

diff --git a/setup/templates/steps/stack/03_expose.html b/setup/templates/steps/stack/03_expose.html index a9cffc1c..d47390be 100644 --- a/setup/templates/steps/stack/03_expose.html +++ b/setup/templates/steps/stack/03_expose.html @@ -3,9 +3,17 @@ and let users access their mailboxes. Mailu has some flexibility in the way you expose it to the world.

+
+ +
+
- +

You server will be available under a main hostname but may expose multiple public diff --git a/tests/build.yml b/tests/build.yml index dc259608..8603e4af 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -6,6 +6,10 @@ services: image: ${DOCKER_ORG:-mailu}/nginx:${MAILU_VERSION:-local} build: ../core/nginx + resolver: + image: ${DOCKER_ORG:-mailu}/unbound:${VERSION:-local} + build: ../services/unbound + imap: image: ${DOCKER_ORG:-mailu}/dovecot:${MAILU_VERSION:-local} build: ../core/dovecot diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index 6250e6df..69e0f334 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -1,6 +1,6 @@ FROM php:7.2-apache -ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.7/roundcubemail-1.3.7-complete.tar.gz +ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.8/roundcubemail-1.3.8-complete.tar.gz RUN apt-get update && apt-get install -y \ zlib1g-dev curl \