From 12294a6e5aa3c57c8c603d97a35fa443a962a8e2 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Wed, 6 Jun 2018 18:40:51 +0000 Subject: [PATCH 01/21] Trying to enable fuzzy hashes for rspamd --- services/rspamd/Dockerfile | 5 +--- services/rspamd/conf/fuzzy_check.conf | 34 ++++++++++++++++++++++ services/rspamd/conf/metrics.conf | 19 ++++++++++++ services/rspamd/conf/worker-controller.inc | 1 + services/rspamd/conf/worker-fuzzy.inc | 5 ++++ services/rspamd/conf/worker-normal.inc | 1 + 6 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 services/rspamd/conf/fuzzy_check.conf create mode 100644 services/rspamd/conf/metrics.conf create mode 100644 services/rspamd/conf/worker-fuzzy.inc diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index c6c2afdd..1b8d7e6b 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -1,15 +1,12 @@ FROM alpine:edge -RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates +RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates RUN mkdir /run/rspamd COPY conf/ /conf COPY start.py /start.py -# Temporary fix to remove references to rspamd-fuzzy for now -RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf - EXPOSE 11332/tcp 11334/tcp CMD /start.py diff --git a/services/rspamd/conf/fuzzy_check.conf b/services/rspamd/conf/fuzzy_check.conf new file mode 100644 index 00000000..7c87e1c3 --- /dev/null +++ b/services/rspamd/conf/fuzzy_check.conf @@ -0,0 +1,34 @@ +rule "local" { + # Fuzzy storage server list + servers = "localhost:11335"; + # Default symbol for unknown flags + symbol = "LOCAL_FUZZY_UNKNOWN"; + # Additional mime types to store/check + mime_types = ["application/*"]; + # Hash weight threshold for all maps + max_score = 20.0; + # Whether we can learn this storage + read_only = no; + # Ignore unknown flags + skip_unknown = yes; + # Hash generation algorithm + algorithm = "mumhash"; + + # Map flags to symbols + fuzzy_map = { + LOCAL_FUZZY_DENIED { + # Local threshold + max_score = 20.0; + # Flag to match + flag = 11; + } + LOCAL_FUZZY_PROB { + max_score = 10.0; + flag = 12; + } + LOCAL_FUZZY_WHITE { + max_score = 2.0; + flag = 13; + } + } +} diff --git a/services/rspamd/conf/metrics.conf b/services/rspamd/conf/metrics.conf new file mode 100644 index 00000000..6a31964f --- /dev/null +++ b/services/rspamd/conf/metrics.conf @@ -0,0 +1,19 @@ +group "fuzzy" { + max_score = 12.0; + symbol "LOCAL_FUZZY_UNKNOWN" { + weight = 5.0; + description = "Generic fuzzy hash match"; + } + symbol "LOCAL_FUZZY_DENIED" { + weight = 12.0; + description = "Denied fuzzy hash"; + } + symbol "LOCAL_FUZZY_PROB" { + weight = 5.0; + description = "Probable fuzzy hash"; + } + symbol "LOCAL_FUZZY_WHITE" { + weight = -2.1; + description = "Whitelisted fuzzy hash"; + } +} diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index 6a020672..dd143942 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,3 +1,4 @@ +type = "controller"; bind_socket = "*:11334"; password = "mailu"; secure_ip = "{{ FRONT_ADDRESS }}"; diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc new file mode 100644 index 00000000..a0021a03 --- /dev/null +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -0,0 +1,5 @@ +type = "fuzzy"; +count = 1; +backend = "redis"; +expire = 90d; +allow_update = ["127.0.0.1"]; diff --git a/services/rspamd/conf/worker-normal.inc b/services/rspamd/conf/worker-normal.inc index a6ee8317..ab996fb8 100644 --- a/services/rspamd/conf/worker-normal.inc +++ b/services/rspamd/conf/worker-normal.inc @@ -1 +1,2 @@ +type = "normal"; enabled = false; From 23e288aadcf7d5df941c8516fae9d0cbd6b1e054 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Mon, 24 Sep 2018 17:29:31 +0000 Subject: [PATCH 02/21] Enabling swarm deployment on master branch: -Extends the usage of POD_ADDRESS_RANGE -Provides documentation --- core/dovecot/conf/dovecot.conf | 2 +- core/postfix/conf/main.cf | 2 +- docs/swarm/master/README.md | 349 +++++++++++++++++++++ services/rspamd/conf/worker-controller.inc | 2 +- 4 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 docs/swarm/master/README.md diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index a5973bf8..7cf10774 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -5,7 +5,7 @@ log_path = /dev/stderr protocols = imap pop3 lmtp sieve postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }} hostname = {{ HOSTNAMES.split(",")[0] }} -submission_host = {{ FRONT_ADDRESS }} +submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }} service dict { unix_listener dict { diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 2f2c6990..bde42dd1 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }} recipient_delimiter = {{ RECIPIENT_DELIMITER }} # Only the front server is allowed to perform xclient -smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} +smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }} ############### # TLS diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md new file mode 100644 index 00000000..1406e05c --- /dev/null +++ b/docs/swarm/master/README.md @@ -0,0 +1,349 @@ +# Install Mailu on a docker swarm + +## Prequisites + +### Swarm + +In order to deploy Mailu on a swarm, you will first need to initialize the swarm: + +The main command will be: +```bash +docker swarm init --advertise-addr +``` +See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/ + +If you want to add other managers or workers, please use: +```bash +docker swarm join --token xxxxx +``` +See https://docs.docker.com/engine/swarm/join-nodes/ + +You have now a working swarm, and you can check its status with: +```bash +core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls +ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION +xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce +sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce +mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce +``` + +### Volume definition +For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. +Hereafter we will use a NFS share: +```bash +core@coreos-01 ~ $ showmount -e 192.168.0.30 +Export list for 192.168.0.30: +/mnt/Pool1/pv 192.168.0.0 +``` + +on the nfs server, I am using the following /etc/exports +```bash +$more /etc/exports +/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 +``` +on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up) +```bash +$mkdir /mnt/Pool1/pv/mailu +``` + +On your manager node, mount the nfs share to check that the share is available: +```bash +core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/ +``` +If this is ok, you can umount it: +```bash +core@coreos-01 ~ $ sudo umount /mnt/local/ +``` + + +### Networking mode +On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. + +In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE. + +Let's create the mailu_default network: +```bash +core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default +core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet + "Subnet": "10.0.1.0/24", +``` +In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 + +Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network) + +### Scalability +- smtp and imap are scalable +- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) +- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) + +### Variable substitution and docker-compose.yml +The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to : +- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc +- change the way we define the volumes (nfs share in our case) +- add a deploy section for every service + +### Docker compose +An example of docker-compose-stack.yml file is available here: + +```yaml + +version: '3.2' + +services: + + front: + image: mailu/nginx:master + restart: always + env_file: .env + ports: + - target: 80 + published: 80 + - target: 443 + published: 443 + - target: 110 + published: 110 + - target: 143 + published: 143 + - target: 993 + published: 993 + - target: 995 + published: 995 + - target: 25 + published: 25 + - target: 465 + published: 465 + - target: 587 + published: 587 + volumes: +# - "$ROOT/certs:/certs" + - type: volume + source: mailu_certs + target: /certs + deploy: + replicas: 2 + + redis: + image: redis:alpine + restart: always + volumes: +# - "$ROOT/redis:/data" + - type: volume + source: mailu_redis + target: /data + deploy: + replicas: 1 + + imap: + image: mailu/dovecot:master + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data +# - "$ROOT/mail:/mail" + - type: volume + source: mailu_mail + target: /mail +# - "$ROOT/overrides:/overrides" + - type: volume + source: mailu_overrides + target: /overrides + depends_on: + - front + deploy: + replicas: 2 + + smtp: + image: mailu/postfix:master + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data +# - "$ROOT/overrides:/overrides" + - type: volume + source: mailu_overrides + target: /overrides + depends_on: + - front + deploy: + replicas: 2 + + antispam: + image: mailu/rspamd:master + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + depends_on: + - front + volumes: +# - "$ROOT/filter:/var/lib/rspamd" + - type: volume + source: mailu_filter + target: /var/lib/rspamd +# - "$ROOT/dkim:/dkim" + - type: volume + source: mailu_dkim + target: /dkim +# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" + - type: volume + source: mailu_overrides_rspamd + target: /etc/rspamd/override.d + deploy: + replicas: 1 + + antivirus: + image: mailu/none:master + restart: always + env_file: .env + volumes: +# - "$ROOT/filter:/data" + - type: volume + source: mailu_filter + target: /data + deploy: + replicas: 1 + + webdav: + image: mailu/none:master + restart: always + env_file: .env + volumes: +# - "$ROOT/dav:/data" + - type: volume + source: mailu_dav + target: /data + deploy: + replicas: 1 + + admin: + image: mailu/admin:master + restart: always + env_file: .env + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data +# - "$ROOT/dkim:/dkim" + - type: volume + source: mailu_dkim + target: /dkim + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - redis + deploy: + replicas: 1 + + webmail: + image: "mailu/roundcube:master" + restart: always + env_file: .env + volumes: +# - "$ROOT/webmail:/data" + - type: volume + source: mailu_data + target: /data + depends_on: + - imap + deploy: + replicas: 2 + + fetchmail: + image: mailu/fetchmail:master + restart: always + env_file: .env + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data + deploy: + replicas: 1 + +networks: + default: + external: + name: mailu_default + +volumes: + mailu_filter: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/filter" + mailu_dkim: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/dkim" + mailu_overrides_rspamd: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/overrides/rspamd" + mailu_data: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/data" + mailu_mail: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/mail" + mailu_overrides: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/overrides" + mailu_dav: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/dav" + mailu_certs: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/certs" + mailu_redis: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/redis" +``` + +### Deploy Mailu on the docker swarm +Run the following command: +```bash +docker stack deploy -c docker-compose-stack.yml mailu +``` +See how the services are being deployed: +```bash +core@coreos-01 ~ $ docker service ls +ID NAME MODE REPLICAS IMAGE PORTS +ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:1.5 +pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:1.5 +``` +check a specific service: +```bash +core@coreos-01 ~ $ docker service ps mailu_fetchmail +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago +``` + +### Remove the stack +Run the follwoing command: +```bash +core@coreos-01 ~ $ docker stack rm mailu +``` diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index 6a020672..0cb0d5c0 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,3 +1,3 @@ bind_socket = "*:11334"; password = "mailu"; -secure_ip = "{{ FRONT_ADDRESS }}"; +secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}"; From f5f09fad6ecc41e64b90dc4044450b61ea19ae4a Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Tue, 25 Sep 2018 18:54:40 +0000 Subject: [PATCH 03/21] Reverting the patch for dovecot.conf, as it is not needed --- core/dovecot/conf/dovecot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 7cf10774..a5973bf8 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -5,7 +5,7 @@ log_path = /dev/stderr protocols = imap pop3 lmtp sieve postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }} hostname = {{ HOSTNAMES.split(",")[0] }} -submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }} +submission_host = {{ FRONT_ADDRESS }} service dict { unix_listener dict { From 6b34b2728ece020918dcb10a901afdd9894a69e3 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Sun, 7 Oct 2018 16:38:41 +0000 Subject: [PATCH 04/21] Declare fuzzy_worker port 11335 in EXPOSE section --- services/rspamd/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index 7dff8c1f..cfb4d0eb 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir /run/rspamd COPY conf/ /conf COPY start.py /start.py -EXPOSE 11332/tcp 11334/tcp +EXPOSE 11332/tcp 11334/tcp 11335/tcp VOLUME ["/var/lib/rspamd"] From 1f71d10899d72571b39a60448a23adc41d5f10f2 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Sun, 7 Oct 2018 16:46:42 +0000 Subject: [PATCH 05/21] Change POD_ADDRESS_RANGE introduction like it is done on deovecot-sql.conf.ext --- services/rspamd/conf/worker-controller.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index 0cb0d5c0..4b23a897 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,3 +1,3 @@ bind_socket = "*:11334"; password = "mailu"; -secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}"; +secure_ip = "{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% endif %}"; From 9d610f56f7ff39232cb15ce02b42f8aca94d846a Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Mon, 8 Oct 2018 18:53:44 +0000 Subject: [PATCH 06/21] Added some lines around ingress mode --- docs/swarm/master/README.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index 1406e05c..c09f1dd3 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -57,8 +57,11 @@ core@coreos-01 ~ $ sudo umount /mnt/local/ ### Networking mode -On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. +On this example, we are using: +- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. +- the default ingress mode. +## Allow authentification with the mesh routing In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE. Let's create the mailu_default network: @@ -68,8 +71,22 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet "Subnet": "10.0.1.0/24", ``` In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 +In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation... + +## Limitation with the ingress mode +With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network) + +This issue is known and discussed here: + +https://github.com/moby/moby/issues/25526 + +A workaround (using network host mode and global deployment) is discussed here: + +https://github.com/moby/moby/issues/25526#issuecomment-336363408 + +## Don't create an open relay ! +As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-( -Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network) ### Scalability - smtp and imap are scalable From 6bd365e7714e765c75d8fdd93bc04065a53eae14 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Mon, 8 Oct 2018 21:00:44 +0000 Subject: [PATCH 07/21] Change title layout --- docs/swarm/master/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index c09f1dd3..2a2a021a 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -56,12 +56,12 @@ core@coreos-01 ~ $ sudo umount /mnt/local/ ``` -### Networking mode +## Networking mode On this example, we are using: - the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. - the default ingress mode. -## Allow authentification with the mesh routing +### Allow authentification with the mesh routing In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE. Let's create the mailu_default network: @@ -73,7 +73,7 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation... -## Limitation with the ingress mode +### Limitation with the ingress mode With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network) This issue is known and discussed here: @@ -84,22 +84,22 @@ A workaround (using network host mode and global deployment) is discussed here: https://github.com/moby/moby/issues/25526#issuecomment-336363408 -## Don't create an open relay ! +### Don't create an open relay ! As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-( -### Scalability +## Scalability - smtp and imap are scalable - front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) - redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) -### Variable substitution and docker-compose.yml +## Variable substitution and docker-compose.yml The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to : - remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc - change the way we define the volumes (nfs share in our case) - add a deploy section for every service -### Docker compose +## Docker compose An example of docker-compose-stack.yml file is available here: ```yaml @@ -340,7 +340,7 @@ volumes: device: ":/mnt/Pool1/pv/mailu/redis" ``` -### Deploy Mailu on the docker swarm +## Deploy Mailu on the docker swarm Run the following command: ```bash docker stack deploy -c docker-compose-stack.yml mailu @@ -359,7 +359,7 @@ ID NAME IMAGE NODE tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago ``` -### Remove the stack +## Remove the stack Run the follwoing command: ```bash core@coreos-01 ~ $ docker stack rm mailu From 0a5dbf6230d9749dbf1a1129b606abe6f4dad0fc Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 17:41:21 +0200 Subject: [PATCH 08/21] Re-enable local dovecot sieve scripts --- core/dovecot/conf/dovecot.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index d6c4eb21..a9ec2676 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -136,7 +136,8 @@ service managesieve { } plugin { - sieve = dict:proxy:/tmp/podop.socket:sieve + sieve = file:~/sieve;active=~/.dovecot.sieve + sieve_before = dict:proxy:/tmp/podop.socket:sieve sieve_plugins = sieve_imapsieve sieve_extprograms sieve_extensions = +spamtest +spamtestplus +editheader sieve_global_extensions = +vnd.dovecot.execute From b9b4a8cd77c4a483d7dbf70314db777f021f0443 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:47:55 +0200 Subject: [PATCH 09/21] Explicitely specify the fuzzy worker listen address --- services/rspamd/conf/worker-fuzzy.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc index a0021a03..0f71ba32 100644 --- a/services/rspamd/conf/worker-fuzzy.inc +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -1,4 +1,5 @@ type = "fuzzy"; +bind_socket = "*:11335"; count = 1; backend = "redis"; expire = 90d; From ce0bf3366d2193b01e34c04ede5f870c94d250c3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:48:28 +0200 Subject: [PATCH 10/21] Learn fuzzy hashes automatically --- core/dovecot/Dockerfile | 2 +- core/dovecot/conf/bin/ham | 4 ++++ core/dovecot/conf/bin/mailtrain | 3 --- core/dovecot/conf/bin/spam | 4 ++++ core/dovecot/conf/report-ham.sieve | 2 +- core/dovecot/conf/report-spam.sieve | 2 +- 6 files changed, 11 insertions(+), 6 deletions(-) create mode 100755 core/dovecot/conf/bin/ham delete mode 100755 core/dovecot/conf/bin/mailtrain create mode 100755 core/dovecot/conf/bin/spam diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index d8d4c55b..41437e23 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.8 RUN apk add --no-cache \ dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ - python3 py3-pip \ + bash python3 py3-pip \ && pip3 install --upgrade pip \ && pip3 install jinja2 podop tenacity diff --git a/core/dovecot/conf/bin/ham b/core/dovecot/conf/bin/ham new file mode 100755 index 00000000..c74a97bd --- /dev/null +++ b/core/dovecot/conf/bin/ham @@ -0,0 +1,4 @@ +#!/bin/bash + +tee >(rspamc -h antispam:11334 -P mailu learn_ham /dev/stdin) \ + | rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /dev/stdin \ No newline at end of file diff --git a/core/dovecot/conf/bin/mailtrain b/core/dovecot/conf/bin/mailtrain deleted file mode 100755 index cfa36398..00000000 --- a/core/dovecot/conf/bin/mailtrain +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -rspamc -h antispam:11334 -P mailu "learn_$1" /dev/stdin <&0 diff --git a/core/dovecot/conf/bin/spam b/core/dovecot/conf/bin/spam new file mode 100755 index 00000000..e6a66f89 --- /dev/null +++ b/core/dovecot/conf/bin/spam @@ -0,0 +1,4 @@ +#!/bin/bash + +tee >(rspamc -h antispam:11334 -P mailu learn_spam /dev/stdin) \ + >(rspamc -h antispam:11334 -P mailu -f 11 fuzzy_add /dev/stdin) \ No newline at end of file diff --git a/core/dovecot/conf/report-ham.sieve b/core/dovecot/conf/report-ham.sieve index 1ad8abdf..0c69d67b 100644 --- a/core/dovecot/conf/report-ham.sieve +++ b/core/dovecot/conf/report-ham.sieve @@ -8,4 +8,4 @@ if string "${mailbox}" "Trash" { stop; } -execute :pipe "mailtrain" "ham"; +execute :pipe "ham"; diff --git a/core/dovecot/conf/report-spam.sieve b/core/dovecot/conf/report-spam.sieve index b2a544a6..108d6210 100644 --- a/core/dovecot/conf/report-spam.sieve +++ b/core/dovecot/conf/report-spam.sieve @@ -1,3 +1,3 @@ require "vnd.dovecot.execute"; -execute :pipe "mailtrain" "spam"; +execute :pipe "spam"; From d5162328eca5e2344c4ca1e432ec8f31bc5b7e1f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:48:59 +0200 Subject: [PATCH 11/21] Allow dovecot to write the source configuration directory for compiling sieve scripts --- core/dovecot/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index afd0513e..b65b4db9 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -36,5 +36,5 @@ for dovecot_file in glob.glob("/conf/*.conf"): # Run Podop, then postfix multiprocessing.Process(target=start_podop).start() -os.system("chown -R mail:mail /mail /var/lib/dovecot") +os.system("chown -R mail:mail /mail /var/lib/dovecot /conf") os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"]) From 86bdce840774f7c7170d4eccfc7dc9e9d3d9ea5e Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Wed, 17 Oct 2018 18:49:19 +0000 Subject: [PATCH 12/21] Explicitely specify the fuzzy worker listen address --- services/rspamd/conf/worker-fuzzy.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc index a0021a03..0f71ba32 100644 --- a/services/rspamd/conf/worker-fuzzy.inc +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -1,4 +1,5 @@ type = "fuzzy"; +bind_socket = "*:11335"; count = 1; backend = "redis"; expire = 90d; From dba8f1810d3d9271ff9ca9dcc21521d3bef7327c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 21:21:00 +0200 Subject: [PATCH 13/21] Do not check the password another time in Dovecot --- core/admin/mailu/internal/views/dovecot.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index c2f53794..036140f0 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -1,14 +1,24 @@ -from mailu import db, models +from mailu import db, models, app from mailu.internal import internal import flask +import socket @internal.route("/dovecot/passdb/") def dovecot_passdb_dict(user_email): user = models.User.query.get(user_email) or flask.abort(404) + allow_nets = [] + allow_nets.append( + app.config.get("POD_ADDRESS_RANGE") or + socket.gethostbyname(app.config["HOST_FRONT"]) + ) + allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"])) + print(allow_nets) return flask.jsonify({ - "password": user.password, + "password": None, + "nopassword": "Y", + "allow_nets": ",".join(allow_nets) }) From 988e09e65eecab6daae5f090556826b0310a434a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 21:21:24 +0200 Subject: [PATCH 14/21] Add a profiler in debug mode for improving performance --- core/admin/mailu/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 167f04ae..a73e6ab9 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -12,7 +12,7 @@ import docker import socket import uuid -from werkzeug.contrib import fixers +from werkzeug.contrib import fixers, profiler # Create application app = flask.Flask(__name__) @@ -62,7 +62,10 @@ default_config = { 'HOST_IMAP': 'imap', 'HOST_POP3': 'imap', 'HOST_SMTP': 'smtp', + 'HOST_WEBMAIL': 'webmail', + 'HOST_FRONT': 'front', 'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'), + 'POD_ADDRESS_RANGE': None } # Load configuration from the environment if available @@ -80,6 +83,10 @@ if app.config.get("DEBUG"): import flask_debugtoolbar toolbar = flask_debugtoolbar.DebugToolbarExtension(app) +# Profiler +if app.config.get("DEBUG"): + app.wsgi_app = profiler.ProfilerMiddleware(app.wsgi_app, restrictions=[30]) + # Manager commnad manager = flask_script.Manager(app) manager.add_command('db', flask_migrate.MigrateCommand) @@ -129,4 +136,5 @@ class PrefixMiddleware(object): environ['SCRIPT_NAME'] = prefix return self.app(environ, start_response) + app.wsgi_app = PrefixMiddleware(fixers.ProxyFix(app.wsgi_app)) From 01fa1797674e2badeaa42292172807107338c41a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 21:21:35 +0200 Subject: [PATCH 15/21] Update the user password in database when needed --- core/admin/mailu/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 0c80fd4f..51d07f24 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -276,7 +276,8 @@ class User(Base, Email): else: return self.email - scheme_dict = {'BLF-CRYPT': "bcrypt", + scheme_dict = {'PBKDF2': "pbkdf2_sha512", + 'BLF-CRYPT': "bcrypt", 'SHA512-CRYPT': "sha512_crypt", 'SHA256-CRYPT': "sha256_crypt", 'MD5-CRYPT': "md5_crypt", @@ -287,8 +288,14 @@ class User(Base, Email): ) def check_password(self, password): + context = User.pw_context reference = re.match('({[^}]+})?(.*)', self.password).group(2) - return User.pw_context.verify(password, reference) + result = context.verify(password, reference) + if result and context.identify(reference) != context.default_scheme(): + self.set_password(password) + db.session.add(self) + db.session.commit() + return result def set_password(self, password, hash_scheme=app.config['PASSWORD_SCHEME'], raw=False): """Set password for user with specified encryption scheme From 828d96f8f01da044611950b3df57137ef1797c11 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 21:26:44 +0200 Subject: [PATCH 16/21] Switch the default password scheme to PBKDF2 --- docs/compose/.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/compose/.env b/docs/compose/.env index 7823bc3e..ce8e9eb9 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -130,8 +130,8 @@ LOG_DRIVER=json-file COMPOSE_PROJECT_NAME=mailu # Default password scheme used for newly created accounts and changed passwords -# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) -PASSWORD_SCHEME=BLF-CRYPT +# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) +PASSWORD_SCHEME=PBKDF2 # Header to take the real ip from REAL_IP_HEADER= From d4f32c3e7d1d4150cc25a9e930ad7737521a65d3 Mon Sep 17 00:00:00 2001 From: hoellen Date: Thu, 18 Oct 2018 14:27:28 +0200 Subject: [PATCH 17/21] remove rewrite if webmail is on root --- core/nginx/conf/nginx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 8fcda1c3..7eaba003 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -91,8 +91,10 @@ http { {% endif %} location {{ WEB_WEBMAIL }} { + {% if WEB_WEBMAIL != '/' %} rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent; rewrite ^{{ WEB_WEBMAIL }}/(.*) /$1 break; + {% endif %} include /etc/nginx/proxy.conf; client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; proxy_pass http://$webmail; From 9c639eebd48cfc8b49f73345a13bcb33303054d6 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 18 Oct 2018 18:46:59 +0000 Subject: [PATCH 18/21] Made the instructions more simple (moved the nfs-volume example to another file) Removed he part around variable substitution as we can use a "trick" --- docs/swarm/master/README.md | 192 +++---------- docs/swarm/master/README_nfs_example.md | 357 ++++++++++++++++++++++++ 2 files changed, 394 insertions(+), 155 deletions(-) create mode 100644 docs/swarm/master/README_nfs_example.md diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index 2a2a021a..c6120228 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -29,34 +29,7 @@ mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active ### Volume definition For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. -Hereafter we will use a NFS share: -```bash -core@coreos-01 ~ $ showmount -e 192.168.0.30 -Export list for 192.168.0.30: -/mnt/Pool1/pv 192.168.0.0 -``` - -on the nfs server, I am using the following /etc/exports -```bash -$more /etc/exports -/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 -``` -on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up) -```bash -$mkdir /mnt/Pool1/pv/mailu -``` - -On your manager node, mount the nfs share to check that the share is available: -```bash -core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/ -``` -If this is ok, you can umount it: -```bash -core@coreos-01 ~ $ sudo umount /mnt/local/ -``` - - -## Networking mode +Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs". (GlusterFS and nfs shares have been successfully used) On this example, we are using: - the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. - the default ingress mode. @@ -94,10 +67,13 @@ As a side effect of this ingress mode "feature", make sure that the ingress subn - redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) ## Variable substitution and docker-compose.yml -The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to : -- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc +The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around: +``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` + +We need also to: - change the way we define the volumes (nfs share in our case) - add a deploy section for every service +- the way the ports are defined for the front service ## Docker compose An example of docker-compose-stack.yml file is available here: @@ -109,7 +85,7 @@ version: '3.2' services: front: - image: mailu/nginx:master + image: mailu/nginx:$VERSION restart: always env_file: .env ports: @@ -132,10 +108,7 @@ services: - target: 587 published: 587 volumes: -# - "$ROOT/certs:/certs" - - type: volume - source: mailu_certs - target: /certs + - "$ROOT/certs:/certs" deploy: replicas: 2 @@ -143,118 +116,77 @@ services: image: redis:alpine restart: always volumes: -# - "$ROOT/redis:/data" - - type: volume - source: mailu_redis - target: /data + - "$ROOT/redis:/data" deploy: replicas: 1 imap: - image: mailu/dovecot:master + image: mailu/dovecot:$VERSION restart: always env_file: .env environment: - POD_ADDRESS_RANGE=10.0.1.0/24 volumes: -# - "$ROOT/data:/data" - - type: volume - source: mailu_data - target: /data -# - "$ROOT/mail:/mail" - - type: volume - source: mailu_mail - target: /mail -# - "$ROOT/overrides:/overrides" - - type: volume - source: mailu_overrides - target: /overrides + - "$ROOT/mail:/mail" + - "$ROOT/overrides:/overrides" depends_on: - front deploy: replicas: 2 smtp: - image: mailu/postfix:master + image: mailu/postfix:$VERSION restart: always env_file: .env environment: - POD_ADDRESS_RANGE=10.0.1.0/24 volumes: -# - "$ROOT/data:/data" - - type: volume - source: mailu_data - target: /data -# - "$ROOT/overrides:/overrides" - - type: volume - source: mailu_overrides - target: /overrides + - "$ROOT/overrides:/overrides" depends_on: - front deploy: replicas: 2 antispam: - image: mailu/rspamd:master + image: mailu/rspamd:$VERSION restart: always env_file: .env environment: - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: + - "$ROOT/filter:/var/lib/rspamd" + - "$ROOT/dkim:/dkim" + - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" depends_on: - front - volumes: -# - "$ROOT/filter:/var/lib/rspamd" - - type: volume - source: mailu_filter - target: /var/lib/rspamd -# - "$ROOT/dkim:/dkim" - - type: volume - source: mailu_dkim - target: /dkim -# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" - - type: volume - source: mailu_overrides_rspamd - target: /etc/rspamd/override.d deploy: replicas: 1 antivirus: - image: mailu/none:master + image: mailu/none:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/filter:/data" - - type: volume - source: mailu_filter - target: /data + - "$ROOT/filter:/data" deploy: replicas: 1 webdav: - image: mailu/none:master + image: mailu/none:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/dav:/data" - - type: volume - source: mailu_dav - target: /data + - "$ROOT/dav:/data" deploy: replicas: 1 admin: - image: mailu/admin:master + image: mailu/admin:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/data:/data" - - type: volume - source: mailu_data - target: /data -# - "$ROOT/dkim:/dkim" - - type: volume - source: mailu_dkim - target: /dkim + - "$ROOT/data:/data" + - "$ROOT/dkim:/dkim" - /var/run/docker.sock:/var/run/docker.sock:ro depends_on: - redis @@ -262,28 +194,21 @@ services: replicas: 1 webmail: - image: "mailu/roundcube:master" + image: mailu/roundcube:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/webmail:/data" - - type: volume - source: mailu_data - target: /data + - "$ROOT/webmail:/data" depends_on: - imap deploy: replicas: 2 fetchmail: - image: mailu/fetchmail:master + image: mailu/fetchmail:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/data:/data" - - type: volume - source: mailu_data - target: /data deploy: replicas: 1 @@ -291,72 +216,29 @@ networks: default: external: name: mailu_default - -volumes: - mailu_filter: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/filter" - mailu_dkim: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/dkim" - mailu_overrides_rspamd: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/overrides/rspamd" - mailu_data: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/data" - mailu_mail: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/mail" - mailu_overrides: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/overrides" - mailu_dav: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/dav" - mailu_certs: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/certs" - mailu_redis: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/redis" ``` ## Deploy Mailu on the docker swarm Run the following command: ```bash -docker stack deploy -c docker-compose-stack.yml mailu +echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` See how the services are being deployed: ```bash core@coreos-01 ~ $ docker service ls ID NAME MODE REPLICAS IMAGE PORTS -ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:1.5 -pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:1.5 +ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:master +pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:master ``` check a specific service: ```bash core@coreos-01 ~ $ docker service ps mailu_fetchmail ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS -tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago +tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:master coreos-01 Running Running 11 days ago +``` +You might also have a look on the logs: +```bash +core@coreos-01 ~ $ docker service logs -f mailu_fetchmail ``` ## Remove the stack diff --git a/docs/swarm/master/README_nfs_example.md b/docs/swarm/master/README_nfs_example.md new file mode 100644 index 00000000..5cfd0a73 --- /dev/null +++ b/docs/swarm/master/README_nfs_example.md @@ -0,0 +1,357 @@ +# Install Mailu on a docker swarm + +## Prequisites + +### Swarm + +In order to deploy Mailu on a swarm, you will first need to initialize the swarm: + +The main command will be: +```bash +docker swarm init --advertise-addr +``` +See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/ + +If you want to add other managers or workers, please use: +```bash +docker swarm join --token xxxxx +``` +See https://docs.docker.com/engine/swarm/join-nodes/ + +You have now a working swarm, and you can check its status with: +```bash +core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls +ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION +xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce +sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce +mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce +``` + +### Volume definition +For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. +Hereafter we will use a NFS share: +```bash +core@coreos-01 ~ $ showmount -e 192.168.0.30 +Export list for 192.168.0.30: +/mnt/Pool1/pv 192.168.0.0 +``` + +on the nfs server, I am using the following /etc/exports +```bash +$more /etc/exports +/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 +``` +on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up) +```bash +$mkdir /mnt/Pool1/pv/mailu +``` + +On your manager node, mount the nfs share to check that the share is available: +```bash +core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/ +``` +If this is ok, you can umount it: +```bash +core@coreos-01 ~ $ sudo umount /mnt/local/ +``` + + +## Networking mode +On this example, we are using: +- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. +- the default ingress mode. + +### Allow authentification with the mesh routing +In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE. + +Let's create the mailu_default network: +```bash +core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default +core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet + "Subnet": "10.0.1.0/24", +``` +In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 +In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation... + +### Limitation with the ingress mode +With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network) + +This issue is known and discussed here: + +https://github.com/moby/moby/issues/25526 + +A workaround (using network host mode and global deployment) is discussed here: + +https://github.com/moby/moby/issues/25526#issuecomment-336363408 + +### Don't create an open relay ! +As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-( + + +## Scalability +- smtp and imap are scalable +- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) +- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) + +## Variable substitution and docker-compose.yml +The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around: +``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` + +We need also to: +- change the way we define the volumes (nfs share in our case) +- add a deploy section for every service +- the way the ports are defined for the front service + +## Docker compose +An example of docker-compose-stack.yml file is available here: + +```yaml + +version: '3.2' + +services: + + front: + image: mailu/nginx:$VERSION + restart: always + env_file: .env + ports: + - target: 80 + published: 80 + - target: 443 + published: 443 + - target: 110 + published: 110 + - target: 143 + published: 143 + - target: 993 + published: 993 + - target: 995 + published: 995 + - target: 25 + published: 25 + - target: 465 + published: 465 + - target: 587 + published: 587 + volumes: +# - "$ROOT/certs:/certs" + - type: volume + source: mailu_certs + target: /certs + deploy: + replicas: 2 + + redis: + image: redis:alpine + restart: always + volumes: +# - "$ROOT/redis:/data" + - type: volume + source: mailu_redis + target: /data + deploy: + replicas: 1 + + imap: + image: mailu/dovecot:$VERSION + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: +# - "$ROOT/mail:/mail" + - type: volume + source: mailu_mail + target: /mail +# - "$ROOT/overrides:/overrides" + - type: volume + source: mailu_overrides + target: /overrides + depends_on: + - front + deploy: + replicas: 2 + + smtp: + image: mailu/postfix:$VERSION + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: +# - "$ROOT/overrides:/overrides" + - type: volume + source: mailu_overrides + target: /overrides + depends_on: + - front + deploy: + replicas: 2 + + antispam: + image: mailu/rspamd:$VERSION + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + depends_on: + - front + volumes: +# - "$ROOT/filter:/var/lib/rspamd" + - type: volume + source: mailu_filter + target: /var/lib/rspamd +# - "$ROOT/dkim:/dkim" + - type: volume + source: mailu_dkim + target: /dkim +# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" + - type: volume + source: mailu_overrides_rspamd + target: /etc/rspamd/override.d + deploy: + replicas: 1 + + antivirus: + image: mailu/none:$VERSION + restart: always + env_file: .env + volumes: +# - "$ROOT/filter:/data" + - type: volume + source: mailu_filter + target: /data + deploy: + replicas: 1 + + webdav: + image: mailu/none:$VERSION + restart: always + env_file: .env + volumes: +# - "$ROOT/dav:/data" + - type: volume + source: mailu_dav + target: /data + deploy: + replicas: 1 + + admin: + image: mailu/admin:$VERSION + restart: always + env_file: .env + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data +# - "$ROOT/dkim:/dkim" + - type: volume + source: mailu_dkim + target: /dkim + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - redis + deploy: + replicas: 1 + + webmail: + image: mailu/roundcube:$VERSION + restart: always + env_file: .env + volumes: +# - "$ROOT/webmail:/data" + - type: volume + source: mailu_data + target: /data + depends_on: + - imap + deploy: + replicas: 2 + + fetchmail: + image: mailu/fetchmail:$VERSION + restart: always + env_file: .env + volumes: + deploy: + replicas: 1 + +networks: + default: + external: + name: mailu_default + +volumes: + mailu_filter: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/filter" + mailu_dkim: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/dkim" + mailu_overrides_rspamd: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/overrides/rspamd" + mailu_data: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/data" + mailu_mail: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/mail" + mailu_overrides: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/overrides" + mailu_dav: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/dav" + mailu_certs: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/certs" + mailu_redis: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/redis" +``` + +## Deploy Mailu on the docker swarm +Run the following command: +```bash +echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu +``` +See how the services are being deployed: +```bash +core@coreos-01 ~ $ docker service ls +ID NAME MODE REPLICAS IMAGE PORTS +ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:master +pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:master +``` +check a specific service: +```bash +core@coreos-01 ~ $ docker service ps mailu_fetchmail +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:master coreos-01 Running Running 11 days ago +``` + +## Remove the stack +Run the follwoing command: +```bash +core@coreos-01 ~ $ docker stack rm mailu +``` From 5b8deed06ba8e67ab592a20d104033a9ee63445e Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 18 Oct 2018 18:59:22 +0000 Subject: [PATCH 19/21] Made the instructions more simple Updated the volume defintion to the latest master status Removed the part around .yml variable substitution as we can use a "trick" --- docs/swarm/master/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index c6120228..ba5e0011 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -29,7 +29,9 @@ mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active ### Volume definition For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. -Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs". (GlusterFS and nfs shares have been successfully used) + +Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs" (GlusterFS and nfs shares have been successfully used). + On this example, we are using: - the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. - the default ingress mode. @@ -67,13 +69,16 @@ As a side effect of this ingress mode "feature", make sure that the ingress subn - redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) ## Variable substitution and docker-compose.yml -The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around: +The docker stack deploy command doesn't support variable substitution in the .yml file itself. +As a consequence, we cannot simply use ``` docker stack deploy -c docker.compose.yml mailu ``` +Instead, we will use the following work-around: ``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` We need also to: - change the way we define the volumes (nfs share in our case) - add a deploy section for every service - the way the ports are defined for the front service +- add the POD_ADDRESS_RANGE definition imap, smtp and antispam services ## Docker compose An example of docker-compose-stack.yml file is available here: From 7a3922c2e733a9d4c9a58910e86a7dcf032e6788 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 18 Oct 2018 19:14:53 +0000 Subject: [PATCH 20/21] Fixes few typos --- docs/swarm/master/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index ba5e0011..44c19dc7 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -75,10 +75,9 @@ Instead, we will use the following work-around: ``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` We need also to: -- change the way we define the volumes (nfs share in our case) - add a deploy section for every service -- the way the ports are defined for the front service -- add the POD_ADDRESS_RANGE definition imap, smtp and antispam services +- modify the way the ports are defined for the front service +- add the POD_ADDRESS_RANGE definition for imap, smtp and antispam services ## Docker compose An example of docker-compose-stack.yml file is available here: From 771e0ee6a2d21393a83b7eabd343355c03882897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 19 Oct 2018 16:29:47 +0300 Subject: [PATCH 21/21] Remove old crypt settings from .env As per conversation in PR --- docs/compose/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compose/.env b/docs/compose/.env index ce8e9eb9..2100e27a 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -130,7 +130,7 @@ LOG_DRIVER=json-file COMPOSE_PROJECT_NAME=mailu # Default password scheme used for newly created accounts and changed passwords -# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) +# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT) PASSWORD_SCHEME=PBKDF2 # Header to take the real ip from