From d0f759accae612eec2520de97eaad3fa8aebc527 Mon Sep 17 00:00:00 2001
From: Patrick Oberdorf
Date: Wed, 14 Feb 2018 12:15:45 +0100
Subject: [PATCH 01/93] Adding unbound as dns resolver
---
core/unbound/Dockerfile | 14 +++++
core/unbound/unbound.conf | 19 +++++++
docs/compose/.env | 3 ++
docs/compose/docker-compose.yml | 94 ++++++++++++++++++++++++++++++++-
4 files changed, 129 insertions(+), 1 deletion(-)
create mode 100644 core/unbound/Dockerfile
create mode 100644 core/unbound/unbound.conf
diff --git a/core/unbound/Dockerfile b/core/unbound/Dockerfile
new file mode 100644
index 00000000..6ae8a6ee
--- /dev/null
+++ b/core/unbound/Dockerfile
@@ -0,0 +1,14 @@
+FROM alpine:edge
+
+RUN apk add --no-cache unbound curl \
+ && curl -o /etc/unbound/root.hints https://www.internic.net/domain/named.cache \
+ && chown root:unbound /etc/unbound \
+ && chmod 775 /etc/unbound \
+ && apk del --no-cache curl \
+ && /usr/sbin/unbound-anchor -a /etc/unbound/trusted-key.key | true
+
+COPY unbound.conf /etc/unbound/unbound.conf
+
+EXPOSE 53/udp 53/tcp
+
+CMD /usr/sbin/unbound
diff --git a/core/unbound/unbound.conf b/core/unbound/unbound.conf
new file mode 100644
index 00000000..d2d9ce74
--- /dev/null
+++ b/core/unbound/unbound.conf
@@ -0,0 +1,19 @@
+server:
+ verbosity: 1
+ interface: 0.0.0.0
+ interface: ::0
+ logfile: /dev/stdout
+ do-ip4: yes
+ do-ip6: yes
+ do-udp: yes
+ do-tcp: yes
+ do-daemonize: no
+ access-control: 0.0.0.0/0 allow
+ directory: "/etc/unbound"
+ username: unbound
+ auto-trust-anchor-file: trusted-key.key
+ root-hints: "/etc/unbound/root.hints"
+ hide-identity: yes
+ hide-version: yes
+ max-udp-size: 4096
+ msg-buffer-size: 65552
diff --git a/docs/compose/.env b/docs/compose/.env
index 6f330b64..e4c6dff9 100644
--- a/docs/compose/.env
+++ b/docs/compose/.env
@@ -21,6 +21,9 @@ SECRET_KEY=ChangeMeChangeMe
BIND_ADDRESS4=127.0.0.1
BIND_ADDRESS6=::1
+# Internal Docker network
+IPV4_NETWORK=172.22.1
+
# Main mail domain
DOMAIN=mailu.io
diff --git a/docs/compose/docker-compose.yml b/docs/compose/docker-compose.yml
index 740a5ffc..dc674a2b 100644
--- a/docs/compose/docker-compose.yml
+++ b/docs/compose/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '2'
+version: '2.1'
services:
@@ -27,12 +27,37 @@ services:
- "$BIND_ADDRESS6:587:587"
volumes:
- "$ROOT/certs:/certs"
+ depends_on:
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - front
+
+ unbound:
+ image: mailu/unbound:$VERSION
+ restart: always
+ networks:
+ backend:
+ ipv4_address: ${IPV4_NETWORK:-172.22.1}.254
+ aliases:
+ - unbound
redis:
image: redis:alpine
restart: always
volumes:
- "$ROOT/redis:/data"
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ depends_on:
+ - unbound
+ networks:
+ backend:
+ aliases:
+ - redis
imap:
image: mailu/dovecot:$VERSION
@@ -44,6 +69,13 @@ services:
- "$ROOT/overrides:/overrides"
depends_on:
- front
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - imap
smtp:
image: mailu/postfix:$VERSION
@@ -54,6 +86,13 @@ services:
- "$ROOT/overrides:/overrides"
depends_on:
- front
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - smtp
antispam:
image: mailu/rspamd:$VERSION
@@ -65,6 +104,13 @@ services:
- "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
depends_on:
- front
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - antispam
antivirus:
image: mailu/$ANTIVIRUS:$VERSION
@@ -72,6 +118,14 @@ services:
env_file: .env
volumes:
- "$ROOT/filter:/data"
+ depends_on:
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - antivirus
webdav:
image: mailu/$WEBDAV:$VERSION
@@ -79,6 +133,14 @@ services:
env_file: .env
volumes:
- "$ROOT/dav:/data"
+ depends_on:
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - webdav
admin:
image: mailu/admin:$VERSION
@@ -90,6 +152,13 @@ services:
- /var/run/docker.sock:/var/run/docker.sock:ro
depends_on:
- redis
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - admin
webmail:
image: "mailu/$WEBMAIL:$VERSION"
@@ -99,6 +168,13 @@ services:
- "$ROOT/webmail:/data"
depends_on:
- imap
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - webmail
fetchmail:
image: mailu/fetchmail:$VERSION
@@ -106,3 +182,19 @@ services:
env_file: .env
volumes:
- "$ROOT/data:/data"
+ depends_on:
+ - unbound
+ dns:
+ - ${IPV4_NETWORK:-172.22.1}.254
+ networks:
+ backend:
+ aliases:
+ - fetchmail
+
+networks:
+ backend:
+ driver: bridge
+ ipam:
+ driver: default
+ config:
+ - subnet: ${IPV4_NETWORK:-172.22.1}.0/24
From 12294a6e5aa3c57c8c603d97a35fa443a962a8e2 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Wed, 6 Jun 2018 18:40:51 +0000
Subject: [PATCH 02/93] Trying to enable fuzzy hashes for rspamd
---
services/rspamd/Dockerfile | 5 +---
services/rspamd/conf/fuzzy_check.conf | 34 ++++++++++++++++++++++
services/rspamd/conf/metrics.conf | 19 ++++++++++++
services/rspamd/conf/worker-controller.inc | 1 +
services/rspamd/conf/worker-fuzzy.inc | 5 ++++
services/rspamd/conf/worker-normal.inc | 1 +
6 files changed, 61 insertions(+), 4 deletions(-)
create mode 100644 services/rspamd/conf/fuzzy_check.conf
create mode 100644 services/rspamd/conf/metrics.conf
create mode 100644 services/rspamd/conf/worker-fuzzy.inc
diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile
index c6c2afdd..1b8d7e6b 100644
--- a/services/rspamd/Dockerfile
+++ b/services/rspamd/Dockerfile
@@ -1,15 +1,12 @@
FROM alpine:edge
-RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates
+RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates
RUN mkdir /run/rspamd
COPY conf/ /conf
COPY start.py /start.py
-# Temporary fix to remove references to rspamd-fuzzy for now
-RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
-
EXPOSE 11332/tcp 11334/tcp
CMD /start.py
diff --git a/services/rspamd/conf/fuzzy_check.conf b/services/rspamd/conf/fuzzy_check.conf
new file mode 100644
index 00000000..7c87e1c3
--- /dev/null
+++ b/services/rspamd/conf/fuzzy_check.conf
@@ -0,0 +1,34 @@
+rule "local" {
+ # Fuzzy storage server list
+ servers = "localhost:11335";
+ # Default symbol for unknown flags
+ symbol = "LOCAL_FUZZY_UNKNOWN";
+ # Additional mime types to store/check
+ mime_types = ["application/*"];
+ # Hash weight threshold for all maps
+ max_score = 20.0;
+ # Whether we can learn this storage
+ read_only = no;
+ # Ignore unknown flags
+ skip_unknown = yes;
+ # Hash generation algorithm
+ algorithm = "mumhash";
+
+ # Map flags to symbols
+ fuzzy_map = {
+ LOCAL_FUZZY_DENIED {
+ # Local threshold
+ max_score = 20.0;
+ # Flag to match
+ flag = 11;
+ }
+ LOCAL_FUZZY_PROB {
+ max_score = 10.0;
+ flag = 12;
+ }
+ LOCAL_FUZZY_WHITE {
+ max_score = 2.0;
+ flag = 13;
+ }
+ }
+}
diff --git a/services/rspamd/conf/metrics.conf b/services/rspamd/conf/metrics.conf
new file mode 100644
index 00000000..6a31964f
--- /dev/null
+++ b/services/rspamd/conf/metrics.conf
@@ -0,0 +1,19 @@
+group "fuzzy" {
+ max_score = 12.0;
+ symbol "LOCAL_FUZZY_UNKNOWN" {
+ weight = 5.0;
+ description = "Generic fuzzy hash match";
+ }
+ symbol "LOCAL_FUZZY_DENIED" {
+ weight = 12.0;
+ description = "Denied fuzzy hash";
+ }
+ symbol "LOCAL_FUZZY_PROB" {
+ weight = 5.0;
+ description = "Probable fuzzy hash";
+ }
+ symbol "LOCAL_FUZZY_WHITE" {
+ weight = -2.1;
+ description = "Whitelisted fuzzy hash";
+ }
+}
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 6a020672..dd143942 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,4 @@
+type = "controller";
bind_socket = "*:11334";
password = "mailu";
secure_ip = "{{ FRONT_ADDRESS }}";
diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc
new file mode 100644
index 00000000..a0021a03
--- /dev/null
+++ b/services/rspamd/conf/worker-fuzzy.inc
@@ -0,0 +1,5 @@
+type = "fuzzy";
+count = 1;
+backend = "redis";
+expire = 90d;
+allow_update = ["127.0.0.1"];
diff --git a/services/rspamd/conf/worker-normal.inc b/services/rspamd/conf/worker-normal.inc
index a6ee8317..ab996fb8 100644
--- a/services/rspamd/conf/worker-normal.inc
+++ b/services/rspamd/conf/worker-normal.inc
@@ -1 +1,2 @@
+type = "normal";
enabled = false;
From 23e288aadcf7d5df941c8516fae9d0cbd6b1e054 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 24 Sep 2018 17:29:31 +0000
Subject: [PATCH 03/93] Enabling swarm deployment on master branch: -Extends
the usage of POD_ADDRESS_RANGE -Provides documentation
---
core/dovecot/conf/dovecot.conf | 2 +-
core/postfix/conf/main.cf | 2 +-
docs/swarm/master/README.md | 349 +++++++++++++++++++++
services/rspamd/conf/worker-controller.inc | 2 +-
4 files changed, 352 insertions(+), 3 deletions(-)
create mode 100644 docs/swarm/master/README.md
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index a5973bf8..7cf10774 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -5,7 +5,7 @@ log_path = /dev/stderr
protocols = imap pop3 lmtp sieve
postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
hostname = {{ HOSTNAMES.split(",")[0] }}
-submission_host = {{ FRONT_ADDRESS }}
+submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
service dict {
unix_listener dict {
diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf
index 2f2c6990..bde42dd1 100644
--- a/core/postfix/conf/main.cf
+++ b/core/postfix/conf/main.cf
@@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }}
recipient_delimiter = {{ RECIPIENT_DELIMITER }}
# Only the front server is allowed to perform xclient
-smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }}
+smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
###############
# TLS
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
new file mode 100644
index 00000000..1406e05c
--- /dev/null
+++ b/docs/swarm/master/README.md
@@ -0,0 +1,349 @@
+# Install Mailu on a docker swarm
+
+## Prequisites
+
+### Swarm
+
+In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
+
+The main command will be:
+```bash
+docker swarm init --advertise-addr
+```
+See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
+
+If you want to add other managers or workers, please use:
+```bash
+docker swarm join --token xxxxx
+```
+See https://docs.docker.com/engine/swarm/join-nodes/
+
+You have now a working swarm, and you can check its status with:
+```bash
+core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
+ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
+xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce
+sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce
+mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce
+```
+
+### Volume definition
+For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
+Hereafter we will use a NFS share:
+```bash
+core@coreos-01 ~ $ showmount -e 192.168.0.30
+Export list for 192.168.0.30:
+/mnt/Pool1/pv 192.168.0.0
+```
+
+on the nfs server, I am using the following /etc/exports
+```bash
+$more /etc/exports
+/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0
+```
+on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up)
+```bash
+$mkdir /mnt/Pool1/pv/mailu
+```
+
+On your manager node, mount the nfs share to check that the share is available:
+```bash
+core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/
+```
+If this is ok, you can umount it:
+```bash
+core@coreos-01 ~ $ sudo umount /mnt/local/
+```
+
+
+### Networking mode
+On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+
+In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
+
+Let's create the mailu_default network:
+```bash
+core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
+core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
+ "Subnet": "10.0.1.0/24",
+```
+In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
+
+Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network)
+
+### Scalability
+- smtp and imap are scalable
+- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time)
+- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
+
+### Variable substitution and docker-compose.yml
+The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to :
+- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc
+- change the way we define the volumes (nfs share in our case)
+- add a deploy section for every service
+
+### Docker compose
+An example of docker-compose-stack.yml file is available here:
+
+```yaml
+
+version: '3.2'
+
+services:
+
+ front:
+ image: mailu/nginx:master
+ restart: always
+ env_file: .env
+ ports:
+ - target: 80
+ published: 80
+ - target: 443
+ published: 443
+ - target: 110
+ published: 110
+ - target: 143
+ published: 143
+ - target: 993
+ published: 993
+ - target: 995
+ published: 995
+ - target: 25
+ published: 25
+ - target: 465
+ published: 465
+ - target: 587
+ published: 587
+ volumes:
+# - "$ROOT/certs:/certs"
+ - type: volume
+ source: mailu_certs
+ target: /certs
+ deploy:
+ replicas: 2
+
+ redis:
+ image: redis:alpine
+ restart: always
+ volumes:
+# - "$ROOT/redis:/data"
+ - type: volume
+ source: mailu_redis
+ target: /data
+ deploy:
+ replicas: 1
+
+ imap:
+ image: mailu/dovecot:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/mail:/mail"
+ - type: volume
+ source: mailu_mail
+ target: /mail
+# - "$ROOT/overrides:/overrides"
+ - type: volume
+ source: mailu_overrides
+ target: /overrides
+ depends_on:
+ - front
+ deploy:
+ replicas: 2
+
+ smtp:
+ image: mailu/postfix:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/overrides:/overrides"
+ - type: volume
+ source: mailu_overrides
+ target: /overrides
+ depends_on:
+ - front
+ deploy:
+ replicas: 2
+
+ antispam:
+ image: mailu/rspamd:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ depends_on:
+ - front
+ volumes:
+# - "$ROOT/filter:/var/lib/rspamd"
+ - type: volume
+ source: mailu_filter
+ target: /var/lib/rspamd
+# - "$ROOT/dkim:/dkim"
+ - type: volume
+ source: mailu_dkim
+ target: /dkim
+# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+ - type: volume
+ source: mailu_overrides_rspamd
+ target: /etc/rspamd/override.d
+ deploy:
+ replicas: 1
+
+ antivirus:
+ image: mailu/none:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/filter:/data"
+ - type: volume
+ source: mailu_filter
+ target: /data
+ deploy:
+ replicas: 1
+
+ webdav:
+ image: mailu/none:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/dav:/data"
+ - type: volume
+ source: mailu_dav
+ target: /data
+ deploy:
+ replicas: 1
+
+ admin:
+ image: mailu/admin:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/dkim:/dkim"
+ - type: volume
+ source: mailu_dkim
+ target: /dkim
+ - /var/run/docker.sock:/var/run/docker.sock:ro
+ depends_on:
+ - redis
+ deploy:
+ replicas: 1
+
+ webmail:
+ image: "mailu/roundcube:master"
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/webmail:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+ depends_on:
+ - imap
+ deploy:
+ replicas: 2
+
+ fetchmail:
+ image: mailu/fetchmail:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+ deploy:
+ replicas: 1
+
+networks:
+ default:
+ external:
+ name: mailu_default
+
+volumes:
+ mailu_filter:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/filter"
+ mailu_dkim:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/dkim"
+ mailu_overrides_rspamd:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/overrides/rspamd"
+ mailu_data:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/data"
+ mailu_mail:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/mail"
+ mailu_overrides:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/overrides"
+ mailu_dav:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/dav"
+ mailu_certs:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/certs"
+ mailu_redis:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/redis"
+```
+
+### Deploy Mailu on the docker swarm
+Run the following command:
+```bash
+docker stack deploy -c docker-compose-stack.yml mailu
+```
+See how the services are being deployed:
+```bash
+core@coreos-01 ~ $ docker service ls
+ID NAME MODE REPLICAS IMAGE PORTS
+ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:1.5
+pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:1.5
+```
+check a specific service:
+```bash
+core@coreos-01 ~ $ docker service ps mailu_fetchmail
+ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS
+tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago
+```
+
+### Remove the stack
+Run the follwoing command:
+```bash
+core@coreos-01 ~ $ docker stack rm mailu
+```
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 6a020672..0cb0d5c0 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,3 @@
bind_socket = "*:11334";
password = "mailu";
-secure_ip = "{{ FRONT_ADDRESS }}";
+secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}";
From f5f09fad6ecc41e64b90dc4044450b61ea19ae4a Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Tue, 25 Sep 2018 18:54:40 +0000
Subject: [PATCH 04/93] Reverting the patch for dovecot.conf, as it is not
needed
---
core/dovecot/conf/dovecot.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index 7cf10774..a5973bf8 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -5,7 +5,7 @@ log_path = /dev/stderr
protocols = imap pop3 lmtp sieve
postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
hostname = {{ HOSTNAMES.split(",")[0] }}
-submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
+submission_host = {{ FRONT_ADDRESS }}
service dict {
unix_listener dict {
From fcad52b1454aeeb5b11d64fe0186f644922f96b2 Mon Sep 17 00:00:00 2001
From: kaiyou
Date: Thu, 27 Sep 2018 22:45:16 +0200
Subject: [PATCH 05/93] Implement a start date filter for autoreply, fixes #362
---
core/admin/mailu/models.py | 2 ++
core/admin/mailu/ui/forms.py | 1 +
core/admin/mailu/ui/templates/user/reply.html | 7 ++++--
.../migrations/versions/3b281286c7bd_.py | 24 +++++++++++++++++++
core/dovecot/conf/pigeonhole-sieve.dict | 8 +++++++
core/dovecot/sieve/before.sieve | 1 +
6 files changed, 41 insertions(+), 2 deletions(-)
create mode 100644 core/admin/migrations/versions/3b281286c7bd_.py
diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index 71c5f4d7..1be4c8e6 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -249,6 +249,8 @@ class User(Base, Email):
reply_enabled = db.Column(db.Boolean(), nullable=False, default=False)
reply_subject = db.Column(db.String(255), nullable=True, default=None)
reply_body = db.Column(db.Text(), nullable=True, default=None)
+ reply_startdate = db.Column(db.Date, nullable=False,
+ default=date(1900, 1, 1))
reply_enddate = db.Column(db.Date, nullable=False,
default=date(2999, 12, 31))
diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py
index 326d721b..4f7a30ae 100644
--- a/core/admin/mailu/ui/forms.py
+++ b/core/admin/mailu/ui/forms.py
@@ -117,6 +117,7 @@ class UserReplyForm(flask_wtf.FlaskForm):
reply_subject = fields.StringField(_('Reply subject'))
reply_body = fields.StringField(_('Reply body'),
widget=widgets.TextArea())
+ reply_startdate = fields.html5.DateField(_('Start of vacation'))
reply_enddate = fields.html5.DateField(_('End of vacation'))
submit = fields.SubmitField(_('Update'))
diff --git a/core/admin/mailu/ui/templates/user/reply.html b/core/admin/mailu/ui/templates/user/reply.html
index 7906bc42..7225a178 100644
--- a/core/admin/mailu/ui/templates/user/reply.html
+++ b/core/admin/mailu/ui/templates/user/reply.html
@@ -13,14 +13,17 @@
{% endcall %}
diff --git a/core/admin/migrations/versions/3b281286c7bd_.py b/core/admin/migrations/versions/3b281286c7bd_.py
new file mode 100644
index 00000000..78e44a4c
--- /dev/null
+++ b/core/admin/migrations/versions/3b281286c7bd_.py
@@ -0,0 +1,24 @@
+""" Add a start day for vacations
+
+Revision ID: 3b281286c7bd
+Revises: 049fed905da7
+Create Date: 2018-09-27 22:20:08.158553
+
+"""
+
+revision = '3b281286c7bd'
+down_revision = '049fed905da7'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ with op.batch_alter_table('user') as batch:
+ batch.add_column(sa.Column('reply_startdate', sa.Date(), nullable=False,
+ server_default="1900-01-01"))
+
+
+def downgrade():
+ with op.batch_alter_table('user') as batch:
+ batch.drop_column('reply_startdate')
diff --git a/core/dovecot/conf/pigeonhole-sieve.dict b/core/dovecot/conf/pigeonhole-sieve.dict
index 917fce83..604371a8 100644
--- a/core/dovecot/conf/pigeonhole-sieve.dict
+++ b/core/dovecot/conf/pigeonhole-sieve.dict
@@ -41,3 +41,11 @@ map {
username_field = email
value_field = reply_enddate
}
+
+map {
+ pattern = priv/reply_startdate
+ table = user
+ username_field = email
+ value_field = reply_startdate
+}
+
diff --git a/core/dovecot/sieve/before.sieve b/core/dovecot/sieve/before.sieve
index 6ebc20c5..81d20f30 100644
--- a/core/dovecot/sieve/before.sieve
+++ b/core/dovecot/sieve/before.sieve
@@ -34,6 +34,7 @@ if exists "X-Virus" {
}
if allof (string :is "${extdata.reply_enabled}" "1",
+ currentdate :value "ge" "date" "${extdata.reply_startdate}",
currentdate :value "le" "date" "${extdata.reply_enddate}")
{
vacation :days 1 :subject "${extdata.reply_subject}" "${extdata.reply_body}";
From 6b34b2728ece020918dcb10a901afdd9894a69e3 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Sun, 7 Oct 2018 16:38:41 +0000
Subject: [PATCH 06/93] Declare fuzzy_worker port 11335 in EXPOSE section
---
services/rspamd/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile
index 7dff8c1f..cfb4d0eb 100644
--- a/services/rspamd/Dockerfile
+++ b/services/rspamd/Dockerfile
@@ -9,7 +9,7 @@ RUN mkdir /run/rspamd
COPY conf/ /conf
COPY start.py /start.py
-EXPOSE 11332/tcp 11334/tcp
+EXPOSE 11332/tcp 11334/tcp 11335/tcp
VOLUME ["/var/lib/rspamd"]
From 1f71d10899d72571b39a60448a23adc41d5f10f2 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Sun, 7 Oct 2018 16:46:42 +0000
Subject: [PATCH 07/93] Change POD_ADDRESS_RANGE introduction like it is done
on deovecot-sql.conf.ext
---
services/rspamd/conf/worker-controller.inc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 0cb0d5c0..4b23a897 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,3 @@
bind_socket = "*:11334";
password = "mailu";
-secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}";
+secure_ip = "{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% endif %}";
From 9d610f56f7ff39232cb15ce02b42f8aca94d846a Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 8 Oct 2018 18:53:44 +0000
Subject: [PATCH 08/93] Added some lines around ingress mode
---
docs/swarm/master/README.md | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
index 1406e05c..c09f1dd3 100644
--- a/docs/swarm/master/README.md
+++ b/docs/swarm/master/README.md
@@ -57,8 +57,11 @@ core@coreos-01 ~ $ sudo umount /mnt/local/
### Networking mode
-On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+On this example, we are using:
+- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+- the default ingress mode.
+## Allow authentification with the mesh routing
In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
Let's create the mailu_default network:
@@ -68,8 +71,22 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
"Subnet": "10.0.1.0/24",
```
In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
+In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
+
+## Limitation with the ingress mode
+With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
+
+This issue is known and discussed here:
+
+https://github.com/moby/moby/issues/25526
+
+A workaround (using network host mode and global deployment) is discussed here:
+
+https://github.com/moby/moby/issues/25526#issuecomment-336363408
+
+## Don't create an open relay !
+As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
-Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network)
### Scalability
- smtp and imap are scalable
From 6bd365e7714e765c75d8fdd93bc04065a53eae14 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 8 Oct 2018 21:00:44 +0000
Subject: [PATCH 09/93] Change title layout
---
docs/swarm/master/README.md | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
index c09f1dd3..2a2a021a 100644
--- a/docs/swarm/master/README.md
+++ b/docs/swarm/master/README.md
@@ -56,12 +56,12 @@ core@coreos-01 ~ $ sudo umount /mnt/local/
```
-### Networking mode
+## Networking mode
On this example, we are using:
- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
- the default ingress mode.
-## Allow authentification with the mesh routing
+### Allow authentification with the mesh routing
In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
Let's create the mailu_default network:
@@ -73,7 +73,7 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
-## Limitation with the ingress mode
+### Limitation with the ingress mode
With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
This issue is known and discussed here:
@@ -84,22 +84,22 @@ A workaround (using network host mode and global deployment) is discussed here:
https://github.com/moby/moby/issues/25526#issuecomment-336363408
-## Don't create an open relay !
+### Don't create an open relay !
As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
-### Scalability
+## Scalability
- smtp and imap are scalable
- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time)
- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
-### Variable substitution and docker-compose.yml
+## Variable substitution and docker-compose.yml
The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to :
- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc
- change the way we define the volumes (nfs share in our case)
- add a deploy section for every service
-### Docker compose
+## Docker compose
An example of docker-compose-stack.yml file is available here:
```yaml
@@ -340,7 +340,7 @@ volumes:
device: ":/mnt/Pool1/pv/mailu/redis"
```
-### Deploy Mailu on the docker swarm
+## Deploy Mailu on the docker swarm
Run the following command:
```bash
docker stack deploy -c docker-compose-stack.yml mailu
@@ -359,7 +359,7 @@ ID NAME IMAGE NODE
tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago
```
-### Remove the stack
+## Remove the stack
Run the follwoing command:
```bash
core@coreos-01 ~ $ docker stack rm mailu
From ada09f7922dd50656bbaa99f0c624ad2f90eb1d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Tue, 9 Oct 2018 12:35:08 +0300
Subject: [PATCH 10/93] Unbound: Use alpine:3.8
---
core/unbound/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/unbound/Dockerfile b/core/unbound/Dockerfile
index 6ae8a6ee..3c7f0e7a 100644
--- a/core/unbound/Dockerfile
+++ b/core/unbound/Dockerfile
@@ -1,4 +1,4 @@
-FROM alpine:edge
+FROM alpine:3.8
RUN apk add --no-cache unbound curl \
&& curl -o /etc/unbound/root.hints https://www.internic.net/domain/named.cache \
From 013d02d7264c182db459a7d67acc7d941eb68f4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Tue, 9 Oct 2018 14:11:59 +0300
Subject: [PATCH 11/93] Add unbound to the build directive
---
tests/build.yml | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/tests/build.yml b/tests/build.yml
index 0b6858a0..e0123ca7 100644
--- a/tests/build.yml
+++ b/tests/build.yml
@@ -6,6 +6,10 @@ services:
image: $DOCKER_ORG/nginx:$VERSION
build: ../core/nginx
+ unbound:
+ image: $DOCKER_ORG/unbound:$VERSION
+ build: ../core/unbound
+
imap:
image: $DOCKER_ORG/dovecot:$VERSION
build: ../core/dovecot
From cde22be4c9f4dec9889c75ba1dc7f234b586ba0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Tue, 9 Oct 2018 14:50:09 +0300
Subject: [PATCH 12/93] Some cleanup and changes: - Don't upgrade the
docker-compose file. (Not in the scope of this feature) - No need to use
aliases. Docker already resolves to service names - Use a fixed IP range,
which stays clear of the network ranges used by Docker. (172.xx.0.0/16)
---
docs/compose/.env | 3 --
docs/compose/docker-compose.yml | 72 +++++++--------------------------
2 files changed, 15 insertions(+), 60 deletions(-)
diff --git a/docs/compose/.env b/docs/compose/.env
index 5378d37e..721aaf22 100644
--- a/docs/compose/.env
+++ b/docs/compose/.env
@@ -21,9 +21,6 @@ SECRET_KEY=ChangeMeChangeMe
BIND_ADDRESS4=127.0.0.1
BIND_ADDRESS6=::1
-# Internal Docker network
-IPV4_NETWORK=172.22.1
-
# Main mail domain
DOMAIN=mailu.io
diff --git a/docs/compose/docker-compose.yml b/docs/compose/docker-compose.yml
index dc674a2b..8038a0bf 100644
--- a/docs/compose/docker-compose.yml
+++ b/docs/compose/docker-compose.yml
@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2'
services:
@@ -30,20 +30,14 @@ services:
depends_on:
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - front
+ - 10.177.20.254
unbound:
image: mailu/unbound:$VERSION
restart: always
networks:
- backend:
- ipv4_address: ${IPV4_NETWORK:-172.22.1}.254
- aliases:
- - unbound
+ default:
+ ipv4_address: 10.177.20.254
redis:
image: redis:alpine
@@ -51,13 +45,9 @@ services:
volumes:
- "$ROOT/redis:/data"
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
+ - 10.177.20.254
depends_on:
- unbound
- networks:
- backend:
- aliases:
- - redis
imap:
image: mailu/dovecot:$VERSION
@@ -71,11 +61,7 @@ services:
- front
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - imap
+ - 10.177.20.254
smtp:
image: mailu/postfix:$VERSION
@@ -88,11 +74,7 @@ services:
- front
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - smtp
+ - 10.177.20.254
antispam:
image: mailu/rspamd:$VERSION
@@ -106,11 +88,7 @@ services:
- front
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - antispam
+ - 10.177.20.254
antivirus:
image: mailu/$ANTIVIRUS:$VERSION
@@ -121,11 +99,7 @@ services:
depends_on:
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - antivirus
+ - 10.177.20.254
webdav:
image: mailu/$WEBDAV:$VERSION
@@ -136,11 +110,7 @@ services:
depends_on:
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - webdav
+ - 10.177.20.254
admin:
image: mailu/admin:$VERSION
@@ -154,11 +124,7 @@ services:
- redis
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - admin
+ - 10.177.20.254
webmail:
image: "mailu/$WEBMAIL:$VERSION"
@@ -170,11 +136,7 @@ services:
- imap
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - webmail
+ - 10.177.20.254
fetchmail:
image: mailu/fetchmail:$VERSION
@@ -185,16 +147,12 @@ services:
depends_on:
- unbound
dns:
- - ${IPV4_NETWORK:-172.22.1}.254
- networks:
- backend:
- aliases:
- - fetchmail
+ - 10.177.20.254
networks:
- backend:
+ default:
driver: bridge
ipam:
driver: default
config:
- - subnet: ${IPV4_NETWORK:-172.22.1}.0/24
+ - subnet: 10.177.20.0/24
From f4ef0eed09a31a5ee609414046a99c8cf0d02be3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Fri, 12 Oct 2018 20:48:44 +0300
Subject: [PATCH 13/93] Wrote informational section of the FAQ
---
docs/faq.rst | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++
docs/index.rst | 1 +
2 files changed, 98 insertions(+)
create mode 100644 docs/faq.rst
diff --git a/docs/faq.rst b/docs/faq.rst
new file mode 100644
index 00000000..8c5b5598
--- /dev/null
+++ b/docs/faq.rst
@@ -0,0 +1,97 @@
+Frequently asked questions
+==========================
+
+Informational
+-------------
+
+Where to ask questions?
+```````````````````````
+
+First, please read this FAQ to check if your question is listed here.
+Simple questions best fit in our `Matrix`_ room.
+For more complex questions, you can always open a `new issue`_ on GitHub.
+We actively monitor the issues list.
+
+
+My installation is broken!
+``````````````````````````
+
+We're sorry to hear that. Please check for common mistakes and troubleshooting
+advice in the `Technical issues`_ section of this page.
+
+I think I found a bug!
+``````````````````````
+
+If you did not manage to solve the issue using this FAQ and there is not any
+`open issues`_ describing the same problem, you can continue to open a
+`new issue`_ on GitHub.
+
+I want a new feature or enhancement!
+````````````````````````````````````
+
+Great! We are always open for suggestions. We currently maintain two tags:
+
+- `Enhancement issues`_: Typically used for optimization of features in the project.
+- `Feature request issues`_: For implementing new functionality,
+ plugins and applications.
+
+Please check if your idea (or something similar) is already mentioned there.
+If there is one open, you can choose to vote with a thumbs up, so we can
+estimate the popular demand. Please refrain from writing comments like
+*"me too"* as it clobbers the actual discussion.
+
+If you can't find anything similar, you can open a `new issue`_.
+Please also share (where applicable):
+
+- Use case: how does this improve the project?
+- Any research done on the subject. Perhaps some links to upstream website,
+ reference implementations etc.
+
+Why does my feature/bug take so long to solve?
+``````````````````````````````````````````````
+
+You should be aware that creating, maintaining and expanding a mail server
+distribution requires a lot of effort. Mail servers are highly exposed to hacking attempts,
+open relay scanners, spam and malware distributors etc. We need to work in a safe way and
+have to prevent pushing out something quickly.
+
+We currently maintain a strict work flow:
+
+#. Someone writes a solution and sends a pull request;
+#. We use Travis-CI fore some very basic building and testing;
+#. The pull request needs to be code-reviewed and tested by at least two members
+ from the contributors team.
+
+Please consider that this project is mostly developed in people their free time.
+We thank you for your understanding and patience.
+
+I would to donate (for a feature)
+`````````````````````````````````
+
+Donations are welcome at the `patreon`_ account of the project lead. It will be used to pay
+for infra structure and project related costs. If there are leftovers, it will be distributed
+among the developers.
+
+It is not yet possible to pay for a specific feature. We don't have
+any bounty system implemented. Feel free to come with suggestions in
+our ongoing `project management`_ discussion issue.
+
+
+.. _`Matrix`: https://matrix.to/#/#mailu:tedomum.net
+.. _`open issues`: https://github.com/Mailu/Mailu/issues
+.. _`new issue`: https://github.com/Mailu/Mailu/issues/new
+.. _`Enhancement issues`: https://github.com/Mailu/Mailu/issues?q=is%3Aissue+is%3Aopen+label%3Atype%2Fenhancement
+.. _`Feature request issues`: https://github.com/Mailu/Mailu/issues?q=is%3Aopen+is%3Aissue+label%3Atype%2Ffeature
+.. _`patreon`: https://patreon.com/kaiyou
+.. _`project management`: https://github.com/Mailu/Mailu/issues/508
+
+Deployment related
+------------------
+
+
+Technical issues
+----------------
+
+WIP: Link to `troubleshooting`_ related issues will be in the bottom of this section.
+
+.. _`troubleshooting`: https://github.com/Mailu/Mailu/issues?utf8=%E2%9C%93&q=label%3Afaq%2Ftroubleshooting
diff --git a/docs/index.rst b/docs/index.rst
index 5219145f..0a4aadff 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -44,6 +44,7 @@ the version of Mailu that you are running.
general
features
+ faq
releases
demo
From b5693edc63ce11979459c204793a525b4dc57b05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Sat, 13 Oct 2018 20:59:14 +0300
Subject: [PATCH 14/93] Include a doc section for external certbot
---
docs/maintain.rst | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/docs/maintain.rst b/docs/maintain.rst
index d570690e..ffb51a50 100644
--- a/docs/maintain.rst
+++ b/docs/maintain.rst
@@ -28,6 +28,33 @@ Logs are managed by Docker directly. You can easily read your logs using:
Docker is able to forward logs to multiple log engines. Read the following documentation for details: https://docs.docker.com/engine/admin/logging/overview/.
+.. _external_certs:
+
+Managing of external Let's encrypt certificates
+-----------------------------------------------
+
+When you are not using the embedded ``letsencrypt`` option from Mailu,
+you cannot make use of it's symlink functionality in the ``letsencrypt/live`` directory.
+You should take care that after every renewal new certificates are copied to ``/mailu/certs`` and
+the *nginx* process in the ``front`` container is reloaded.
+
+In the case of *certbot* you could write a script to be executed as `deploy hook`_. Example:
+
+.. code-block:: bash
+
+ #!/bin/sh
+ cp /etc/letsencrypt/live/domain.com/privkey.pem /mailu/certs/key.pem || exit 1
+ cp /etc/letsencrypt/live/domain.com/fullchain.pem /mailu/certs/cert.pem || exit 1
+ docker exec mailu_front_1 nginx -s reload
+
+And the certbot command you will use in crontab would look something like:
+
+.. code-block:: bash
+
+ 52 0,12 * * * root /usr/bin/certbot renew --deploy-hook /path/to/script.sh
+
+.. _`deploy hook`: https://certbot.eff.org/docs/using.html#renewing-certificates
+
Migrating an instance
---------------------
From 13949554479383e09a184bac695e7e6e085350ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Sat, 13 Oct 2018 21:13:09 +0300
Subject: [PATCH 15/93] FAQ about TLS issues
---
docs/faq.rst | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 84 insertions(+), 3 deletions(-)
diff --git a/docs/faq.rst b/docs/faq.rst
index 8c5b5598..81d51b86 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -55,6 +55,7 @@ distribution requires a lot of effort. Mail servers are highly exposed to hackin
open relay scanners, spam and malware distributors etc. We need to work in a safe way and
have to prevent pushing out something quickly.
+**TODO: Move the next section into the contributors part of docs**
We currently maintain a strict work flow:
#. Someone writes a solution and sends a pull request;
@@ -65,8 +66,8 @@ We currently maintain a strict work flow:
Please consider that this project is mostly developed in people their free time.
We thank you for your understanding and patience.
-I would to donate (for a feature)
-`````````````````````````````````
+I would like to donate (for a feature)
+``````````````````````````````````````
Donations are welcome at the `patreon`_ account of the project lead. It will be used to pay
for infra structure and project related costs. If there are leftovers, it will be distributed
@@ -88,10 +89,90 @@ our ongoing `project management`_ discussion issue.
Deployment related
------------------
-
Technical issues
----------------
+Changes in .env don't propagate
+```````````````````````````````
+
+Variables are sent to the containers at creation time. This means you need to take the project
+down and up again. A container restart is not sufficient.
+
+.. code-block:: bash
+
+ docker-compose down && \
+ docker-compose up -d
+
+*Issue reference:* `615`_,
+
+TLS certificate issues
+``````````````````````
+
+When there are issues with the TLS/SSL certificates, Mailu denies service on secure ports.
+This is a security precaution. Symptoms are:
+
+- 403 browser errors;
+
+These issues are typically caused by four scenarios:
+
+#. ``TLS_FLAVOR=notls`` in ``.env``;
+#. Certificates expired;
+#. When ``TLS_FLAVOR=letsencrypt``, it might be that the *certbot* script is not capable of
+ obtaining the certificates for your domain. See `letsencrypt issues`_
+#. When ``TLS_FLAVOR=certs``, certificates are supposed to be copied to ``/mailu/certs``.
+ Using an external ``letsencrypt`` program, it tends to happen people copy the whole
+ ``letsencrypt/live`` directory containing symlinks. Symlinks do not resolve inside the
+ container and therefore it breaks the TLS implementation.
+
+letsencrypt issues
+..................
+
+In order to determine the exact problem on TLS / Let's encrypt issues, it might be helpful
+to check the logs.
+
+.. code-block:: bash
+
+ docker-compose logs front | less -R
+ docker-compose exec front less /var/log/letsencrypt/letsencrypt.log
+
+Common problems:
+
+- Port 80 not reachable from outside.
+- Faulty DNS records: make sure that all ``HOSTNAMES`` have **A** (IPv4) and **AAAA** (IPv6)
+ records, pointing the the ``BIND_ADDRESS4`` and ``BIND_ADDRESS6``.
+- DNS cache not yet expired. It might be that old / faulty DNS records are stuck in a cache
+ en-route to letsencrypt's server. The time this takes is set by the ``TTL`` field in the
+ records. You'll have to wait at least this time after changing the DNS entries.
+ Don't keep trying, as you might hit `rate-limits`_.
+
+.. _`rate-limits`: https://letsencrypt.org/docs/rate-limits/
+
+Copying certificates
+....................
+
+As mentioned above, care must be taken not to copy symlinks to the ``/mailu/certs`` location.
+
+**The wrong way!:**
+
+.. code-block:: bash
+
+ cp -r /etc/letsencrypt/live/domain.com /mailu/certs
+
+**The right way!:**
+
+.. code-block:: bash
+
+ mkdir -p /mailu/certs
+ cp /etc/letsencrypt/live/domain.com/privkey.pem /mailu/certs/key.pem
+ cp /etc/letsencrypt/live/domain.com/fullchain.pem /mailu/certs/cert.pem
+
+See also :ref:`external_certs`.
+
+*Issue reference:* `426`_, `615`_.
+
+
WIP: Link to `troubleshooting`_ related issues will be in the bottom of this section.
+.. _`426`: https://github.com/Mailu/Mailu/issues/426
+.. _`615`: https://github.com/Mailu/Mailu/issues/615
.. _`troubleshooting`: https://github.com/Mailu/Mailu/issues?utf8=%E2%9C%93&q=label%3Afaq%2Ftroubleshooting
From 11a8e49f059c4eb71a8f931edcbfa238d90873a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Tue, 16 Oct 2018 11:09:42 +0300
Subject: [PATCH 16/93] Compose file upgrade and define more variables for
setup
---
setup/flavors/compose/docker-compose.yml | 38 ++++++++++++------------
setup/flavors/compose/mailu.env | 16 +++++-----
2 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index fcf0c092..a82817af 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -2,7 +2,7 @@
# This file is auto-generated by the Mailu configuration wizard.
# Please read the documentation before attempting any change.
-version: '2'
+version: '3.7'
services:
@@ -11,7 +11,7 @@ services:
image: redis:alpine
restart: always
volumes:
- - "$ROOT/redis:/data"
+ - "{{ root }}/redis:/data"
# Core services
front:
@@ -24,15 +24,15 @@ services:
ports:
{% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
{% if bind4 %}
- - "$PUBLIC_IPV4:{{ port }}:{{ port }}"
+ - "{{ bind4}}:{{ port }}:{{ port }}"
{% endif %}
{% if bind6 %}
- - "$PUBLIC_IPV6:{{ port }}:{{ port }}"
+ - "{{ bind6 }}:{{ port }}:{{ port }}"
{% endif %}
{% endfor %}
{% if flavor in ('cert', 'mail') %}
volumes:
- - "$ROOT/certs:/certs"
+ - "{{ root }}/certs:/certs"
{% endif %}
admin:
@@ -44,8 +44,8 @@ services:
- 127.0.0.1:8080:80
{% endif %}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/dkim:/dkim"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/dkim:/dkim"
depends_on:
- redis
@@ -54,9 +54,9 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/mail:/mail"
- - "$ROOT/overrides:/overrides"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/mail:/mail"
+ - "{{ root }}/overrides:/overrides"
depends_on:
- front
@@ -65,8 +65,8 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/overrides:/overrides"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/overrides:/overrides"
depends_on:
- front
@@ -77,9 +77,9 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/filter:/var/lib/rspamd"
- - "$ROOT/dkim:/dkim"
- - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+ - "{{ root }}/filter:/var/lib/rspamd"
+ - "{{ root }}/dkim:/dkim"
+ - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d"
depends_on:
- front
{% endif %}
@@ -90,7 +90,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/filter:/data"
+ - "{{ root }}/filter:/data"
{% endif %}
{% if enable_webdav %}
@@ -99,7 +99,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/dav:/data"
+ - "{{ root }}/dav:/data"
{% endif %}
{% if enable_fetchmail %}
@@ -108,7 +108,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
+ - "{{ root }}/data:/data"
{% endif %}
# Webmail
@@ -118,7 +118,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/webmail:/data"
+ - "{{ root }}/webmail:/data"
depends_on:
- imap
{% endif %}
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 24d7b247..2de771f3 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -9,14 +9,16 @@
###################################
# Set this to the path where Mailu data and configuration is stored
-ROOT=/mailu
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
# Set to a randomly generated 16 bytes string
SECRET_KEY={{ secret(16) }}
# Address where listening ports should bind
-{% if bind4 %}PUBLIC_IPV4={{ bind4 }}{% endif %}
-{% if bind6 %}PUBLIC_IPV6={{ bind6 }}{% endif %}
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4=127.0.0.1
+# PUBLIC_IPV6=::1
# Mail address of the postmaster
POSTMASTER={{ postmaster }}
@@ -75,16 +77,16 @@ DOMAIN_REGISTRATION=true
###################################
# Path to the admin interface if enabled
-WEB_ADMIN=/admin
+WEB_ADMIN={{ admin_path }}
# Path to the webmail if enabled
-WEB_WEBMAIL=/webmail
+WEB_WEBMAIL={{ webmail_path }}
# Website name
-SITENAME=Mailu
+SITENAME={{ site_name }{
# Linked Website URL
-WEBSITE=https://mailu.io
+WEBSITE={{ website }}
{% if recaptcha_public_key and recaptcha_private_key %}
# Registration reCaptcha settings (warning, this has some privacy impact)
From 0d164486b45b194de81905089b37a4557a828a8f Mon Sep 17 00:00:00 2001
From: Ionut Filip
Date: Tue, 16 Oct 2018 12:34:55 +0300
Subject: [PATCH 17/93] docker-compose variables and setup
---
setup/Dockerfile | 4 +++-
setup/docker-compose.yml | 1 +
setup/flavors/compose/docker-compose.yml | 2 +-
setup/flavors/compose/mailu.env | 8 ++++----
setup/flavors/compose/setup.html | 6 +++---
setup/server.py | 6 ++++--
setup/templates/steps/expose.html | 4 ++--
setup/templates/steps/root.html | 8 ++++++++
setup/templates/wizard.html | 1 +
9 files changed, 27 insertions(+), 13 deletions(-)
create mode 100644 setup/templates/steps/root.html
diff --git a/setup/Dockerfile b/setup/Dockerfile
index 1fc808f1..c970e57d 100644
--- a/setup/Dockerfile
+++ b/setup/Dockerfile
@@ -10,8 +10,10 @@ RUN apk add --no-cache git \
COPY server.py ./server.py
COPY setup.py ./setup.py
COPY main.py ./main.py
+COPY flavors /data/master/flavors
+COPY templates /data/master/templates
-RUN python setup.py https://github.com/mailu/mailu /data
+#RUN python setup.py https://github.com/mailu/mailu /data
EXPOSE 80/tcp
diff --git a/setup/docker-compose.yml b/setup/docker-compose.yml
index 9288bb7e..30966167 100644
--- a/setup/docker-compose.yml
+++ b/setup/docker-compose.yml
@@ -10,4 +10,5 @@ services:
image: mailu/setup
ports:
- "80:80"
+ build: .
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index a82817af..3dcfa5a2 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -24,7 +24,7 @@ services:
ports:
{% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
{% if bind4 %}
- - "{{ bind4}}:{{ port }}:{{ port }}"
+ - "{{ bind4 }}:{{ port }}:{{ port }}"
{% endif %}
{% if bind6 %}
- "{{ bind6 }}:{{ port }}:{{ port }}"
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 2de771f3..1512e75b 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -10,15 +10,15 @@
# Set this to the path where Mailu data and configuration is stored
# This variable is now set directly in `docker-compose.yml by the setup utility
-# ROOT=/mailu
+# ROOT= {{ root }}
# Set to a randomly generated 16 bytes string
SECRET_KEY={{ secret(16) }}
# Address where listening ports should bind
# This variables are now set directly in `docker-compose.yml by the setup utility
-# PUBLIC_IPV4=127.0.0.1
-# PUBLIC_IPV6=::1
+# PUBLIC_IPV4= {{ bind4 }} (default: 127.0.0.1)
+# PUBLIC_IPV6= {{ bind6 }} (default: ::1)
# Mail address of the postmaster
POSTMASTER={{ postmaster }}
@@ -83,7 +83,7 @@ WEB_ADMIN={{ admin_path }}
WEB_WEBMAIL={{ webmail_path }}
# Website name
-SITENAME={{ site_name }{
+SITENAME={{ site_name }}
# Linked Website URL
WEBSITE={{ website }}
diff --git a/setup/flavors/compose/setup.html b/setup/flavors/compose/setup.html
index e4506e6d..3c190c9c 100644
--- a/setup/flavors/compose/setup.html
+++ b/setup/flavors/compose/setup.html
@@ -4,13 +4,13 @@
Docker Compose expects a project file, named docker-compose.yml
in a project directory. First create your project directory.
-
mkdir /mailu
+
mkdir {{ root }}
Then download the project file. A side configuration file makes it easier
to read and check the configuration variables generated by the wizard.
To start your compose project, simply run the Docker Compose up
command.
-
cd /mailu
+
cd {{ root }}
docker-compose up -d
{% endcall %}
diff --git a/setup/server.py b/setup/server.py
index 108f5043..ddeafd90 100644
--- a/setup/server.py
+++ b/setup/server.py
@@ -32,9 +32,11 @@ def secret(length=16):
def build_app(path):
+ #Hardcoded master as the only version for test purposes
versions = [
- version for version in os.listdir(path)
- if os.path.isdir(os.path.join(path, version))
+ # version for version in os.listdir(path)
+ # if os.path.isdir(os.path.join(path, version))
+ "master"
]
app.jinja_env.trim_blocks = True
diff --git a/setup/templates/steps/expose.html b/setup/templates/steps/expose.html
index 665b08a2..372ebddc 100644
--- a/setup/templates/steps/expose.html
+++ b/setup/templates/steps/expose.html
@@ -14,12 +14,12 @@ avoid generic all-interfaces addresses like 0.0.0.0 or ::
-
+
-
+
You server will be available under a main hostname but may expose multiple public
diff --git a/setup/templates/steps/root.html b/setup/templates/steps/root.html
new file mode 100644
index 00000000..f32c2250
--- /dev/null
+++ b/setup/templates/steps/root.html
@@ -0,0 +1,8 @@
+{% call macros.panel("info", "Step 0 - Set root path") %}
+