From 12294a6e5aa3c57c8c603d97a35fa443a962a8e2 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Wed, 6 Jun 2018 18:40:51 +0000
Subject: [PATCH 01/53] Trying to enable fuzzy hashes for rspamd
---
services/rspamd/Dockerfile | 5 +---
services/rspamd/conf/fuzzy_check.conf | 34 ++++++++++++++++++++++
services/rspamd/conf/metrics.conf | 19 ++++++++++++
services/rspamd/conf/worker-controller.inc | 1 +
services/rspamd/conf/worker-fuzzy.inc | 5 ++++
services/rspamd/conf/worker-normal.inc | 1 +
6 files changed, 61 insertions(+), 4 deletions(-)
create mode 100644 services/rspamd/conf/fuzzy_check.conf
create mode 100644 services/rspamd/conf/metrics.conf
create mode 100644 services/rspamd/conf/worker-fuzzy.inc
diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile
index c6c2afdd..1b8d7e6b 100644
--- a/services/rspamd/Dockerfile
+++ b/services/rspamd/Dockerfile
@@ -1,15 +1,12 @@
FROM alpine:edge
-RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates
+RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates
RUN mkdir /run/rspamd
COPY conf/ /conf
COPY start.py /start.py
-# Temporary fix to remove references to rspamd-fuzzy for now
-RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
-
EXPOSE 11332/tcp 11334/tcp
CMD /start.py
diff --git a/services/rspamd/conf/fuzzy_check.conf b/services/rspamd/conf/fuzzy_check.conf
new file mode 100644
index 00000000..7c87e1c3
--- /dev/null
+++ b/services/rspamd/conf/fuzzy_check.conf
@@ -0,0 +1,34 @@
+rule "local" {
+ # Fuzzy storage server list
+ servers = "localhost:11335";
+ # Default symbol for unknown flags
+ symbol = "LOCAL_FUZZY_UNKNOWN";
+ # Additional mime types to store/check
+ mime_types = ["application/*"];
+ # Hash weight threshold for all maps
+ max_score = 20.0;
+ # Whether we can learn this storage
+ read_only = no;
+ # Ignore unknown flags
+ skip_unknown = yes;
+ # Hash generation algorithm
+ algorithm = "mumhash";
+
+ # Map flags to symbols
+ fuzzy_map = {
+ LOCAL_FUZZY_DENIED {
+ # Local threshold
+ max_score = 20.0;
+ # Flag to match
+ flag = 11;
+ }
+ LOCAL_FUZZY_PROB {
+ max_score = 10.0;
+ flag = 12;
+ }
+ LOCAL_FUZZY_WHITE {
+ max_score = 2.0;
+ flag = 13;
+ }
+ }
+}
diff --git a/services/rspamd/conf/metrics.conf b/services/rspamd/conf/metrics.conf
new file mode 100644
index 00000000..6a31964f
--- /dev/null
+++ b/services/rspamd/conf/metrics.conf
@@ -0,0 +1,19 @@
+group "fuzzy" {
+ max_score = 12.0;
+ symbol "LOCAL_FUZZY_UNKNOWN" {
+ weight = 5.0;
+ description = "Generic fuzzy hash match";
+ }
+ symbol "LOCAL_FUZZY_DENIED" {
+ weight = 12.0;
+ description = "Denied fuzzy hash";
+ }
+ symbol "LOCAL_FUZZY_PROB" {
+ weight = 5.0;
+ description = "Probable fuzzy hash";
+ }
+ symbol "LOCAL_FUZZY_WHITE" {
+ weight = -2.1;
+ description = "Whitelisted fuzzy hash";
+ }
+}
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 6a020672..dd143942 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,4 @@
+type = "controller";
bind_socket = "*:11334";
password = "mailu";
secure_ip = "{{ FRONT_ADDRESS }}";
diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc
new file mode 100644
index 00000000..a0021a03
--- /dev/null
+++ b/services/rspamd/conf/worker-fuzzy.inc
@@ -0,0 +1,5 @@
+type = "fuzzy";
+count = 1;
+backend = "redis";
+expire = 90d;
+allow_update = ["127.0.0.1"];
diff --git a/services/rspamd/conf/worker-normal.inc b/services/rspamd/conf/worker-normal.inc
index a6ee8317..ab996fb8 100644
--- a/services/rspamd/conf/worker-normal.inc
+++ b/services/rspamd/conf/worker-normal.inc
@@ -1 +1,2 @@
+type = "normal";
enabled = false;
From 23e288aadcf7d5df941c8516fae9d0cbd6b1e054 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 24 Sep 2018 17:29:31 +0000
Subject: [PATCH 02/53] Enabling swarm deployment on master branch: -Extends
the usage of POD_ADDRESS_RANGE -Provides documentation
---
core/dovecot/conf/dovecot.conf | 2 +-
core/postfix/conf/main.cf | 2 +-
docs/swarm/master/README.md | 349 +++++++++++++++++++++
services/rspamd/conf/worker-controller.inc | 2 +-
4 files changed, 352 insertions(+), 3 deletions(-)
create mode 100644 docs/swarm/master/README.md
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index a5973bf8..7cf10774 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -5,7 +5,7 @@ log_path = /dev/stderr
protocols = imap pop3 lmtp sieve
postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
hostname = {{ HOSTNAMES.split(",")[0] }}
-submission_host = {{ FRONT_ADDRESS }}
+submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
service dict {
unix_listener dict {
diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf
index 2f2c6990..bde42dd1 100644
--- a/core/postfix/conf/main.cf
+++ b/core/postfix/conf/main.cf
@@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }}
recipient_delimiter = {{ RECIPIENT_DELIMITER }}
# Only the front server is allowed to perform xclient
-smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }}
+smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
###############
# TLS
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
new file mode 100644
index 00000000..1406e05c
--- /dev/null
+++ b/docs/swarm/master/README.md
@@ -0,0 +1,349 @@
+# Install Mailu on a docker swarm
+
+## Prequisites
+
+### Swarm
+
+In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
+
+The main command will be:
+```bash
+docker swarm init --advertise-addr
+```
+See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
+
+If you want to add other managers or workers, please use:
+```bash
+docker swarm join --token xxxxx
+```
+See https://docs.docker.com/engine/swarm/join-nodes/
+
+You have now a working swarm, and you can check its status with:
+```bash
+core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
+ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
+xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce
+sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce
+mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce
+```
+
+### Volume definition
+For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
+Hereafter we will use a NFS share:
+```bash
+core@coreos-01 ~ $ showmount -e 192.168.0.30
+Export list for 192.168.0.30:
+/mnt/Pool1/pv 192.168.0.0
+```
+
+on the nfs server, I am using the following /etc/exports
+```bash
+$more /etc/exports
+/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0
+```
+on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up)
+```bash
+$mkdir /mnt/Pool1/pv/mailu
+```
+
+On your manager node, mount the nfs share to check that the share is available:
+```bash
+core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/
+```
+If this is ok, you can umount it:
+```bash
+core@coreos-01 ~ $ sudo umount /mnt/local/
+```
+
+
+### Networking mode
+On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+
+In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
+
+Let's create the mailu_default network:
+```bash
+core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
+core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
+ "Subnet": "10.0.1.0/24",
+```
+In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
+
+Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network)
+
+### Scalability
+- smtp and imap are scalable
+- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time)
+- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
+
+### Variable substitution and docker-compose.yml
+The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to :
+- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc
+- change the way we define the volumes (nfs share in our case)
+- add a deploy section for every service
+
+### Docker compose
+An example of docker-compose-stack.yml file is available here:
+
+```yaml
+
+version: '3.2'
+
+services:
+
+ front:
+ image: mailu/nginx:master
+ restart: always
+ env_file: .env
+ ports:
+ - target: 80
+ published: 80
+ - target: 443
+ published: 443
+ - target: 110
+ published: 110
+ - target: 143
+ published: 143
+ - target: 993
+ published: 993
+ - target: 995
+ published: 995
+ - target: 25
+ published: 25
+ - target: 465
+ published: 465
+ - target: 587
+ published: 587
+ volumes:
+# - "$ROOT/certs:/certs"
+ - type: volume
+ source: mailu_certs
+ target: /certs
+ deploy:
+ replicas: 2
+
+ redis:
+ image: redis:alpine
+ restart: always
+ volumes:
+# - "$ROOT/redis:/data"
+ - type: volume
+ source: mailu_redis
+ target: /data
+ deploy:
+ replicas: 1
+
+ imap:
+ image: mailu/dovecot:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/mail:/mail"
+ - type: volume
+ source: mailu_mail
+ target: /mail
+# - "$ROOT/overrides:/overrides"
+ - type: volume
+ source: mailu_overrides
+ target: /overrides
+ depends_on:
+ - front
+ deploy:
+ replicas: 2
+
+ smtp:
+ image: mailu/postfix:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/overrides:/overrides"
+ - type: volume
+ source: mailu_overrides
+ target: /overrides
+ depends_on:
+ - front
+ deploy:
+ replicas: 2
+
+ antispam:
+ image: mailu/rspamd:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ depends_on:
+ - front
+ volumes:
+# - "$ROOT/filter:/var/lib/rspamd"
+ - type: volume
+ source: mailu_filter
+ target: /var/lib/rspamd
+# - "$ROOT/dkim:/dkim"
+ - type: volume
+ source: mailu_dkim
+ target: /dkim
+# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+ - type: volume
+ source: mailu_overrides_rspamd
+ target: /etc/rspamd/override.d
+ deploy:
+ replicas: 1
+
+ antivirus:
+ image: mailu/none:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/filter:/data"
+ - type: volume
+ source: mailu_filter
+ target: /data
+ deploy:
+ replicas: 1
+
+ webdav:
+ image: mailu/none:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/dav:/data"
+ - type: volume
+ source: mailu_dav
+ target: /data
+ deploy:
+ replicas: 1
+
+ admin:
+ image: mailu/admin:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/dkim:/dkim"
+ - type: volume
+ source: mailu_dkim
+ target: /dkim
+ - /var/run/docker.sock:/var/run/docker.sock:ro
+ depends_on:
+ - redis
+ deploy:
+ replicas: 1
+
+ webmail:
+ image: "mailu/roundcube:master"
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/webmail:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+ depends_on:
+ - imap
+ deploy:
+ replicas: 2
+
+ fetchmail:
+ image: mailu/fetchmail:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+ deploy:
+ replicas: 1
+
+networks:
+ default:
+ external:
+ name: mailu_default
+
+volumes:
+ mailu_filter:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/filter"
+ mailu_dkim:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/dkim"
+ mailu_overrides_rspamd:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/overrides/rspamd"
+ mailu_data:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/data"
+ mailu_mail:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/mail"
+ mailu_overrides:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/overrides"
+ mailu_dav:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/dav"
+ mailu_certs:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/certs"
+ mailu_redis:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/redis"
+```
+
+### Deploy Mailu on the docker swarm
+Run the following command:
+```bash
+docker stack deploy -c docker-compose-stack.yml mailu
+```
+See how the services are being deployed:
+```bash
+core@coreos-01 ~ $ docker service ls
+ID NAME MODE REPLICAS IMAGE PORTS
+ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:1.5
+pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:1.5
+```
+check a specific service:
+```bash
+core@coreos-01 ~ $ docker service ps mailu_fetchmail
+ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS
+tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago
+```
+
+### Remove the stack
+Run the follwoing command:
+```bash
+core@coreos-01 ~ $ docker stack rm mailu
+```
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 6a020672..0cb0d5c0 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,3 @@
bind_socket = "*:11334";
password = "mailu";
-secure_ip = "{{ FRONT_ADDRESS }}";
+secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}";
From f5f09fad6ecc41e64b90dc4044450b61ea19ae4a Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Tue, 25 Sep 2018 18:54:40 +0000
Subject: [PATCH 03/53] Reverting the patch for dovecot.conf, as it is not
needed
---
core/dovecot/conf/dovecot.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index 7cf10774..a5973bf8 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -5,7 +5,7 @@ log_path = /dev/stderr
protocols = imap pop3 lmtp sieve
postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
hostname = {{ HOSTNAMES.split(",")[0] }}
-submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
+submission_host = {{ FRONT_ADDRESS }}
service dict {
unix_listener dict {
From fcad52b1454aeeb5b11d64fe0186f644922f96b2 Mon Sep 17 00:00:00 2001
From: kaiyou
Date: Thu, 27 Sep 2018 22:45:16 +0200
Subject: [PATCH 04/53] Implement a start date filter for autoreply, fixes #362
---
core/admin/mailu/models.py | 2 ++
core/admin/mailu/ui/forms.py | 1 +
core/admin/mailu/ui/templates/user/reply.html | 7 ++++--
.../migrations/versions/3b281286c7bd_.py | 24 +++++++++++++++++++
core/dovecot/conf/pigeonhole-sieve.dict | 8 +++++++
core/dovecot/sieve/before.sieve | 1 +
6 files changed, 41 insertions(+), 2 deletions(-)
create mode 100644 core/admin/migrations/versions/3b281286c7bd_.py
diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index 71c5f4d7..1be4c8e6 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -249,6 +249,8 @@ class User(Base, Email):
reply_enabled = db.Column(db.Boolean(), nullable=False, default=False)
reply_subject = db.Column(db.String(255), nullable=True, default=None)
reply_body = db.Column(db.Text(), nullable=True, default=None)
+ reply_startdate = db.Column(db.Date, nullable=False,
+ default=date(1900, 1, 1))
reply_enddate = db.Column(db.Date, nullable=False,
default=date(2999, 12, 31))
diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py
index 326d721b..4f7a30ae 100644
--- a/core/admin/mailu/ui/forms.py
+++ b/core/admin/mailu/ui/forms.py
@@ -117,6 +117,7 @@ class UserReplyForm(flask_wtf.FlaskForm):
reply_subject = fields.StringField(_('Reply subject'))
reply_body = fields.StringField(_('Reply body'),
widget=widgets.TextArea())
+ reply_startdate = fields.html5.DateField(_('Start of vacation'))
reply_enddate = fields.html5.DateField(_('End of vacation'))
submit = fields.SubmitField(_('Update'))
diff --git a/core/admin/mailu/ui/templates/user/reply.html b/core/admin/mailu/ui/templates/user/reply.html
index 7906bc42..7225a178 100644
--- a/core/admin/mailu/ui/templates/user/reply.html
+++ b/core/admin/mailu/ui/templates/user/reply.html
@@ -13,14 +13,17 @@
{% endcall %}
diff --git a/core/admin/migrations/versions/3b281286c7bd_.py b/core/admin/migrations/versions/3b281286c7bd_.py
new file mode 100644
index 00000000..78e44a4c
--- /dev/null
+++ b/core/admin/migrations/versions/3b281286c7bd_.py
@@ -0,0 +1,24 @@
+""" Add a start day for vacations
+
+Revision ID: 3b281286c7bd
+Revises: 049fed905da7
+Create Date: 2018-09-27 22:20:08.158553
+
+"""
+
+revision = '3b281286c7bd'
+down_revision = '049fed905da7'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ with op.batch_alter_table('user') as batch:
+ batch.add_column(sa.Column('reply_startdate', sa.Date(), nullable=False,
+ server_default="1900-01-01"))
+
+
+def downgrade():
+ with op.batch_alter_table('user') as batch:
+ batch.drop_column('reply_startdate')
diff --git a/core/dovecot/conf/pigeonhole-sieve.dict b/core/dovecot/conf/pigeonhole-sieve.dict
index 917fce83..604371a8 100644
--- a/core/dovecot/conf/pigeonhole-sieve.dict
+++ b/core/dovecot/conf/pigeonhole-sieve.dict
@@ -41,3 +41,11 @@ map {
username_field = email
value_field = reply_enddate
}
+
+map {
+ pattern = priv/reply_startdate
+ table = user
+ username_field = email
+ value_field = reply_startdate
+}
+
diff --git a/core/dovecot/sieve/before.sieve b/core/dovecot/sieve/before.sieve
index 6ebc20c5..81d20f30 100644
--- a/core/dovecot/sieve/before.sieve
+++ b/core/dovecot/sieve/before.sieve
@@ -34,6 +34,7 @@ if exists "X-Virus" {
}
if allof (string :is "${extdata.reply_enabled}" "1",
+ currentdate :value "ge" "date" "${extdata.reply_startdate}",
currentdate :value "le" "date" "${extdata.reply_enddate}")
{
vacation :days 1 :subject "${extdata.reply_subject}" "${extdata.reply_body}";
From 6b34b2728ece020918dcb10a901afdd9894a69e3 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Sun, 7 Oct 2018 16:38:41 +0000
Subject: [PATCH 05/53] Declare fuzzy_worker port 11335 in EXPOSE section
---
services/rspamd/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile
index 7dff8c1f..cfb4d0eb 100644
--- a/services/rspamd/Dockerfile
+++ b/services/rspamd/Dockerfile
@@ -9,7 +9,7 @@ RUN mkdir /run/rspamd
COPY conf/ /conf
COPY start.py /start.py
-EXPOSE 11332/tcp 11334/tcp
+EXPOSE 11332/tcp 11334/tcp 11335/tcp
VOLUME ["/var/lib/rspamd"]
From 1f71d10899d72571b39a60448a23adc41d5f10f2 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Sun, 7 Oct 2018 16:46:42 +0000
Subject: [PATCH 06/53] Change POD_ADDRESS_RANGE introduction like it is done
on deovecot-sql.conf.ext
---
services/rspamd/conf/worker-controller.inc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 0cb0d5c0..4b23a897 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,3 @@
bind_socket = "*:11334";
password = "mailu";
-secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}";
+secure_ip = "{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% endif %}";
From 9d610f56f7ff39232cb15ce02b42f8aca94d846a Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 8 Oct 2018 18:53:44 +0000
Subject: [PATCH 07/53] Added some lines around ingress mode
---
docs/swarm/master/README.md | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
index 1406e05c..c09f1dd3 100644
--- a/docs/swarm/master/README.md
+++ b/docs/swarm/master/README.md
@@ -57,8 +57,11 @@ core@coreos-01 ~ $ sudo umount /mnt/local/
### Networking mode
-On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+On this example, we are using:
+- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+- the default ingress mode.
+## Allow authentification with the mesh routing
In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
Let's create the mailu_default network:
@@ -68,8 +71,22 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
"Subnet": "10.0.1.0/24",
```
In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
+In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
+
+## Limitation with the ingress mode
+With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
+
+This issue is known and discussed here:
+
+https://github.com/moby/moby/issues/25526
+
+A workaround (using network host mode and global deployment) is discussed here:
+
+https://github.com/moby/moby/issues/25526#issuecomment-336363408
+
+## Don't create an open relay !
+As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
-Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network)
### Scalability
- smtp and imap are scalable
From 6bd365e7714e765c75d8fdd93bc04065a53eae14 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 8 Oct 2018 21:00:44 +0000
Subject: [PATCH 08/53] Change title layout
---
docs/swarm/master/README.md | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
index c09f1dd3..2a2a021a 100644
--- a/docs/swarm/master/README.md
+++ b/docs/swarm/master/README.md
@@ -56,12 +56,12 @@ core@coreos-01 ~ $ sudo umount /mnt/local/
```
-### Networking mode
+## Networking mode
On this example, we are using:
- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
- the default ingress mode.
-## Allow authentification with the mesh routing
+### Allow authentification with the mesh routing
In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
Let's create the mailu_default network:
@@ -73,7 +73,7 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
-## Limitation with the ingress mode
+### Limitation with the ingress mode
With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
This issue is known and discussed here:
@@ -84,22 +84,22 @@ A workaround (using network host mode and global deployment) is discussed here:
https://github.com/moby/moby/issues/25526#issuecomment-336363408
-## Don't create an open relay !
+### Don't create an open relay !
As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
-### Scalability
+## Scalability
- smtp and imap are scalable
- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time)
- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
-### Variable substitution and docker-compose.yml
+## Variable substitution and docker-compose.yml
The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to :
- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc
- change the way we define the volumes (nfs share in our case)
- add a deploy section for every service
-### Docker compose
+## Docker compose
An example of docker-compose-stack.yml file is available here:
```yaml
@@ -340,7 +340,7 @@ volumes:
device: ":/mnt/Pool1/pv/mailu/redis"
```
-### Deploy Mailu on the docker swarm
+## Deploy Mailu on the docker swarm
Run the following command:
```bash
docker stack deploy -c docker-compose-stack.yml mailu
@@ -359,7 +359,7 @@ ID NAME IMAGE NODE
tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago
```
-### Remove the stack
+## Remove the stack
Run the follwoing command:
```bash
core@coreos-01 ~ $ docker stack rm mailu
From 5035975c4168980c7dacc7f80c49b062c89eb7c6 Mon Sep 17 00:00:00 2001
From: kaiyou
Date: Mon, 15 Oct 2018 22:07:38 +0200
Subject: [PATCH 09/53] Remove Postfix debugging
---
core/postfix/conf/main.cf | 2 --
1 file changed, 2 deletions(-)
diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf
index cd052d46..1f5c8f7c 100644
--- a/core/postfix/conf/main.cf
+++ b/core/postfix/conf/main.cf
@@ -2,8 +2,6 @@
# General
###############
-debug_peer_list = 0.0.0.0/0
-
# Main domain and hostname
mydomain = {{ DOMAIN }}
myhostname = {{ HOSTNAMES.split(",")[0] }}
From 11a8e49f059c4eb71a8f931edcbfa238d90873a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Tue, 16 Oct 2018 11:09:42 +0300
Subject: [PATCH 10/53] Compose file upgrade and define more variables for
setup
---
setup/flavors/compose/docker-compose.yml | 38 ++++++++++++------------
setup/flavors/compose/mailu.env | 16 +++++-----
2 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index fcf0c092..a82817af 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -2,7 +2,7 @@
# This file is auto-generated by the Mailu configuration wizard.
# Please read the documentation before attempting any change.
-version: '2'
+version: '3.7'
services:
@@ -11,7 +11,7 @@ services:
image: redis:alpine
restart: always
volumes:
- - "$ROOT/redis:/data"
+ - "{{ root }}/redis:/data"
# Core services
front:
@@ -24,15 +24,15 @@ services:
ports:
{% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
{% if bind4 %}
- - "$PUBLIC_IPV4:{{ port }}:{{ port }}"
+ - "{{ bind4}}:{{ port }}:{{ port }}"
{% endif %}
{% if bind6 %}
- - "$PUBLIC_IPV6:{{ port }}:{{ port }}"
+ - "{{ bind6 }}:{{ port }}:{{ port }}"
{% endif %}
{% endfor %}
{% if flavor in ('cert', 'mail') %}
volumes:
- - "$ROOT/certs:/certs"
+ - "{{ root }}/certs:/certs"
{% endif %}
admin:
@@ -44,8 +44,8 @@ services:
- 127.0.0.1:8080:80
{% endif %}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/dkim:/dkim"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/dkim:/dkim"
depends_on:
- redis
@@ -54,9 +54,9 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/mail:/mail"
- - "$ROOT/overrides:/overrides"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/mail:/mail"
+ - "{{ root }}/overrides:/overrides"
depends_on:
- front
@@ -65,8 +65,8 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/overrides:/overrides"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/overrides:/overrides"
depends_on:
- front
@@ -77,9 +77,9 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/filter:/var/lib/rspamd"
- - "$ROOT/dkim:/dkim"
- - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+ - "{{ root }}/filter:/var/lib/rspamd"
+ - "{{ root }}/dkim:/dkim"
+ - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d"
depends_on:
- front
{% endif %}
@@ -90,7 +90,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/filter:/data"
+ - "{{ root }}/filter:/data"
{% endif %}
{% if enable_webdav %}
@@ -99,7 +99,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/dav:/data"
+ - "{{ root }}/dav:/data"
{% endif %}
{% if enable_fetchmail %}
@@ -108,7 +108,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
+ - "{{ root }}/data:/data"
{% endif %}
# Webmail
@@ -118,7 +118,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/webmail:/data"
+ - "{{ root }}/webmail:/data"
depends_on:
- imap
{% endif %}
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 24d7b247..2de771f3 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -9,14 +9,16 @@
###################################
# Set this to the path where Mailu data and configuration is stored
-ROOT=/mailu
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
# Set to a randomly generated 16 bytes string
SECRET_KEY={{ secret(16) }}
# Address where listening ports should bind
-{% if bind4 %}PUBLIC_IPV4={{ bind4 }}{% endif %}
-{% if bind6 %}PUBLIC_IPV6={{ bind6 }}{% endif %}
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4=127.0.0.1
+# PUBLIC_IPV6=::1
# Mail address of the postmaster
POSTMASTER={{ postmaster }}
@@ -75,16 +77,16 @@ DOMAIN_REGISTRATION=true
###################################
# Path to the admin interface if enabled
-WEB_ADMIN=/admin
+WEB_ADMIN={{ admin_path }}
# Path to the webmail if enabled
-WEB_WEBMAIL=/webmail
+WEB_WEBMAIL={{ webmail_path }}
# Website name
-SITENAME=Mailu
+SITENAME={{ site_name }{
# Linked Website URL
-WEBSITE=https://mailu.io
+WEBSITE={{ website }}
{% if recaptcha_public_key and recaptcha_private_key %}
# Registration reCaptcha settings (warning, this has some privacy impact)
From 0d164486b45b194de81905089b37a4557a828a8f Mon Sep 17 00:00:00 2001
From: Ionut Filip
Date: Tue, 16 Oct 2018 12:34:55 +0300
Subject: [PATCH 11/53] docker-compose variables and setup
---
setup/Dockerfile | 4 +++-
setup/docker-compose.yml | 1 +
setup/flavors/compose/docker-compose.yml | 2 +-
setup/flavors/compose/mailu.env | 8 ++++----
setup/flavors/compose/setup.html | 6 +++---
setup/server.py | 6 ++++--
setup/templates/steps/expose.html | 4 ++--
setup/templates/steps/root.html | 8 ++++++++
setup/templates/wizard.html | 1 +
9 files changed, 27 insertions(+), 13 deletions(-)
create mode 100644 setup/templates/steps/root.html
diff --git a/setup/Dockerfile b/setup/Dockerfile
index 1fc808f1..c970e57d 100644
--- a/setup/Dockerfile
+++ b/setup/Dockerfile
@@ -10,8 +10,10 @@ RUN apk add --no-cache git \
COPY server.py ./server.py
COPY setup.py ./setup.py
COPY main.py ./main.py
+COPY flavors /data/master/flavors
+COPY templates /data/master/templates
-RUN python setup.py https://github.com/mailu/mailu /data
+#RUN python setup.py https://github.com/mailu/mailu /data
EXPOSE 80/tcp
diff --git a/setup/docker-compose.yml b/setup/docker-compose.yml
index 9288bb7e..30966167 100644
--- a/setup/docker-compose.yml
+++ b/setup/docker-compose.yml
@@ -10,4 +10,5 @@ services:
image: mailu/setup
ports:
- "80:80"
+ build: .
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index a82817af..3dcfa5a2 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -24,7 +24,7 @@ services:
ports:
{% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
{% if bind4 %}
- - "{{ bind4}}:{{ port }}:{{ port }}"
+ - "{{ bind4 }}:{{ port }}:{{ port }}"
{% endif %}
{% if bind6 %}
- "{{ bind6 }}:{{ port }}:{{ port }}"
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 2de771f3..1512e75b 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -10,15 +10,15 @@
# Set this to the path where Mailu data and configuration is stored
# This variable is now set directly in `docker-compose.yml by the setup utility
-# ROOT=/mailu
+# ROOT= {{ root }}
# Set to a randomly generated 16 bytes string
SECRET_KEY={{ secret(16) }}
# Address where listening ports should bind
# This variables are now set directly in `docker-compose.yml by the setup utility
-# PUBLIC_IPV4=127.0.0.1
-# PUBLIC_IPV6=::1
+# PUBLIC_IPV4= {{ bind4 }} (default: 127.0.0.1)
+# PUBLIC_IPV6= {{ bind6 }} (default: ::1)
# Mail address of the postmaster
POSTMASTER={{ postmaster }}
@@ -83,7 +83,7 @@ WEB_ADMIN={{ admin_path }}
WEB_WEBMAIL={{ webmail_path }}
# Website name
-SITENAME={{ site_name }{
+SITENAME={{ site_name }}
# Linked Website URL
WEBSITE={{ website }}
diff --git a/setup/flavors/compose/setup.html b/setup/flavors/compose/setup.html
index e4506e6d..3c190c9c 100644
--- a/setup/flavors/compose/setup.html
+++ b/setup/flavors/compose/setup.html
@@ -4,13 +4,13 @@
Docker Compose expects a project file, named docker-compose.yml
in a project directory. First create your project directory.
-mkdir /mailu
+mkdir {{ root }}
Then download the project file. A side configuration file makes it easier
to read and check the configuration variables generated by the wizard.
-cd /mailu
+cd {{ root }}
wget {{ url_for('.file', uid=uid, filepath='docker-compose.yml', _external=True) }}
wget {{ url_for('.file', uid=uid, filepath='mailu.env', _external=True) }}
@@ -30,7 +30,7 @@ files before going any further.
To start your compose project, simply run the Docker Compose up
command.
-cd /mailu
+cd {{ root }}
docker-compose up -d
{% endcall %}
diff --git a/setup/server.py b/setup/server.py
index 108f5043..ddeafd90 100644
--- a/setup/server.py
+++ b/setup/server.py
@@ -32,9 +32,11 @@ def secret(length=16):
def build_app(path):
+ #Hardcoded master as the only version for test purposes
versions = [
- version for version in os.listdir(path)
- if os.path.isdir(os.path.join(path, version))
+ # version for version in os.listdir(path)
+ # if os.path.isdir(os.path.join(path, version))
+ "master"
]
app.jinja_env.trim_blocks = True
diff --git a/setup/templates/steps/expose.html b/setup/templates/steps/expose.html
index 665b08a2..372ebddc 100644
--- a/setup/templates/steps/expose.html
+++ b/setup/templates/steps/expose.html
@@ -14,12 +14,12 @@ avoid generic all-interfaces addresses like 0.0.0.0
or ::
IPv4 listen address
-
+
IPv6 listen address
-
+
You server will be available under a main hostname but may expose multiple public
diff --git a/setup/templates/steps/root.html b/setup/templates/steps/root.html
new file mode 100644
index 00000000..f32c2250
--- /dev/null
+++ b/setup/templates/steps/root.html
@@ -0,0 +1,8 @@
+{% call macros.panel("info", "Step 0 - Set root path") %}
+
Before starting root path must be set
+
+
+ Root path:
+
+
+{% endcall %}
diff --git a/setup/templates/wizard.html b/setup/templates/wizard.html
index da9d5134..bbebe569 100644
--- a/setup/templates/wizard.html
+++ b/setup/templates/wizard.html
@@ -9,6 +9,7 @@
{% endcall %}