From 12294a6e5aa3c57c8c603d97a35fa443a962a8e2 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Wed, 6 Jun 2018 18:40:51 +0000
Subject: [PATCH 01/62] Trying to enable fuzzy hashes for rspamd
---
services/rspamd/Dockerfile | 5 +---
services/rspamd/conf/fuzzy_check.conf | 34 ++++++++++++++++++++++
services/rspamd/conf/metrics.conf | 19 ++++++++++++
services/rspamd/conf/worker-controller.inc | 1 +
services/rspamd/conf/worker-fuzzy.inc | 5 ++++
services/rspamd/conf/worker-normal.inc | 1 +
6 files changed, 61 insertions(+), 4 deletions(-)
create mode 100644 services/rspamd/conf/fuzzy_check.conf
create mode 100644 services/rspamd/conf/metrics.conf
create mode 100644 services/rspamd/conf/worker-fuzzy.inc
diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile
index c6c2afdd..1b8d7e6b 100644
--- a/services/rspamd/Dockerfile
+++ b/services/rspamd/Dockerfile
@@ -1,15 +1,12 @@
FROM alpine:edge
-RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates
+RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates
RUN mkdir /run/rspamd
COPY conf/ /conf
COPY start.py /start.py
-# Temporary fix to remove references to rspamd-fuzzy for now
-RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
-
EXPOSE 11332/tcp 11334/tcp
CMD /start.py
diff --git a/services/rspamd/conf/fuzzy_check.conf b/services/rspamd/conf/fuzzy_check.conf
new file mode 100644
index 00000000..7c87e1c3
--- /dev/null
+++ b/services/rspamd/conf/fuzzy_check.conf
@@ -0,0 +1,34 @@
+rule "local" {
+ # Fuzzy storage server list
+ servers = "localhost:11335";
+ # Default symbol for unknown flags
+ symbol = "LOCAL_FUZZY_UNKNOWN";
+ # Additional mime types to store/check
+ mime_types = ["application/*"];
+ # Hash weight threshold for all maps
+ max_score = 20.0;
+ # Whether we can learn this storage
+ read_only = no;
+ # Ignore unknown flags
+ skip_unknown = yes;
+ # Hash generation algorithm
+ algorithm = "mumhash";
+
+ # Map flags to symbols
+ fuzzy_map = {
+ LOCAL_FUZZY_DENIED {
+ # Local threshold
+ max_score = 20.0;
+ # Flag to match
+ flag = 11;
+ }
+ LOCAL_FUZZY_PROB {
+ max_score = 10.0;
+ flag = 12;
+ }
+ LOCAL_FUZZY_WHITE {
+ max_score = 2.0;
+ flag = 13;
+ }
+ }
+}
diff --git a/services/rspamd/conf/metrics.conf b/services/rspamd/conf/metrics.conf
new file mode 100644
index 00000000..6a31964f
--- /dev/null
+++ b/services/rspamd/conf/metrics.conf
@@ -0,0 +1,19 @@
+group "fuzzy" {
+ max_score = 12.0;
+ symbol "LOCAL_FUZZY_UNKNOWN" {
+ weight = 5.0;
+ description = "Generic fuzzy hash match";
+ }
+ symbol "LOCAL_FUZZY_DENIED" {
+ weight = 12.0;
+ description = "Denied fuzzy hash";
+ }
+ symbol "LOCAL_FUZZY_PROB" {
+ weight = 5.0;
+ description = "Probable fuzzy hash";
+ }
+ symbol "LOCAL_FUZZY_WHITE" {
+ weight = -2.1;
+ description = "Whitelisted fuzzy hash";
+ }
+}
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 6a020672..dd143942 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,4 @@
+type = "controller";
bind_socket = "*:11334";
password = "mailu";
secure_ip = "{{ FRONT_ADDRESS }}";
diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc
new file mode 100644
index 00000000..a0021a03
--- /dev/null
+++ b/services/rspamd/conf/worker-fuzzy.inc
@@ -0,0 +1,5 @@
+type = "fuzzy";
+count = 1;
+backend = "redis";
+expire = 90d;
+allow_update = ["127.0.0.1"];
diff --git a/services/rspamd/conf/worker-normal.inc b/services/rspamd/conf/worker-normal.inc
index a6ee8317..ab996fb8 100644
--- a/services/rspamd/conf/worker-normal.inc
+++ b/services/rspamd/conf/worker-normal.inc
@@ -1 +1,2 @@
+type = "normal";
enabled = false;
From 23e288aadcf7d5df941c8516fae9d0cbd6b1e054 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 24 Sep 2018 17:29:31 +0000
Subject: [PATCH 02/62] Enabling swarm deployment on master branch: -Extends
the usage of POD_ADDRESS_RANGE -Provides documentation
---
core/dovecot/conf/dovecot.conf | 2 +-
core/postfix/conf/main.cf | 2 +-
docs/swarm/master/README.md | 349 +++++++++++++++++++++
services/rspamd/conf/worker-controller.inc | 2 +-
4 files changed, 352 insertions(+), 3 deletions(-)
create mode 100644 docs/swarm/master/README.md
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index a5973bf8..7cf10774 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -5,7 +5,7 @@ log_path = /dev/stderr
protocols = imap pop3 lmtp sieve
postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
hostname = {{ HOSTNAMES.split(",")[0] }}
-submission_host = {{ FRONT_ADDRESS }}
+submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
service dict {
unix_listener dict {
diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf
index 2f2c6990..bde42dd1 100644
--- a/core/postfix/conf/main.cf
+++ b/core/postfix/conf/main.cf
@@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }}
recipient_delimiter = {{ RECIPIENT_DELIMITER }}
# Only the front server is allowed to perform xclient
-smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }}
+smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
###############
# TLS
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
new file mode 100644
index 00000000..1406e05c
--- /dev/null
+++ b/docs/swarm/master/README.md
@@ -0,0 +1,349 @@
+# Install Mailu on a docker swarm
+
+## Prequisites
+
+### Swarm
+
+In order to deploy Mailu on a swarm, you will first need to initialize the swarm:
+
+The main command will be:
+```bash
+docker swarm init --advertise-addr
+```
+See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
+
+If you want to add other managers or workers, please use:
+```bash
+docker swarm join --token xxxxx
+```
+See https://docs.docker.com/engine/swarm/join-nodes/
+
+You have now a working swarm, and you can check its status with:
+```bash
+core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls
+ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
+xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce
+sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce
+mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce
+```
+
+### Volume definition
+For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm.
+Hereafter we will use a NFS share:
+```bash
+core@coreos-01 ~ $ showmount -e 192.168.0.30
+Export list for 192.168.0.30:
+/mnt/Pool1/pv 192.168.0.0
+```
+
+on the nfs server, I am using the following /etc/exports
+```bash
+$more /etc/exports
+/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0
+```
+on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up)
+```bash
+$mkdir /mnt/Pool1/pv/mailu
+```
+
+On your manager node, mount the nfs share to check that the share is available:
+```bash
+core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/
+```
+If this is ok, you can umount it:
+```bash
+core@coreos-01 ~ $ sudo umount /mnt/local/
+```
+
+
+### Networking mode
+On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+
+In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
+
+Let's create the mailu_default network:
+```bash
+core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default
+core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
+ "Subnet": "10.0.1.0/24",
+```
+In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
+
+Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network)
+
+### Scalability
+- smtp and imap are scalable
+- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time)
+- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
+
+### Variable substitution and docker-compose.yml
+The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to :
+- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc
+- change the way we define the volumes (nfs share in our case)
+- add a deploy section for every service
+
+### Docker compose
+An example of docker-compose-stack.yml file is available here:
+
+```yaml
+
+version: '3.2'
+
+services:
+
+ front:
+ image: mailu/nginx:master
+ restart: always
+ env_file: .env
+ ports:
+ - target: 80
+ published: 80
+ - target: 443
+ published: 443
+ - target: 110
+ published: 110
+ - target: 143
+ published: 143
+ - target: 993
+ published: 993
+ - target: 995
+ published: 995
+ - target: 25
+ published: 25
+ - target: 465
+ published: 465
+ - target: 587
+ published: 587
+ volumes:
+# - "$ROOT/certs:/certs"
+ - type: volume
+ source: mailu_certs
+ target: /certs
+ deploy:
+ replicas: 2
+
+ redis:
+ image: redis:alpine
+ restart: always
+ volumes:
+# - "$ROOT/redis:/data"
+ - type: volume
+ source: mailu_redis
+ target: /data
+ deploy:
+ replicas: 1
+
+ imap:
+ image: mailu/dovecot:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/mail:/mail"
+ - type: volume
+ source: mailu_mail
+ target: /mail
+# - "$ROOT/overrides:/overrides"
+ - type: volume
+ source: mailu_overrides
+ target: /overrides
+ depends_on:
+ - front
+ deploy:
+ replicas: 2
+
+ smtp:
+ image: mailu/postfix:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/overrides:/overrides"
+ - type: volume
+ source: mailu_overrides
+ target: /overrides
+ depends_on:
+ - front
+ deploy:
+ replicas: 2
+
+ antispam:
+ image: mailu/rspamd:master
+ restart: always
+ env_file: .env
+ environment:
+ - POD_ADDRESS_RANGE=10.0.1.0/24
+ depends_on:
+ - front
+ volumes:
+# - "$ROOT/filter:/var/lib/rspamd"
+ - type: volume
+ source: mailu_filter
+ target: /var/lib/rspamd
+# - "$ROOT/dkim:/dkim"
+ - type: volume
+ source: mailu_dkim
+ target: /dkim
+# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+ - type: volume
+ source: mailu_overrides_rspamd
+ target: /etc/rspamd/override.d
+ deploy:
+ replicas: 1
+
+ antivirus:
+ image: mailu/none:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/filter:/data"
+ - type: volume
+ source: mailu_filter
+ target: /data
+ deploy:
+ replicas: 1
+
+ webdav:
+ image: mailu/none:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/dav:/data"
+ - type: volume
+ source: mailu_dav
+ target: /data
+ deploy:
+ replicas: 1
+
+ admin:
+ image: mailu/admin:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+# - "$ROOT/dkim:/dkim"
+ - type: volume
+ source: mailu_dkim
+ target: /dkim
+ - /var/run/docker.sock:/var/run/docker.sock:ro
+ depends_on:
+ - redis
+ deploy:
+ replicas: 1
+
+ webmail:
+ image: "mailu/roundcube:master"
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/webmail:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+ depends_on:
+ - imap
+ deploy:
+ replicas: 2
+
+ fetchmail:
+ image: mailu/fetchmail:master
+ restart: always
+ env_file: .env
+ volumes:
+# - "$ROOT/data:/data"
+ - type: volume
+ source: mailu_data
+ target: /data
+ deploy:
+ replicas: 1
+
+networks:
+ default:
+ external:
+ name: mailu_default
+
+volumes:
+ mailu_filter:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/filter"
+ mailu_dkim:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/dkim"
+ mailu_overrides_rspamd:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/overrides/rspamd"
+ mailu_data:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/data"
+ mailu_mail:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/mail"
+ mailu_overrides:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/overrides"
+ mailu_dav:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/dav"
+ mailu_certs:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/certs"
+ mailu_redis:
+ driver_opts:
+ type: "nfs"
+ o: "addr=192.168.0.30,nolock,soft,rw"
+ device: ":/mnt/Pool1/pv/mailu/redis"
+```
+
+### Deploy Mailu on the docker swarm
+Run the following command:
+```bash
+docker stack deploy -c docker-compose-stack.yml mailu
+```
+See how the services are being deployed:
+```bash
+core@coreos-01 ~ $ docker service ls
+ID NAME MODE REPLICAS IMAGE PORTS
+ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:1.5
+pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:1.5
+```
+check a specific service:
+```bash
+core@coreos-01 ~ $ docker service ps mailu_fetchmail
+ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS
+tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago
+```
+
+### Remove the stack
+Run the follwoing command:
+```bash
+core@coreos-01 ~ $ docker stack rm mailu
+```
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 6a020672..0cb0d5c0 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,3 @@
bind_socket = "*:11334";
password = "mailu";
-secure_ip = "{{ FRONT_ADDRESS }}";
+secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}";
From f5f09fad6ecc41e64b90dc4044450b61ea19ae4a Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Tue, 25 Sep 2018 18:54:40 +0000
Subject: [PATCH 03/62] Reverting the patch for dovecot.conf, as it is not
needed
---
core/dovecot/conf/dovecot.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf
index 7cf10774..a5973bf8 100644
--- a/core/dovecot/conf/dovecot.conf
+++ b/core/dovecot/conf/dovecot.conf
@@ -5,7 +5,7 @@ log_path = /dev/stderr
protocols = imap pop3 lmtp sieve
postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
hostname = {{ HOSTNAMES.split(",")[0] }}
-submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}
+submission_host = {{ FRONT_ADDRESS }}
service dict {
unix_listener dict {
From fcad52b1454aeeb5b11d64fe0186f644922f96b2 Mon Sep 17 00:00:00 2001
From: kaiyou
Date: Thu, 27 Sep 2018 22:45:16 +0200
Subject: [PATCH 04/62] Implement a start date filter for autoreply, fixes #362
---
core/admin/mailu/models.py | 2 ++
core/admin/mailu/ui/forms.py | 1 +
core/admin/mailu/ui/templates/user/reply.html | 7 ++++--
.../migrations/versions/3b281286c7bd_.py | 24 +++++++++++++++++++
core/dovecot/conf/pigeonhole-sieve.dict | 8 +++++++
core/dovecot/sieve/before.sieve | 1 +
6 files changed, 41 insertions(+), 2 deletions(-)
create mode 100644 core/admin/migrations/versions/3b281286c7bd_.py
diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index 71c5f4d7..1be4c8e6 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -249,6 +249,8 @@ class User(Base, Email):
reply_enabled = db.Column(db.Boolean(), nullable=False, default=False)
reply_subject = db.Column(db.String(255), nullable=True, default=None)
reply_body = db.Column(db.Text(), nullable=True, default=None)
+ reply_startdate = db.Column(db.Date, nullable=False,
+ default=date(1900, 1, 1))
reply_enddate = db.Column(db.Date, nullable=False,
default=date(2999, 12, 31))
diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py
index 326d721b..4f7a30ae 100644
--- a/core/admin/mailu/ui/forms.py
+++ b/core/admin/mailu/ui/forms.py
@@ -117,6 +117,7 @@ class UserReplyForm(flask_wtf.FlaskForm):
reply_subject = fields.StringField(_('Reply subject'))
reply_body = fields.StringField(_('Reply body'),
widget=widgets.TextArea())
+ reply_startdate = fields.html5.DateField(_('Start of vacation'))
reply_enddate = fields.html5.DateField(_('End of vacation'))
submit = fields.SubmitField(_('Update'))
diff --git a/core/admin/mailu/ui/templates/user/reply.html b/core/admin/mailu/ui/templates/user/reply.html
index 7906bc42..7225a178 100644
--- a/core/admin/mailu/ui/templates/user/reply.html
+++ b/core/admin/mailu/ui/templates/user/reply.html
@@ -13,14 +13,17 @@
{% endcall %}
diff --git a/core/admin/migrations/versions/3b281286c7bd_.py b/core/admin/migrations/versions/3b281286c7bd_.py
new file mode 100644
index 00000000..78e44a4c
--- /dev/null
+++ b/core/admin/migrations/versions/3b281286c7bd_.py
@@ -0,0 +1,24 @@
+""" Add a start day for vacations
+
+Revision ID: 3b281286c7bd
+Revises: 049fed905da7
+Create Date: 2018-09-27 22:20:08.158553
+
+"""
+
+revision = '3b281286c7bd'
+down_revision = '049fed905da7'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ with op.batch_alter_table('user') as batch:
+ batch.add_column(sa.Column('reply_startdate', sa.Date(), nullable=False,
+ server_default="1900-01-01"))
+
+
+def downgrade():
+ with op.batch_alter_table('user') as batch:
+ batch.drop_column('reply_startdate')
diff --git a/core/dovecot/conf/pigeonhole-sieve.dict b/core/dovecot/conf/pigeonhole-sieve.dict
index 917fce83..604371a8 100644
--- a/core/dovecot/conf/pigeonhole-sieve.dict
+++ b/core/dovecot/conf/pigeonhole-sieve.dict
@@ -41,3 +41,11 @@ map {
username_field = email
value_field = reply_enddate
}
+
+map {
+ pattern = priv/reply_startdate
+ table = user
+ username_field = email
+ value_field = reply_startdate
+}
+
diff --git a/core/dovecot/sieve/before.sieve b/core/dovecot/sieve/before.sieve
index 6ebc20c5..81d20f30 100644
--- a/core/dovecot/sieve/before.sieve
+++ b/core/dovecot/sieve/before.sieve
@@ -34,6 +34,7 @@ if exists "X-Virus" {
}
if allof (string :is "${extdata.reply_enabled}" "1",
+ currentdate :value "ge" "date" "${extdata.reply_startdate}",
currentdate :value "le" "date" "${extdata.reply_enddate}")
{
vacation :days 1 :subject "${extdata.reply_subject}" "${extdata.reply_body}";
From 6b34b2728ece020918dcb10a901afdd9894a69e3 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Sun, 7 Oct 2018 16:38:41 +0000
Subject: [PATCH 05/62] Declare fuzzy_worker port 11335 in EXPOSE section
---
services/rspamd/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile
index 7dff8c1f..cfb4d0eb 100644
--- a/services/rspamd/Dockerfile
+++ b/services/rspamd/Dockerfile
@@ -9,7 +9,7 @@ RUN mkdir /run/rspamd
COPY conf/ /conf
COPY start.py /start.py
-EXPOSE 11332/tcp 11334/tcp
+EXPOSE 11332/tcp 11334/tcp 11335/tcp
VOLUME ["/var/lib/rspamd"]
From 1f71d10899d72571b39a60448a23adc41d5f10f2 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Sun, 7 Oct 2018 16:46:42 +0000
Subject: [PATCH 06/62] Change POD_ADDRESS_RANGE introduction like it is done
on deovecot-sql.conf.ext
---
services/rspamd/conf/worker-controller.inc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc
index 0cb0d5c0..4b23a897 100644
--- a/services/rspamd/conf/worker-controller.inc
+++ b/services/rspamd/conf/worker-controller.inc
@@ -1,3 +1,3 @@
bind_socket = "*:11334";
password = "mailu";
-secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}";
+secure_ip = "{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% endif %}";
From 9d610f56f7ff39232cb15ce02b42f8aca94d846a Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 8 Oct 2018 18:53:44 +0000
Subject: [PATCH 07/62] Added some lines around ingress mode
---
docs/swarm/master/README.md | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
index 1406e05c..c09f1dd3 100644
--- a/docs/swarm/master/README.md
+++ b/docs/swarm/master/README.md
@@ -57,8 +57,11 @@ core@coreos-01 ~ $ sudo umount /mnt/local/
### Networking mode
-On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+On this example, we are using:
+- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
+- the default ingress mode.
+## Allow authentification with the mesh routing
In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
Let's create the mailu_default network:
@@ -68,8 +71,22 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
"Subnet": "10.0.1.0/24",
```
In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
+In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
+
+## Limitation with the ingress mode
+With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
+
+This issue is known and discussed here:
+
+https://github.com/moby/moby/issues/25526
+
+A workaround (using network host mode and global deployment) is discussed here:
+
+https://github.com/moby/moby/issues/25526#issuecomment-336363408
+
+## Don't create an open relay !
+As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
-Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network)
### Scalability
- smtp and imap are scalable
From 6bd365e7714e765c75d8fdd93bc04065a53eae14 Mon Sep 17 00:00:00 2001
From: ofthesun9
Date: Mon, 8 Oct 2018 21:00:44 +0000
Subject: [PATCH 08/62] Change title layout
---
docs/swarm/master/README.md | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md
index c09f1dd3..2a2a021a 100644
--- a/docs/swarm/master/README.md
+++ b/docs/swarm/master/README.md
@@ -56,12 +56,12 @@ core@coreos-01 ~ $ sudo umount /mnt/local/
```
-### Networking mode
+## Networking mode
On this example, we are using:
- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service.
- the default ingress mode.
-## Allow authentification with the mesh routing
+### Allow authentification with the mesh routing
In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE.
Let's create the mailu_default network:
@@ -73,7 +73,7 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet
In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24
In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation...
-## Limitation with the ingress mode
+### Limitation with the ingress mode
With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network)
This issue is known and discussed here:
@@ -84,22 +84,22 @@ A workaround (using network host mode and global deployment) is discussed here:
https://github.com/moby/moby/issues/25526#issuecomment-336363408
-## Don't create an open relay !
+### Don't create an open relay !
As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-(
-### Scalability
+## Scalability
- smtp and imap are scalable
- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time)
- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file)
-### Variable substitution and docker-compose.yml
+## Variable substitution and docker-compose.yml
The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to :
- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc
- change the way we define the volumes (nfs share in our case)
- add a deploy section for every service
-### Docker compose
+## Docker compose
An example of docker-compose-stack.yml file is available here:
```yaml
@@ -340,7 +340,7 @@ volumes:
device: ":/mnt/Pool1/pv/mailu/redis"
```
-### Deploy Mailu on the docker swarm
+## Deploy Mailu on the docker swarm
Run the following command:
```bash
docker stack deploy -c docker-compose-stack.yml mailu
@@ -359,7 +359,7 @@ ID NAME IMAGE NODE
tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago
```
-### Remove the stack
+## Remove the stack
Run the follwoing command:
```bash
core@coreos-01 ~ $ docker stack rm mailu
From c8b39c5d4a4eb30b59a88342b4b428cb0f02c426 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20S=C3=A4nger?=
Date: Wed, 10 Oct 2018 19:29:23 +0200
Subject: [PATCH 09/62] support bcrypt and use it as default
---
core/admin/mailu/__init__.py | 2 +-
core/admin/mailu/models.py | 3 ++-
core/admin/requirements-prod.txt | 1 +
core/admin/requirements.txt | 1 +
docs/compose/.env | 4 ++--
5 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py
index d77420e6..167f04ae 100644
--- a/core/admin/mailu/__init__.py
+++ b/core/admin/mailu/__init__.py
@@ -57,7 +57,7 @@ default_config = {
'RECAPTCHA_PUBLIC_KEY': '',
'RECAPTCHA_PRIVATE_KEY': '',
# Advanced settings
- 'PASSWORD_SCHEME': 'SHA512-CRYPT',
+ 'PASSWORD_SCHEME': 'BLF-CRYPT',
# Host settings
'HOST_IMAP': 'imap',
'HOST_POP3': 'imap',
diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index 1bcc4e9f..0c80fd4f 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -276,7 +276,8 @@ class User(Base, Email):
else:
return self.email
- scheme_dict = {'SHA512-CRYPT': "sha512_crypt",
+ scheme_dict = {'BLF-CRYPT': "bcrypt",
+ 'SHA512-CRYPT': "sha512_crypt",
'SHA256-CRYPT': "sha256_crypt",
'MD5-CRYPT': "md5_crypt",
'CRYPT': "des_crypt"}
diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt
index 94c28177..e321a4d6 100644
--- a/core/admin/requirements-prod.txt
+++ b/core/admin/requirements-prod.txt
@@ -1,6 +1,7 @@
alembic==0.9.9
asn1crypto==0.24.0
Babel==2.5.3
+bcrypt==3.1.4
blinker==1.4
certifi==2018.4.16
cffi==1.11.5
diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt
index a40e6eb5..d6e7adb1 100644
--- a/core/admin/requirements.txt
+++ b/core/admin/requirements.txt
@@ -17,3 +17,4 @@ tabulate
PyYAML
PyOpenSSL
dnspython
+bcrypt
diff --git a/docs/compose/.env b/docs/compose/.env
index 721aaf22..cdb48310 100644
--- a/docs/compose/.env
+++ b/docs/compose/.env
@@ -124,8 +124,8 @@ WEBSITE=https://mailu.io
COMPOSE_PROJECT_NAME=mailu
# Default password scheme used for newly created accounts and changed passwords
-# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
-PASSWORD_SCHEME=SHA512-CRYPT
+# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME=BLF-CRYPT
# Header to take the real ip from
REAL_IP_HEADER=
From 6aafef88bdc2bae1954d7c7baa9566e2f3807da4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20S=C3=A4nger?=
Date: Thu, 11 Oct 2018 02:57:13 +0200
Subject: [PATCH 10/62] remove apk-warning about cache
---
core/admin/Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile
index 08de0e88..2e637206 100644
--- a/core/admin/Dockerfile
+++ b/core/admin/Dockerfile
@@ -7,7 +7,7 @@ COPY requirements-prod.txt requirements.txt
RUN apk add --no-cache openssl \
&& apk add --no-cache --virtual build-dep openssl-dev libffi-dev python-dev build-base \
&& pip install -r requirements.txt \
- && apk del build-dep
+ && apk del --no-cache build-dep
COPY mailu ./mailu
COPY migrations ./migrations
From f2259c3302ccafe17dc9e8f39ce177ffd19a2b18 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20S=C3=A4nger?=
Date: Thu, 11 Oct 2018 03:18:35 +0200
Subject: [PATCH 11/62] reduce webmail image-layers/sizes
---
webmails/rainloop/Dockerfile | 13 +++++++------
webmails/roundcube/Dockerfile | 15 +++++++--------
2 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile
index 2da4e672..889c8486 100644
--- a/webmails/rainloop/Dockerfile
+++ b/webmails/rainloop/Dockerfile
@@ -1,20 +1,21 @@
FROM php:7.2-apache
-RUN apt-get update && apt-get install -y \
- unzip python3 python3-jinja2
-
ENV RAINLOOP_URL https://github.com/RainLoop/rainloop-webmail/releases/download/v1.12.1/rainloop-community-1.12.1.zip
-RUN rm -rf /var/www/html/ \
+RUN apt-get update && apt-get install -y \
+ unzip python3 python3-jinja2 \
+ && rm -rf /var/www/html/ \
&& mkdir /var/www/html \
&& cd /var/www/html \
&& curl -L -O ${RAINLOOP_URL} \
- && unzip *.zip \
+ && unzip -q *.zip \
&& rm -f *.zip \
&& rm -rf data/ \
&& find . -type d -exec chmod 755 {} \; \
&& find . -type f -exec chmod 644 {} \; \
- && chown -R www-data: *
+ && chown -R www-data: * \
+ && apt-get purge -y unzip \
+ && rm -rf /var/lib/apt/lists
COPY include.php /var/www/html/include.php
COPY php.ini /usr/local/etc/php/conf.d/rainloop.ini
diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile
index ad198236..50a58a4f 100644
--- a/webmails/roundcube/Dockerfile
+++ b/webmails/roundcube/Dockerfile
@@ -1,14 +1,12 @@
FROM php:7.2-apache
-RUN apt-get update && apt-get install -y \
- zlib1g-dev \
- && docker-php-ext-install zip
-
ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.7/roundcubemail-1.3.7-complete.tar.gz
-RUN echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini
-
-RUN rm -rf /var/www/html/ \
+RUN apt-get update && apt-get install -y \
+ zlib1g-dev \
+ && docker-php-ext-install zip \
+ && echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini \
+ && rm -rf /var/www/html/ \
&& cd /var/www \
&& curl -L -O ${ROUNDCUBE_URL} \
&& tar -xf *.tar.gz \
@@ -17,7 +15,8 @@ RUN rm -rf /var/www/html/ \
&& cd html \
&& rm -rf CHANGELOG INSTALL LICENSE README.md UPGRADING composer.json-dist installer \
&& sed -i 's,mod_php5.c,mod_php7.c,g' .htaccess \
- && chown -R www-data: logs temp
+ && chown -R www-data: logs temp \
+ && rm -rf /var/lib/apt/lists
COPY php.ini /usr/local/etc/php/conf.d/roundcube.ini
From 77e3fc0ebcd7ac4970e8b05f5e8fa9cd5d97992c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Thu, 11 Oct 2018 18:14:37 +0300
Subject: [PATCH 12/62] Some documentation flow refactoring and updates: -
Improve advice on IP binding; Follow up on issue #641 - mailradar is dead.
Found mxtoolbox instead - Fix some internal linking
---
docs/compose/setup.rst | 57 ++++++++++++++++++++++++++++++------------
docs/configuration.rst | 10 +++++++-
docs/dns.rst | 2 ++
docs/setup.rst | 7 +++---
4 files changed, 55 insertions(+), 21 deletions(-)
diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst
index 8759e2f1..d4fb8014 100644
--- a/docs/compose/setup.rst
+++ b/docs/compose/setup.rst
@@ -26,35 +26,60 @@ for the ``VERSION_TAG`` branch, use:
wget https://mailu.io/VERSION_TAG/_downloads/docker-compose.yml
wget https://mailu.io/VERSION_TAG/_downloads/.env
-Then open the ``.env`` file to setup the mail server. Modify the ``ROOT`` setting
-to match your setup directory if different from ``/mailu``.
+Important configuration variables
+---------------------------------
-Modify the ``VERSION`` configuration in the ``.env`` file to reflect the version you picked.
+Open the ``.env`` file and review the following variable settings:
-Set the common configuration values
------------------------------------
+- Change ``ROOT`` if you have your setup directory in a different location then ``/mailu``.
+- Check ``VERSION`` to reflect the version you picked. (``master`` or ``1.5``).
-Open the ``.env`` file and set configuration settings after reading the configuration
-documentation. Some settings are specific to the Docker Compose setup.
+Make sure to read the comments in the file and instructions from the :ref:`common_cfg` section.
-Modify ``BIND_ADDRESS4`` to match the public IP address assigned to your server.
-This address should be configured on one of the network interfaces of the server.
-If the address is not configured directly (NAT) on any of the network interfaces or if
-you would simply like the server to listen on all interfaces, use ``0.0.0.0``.
-
-Modify ``BIND_ADDRESS6`` to match the public IPv6 address assigned to your server.
-The behavior is identical to ``BIND_ADDRESS4``.
+TLS certificates
+````````````````
Set the ``TLS_FLAVOR`` to one of the following
values:
- ``cert`` is the default and requires certificates to be setup manually;
-- ``letsencrypt`` will use the Letsencrypt! CA to generate automatic ceriticates;
+- ``letsencrypt`` will use the *Letsencrypt!* CA to generate automatic ceriticates;
- ``mail`` is similar to ``cert`` except that TLS will only be served for
emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
- ``mail-letsencrypt`` is similar to ``letsencrypt`` except that TLS will only be served for
emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
-- ``notls`` will disable TLS, this is not recommended except for testing.
+- ``notls`` will disable TLS, this is not recommended except for testing
+
+.. note::
+
+ When using *Letsencrypt!* you have to make sure that the DNS ``A`` and ``AAAA`` records for the
+ all hostnames mentioned in the ``HOSTNAMES`` variable match with the ip adresses of you server.
+ Or else certificate generation will fail! See also: :ref:`dns_setup`.
+
+Bind address
+````````````
+
+Modify ``BIND_ADDRESS4`` and ``BIND_ADDRESS6`` to match the public IP addresses assigned to your server. For IPv6 you will need the ```` scope address.
+
+You can find those addresses by running the following:
+
+.. code-block:: bash
+
+ [root@mailu ~]$ ifconfig eth0
+ eth0: flags=4163 mtu 1500
+ inet 125.189.138.127 netmask 255.255.255.0 broadcast 5.189.138.255
+ inet6 fd21:aab2:717c:cc5a::1 prefixlen 64 scopeid 0x0
+ inet6 fe2f:2a73:43a8:7a1b::1 prefixlen 64 scopeid 0x20
+ ether 00:50:56:3c:b2:23 txqueuelen 1000 (Ethernet)
+ RX packets 174866612 bytes 127773819607 (118.9 GiB)
+ RX errors 0 dropped 0 overruns 0 frame 0
+ TX packets 19905110 bytes 2191519656 (2.0 GiB)
+ TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
+
+If the address is not configured directly (NAT) on any of the network interfaces or if
+you would simply like the server to listen on all interfaces, use ``0.0.0.0`` and ``::``. Note that running is this mode is not supported and can lead to `issues`_.
+
+.. _issues: https://github.com/Mailu/Mailu/issues/641
Enable optional features
------------------------
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 62b6f34e..cab30072 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -1,12 +1,20 @@
Mailu configuration settings
============================
+.. _common_cfg:
+
Common configuration
--------------------
The ``SECRET_KEY`` **must** be changed for every setup and set to a 16 bytes
randomly generated value. It is intended to secure authentication cookies
-among other critical uses.
+among other critical uses. This can be generated with a utility such as *pwgen*,
+which can be installed on most Linux systems:
+
+.. code-block:: bash
+
+ apt-get install pwgen
+ pwgen 16 1
The ``DOMAIN`` holds the main e-mail domain for the server. This email domain
is used for bounce emails, for generating the postmaster email and other
diff --git a/docs/dns.rst b/docs/dns.rst
index 3d94aecb..d2fd4131 100644
--- a/docs/dns.rst
+++ b/docs/dns.rst
@@ -1,3 +1,5 @@
+.. _dns_setup:
+
Setting up your DNS
===================
diff --git a/docs/setup.rst b/docs/setup.rst
index d7d0cc13..9771f886 100644
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -32,7 +32,7 @@ user. Make sure you complete the requirements for the flavor you chose.
You should also have at least a DNS hostname and a DNS name for receiving
emails. Some instructions are provided on the matter in the article
-[Setup your DNS](dns).
+:ref:`dns_setup`.
.. _`MFAshby's fork`: https://github.com/MFAshby/Mailu
@@ -68,10 +68,9 @@ Make sure that you test properly before going live!
- Try to receive an email from an external service
- Check the logs (``docker-compose logs -f servicenamehere``) to look for
warnings or errors
-- Use an open relay checker like `mailradar`_
+- Use an open relay checker like `mxtoolbox`_
to ensure you're not contributing to the spam problem on the internet.
- All tests there should result in "Relay denied".
- If using DMARC, be sure to check the reports you get to verify that legitimate
email is getting through and forgeries are being properly blocked.
- .. _mailradar: http://www.mailradar.com/openrelay/
+ .. _mxtoolbox: https://mxtoolbox.com/diagnostic.aspx
From 70c4e42f74d7236abc53b84de1bb4e90d5d30068 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Thu, 11 Oct 2018 18:33:58 +0300
Subject: [PATCH 13/62] Fix small typo
---
docs/compose/setup.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst
index d4fb8014..64ad7b25 100644
--- a/docs/compose/setup.rst
+++ b/docs/compose/setup.rst
@@ -48,7 +48,7 @@ values:
emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
- ``mail-letsencrypt`` is similar to ``letsencrypt`` except that TLS will only be served for
emails (IMAP and SMTP), not HTTP (use it behind reverse proxies);
-- ``notls`` will disable TLS, this is not recommended except for testing
+- ``notls`` will disable TLS, this is not recommended except for testing.
.. note::
From 3f7e7ca3a6fce6041a55f80c0ecd1a8b03633754 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Mon, 15 Oct 2018 19:36:37 +0300
Subject: [PATCH 14/62] Use defaults in variables, to allow for local builds
---
tests/build.yml | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/tests/build.yml b/tests/build.yml
index 0b6858a0..5f360ece 100644
--- a/tests/build.yml
+++ b/tests/build.yml
@@ -3,54 +3,54 @@ version: '3'
services:
front:
- image: $DOCKER_ORG/nginx:$VERSION
+ image: ${DOCKER_ORG:-mailu}/nginx:${VERSION:-local}
build: ../core/nginx
imap:
- image: $DOCKER_ORG/dovecot:$VERSION
+ image: ${DOCKER_ORG:-mailu}/dovecot:${VERSION:-local}
build: ../core/dovecot
smtp:
- image: $DOCKER_ORG/postfix:$VERSION
+ image: ${DOCKER_ORG:-mailu}/postfix:${VERSION:-local}
build: ../core/postfix
antispam:
- image: $DOCKER_ORG/rspamd:$VERSION
+ image: ${DOCKER_ORG:-mailu}/rspamd:${VERSION:-local}
build: ../services/rspamd
antivirus:
- image: $DOCKER_ORG/clamav:$VERSION
+ image: ${DOCKER_ORG:-mailu}/clamav:${VERSION:-local}
build: ../optional/clamav
webdav:
- image: $DOCKER_ORG/radicale:$VERSION
+ image: ${DOCKER_ORG:-mailu}/radicale:${VERSION:-local}
build: ../optional/radicale
admin:
- image: $DOCKER_ORG/admin:$VERSION
+ image: ${DOCKER_ORG:-mailu}/admin:${VERSION:-local}
build: ../core/admin
roundcube:
- image: $DOCKER_ORG/roundcube:$VERSION
+ image: ${DOCKER_ORG:-mailu}/roundcube:${VERSION:-local}
build: ../webmails/roundcube
rainloop:
- image: $DOCKER_ORG/rainloop:$VERSION
+ image: ${DOCKER_ORG:-mailu}/rainloop:${VERSION:-local}
build: ../webmails/rainloop
fetchmail:
- image: $DOCKER_ORG/fetchmail:$VERSION
+ image: ${DOCKER_ORG:-mailu}/fetchmail:${VERSION:-local}
build: ../services/fetchmail
none:
- image: $DOCKER_ORG/none:$VERSION
+ image: ${DOCKER_ORG:-mailu}/none:${VERSION:-local}
build: ../core/none
docs:
- image: $DOCKER_ORG/docs:$VERSION
+ image: ${DOCKER_ORG:-mailu}/docs:${VERSION:-local}
build: ../docs
setup:
- image: $DOCKER_ORG/setup:$VERSION
+ image: ${DOCKER_ORG:-mailu}/setup:${VERSION:-local}
build: ../setup
From b38deb18cffe35e3f2e445bd2c3635e11dee6a52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Mon, 15 Oct 2018 19:37:24 +0300
Subject: [PATCH 15/62] Update Dev-docs to use build.yml for building
---
docs/contributors/environment.rst | 66 ++++++++++++++++++-------------
1 file changed, 39 insertions(+), 27 deletions(-)
diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst
index a1cce193..f1f447e2 100644
--- a/docs/contributors/environment.rst
+++ b/docs/contributors/environment.rst
@@ -5,39 +5,51 @@ Docker containers
-----------------
The development environment is quite similar to the production one. You should always use
-the ``master`` version when developing. Simply add a build directive to the images
-you are working on in the ``docker-compose.yml``:
+the ``master`` version when developing.
-.. code-block:: yaml
+Building images
+```````````````
- webdav:
- build: ./optional/radicale
- image: mailu/$WEBDAV:$VERSION
- restart: always
- env_file: .env
- volumes:
- - "$ROOT/dav:/data"
-
- admin:
- build: ./core/admin
- image: mailu/admin:$VERSION
- restart: always
- env_file: .env
- volumes:
- - "$ROOT/data:/data"
- - "$ROOT/dkim:/dkim"
- - /var/run/docker.sock:/var/run/docker.sock:ro
- depends_on:
- - redis
-
-
-The build these containers.
+We supply a separate ``test/build.yml`` file for
+convenience. To build all Mailu containers:
.. code-block:: bash
- docker-compose build admin webdav
+ docker-compose -f tests/build.yml build
-Then you can simply start the stack as normal, newly-built images will be used.
+The ``build.yml`` file has two variables:
+
+#. ``$DOCKER_ORG``: First part of the image tag. Defaults to *mailu* and needs to be changed
+ only when pushing to your own Docker hub account.
+#. ``$VERSION``: Last part of the image tag. Defaults to *local* to differentiate from pulled
+ images.
+
+To re-build only specific containers at a later time.
+
+.. code-block:: bash
+
+ docker-compose -f tests/build.yml build admin webdav
+
+If you have to push the images to Docker Hub for testing in Docker Swarm or a remote
+host, you have to define ``DOCKER_ORG`` (usually your Docker user-name) and login to
+the hub.
+
+.. code-block:: bash
+
+ docker login
+ Username: Foo
+ Password: Bar
+ export DOCKER_ORG="Foo"
+ export VERSION="feat-extra-app"
+ docker-compose -f tests/build.yml build
+ docker-compose -f tests/build.yml push
+
+Running containers
+``````````````````
+
+To run the newly created images: ``cd`` to your project directory. Edit ``.env`` to set
+``VERSION`` to the same value as used during the build, which defaults to ``local``.
+After that you can run:
.. code-block:: bash
From 5035975c4168980c7dacc7f80c49b062c89eb7c6 Mon Sep 17 00:00:00 2001
From: kaiyou
Date: Mon, 15 Oct 2018 22:07:38 +0200
Subject: [PATCH 16/62] Remove Postfix debugging
---
core/postfix/conf/main.cf | 2 --
1 file changed, 2 deletions(-)
diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf
index cd052d46..1f5c8f7c 100644
--- a/core/postfix/conf/main.cf
+++ b/core/postfix/conf/main.cf
@@ -2,8 +2,6 @@
# General
###############
-debug_peer_list = 0.0.0.0/0
-
# Main domain and hostname
mydomain = {{ DOMAIN }}
myhostname = {{ HOSTNAMES.split(",")[0] }}
From 11a8e49f059c4eb71a8f931edcbfa238d90873a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tim=20M=C3=B6hlmann?=
Date: Tue, 16 Oct 2018 11:09:42 +0300
Subject: [PATCH 17/62] Compose file upgrade and define more variables for
setup
---
setup/flavors/compose/docker-compose.yml | 38 ++++++++++++------------
setup/flavors/compose/mailu.env | 16 +++++-----
2 files changed, 28 insertions(+), 26 deletions(-)
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index fcf0c092..a82817af 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -2,7 +2,7 @@
# This file is auto-generated by the Mailu configuration wizard.
# Please read the documentation before attempting any change.
-version: '2'
+version: '3.7'
services:
@@ -11,7 +11,7 @@ services:
image: redis:alpine
restart: always
volumes:
- - "$ROOT/redis:/data"
+ - "{{ root }}/redis:/data"
# Core services
front:
@@ -24,15 +24,15 @@ services:
ports:
{% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
{% if bind4 %}
- - "$PUBLIC_IPV4:{{ port }}:{{ port }}"
+ - "{{ bind4}}:{{ port }}:{{ port }}"
{% endif %}
{% if bind6 %}
- - "$PUBLIC_IPV6:{{ port }}:{{ port }}"
+ - "{{ bind6 }}:{{ port }}:{{ port }}"
{% endif %}
{% endfor %}
{% if flavor in ('cert', 'mail') %}
volumes:
- - "$ROOT/certs:/certs"
+ - "{{ root }}/certs:/certs"
{% endif %}
admin:
@@ -44,8 +44,8 @@ services:
- 127.0.0.1:8080:80
{% endif %}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/dkim:/dkim"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/dkim:/dkim"
depends_on:
- redis
@@ -54,9 +54,9 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/mail:/mail"
- - "$ROOT/overrides:/overrides"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/mail:/mail"
+ - "{{ root }}/overrides:/overrides"
depends_on:
- front
@@ -65,8 +65,8 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
- - "$ROOT/overrides:/overrides"
+ - "{{ root }}/data:/data"
+ - "{{ root }}/overrides:/overrides"
depends_on:
- front
@@ -77,9 +77,9 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/filter:/var/lib/rspamd"
- - "$ROOT/dkim:/dkim"
- - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+ - "{{ root }}/filter:/var/lib/rspamd"
+ - "{{ root }}/dkim:/dkim"
+ - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d"
depends_on:
- front
{% endif %}
@@ -90,7 +90,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/filter:/data"
+ - "{{ root }}/filter:/data"
{% endif %}
{% if enable_webdav %}
@@ -99,7 +99,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/dav:/data"
+ - "{{ root }}/dav:/data"
{% endif %}
{% if enable_fetchmail %}
@@ -108,7 +108,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/data:/data"
+ - "{{ root }}/data:/data"
{% endif %}
# Webmail
@@ -118,7 +118,7 @@ services:
restart: always
env_file: {{ env }}
volumes:
- - "$ROOT/webmail:/data"
+ - "{{ root }}/webmail:/data"
depends_on:
- imap
{% endif %}
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 24d7b247..2de771f3 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -9,14 +9,16 @@
###################################
# Set this to the path where Mailu data and configuration is stored
-ROOT=/mailu
+# This variable is now set directly in `docker-compose.yml by the setup utility
+# ROOT=/mailu
# Set to a randomly generated 16 bytes string
SECRET_KEY={{ secret(16) }}
# Address where listening ports should bind
-{% if bind4 %}PUBLIC_IPV4={{ bind4 }}{% endif %}
-{% if bind6 %}PUBLIC_IPV6={{ bind6 }}{% endif %}
+# This variables are now set directly in `docker-compose.yml by the setup utility
+# PUBLIC_IPV4=127.0.0.1
+# PUBLIC_IPV6=::1
# Mail address of the postmaster
POSTMASTER={{ postmaster }}
@@ -75,16 +77,16 @@ DOMAIN_REGISTRATION=true
###################################
# Path to the admin interface if enabled
-WEB_ADMIN=/admin
+WEB_ADMIN={{ admin_path }}
# Path to the webmail if enabled
-WEB_WEBMAIL=/webmail
+WEB_WEBMAIL={{ webmail_path }}
# Website name
-SITENAME=Mailu
+SITENAME={{ site_name }{
# Linked Website URL
-WEBSITE=https://mailu.io
+WEBSITE={{ website }}
{% if recaptcha_public_key and recaptcha_private_key %}
# Registration reCaptcha settings (warning, this has some privacy impact)
From 0d164486b45b194de81905089b37a4557a828a8f Mon Sep 17 00:00:00 2001
From: Ionut Filip
Date: Tue, 16 Oct 2018 12:34:55 +0300
Subject: [PATCH 18/62] docker-compose variables and setup
---
setup/Dockerfile | 4 +++-
setup/docker-compose.yml | 1 +
setup/flavors/compose/docker-compose.yml | 2 +-
setup/flavors/compose/mailu.env | 8 ++++----
setup/flavors/compose/setup.html | 6 +++---
setup/server.py | 6 ++++--
setup/templates/steps/expose.html | 4 ++--
setup/templates/steps/root.html | 8 ++++++++
setup/templates/wizard.html | 1 +
9 files changed, 27 insertions(+), 13 deletions(-)
create mode 100644 setup/templates/steps/root.html
diff --git a/setup/Dockerfile b/setup/Dockerfile
index 1fc808f1..c970e57d 100644
--- a/setup/Dockerfile
+++ b/setup/Dockerfile
@@ -10,8 +10,10 @@ RUN apk add --no-cache git \
COPY server.py ./server.py
COPY setup.py ./setup.py
COPY main.py ./main.py
+COPY flavors /data/master/flavors
+COPY templates /data/master/templates
-RUN python setup.py https://github.com/mailu/mailu /data
+#RUN python setup.py https://github.com/mailu/mailu /data
EXPOSE 80/tcp
diff --git a/setup/docker-compose.yml b/setup/docker-compose.yml
index 9288bb7e..30966167 100644
--- a/setup/docker-compose.yml
+++ b/setup/docker-compose.yml
@@ -10,4 +10,5 @@ services:
image: mailu/setup
ports:
- "80:80"
+ build: .
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index a82817af..3dcfa5a2 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -24,7 +24,7 @@ services:
ports:
{% for port in (80, 443, 25, 465, 587, 110, 995, 143, 993) %}
{% if bind4 %}
- - "{{ bind4}}:{{ port }}:{{ port }}"
+ - "{{ bind4 }}:{{ port }}:{{ port }}"
{% endif %}
{% if bind6 %}
- "{{ bind6 }}:{{ port }}:{{ port }}"
diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env
index 2de771f3..1512e75b 100644
--- a/setup/flavors/compose/mailu.env
+++ b/setup/flavors/compose/mailu.env
@@ -10,15 +10,15 @@
# Set this to the path where Mailu data and configuration is stored
# This variable is now set directly in `docker-compose.yml by the setup utility
-# ROOT=/mailu
+# ROOT= {{ root }}
# Set to a randomly generated 16 bytes string
SECRET_KEY={{ secret(16) }}
# Address where listening ports should bind
# This variables are now set directly in `docker-compose.yml by the setup utility
-# PUBLIC_IPV4=127.0.0.1
-# PUBLIC_IPV6=::1
+# PUBLIC_IPV4= {{ bind4 }} (default: 127.0.0.1)
+# PUBLIC_IPV6= {{ bind6 }} (default: ::1)
# Mail address of the postmaster
POSTMASTER={{ postmaster }}
@@ -83,7 +83,7 @@ WEB_ADMIN={{ admin_path }}
WEB_WEBMAIL={{ webmail_path }}
# Website name
-SITENAME={{ site_name }{
+SITENAME={{ site_name }}
# Linked Website URL
WEBSITE={{ website }}
diff --git a/setup/flavors/compose/setup.html b/setup/flavors/compose/setup.html
index e4506e6d..3c190c9c 100644
--- a/setup/flavors/compose/setup.html
+++ b/setup/flavors/compose/setup.html
@@ -4,13 +4,13 @@
Docker Compose expects a project file, named docker-compose.yml
in a project directory. First create your project directory.
-
mkdir /mailu
+
mkdir {{ root }}
Then download the project file. A side configuration file makes it easier
to read and check the configuration variables generated by the wizard.
To start your compose project, simply run the Docker Compose up
command.
-
cd /mailu
+
cd {{ root }}
docker-compose up -d
{% endcall %}
diff --git a/setup/server.py b/setup/server.py
index 108f5043..ddeafd90 100644
--- a/setup/server.py
+++ b/setup/server.py
@@ -32,9 +32,11 @@ def secret(length=16):
def build_app(path):
+ #Hardcoded master as the only version for test purposes
versions = [
- version for version in os.listdir(path)
- if os.path.isdir(os.path.join(path, version))
+ # version for version in os.listdir(path)
+ # if os.path.isdir(os.path.join(path, version))
+ "master"
]
app.jinja_env.trim_blocks = True
diff --git a/setup/templates/steps/expose.html b/setup/templates/steps/expose.html
index 665b08a2..372ebddc 100644
--- a/setup/templates/steps/expose.html
+++ b/setup/templates/steps/expose.html
@@ -14,12 +14,12 @@ avoid generic all-interfaces addresses like 0.0.0.0 or ::
-
+
-
+
You server will be available under a main hostname but may expose multiple public
diff --git a/setup/templates/steps/root.html b/setup/templates/steps/root.html
new file mode 100644
index 00000000..f32c2250
--- /dev/null
+++ b/setup/templates/steps/root.html
@@ -0,0 +1,8 @@
+{% call macros.panel("info", "Step 0 - Set root path") %}
+