From 12294a6e5aa3c57c8c603d97a35fa443a962a8e2 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Wed, 6 Jun 2018 18:40:51 +0000 Subject: [PATCH 01/51] Trying to enable fuzzy hashes for rspamd --- services/rspamd/Dockerfile | 5 +--- services/rspamd/conf/fuzzy_check.conf | 34 ++++++++++++++++++++++ services/rspamd/conf/metrics.conf | 19 ++++++++++++ services/rspamd/conf/worker-controller.inc | 1 + services/rspamd/conf/worker-fuzzy.inc | 5 ++++ services/rspamd/conf/worker-normal.inc | 1 + 6 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 services/rspamd/conf/fuzzy_check.conf create mode 100644 services/rspamd/conf/metrics.conf create mode 100644 services/rspamd/conf/worker-fuzzy.inc diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index c6c2afdd..1b8d7e6b 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -1,15 +1,12 @@ FROM alpine:edge -RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates +RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates RUN mkdir /run/rspamd COPY conf/ /conf COPY start.py /start.py -# Temporary fix to remove references to rspamd-fuzzy for now -RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf - EXPOSE 11332/tcp 11334/tcp CMD /start.py diff --git a/services/rspamd/conf/fuzzy_check.conf b/services/rspamd/conf/fuzzy_check.conf new file mode 100644 index 00000000..7c87e1c3 --- /dev/null +++ b/services/rspamd/conf/fuzzy_check.conf @@ -0,0 +1,34 @@ +rule "local" { + # Fuzzy storage server list + servers = "localhost:11335"; + # Default symbol for unknown flags + symbol = "LOCAL_FUZZY_UNKNOWN"; + # Additional mime types to store/check + mime_types = ["application/*"]; + # Hash weight threshold for all maps + max_score = 20.0; + # Whether we can learn this storage + read_only = no; + # Ignore unknown flags + skip_unknown = yes; + # Hash generation algorithm + algorithm = "mumhash"; + + # Map flags to symbols + fuzzy_map = { + LOCAL_FUZZY_DENIED { + # Local threshold + max_score = 20.0; + # Flag to match + flag = 11; + } + LOCAL_FUZZY_PROB { + max_score = 10.0; + flag = 12; + } + LOCAL_FUZZY_WHITE { + max_score = 2.0; + flag = 13; + } + } +} diff --git a/services/rspamd/conf/metrics.conf b/services/rspamd/conf/metrics.conf new file mode 100644 index 00000000..6a31964f --- /dev/null +++ b/services/rspamd/conf/metrics.conf @@ -0,0 +1,19 @@ +group "fuzzy" { + max_score = 12.0; + symbol "LOCAL_FUZZY_UNKNOWN" { + weight = 5.0; + description = "Generic fuzzy hash match"; + } + symbol "LOCAL_FUZZY_DENIED" { + weight = 12.0; + description = "Denied fuzzy hash"; + } + symbol "LOCAL_FUZZY_PROB" { + weight = 5.0; + description = "Probable fuzzy hash"; + } + symbol "LOCAL_FUZZY_WHITE" { + weight = -2.1; + description = "Whitelisted fuzzy hash"; + } +} diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index 6a020672..dd143942 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,3 +1,4 @@ +type = "controller"; bind_socket = "*:11334"; password = "mailu"; secure_ip = "{{ FRONT_ADDRESS }}"; diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc new file mode 100644 index 00000000..a0021a03 --- /dev/null +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -0,0 +1,5 @@ +type = "fuzzy"; +count = 1; +backend = "redis"; +expire = 90d; +allow_update = ["127.0.0.1"]; diff --git a/services/rspamd/conf/worker-normal.inc b/services/rspamd/conf/worker-normal.inc index a6ee8317..ab996fb8 100644 --- a/services/rspamd/conf/worker-normal.inc +++ b/services/rspamd/conf/worker-normal.inc @@ -1 +1,2 @@ +type = "normal"; enabled = false; From 23e288aadcf7d5df941c8516fae9d0cbd6b1e054 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Mon, 24 Sep 2018 17:29:31 +0000 Subject: [PATCH 02/51] Enabling swarm deployment on master branch: -Extends the usage of POD_ADDRESS_RANGE -Provides documentation --- core/dovecot/conf/dovecot.conf | 2 +- core/postfix/conf/main.cf | 2 +- docs/swarm/master/README.md | 349 +++++++++++++++++++++ services/rspamd/conf/worker-controller.inc | 2 +- 4 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 docs/swarm/master/README.md diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index a5973bf8..7cf10774 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -5,7 +5,7 @@ log_path = /dev/stderr protocols = imap pop3 lmtp sieve postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }} hostname = {{ HOSTNAMES.split(",")[0] }} -submission_host = {{ FRONT_ADDRESS }} +submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }} service dict { unix_listener dict { diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 2f2c6990..bde42dd1 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }} recipient_delimiter = {{ RECIPIENT_DELIMITER }} # Only the front server is allowed to perform xclient -smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} +smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }} ############### # TLS diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md new file mode 100644 index 00000000..1406e05c --- /dev/null +++ b/docs/swarm/master/README.md @@ -0,0 +1,349 @@ +# Install Mailu on a docker swarm + +## Prequisites + +### Swarm + +In order to deploy Mailu on a swarm, you will first need to initialize the swarm: + +The main command will be: +```bash +docker swarm init --advertise-addr +``` +See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/ + +If you want to add other managers or workers, please use: +```bash +docker swarm join --token xxxxx +``` +See https://docs.docker.com/engine/swarm/join-nodes/ + +You have now a working swarm, and you can check its status with: +```bash +core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls +ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION +xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce +sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce +mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce +``` + +### Volume definition +For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. +Hereafter we will use a NFS share: +```bash +core@coreos-01 ~ $ showmount -e 192.168.0.30 +Export list for 192.168.0.30: +/mnt/Pool1/pv 192.168.0.0 +``` + +on the nfs server, I am using the following /etc/exports +```bash +$more /etc/exports +/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 +``` +on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up) +```bash +$mkdir /mnt/Pool1/pv/mailu +``` + +On your manager node, mount the nfs share to check that the share is available: +```bash +core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/ +``` +If this is ok, you can umount it: +```bash +core@coreos-01 ~ $ sudo umount /mnt/local/ +``` + + +### Networking mode +On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. + +In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE. + +Let's create the mailu_default network: +```bash +core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default +core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet + "Subnet": "10.0.1.0/24", +``` +In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 + +Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network) + +### Scalability +- smtp and imap are scalable +- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) +- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) + +### Variable substitution and docker-compose.yml +The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to : +- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc +- change the way we define the volumes (nfs share in our case) +- add a deploy section for every service + +### Docker compose +An example of docker-compose-stack.yml file is available here: + +```yaml + +version: '3.2' + +services: + + front: + image: mailu/nginx:master + restart: always + env_file: .env + ports: + - target: 80 + published: 80 + - target: 443 + published: 443 + - target: 110 + published: 110 + - target: 143 + published: 143 + - target: 993 + published: 993 + - target: 995 + published: 995 + - target: 25 + published: 25 + - target: 465 + published: 465 + - target: 587 + published: 587 + volumes: +# - "$ROOT/certs:/certs" + - type: volume + source: mailu_certs + target: /certs + deploy: + replicas: 2 + + redis: + image: redis:alpine + restart: always + volumes: +# - "$ROOT/redis:/data" + - type: volume + source: mailu_redis + target: /data + deploy: + replicas: 1 + + imap: + image: mailu/dovecot:master + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data +# - "$ROOT/mail:/mail" + - type: volume + source: mailu_mail + target: /mail +# - "$ROOT/overrides:/overrides" + - type: volume + source: mailu_overrides + target: /overrides + depends_on: + - front + deploy: + replicas: 2 + + smtp: + image: mailu/postfix:master + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data +# - "$ROOT/overrides:/overrides" + - type: volume + source: mailu_overrides + target: /overrides + depends_on: + - front + deploy: + replicas: 2 + + antispam: + image: mailu/rspamd:master + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + depends_on: + - front + volumes: +# - "$ROOT/filter:/var/lib/rspamd" + - type: volume + source: mailu_filter + target: /var/lib/rspamd +# - "$ROOT/dkim:/dkim" + - type: volume + source: mailu_dkim + target: /dkim +# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" + - type: volume + source: mailu_overrides_rspamd + target: /etc/rspamd/override.d + deploy: + replicas: 1 + + antivirus: + image: mailu/none:master + restart: always + env_file: .env + volumes: +# - "$ROOT/filter:/data" + - type: volume + source: mailu_filter + target: /data + deploy: + replicas: 1 + + webdav: + image: mailu/none:master + restart: always + env_file: .env + volumes: +# - "$ROOT/dav:/data" + - type: volume + source: mailu_dav + target: /data + deploy: + replicas: 1 + + admin: + image: mailu/admin:master + restart: always + env_file: .env + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data +# - "$ROOT/dkim:/dkim" + - type: volume + source: mailu_dkim + target: /dkim + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - redis + deploy: + replicas: 1 + + webmail: + image: "mailu/roundcube:master" + restart: always + env_file: .env + volumes: +# - "$ROOT/webmail:/data" + - type: volume + source: mailu_data + target: /data + depends_on: + - imap + deploy: + replicas: 2 + + fetchmail: + image: mailu/fetchmail:master + restart: always + env_file: .env + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data + deploy: + replicas: 1 + +networks: + default: + external: + name: mailu_default + +volumes: + mailu_filter: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/filter" + mailu_dkim: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/dkim" + mailu_overrides_rspamd: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/overrides/rspamd" + mailu_data: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/data" + mailu_mail: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/mail" + mailu_overrides: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/overrides" + mailu_dav: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/dav" + mailu_certs: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/certs" + mailu_redis: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,nolock,soft,rw" + device: ":/mnt/Pool1/pv/mailu/redis" +``` + +### Deploy Mailu on the docker swarm +Run the following command: +```bash +docker stack deploy -c docker-compose-stack.yml mailu +``` +See how the services are being deployed: +```bash +core@coreos-01 ~ $ docker service ls +ID NAME MODE REPLICAS IMAGE PORTS +ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:1.5 +pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:1.5 +``` +check a specific service: +```bash +core@coreos-01 ~ $ docker service ps mailu_fetchmail +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago +``` + +### Remove the stack +Run the follwoing command: +```bash +core@coreos-01 ~ $ docker stack rm mailu +``` diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index 6a020672..0cb0d5c0 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,3 +1,3 @@ bind_socket = "*:11334"; password = "mailu"; -secure_ip = "{{ FRONT_ADDRESS }}"; +secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}"; From f5f09fad6ecc41e64b90dc4044450b61ea19ae4a Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Tue, 25 Sep 2018 18:54:40 +0000 Subject: [PATCH 03/51] Reverting the patch for dovecot.conf, as it is not needed --- core/dovecot/conf/dovecot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 7cf10774..a5973bf8 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -5,7 +5,7 @@ log_path = /dev/stderr protocols = imap pop3 lmtp sieve postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }} hostname = {{ HOSTNAMES.split(",")[0] }} -submission_host = {{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }} +submission_host = {{ FRONT_ADDRESS }} service dict { unix_listener dict { From fcad52b1454aeeb5b11d64fe0186f644922f96b2 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 27 Sep 2018 22:45:16 +0200 Subject: [PATCH 04/51] Implement a start date filter for autoreply, fixes #362 --- core/admin/mailu/models.py | 2 ++ core/admin/mailu/ui/forms.py | 1 + core/admin/mailu/ui/templates/user/reply.html | 7 ++++-- .../migrations/versions/3b281286c7bd_.py | 24 +++++++++++++++++++ core/dovecot/conf/pigeonhole-sieve.dict | 8 +++++++ core/dovecot/sieve/before.sieve | 1 + 6 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 core/admin/migrations/versions/3b281286c7bd_.py diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 71c5f4d7..1be4c8e6 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -249,6 +249,8 @@ class User(Base, Email): reply_enabled = db.Column(db.Boolean(), nullable=False, default=False) reply_subject = db.Column(db.String(255), nullable=True, default=None) reply_body = db.Column(db.Text(), nullable=True, default=None) + reply_startdate = db.Column(db.Date, nullable=False, + default=date(1900, 1, 1)) reply_enddate = db.Column(db.Date, nullable=False, default=date(2999, 12, 31)) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index 326d721b..4f7a30ae 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -117,6 +117,7 @@ class UserReplyForm(flask_wtf.FlaskForm): reply_subject = fields.StringField(_('Reply subject')) reply_body = fields.StringField(_('Reply body'), widget=widgets.TextArea()) + reply_startdate = fields.html5.DateField(_('Start of vacation')) reply_enddate = fields.html5.DateField(_('End of vacation')) submit = fields.SubmitField(_('Update')) diff --git a/core/admin/mailu/ui/templates/user/reply.html b/core/admin/mailu/ui/templates/user/reply.html index 7906bc42..7225a178 100644 --- a/core/admin/mailu/ui/templates/user/reply.html +++ b/core/admin/mailu/ui/templates/user/reply.html @@ -13,14 +13,17 @@
{{ form.hidden_tag() }} {{ macros.form_field(form.reply_enabled, - onchange="if(this.checked){$('#reply_subject,#reply_body,#reply_enddate').removeAttr('readonly')} + onchange="if(this.checked){$('#reply_subject,#reply_body,#reply_enddate,#reply_startdate').removeAttr('readonly')} else{$('#reply_subject,#reply_body,#reply_enddate').attr('readonly', '')}") }} {{ macros.form_field(form.reply_subject, **{("rw" if user.reply_enabled else "readonly"): ""}) }} {{ macros.form_field(form.reply_body, rows=10, **{("rw" if user.reply_enabled else "readonly"): ""}) }} {{ macros.form_field(form.reply_enddate, - **{("rw" if user.reply_enabled else "readonly"): ""}) }} + **{("rw" if user.reply_enabled else "readonly"): ""}) }} + {{ macros.form_field(form.reply_startdate, + **{("rw" if user.reply_enabled else "readonly"): ""}) }} + {{ macros.form_field(form.submit) }}
{% endcall %} diff --git a/core/admin/migrations/versions/3b281286c7bd_.py b/core/admin/migrations/versions/3b281286c7bd_.py new file mode 100644 index 00000000..78e44a4c --- /dev/null +++ b/core/admin/migrations/versions/3b281286c7bd_.py @@ -0,0 +1,24 @@ +""" Add a start day for vacations + +Revision ID: 3b281286c7bd +Revises: 049fed905da7 +Create Date: 2018-09-27 22:20:08.158553 + +""" + +revision = '3b281286c7bd' +down_revision = '049fed905da7' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + with op.batch_alter_table('user') as batch: + batch.add_column(sa.Column('reply_startdate', sa.Date(), nullable=False, + server_default="1900-01-01")) + + +def downgrade(): + with op.batch_alter_table('user') as batch: + batch.drop_column('reply_startdate') diff --git a/core/dovecot/conf/pigeonhole-sieve.dict b/core/dovecot/conf/pigeonhole-sieve.dict index 917fce83..604371a8 100644 --- a/core/dovecot/conf/pigeonhole-sieve.dict +++ b/core/dovecot/conf/pigeonhole-sieve.dict @@ -41,3 +41,11 @@ map { username_field = email value_field = reply_enddate } + +map { + pattern = priv/reply_startdate + table = user + username_field = email + value_field = reply_startdate +} + diff --git a/core/dovecot/sieve/before.sieve b/core/dovecot/sieve/before.sieve index 6ebc20c5..81d20f30 100644 --- a/core/dovecot/sieve/before.sieve +++ b/core/dovecot/sieve/before.sieve @@ -34,6 +34,7 @@ if exists "X-Virus" { } if allof (string :is "${extdata.reply_enabled}" "1", + currentdate :value "ge" "date" "${extdata.reply_startdate}", currentdate :value "le" "date" "${extdata.reply_enddate}") { vacation :days 1 :subject "${extdata.reply_subject}" "${extdata.reply_body}"; From 6b34b2728ece020918dcb10a901afdd9894a69e3 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Sun, 7 Oct 2018 16:38:41 +0000 Subject: [PATCH 05/51] Declare fuzzy_worker port 11335 in EXPOSE section --- services/rspamd/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index 7dff8c1f..cfb4d0eb 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir /run/rspamd COPY conf/ /conf COPY start.py /start.py -EXPOSE 11332/tcp 11334/tcp +EXPOSE 11332/tcp 11334/tcp 11335/tcp VOLUME ["/var/lib/rspamd"] From 1f71d10899d72571b39a60448a23adc41d5f10f2 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Sun, 7 Oct 2018 16:46:42 +0000 Subject: [PATCH 06/51] Change POD_ADDRESS_RANGE introduction like it is done on deovecot-sql.conf.ext --- services/rspamd/conf/worker-controller.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index 0cb0d5c0..4b23a897 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,3 +1,3 @@ bind_socket = "*:11334"; password = "mailu"; -secure_ip = "{{ FRONT_ADDRESS }} {{ POD_ADDRESS_RANGE }}"; +secure_ip = "{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% endif %}"; From 9d610f56f7ff39232cb15ce02b42f8aca94d846a Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Mon, 8 Oct 2018 18:53:44 +0000 Subject: [PATCH 07/51] Added some lines around ingress mode --- docs/swarm/master/README.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index 1406e05c..c09f1dd3 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -57,8 +57,11 @@ core@coreos-01 ~ $ sudo umount /mnt/local/ ### Networking mode -On a swarm, the services are available (default mode) through a routing mesh managed by docker itself. With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. +On this example, we are using: +- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. +- the default ingress mode. +## Allow authentification with the mesh routing In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE. Let's create the mailu_default network: @@ -68,8 +71,22 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet "Subnet": "10.0.1.0/24", ``` In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 +In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation... + +## Limitation with the ingress mode +With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network) + +This issue is known and discussed here: + +https://github.com/moby/moby/issues/25526 + +A workaround (using network host mode and global deployment) is discussed here: + +https://github.com/moby/moby/issues/25526#issuecomment-336363408 + +## Don't create an open relay ! +As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-( -Nota: on my setup, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it might be sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network) ### Scalability - smtp and imap are scalable From 6bd365e7714e765c75d8fdd93bc04065a53eae14 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Mon, 8 Oct 2018 21:00:44 +0000 Subject: [PATCH 08/51] Change title layout --- docs/swarm/master/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index c09f1dd3..2a2a021a 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -56,12 +56,12 @@ core@coreos-01 ~ $ sudo umount /mnt/local/ ``` -### Networking mode +## Networking mode On this example, we are using: - the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. - the default ingress mode. -## Allow authentification with the mesh routing +### Allow authentification with the mesh routing In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE. Let's create the mailu_default network: @@ -73,7 +73,7 @@ core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation... -## Limitation with the ingress mode +### Limitation with the ingress mode With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network) This issue is known and discussed here: @@ -84,22 +84,22 @@ A workaround (using network host mode and global deployment) is discussed here: https://github.com/moby/moby/issues/25526#issuecomment-336363408 -## Don't create an open relay ! +### Don't create an open relay ! As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-( -### Scalability +## Scalability - smtp and imap are scalable - front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) - redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) -### Variable substitution and docker-compose.yml +## Variable substitution and docker-compose.yml The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to : - remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc - change the way we define the volumes (nfs share in our case) - add a deploy section for every service -### Docker compose +## Docker compose An example of docker-compose-stack.yml file is available here: ```yaml @@ -340,7 +340,7 @@ volumes: device: ":/mnt/Pool1/pv/mailu/redis" ``` -### Deploy Mailu on the docker swarm +## Deploy Mailu on the docker swarm Run the following command: ```bash docker stack deploy -c docker-compose-stack.yml mailu @@ -359,7 +359,7 @@ ID NAME IMAGE NODE tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago ``` -### Remove the stack +## Remove the stack Run the follwoing command: ```bash core@coreos-01 ~ $ docker stack rm mailu From c8b39c5d4a4eb30b59a88342b4b428cb0f02c426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Wed, 10 Oct 2018 19:29:23 +0200 Subject: [PATCH 09/51] support bcrypt and use it as default --- core/admin/mailu/__init__.py | 2 +- core/admin/mailu/models.py | 3 ++- core/admin/requirements-prod.txt | 1 + core/admin/requirements.txt | 1 + docs/compose/.env | 4 ++-- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index d77420e6..167f04ae 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -57,7 +57,7 @@ default_config = { 'RECAPTCHA_PUBLIC_KEY': '', 'RECAPTCHA_PRIVATE_KEY': '', # Advanced settings - 'PASSWORD_SCHEME': 'SHA512-CRYPT', + 'PASSWORD_SCHEME': 'BLF-CRYPT', # Host settings 'HOST_IMAP': 'imap', 'HOST_POP3': 'imap', diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 1bcc4e9f..0c80fd4f 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -276,7 +276,8 @@ class User(Base, Email): else: return self.email - scheme_dict = {'SHA512-CRYPT': "sha512_crypt", + scheme_dict = {'BLF-CRYPT': "bcrypt", + 'SHA512-CRYPT': "sha512_crypt", 'SHA256-CRYPT': "sha256_crypt", 'MD5-CRYPT': "md5_crypt", 'CRYPT': "des_crypt"} diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index 94c28177..e321a4d6 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -1,6 +1,7 @@ alembic==0.9.9 asn1crypto==0.24.0 Babel==2.5.3 +bcrypt==3.1.4 blinker==1.4 certifi==2018.4.16 cffi==1.11.5 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index a40e6eb5..d6e7adb1 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -17,3 +17,4 @@ tabulate PyYAML PyOpenSSL dnspython +bcrypt diff --git a/docs/compose/.env b/docs/compose/.env index 721aaf22..cdb48310 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -124,8 +124,8 @@ WEBSITE=https://mailu.io COMPOSE_PROJECT_NAME=mailu # Default password scheme used for newly created accounts and changed passwords -# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) -PASSWORD_SCHEME=SHA512-CRYPT +# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) +PASSWORD_SCHEME=BLF-CRYPT # Header to take the real ip from REAL_IP_HEADER= From 6aafef88bdc2bae1954d7c7baa9566e2f3807da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Thu, 11 Oct 2018 02:57:13 +0200 Subject: [PATCH 10/51] remove apk-warning about cache --- core/admin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 08de0e88..2e637206 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -7,7 +7,7 @@ COPY requirements-prod.txt requirements.txt RUN apk add --no-cache openssl \ && apk add --no-cache --virtual build-dep openssl-dev libffi-dev python-dev build-base \ && pip install -r requirements.txt \ - && apk del build-dep + && apk del --no-cache build-dep COPY mailu ./mailu COPY migrations ./migrations From f2259c3302ccafe17dc9e8f39ce177ffd19a2b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Thu, 11 Oct 2018 03:18:35 +0200 Subject: [PATCH 11/51] reduce webmail image-layers/sizes --- webmails/rainloop/Dockerfile | 13 +++++++------ webmails/roundcube/Dockerfile | 15 +++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile index 2da4e672..889c8486 100644 --- a/webmails/rainloop/Dockerfile +++ b/webmails/rainloop/Dockerfile @@ -1,20 +1,21 @@ FROM php:7.2-apache -RUN apt-get update && apt-get install -y \ - unzip python3 python3-jinja2 - ENV RAINLOOP_URL https://github.com/RainLoop/rainloop-webmail/releases/download/v1.12.1/rainloop-community-1.12.1.zip -RUN rm -rf /var/www/html/ \ +RUN apt-get update && apt-get install -y \ + unzip python3 python3-jinja2 \ + && rm -rf /var/www/html/ \ && mkdir /var/www/html \ && cd /var/www/html \ && curl -L -O ${RAINLOOP_URL} \ - && unzip *.zip \ + && unzip -q *.zip \ && rm -f *.zip \ && rm -rf data/ \ && find . -type d -exec chmod 755 {} \; \ && find . -type f -exec chmod 644 {} \; \ - && chown -R www-data: * + && chown -R www-data: * \ + && apt-get purge -y unzip \ + && rm -rf /var/lib/apt/lists COPY include.php /var/www/html/include.php COPY php.ini /usr/local/etc/php/conf.d/rainloop.ini diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index ad198236..50a58a4f 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -1,14 +1,12 @@ FROM php:7.2-apache -RUN apt-get update && apt-get install -y \ - zlib1g-dev \ - && docker-php-ext-install zip - ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.7/roundcubemail-1.3.7-complete.tar.gz -RUN echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini - -RUN rm -rf /var/www/html/ \ +RUN apt-get update && apt-get install -y \ + zlib1g-dev \ + && docker-php-ext-install zip \ + && echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini \ + && rm -rf /var/www/html/ \ && cd /var/www \ && curl -L -O ${ROUNDCUBE_URL} \ && tar -xf *.tar.gz \ @@ -17,7 +15,8 @@ RUN rm -rf /var/www/html/ \ && cd html \ && rm -rf CHANGELOG INSTALL LICENSE README.md UPGRADING composer.json-dist installer \ && sed -i 's,mod_php5.c,mod_php7.c,g' .htaccess \ - && chown -R www-data: logs temp + && chown -R www-data: logs temp \ + && rm -rf /var/lib/apt/lists COPY php.ini /usr/local/etc/php/conf.d/roundcube.ini From 77e3fc0ebcd7ac4970e8b05f5e8fa9cd5d97992c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Thu, 11 Oct 2018 18:14:37 +0300 Subject: [PATCH 12/51] Some documentation flow refactoring and updates: - Improve advice on IP binding; Follow up on issue #641 - mailradar is dead. Found mxtoolbox instead - Fix some internal linking --- docs/compose/setup.rst | 57 ++++++++++++++++++++++++++++++------------ docs/configuration.rst | 10 +++++++- docs/dns.rst | 2 ++ docs/setup.rst | 7 +++--- 4 files changed, 55 insertions(+), 21 deletions(-) diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst index 8759e2f1..d4fb8014 100644 --- a/docs/compose/setup.rst +++ b/docs/compose/setup.rst @@ -26,35 +26,60 @@ for the ``VERSION_TAG`` branch, use: wget https://mailu.io/VERSION_TAG/_downloads/docker-compose.yml wget https://mailu.io/VERSION_TAG/_downloads/.env -Then open the ``.env`` file to setup the mail server. Modify the ``ROOT`` setting -to match your setup directory if different from ``/mailu``. +Important configuration variables +--------------------------------- -Modify the ``VERSION`` configuration in the ``.env`` file to reflect the version you picked. +Open the ``.env`` file and review the following variable settings: -Set the common configuration values ------------------------------------ +- Change ``ROOT`` if you have your setup directory in a different location then ``/mailu``. +- Check ``VERSION`` to reflect the version you picked. (``master`` or ``1.5``). -Open the ``.env`` file and set configuration settings after reading the configuration -documentation. Some settings are specific to the Docker Compose setup. +Make sure to read the comments in the file and instructions from the :ref:`common_cfg` section. -Modify ``BIND_ADDRESS4`` to match the public IP address assigned to your server. -This address should be configured on one of the network interfaces of the server. -If the address is not configured directly (NAT) on any of the network interfaces or if -you would simply like the server to listen on all interfaces, use ``0.0.0.0``. - -Modify ``BIND_ADDRESS6`` to match the public IPv6 address assigned to your server. -The behavior is identical to ``BIND_ADDRESS4``. +TLS certificates +```````````````` Set the ``TLS_FLAVOR`` to one of the following values: - ``cert`` is the default and requires certificates to be setup manually; -- ``letsencrypt`` will use the Letsencrypt! CA to generate automatic ceriticates; +- ``letsencrypt`` will use the *Letsencrypt!* CA to generate automatic ceriticates; - ``mail`` is similar to ``cert`` except that TLS will only be served for emails (IMAP and SMTP), not HTTP (use it behind reverse proxies); - ``mail-letsencrypt`` is similar to ``letsencrypt`` except that TLS will only be served for emails (IMAP and SMTP), not HTTP (use it behind reverse proxies); -- ``notls`` will disable TLS, this is not recommended except for testing. +- ``notls`` will disable TLS, this is not recommended except for testing + +.. note:: + + When using *Letsencrypt!* you have to make sure that the DNS ``A`` and ``AAAA`` records for the + all hostnames mentioned in the ``HOSTNAMES`` variable match with the ip adresses of you server. + Or else certificate generation will fail! See also: :ref:`dns_setup`. + +Bind address +```````````` + +Modify ``BIND_ADDRESS4`` and ``BIND_ADDRESS6`` to match the public IP addresses assigned to your server. For IPv6 you will need the ```` scope address. + +You can find those addresses by running the following: + +.. code-block:: bash + + [root@mailu ~]$ ifconfig eth0 + eth0: flags=4163 mtu 1500 + inet 125.189.138.127 netmask 255.255.255.0 broadcast 5.189.138.255 + inet6 fd21:aab2:717c:cc5a::1 prefixlen 64 scopeid 0x0 + inet6 fe2f:2a73:43a8:7a1b::1 prefixlen 64 scopeid 0x20 + ether 00:50:56:3c:b2:23 txqueuelen 1000 (Ethernet) + RX packets 174866612 bytes 127773819607 (118.9 GiB) + RX errors 0 dropped 0 overruns 0 frame 0 + TX packets 19905110 bytes 2191519656 (2.0 GiB) + TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 + +If the address is not configured directly (NAT) on any of the network interfaces or if +you would simply like the server to listen on all interfaces, use ``0.0.0.0`` and ``::``. Note that running is this mode is not supported and can lead to `issues`_. + +.. _issues: https://github.com/Mailu/Mailu/issues/641 Enable optional features ------------------------ diff --git a/docs/configuration.rst b/docs/configuration.rst index 62b6f34e..cab30072 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,12 +1,20 @@ Mailu configuration settings ============================ +.. _common_cfg: + Common configuration -------------------- The ``SECRET_KEY`` **must** be changed for every setup and set to a 16 bytes randomly generated value. It is intended to secure authentication cookies -among other critical uses. +among other critical uses. This can be generated with a utility such as *pwgen*, +which can be installed on most Linux systems: + +.. code-block:: bash + + apt-get install pwgen + pwgen 16 1 The ``DOMAIN`` holds the main e-mail domain for the server. This email domain is used for bounce emails, for generating the postmaster email and other diff --git a/docs/dns.rst b/docs/dns.rst index 3d94aecb..d2fd4131 100644 --- a/docs/dns.rst +++ b/docs/dns.rst @@ -1,3 +1,5 @@ +.. _dns_setup: + Setting up your DNS =================== diff --git a/docs/setup.rst b/docs/setup.rst index d7d0cc13..9771f886 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -32,7 +32,7 @@ user. Make sure you complete the requirements for the flavor you chose. You should also have at least a DNS hostname and a DNS name for receiving emails. Some instructions are provided on the matter in the article -[Setup your DNS](dns). +:ref:`dns_setup`. .. _`MFAshby's fork`: https://github.com/MFAshby/Mailu @@ -68,10 +68,9 @@ Make sure that you test properly before going live! - Try to receive an email from an external service - Check the logs (``docker-compose logs -f servicenamehere``) to look for warnings or errors -- Use an open relay checker like `mailradar`_ +- Use an open relay checker like `mxtoolbox`_ to ensure you're not contributing to the spam problem on the internet. - All tests there should result in "Relay denied". - If using DMARC, be sure to check the reports you get to verify that legitimate email is getting through and forgeries are being properly blocked. - .. _mailradar: http://www.mailradar.com/openrelay/ + .. _mxtoolbox: https://mxtoolbox.com/diagnostic.aspx From 70c4e42f74d7236abc53b84de1bb4e90d5d30068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Thu, 11 Oct 2018 18:33:58 +0300 Subject: [PATCH 13/51] Fix small typo --- docs/compose/setup.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compose/setup.rst b/docs/compose/setup.rst index d4fb8014..64ad7b25 100644 --- a/docs/compose/setup.rst +++ b/docs/compose/setup.rst @@ -48,7 +48,7 @@ values: emails (IMAP and SMTP), not HTTP (use it behind reverse proxies); - ``mail-letsencrypt`` is similar to ``letsencrypt`` except that TLS will only be served for emails (IMAP and SMTP), not HTTP (use it behind reverse proxies); -- ``notls`` will disable TLS, this is not recommended except for testing +- ``notls`` will disable TLS, this is not recommended except for testing. .. note:: From 3f7e7ca3a6fce6041a55f80c0ecd1a8b03633754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Mon, 15 Oct 2018 19:36:37 +0300 Subject: [PATCH 14/51] Use defaults in variables, to allow for local builds --- tests/build.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/build.yml b/tests/build.yml index 0b6858a0..5f360ece 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -3,54 +3,54 @@ version: '3' services: front: - image: $DOCKER_ORG/nginx:$VERSION + image: ${DOCKER_ORG:-mailu}/nginx:${VERSION:-local} build: ../core/nginx imap: - image: $DOCKER_ORG/dovecot:$VERSION + image: ${DOCKER_ORG:-mailu}/dovecot:${VERSION:-local} build: ../core/dovecot smtp: - image: $DOCKER_ORG/postfix:$VERSION + image: ${DOCKER_ORG:-mailu}/postfix:${VERSION:-local} build: ../core/postfix antispam: - image: $DOCKER_ORG/rspamd:$VERSION + image: ${DOCKER_ORG:-mailu}/rspamd:${VERSION:-local} build: ../services/rspamd antivirus: - image: $DOCKER_ORG/clamav:$VERSION + image: ${DOCKER_ORG:-mailu}/clamav:${VERSION:-local} build: ../optional/clamav webdav: - image: $DOCKER_ORG/radicale:$VERSION + image: ${DOCKER_ORG:-mailu}/radicale:${VERSION:-local} build: ../optional/radicale admin: - image: $DOCKER_ORG/admin:$VERSION + image: ${DOCKER_ORG:-mailu}/admin:${VERSION:-local} build: ../core/admin roundcube: - image: $DOCKER_ORG/roundcube:$VERSION + image: ${DOCKER_ORG:-mailu}/roundcube:${VERSION:-local} build: ../webmails/roundcube rainloop: - image: $DOCKER_ORG/rainloop:$VERSION + image: ${DOCKER_ORG:-mailu}/rainloop:${VERSION:-local} build: ../webmails/rainloop fetchmail: - image: $DOCKER_ORG/fetchmail:$VERSION + image: ${DOCKER_ORG:-mailu}/fetchmail:${VERSION:-local} build: ../services/fetchmail none: - image: $DOCKER_ORG/none:$VERSION + image: ${DOCKER_ORG:-mailu}/none:${VERSION:-local} build: ../core/none docs: - image: $DOCKER_ORG/docs:$VERSION + image: ${DOCKER_ORG:-mailu}/docs:${VERSION:-local} build: ../docs setup: - image: $DOCKER_ORG/setup:$VERSION + image: ${DOCKER_ORG:-mailu}/setup:${VERSION:-local} build: ../setup From b38deb18cffe35e3f2e445bd2c3635e11dee6a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Mon, 15 Oct 2018 19:37:24 +0300 Subject: [PATCH 15/51] Update Dev-docs to use build.yml for building --- docs/contributors/environment.rst | 66 ++++++++++++++++++------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index a1cce193..f1f447e2 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -5,39 +5,51 @@ Docker containers ----------------- The development environment is quite similar to the production one. You should always use -the ``master`` version when developing. Simply add a build directive to the images -you are working on in the ``docker-compose.yml``: +the ``master`` version when developing. -.. code-block:: yaml +Building images +``````````````` - webdav: - build: ./optional/radicale - image: mailu/$WEBDAV:$VERSION - restart: always - env_file: .env - volumes: - - "$ROOT/dav:/data" - - admin: - build: ./core/admin - image: mailu/admin:$VERSION - restart: always - env_file: .env - volumes: - - "$ROOT/data:/data" - - "$ROOT/dkim:/dkim" - - /var/run/docker.sock:/var/run/docker.sock:ro - depends_on: - - redis - - -The build these containers. +We supply a separate ``test/build.yml`` file for +convenience. To build all Mailu containers: .. code-block:: bash - docker-compose build admin webdav + docker-compose -f tests/build.yml build -Then you can simply start the stack as normal, newly-built images will be used. +The ``build.yml`` file has two variables: + +#. ``$DOCKER_ORG``: First part of the image tag. Defaults to *mailu* and needs to be changed + only when pushing to your own Docker hub account. +#. ``$VERSION``: Last part of the image tag. Defaults to *local* to differentiate from pulled + images. + +To re-build only specific containers at a later time. + +.. code-block:: bash + + docker-compose -f tests/build.yml build admin webdav + +If you have to push the images to Docker Hub for testing in Docker Swarm or a remote +host, you have to define ``DOCKER_ORG`` (usually your Docker user-name) and login to +the hub. + +.. code-block:: bash + + docker login + Username: Foo + Password: Bar + export DOCKER_ORG="Foo" + export VERSION="feat-extra-app" + docker-compose -f tests/build.yml build + docker-compose -f tests/build.yml push + +Running containers +`````````````````` + +To run the newly created images: ``cd`` to your project directory. Edit ``.env`` to set +``VERSION`` to the same value as used during the build, which defaults to ``local``. +After that you can run: .. code-block:: bash From 5035975c4168980c7dacc7f80c49b062c89eb7c6 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 15 Oct 2018 22:07:38 +0200 Subject: [PATCH 16/51] Remove Postfix debugging --- core/postfix/conf/main.cf | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index cd052d46..1f5c8f7c 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -2,8 +2,6 @@ # General ############### -debug_peer_list = 0.0.0.0/0 - # Main domain and hostname mydomain = {{ DOMAIN }} myhostname = {{ HOSTNAMES.split(",")[0] }} From acbb586e71cc148a634961c31709bf36e52b5d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 16 Oct 2018 12:38:17 +0300 Subject: [PATCH 17/51] Option to send logs to journald or syslog --- docs/compose/.env | 6 ++++++ docs/compose/docker-compose.yml | 2 ++ tests/compose/core.env | 6 ++++++ tests/compose/run.yml | 2 ++ 4 files changed, 16 insertions(+) diff --git a/docs/compose/.env b/docs/compose/.env index 721aaf22..86896c43 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -120,6 +120,12 @@ WEBSITE=https://mailu.io # Advanced settings ################################### +# Log driver for front service. Possible values: +# json-file (default) +# journald (On systemd platforms, useful for Fail2Ban integration) +# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!) +LOG_DRIVER=json-file + # Docker-compose project name, this will prepended to containers names. COMPOSE_PROJECT_NAME=mailu diff --git a/docs/compose/docker-compose.yml b/docs/compose/docker-compose.yml index 6f2da078..b8d15587 100644 --- a/docs/compose/docker-compose.yml +++ b/docs/compose/docker-compose.yml @@ -6,6 +6,8 @@ services: image: mailu/nginx:$VERSION restart: always env_file: .env + logging: + driver: $LOG_DRIVER ports: - "$BIND_ADDRESS4:80:80" - "$BIND_ADDRESS4:443:443" diff --git a/tests/compose/core.env b/tests/compose/core.env index 89120d4f..78c307c0 100644 --- a/tests/compose/core.env +++ b/tests/compose/core.env @@ -120,6 +120,12 @@ WEBSITE=https://mailu.io # Advanced settings ################################### +# Log driver for front service. Possible values: +# json-file (default) +# journald (On systemd platforms, useful for Fail2Ban integration) +# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!) +LOG_DRIVER=json-file + # Docker-compose project name, this will prepended to containers names. #COMPOSE_PROJECT_NAME=mailu diff --git a/tests/compose/run.yml b/tests/compose/run.yml index 56ea1627..39ad3dc2 100644 --- a/tests/compose/run.yml +++ b/tests/compose/run.yml @@ -6,6 +6,8 @@ services: image: $DOCKER_ORG/nginx:$VERSION restart: 'no' env_file: $PWD/.env + logging: + driver: journald ports: - "$BIND_ADDRESS4:80:80" - "$BIND_ADDRESS4:443:443" From f3ebf35c75ed2038e01eef4c40863a6a10d29e34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 16 Oct 2018 12:56:12 +0300 Subject: [PATCH 18/51] Fix autotest --- tests/compose/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compose/run.yml b/tests/compose/run.yml index 39ad3dc2..eac35b76 100644 --- a/tests/compose/run.yml +++ b/tests/compose/run.yml @@ -7,7 +7,7 @@ services: restart: 'no' env_file: $PWD/.env logging: - driver: journald + driver: $LOG_DRIVER ports: - "$BIND_ADDRESS4:80:80" - "$BIND_ADDRESS4:443:443" From 5ada669f43b26b35c7f8ca2eab89fd8d2d5e3611 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 16 Oct 2018 20:38:18 +0200 Subject: [PATCH 19/51] Rebase reply startdate on master --- core/admin/mailu/internal/templates/default.sieve | 2 +- core/admin/mailu/models.py | 2 +- core/admin/migrations/versions/3b281286c7bd_.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/internal/templates/default.sieve b/core/admin/mailu/internal/templates/default.sieve index 214b8125..d771ee99 100644 --- a/core/admin/mailu/internal/templates/default.sieve +++ b/core/admin/mailu/internal/templates/default.sieve @@ -32,6 +32,6 @@ if exists "X-Virus" { stop; } -{% if user.reply_active and %} +{% if user.reply_active %} vacation :days 1 :subject "{{ user.reply_subject }}" "{{ user.reply_body }}"; {% endif %} diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index bb8cdc94..f7c3349d 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -280,7 +280,7 @@ class User(Base, Email): @property def reply_active(self): - now = datetime.datetime.now() + now = date.today() return ( self.reply_enabled and self.reply_startdate < now and diff --git a/core/admin/migrations/versions/3b281286c7bd_.py b/core/admin/migrations/versions/3b281286c7bd_.py index 78e44a4c..57b5c327 100644 --- a/core/admin/migrations/versions/3b281286c7bd_.py +++ b/core/admin/migrations/versions/3b281286c7bd_.py @@ -1,13 +1,13 @@ """ Add a start day for vacations Revision ID: 3b281286c7bd -Revises: 049fed905da7 +Revises: 25fd6c7bcb4a Create Date: 2018-09-27 22:20:08.158553 """ revision = '3b281286c7bd' -down_revision = '049fed905da7' +down_revision = '25fd6c7bcb4a' from alembic import op import sqlalchemy as sa From 39272ab05c35d05366c35a8ebfd25e87ea7f41de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Tue, 16 Oct 2018 21:38:12 +0200 Subject: [PATCH 20/51] add healthcheck for http services --- core/admin/Dockerfile | 4 +++- core/nginx/Dockerfile | 4 +++- docs/Dockerfile | 8 ++++++-- optional/radicale/Dockerfile | 4 +++- services/rspamd/Dockerfile | 4 +++- setup/Dockerfile | 4 +++- webmails/rainloop/Dockerfile | 4 +++- webmails/roundcube/Dockerfile | 4 +++- 8 files changed, 27 insertions(+), 9 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 2e637206..083bec80 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -4,7 +4,7 @@ RUN mkdir -p /app WORKDIR /app COPY requirements-prod.txt requirements.txt -RUN apk add --no-cache openssl \ +RUN apk add --no-cache openssl curl \ && apk add --no-cache --virtual build-dep openssl-dev libffi-dev python-dev build-base \ && pip install -r requirements.txt \ && apk del --no-cache build-dep @@ -20,3 +20,5 @@ EXPOSE 80/tcp VOLUME ["/data"] CMD ["/start.sh"] + +HEALTHCHECK CMD curl -f -L http://localhost/ui || exit 1 diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 1b61447a..87951c03 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.8 -RUN apk add --no-cache certbot nginx nginx-mod-mail openssl \ +RUN apk add --no-cache certbot nginx nginx-mod-mail openssl curl \ python py-jinja2 py-requests-toolbelt py-pip \ && pip install --upgrade pip \ && pip install idna @@ -12,3 +12,5 @@ EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 100 VOLUME ["/certs"] CMD /start.py + +HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1 diff --git a/docs/Dockerfile b/docs/Dockerfile index af481a27..b058e0e2 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -3,7 +3,7 @@ FROM python:3-alpine COPY requirements.txt /requirements.txt RUN pip install -r /requirements.txt \ - && apk add --no-cache nginx \ + && apk add --no-cache nginx curl \ && mkdir /run/nginx COPY ./nginx.conf /etc/nginx/conf.d/default.conf @@ -11,4 +11,8 @@ COPY . /docs RUN sphinx-build /docs /build -CMD nginx -g "daemon off;" \ No newline at end of file +EXPOSE 80/tcp + +CMD nginx -g "daemon off;" + +HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1 diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index b82a0804..4616d53d 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:edge RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ - && apk add --no-cache radicale@testing py-dulwich@testing + && apk add --no-cache radicale@testing py-dulwich@testing curl COPY radicale.conf /radicale.conf @@ -9,3 +9,5 @@ EXPOSE 5232/tcp VOLUME ["/data"] CMD radicale -f -S -C /radicale.conf + +HEALTHCHECK CMD curl -f -L http://localhost:5232/ || exit 1 diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index 7239ddaf..d87a64f5 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:3.8 -RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \ +RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip curl \ && pip install --upgrade pip \ && pip install tenacity @@ -17,3 +17,5 @@ EXPOSE 11332/tcp 11334/tcp VOLUME ["/var/lib/rspamd"] CMD /start.py + +HEALTHCHECK CMD curl -f -L http://localhost:11334/ || exit 1 diff --git a/setup/Dockerfile b/setup/Dockerfile index 1fc808f1..0da1eb7b 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -4,7 +4,7 @@ RUN mkdir -p /app WORKDIR /app COPY requirements.txt requirements.txt -RUN apk add --no-cache git \ +RUN apk add --no-cache git curl \ && pip install -r requirements.txt COPY server.py ./server.py @@ -16,3 +16,5 @@ RUN python setup.py https://github.com/mailu/mailu /data EXPOSE 80/tcp CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app + +HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1 diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile index 889c8486..5d751716 100644 --- a/webmails/rainloop/Dockerfile +++ b/webmails/rainloop/Dockerfile @@ -3,7 +3,7 @@ FROM php:7.2-apache ENV RAINLOOP_URL https://github.com/RainLoop/rainloop-webmail/releases/download/v1.12.1/rainloop-community-1.12.1.zip RUN apt-get update && apt-get install -y \ - unzip python3 python3-jinja2 \ + unzip python3 python3-jinja2 curl \ && rm -rf /var/www/html/ \ && mkdir /var/www/html \ && cd /var/www/html \ @@ -29,3 +29,5 @@ EXPOSE 80/tcp VOLUME ["/data"] CMD /start.py + +HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1 diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index 50a58a4f..6250e6df 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -3,7 +3,7 @@ FROM php:7.2-apache ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.7/roundcubemail-1.3.7-complete.tar.gz RUN apt-get update && apt-get install -y \ - zlib1g-dev \ + zlib1g-dev curl \ && docker-php-ext-install zip \ && echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini \ && rm -rf /var/www/html/ \ @@ -28,3 +28,5 @@ EXPOSE 80/tcp VOLUME ["/data"] CMD ["/start.sh"] + +HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1 From 040c1e529ac293b26bccd5d1be86e52a9561bb2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Tue, 16 Oct 2018 22:20:16 +0200 Subject: [PATCH 21/51] add healthcheck for antivirus --- optional/clamav/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/optional/clamav/Dockerfile b/optional/clamav/Dockerfile index a27c0eb2..fd5de38b 100644 --- a/optional/clamav/Dockerfile +++ b/optional/clamav/Dockerfile @@ -9,3 +9,5 @@ EXPOSE 3310/tcp VOLUME ["/data"] CMD ["/start.sh"] + +HEALTHCHECK CMD [ "$(echo PING | nc localhost 3310)" = "PONG" ] From f0f5cea5d1dba49d966d4e61d88e5dfae889c09a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Tue, 16 Oct 2018 22:30:15 +0200 Subject: [PATCH 22/51] make antivirus-healthcheck more understandable --- optional/clamav/Dockerfile | 3 ++- optional/clamav/health.sh | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100755 optional/clamav/health.sh diff --git a/optional/clamav/Dockerfile b/optional/clamav/Dockerfile index fd5de38b..3a9bf4aa 100644 --- a/optional/clamav/Dockerfile +++ b/optional/clamav/Dockerfile @@ -4,10 +4,11 @@ RUN apk add --no-cache clamav rsyslog wget clamav-libunrar COPY conf /etc/clamav COPY start.sh /start.sh +COPY health.sh /health.sh EXPOSE 3310/tcp VOLUME ["/data"] CMD ["/start.sh"] -HEALTHCHECK CMD [ "$(echo PING | nc localhost 3310)" = "PONG" ] +HEALTHCHECK CMD /health.sh diff --git a/optional/clamav/health.sh b/optional/clamav/health.sh new file mode 100755 index 00000000..c4c55044 --- /dev/null +++ b/optional/clamav/health.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +if [ "$(echo PING | nc localhost 3310)" = "PONG" ]; then + echo "ping successful" +else + echo "ping failed" + exit 1 +fi From 8fa80c15894da57131cea00f8274737d37d8054f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 16 Oct 2018 22:52:21 +0200 Subject: [PATCH 23/51] Support multiple docs versions --- docs/Dockerfile | 6 +++++- docs/_templates/page.html | 4 ---- docs/_templates/versions.html | 16 ++++++++++++++++ docs/conf.py | 8 +++++++- docs/docker-compose.yml | 21 +++++++++++++++++++++ 5 files changed, 49 insertions(+), 6 deletions(-) delete mode 100644 docs/_templates/page.html create mode 100644 docs/_templates/versions.html create mode 100644 docs/docker-compose.yml diff --git a/docs/Dockerfile b/docs/Dockerfile index af481a27..c1992f32 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -2,6 +2,9 @@ FROM python:3-alpine COPY requirements.txt /requirements.txt +ARG version=master +ENV VERSION=$version + RUN pip install -r /requirements.txt \ && apk add --no-cache nginx \ && mkdir /run/nginx @@ -9,6 +12,7 @@ RUN pip install -r /requirements.txt \ COPY ./nginx.conf /etc/nginx/conf.d/default.conf COPY . /docs -RUN sphinx-build /docs /build +RUN mkdir -p /build/$VERSION \ + && sphinx-build /docs /build/$VERSION CMD nginx -g "daemon off;" \ No newline at end of file diff --git a/docs/_templates/page.html b/docs/_templates/page.html deleted file mode 100644 index 97296793..00000000 --- a/docs/_templates/page.html +++ /dev/null @@ -1,4 +0,0 @@ -{%- extends "layout.html" %} -{% block body %} - {{ body|replace("VERSION_TAG", version) }} -{% endblock %} diff --git a/docs/_templates/versions.html b/docs/_templates/versions.html new file mode 100644 index 00000000..db4d332d --- /dev/null +++ b/docs/_templates/versions.html @@ -0,0 +1,16 @@ +
+ + Versions + v: {{ version }} + + +
+
+
{{ _('Versions') }}
+ {% for slug, url in versions %} +
{{ slug }}
+ {% endfor %} +
+
+
+ diff --git a/docs/conf.py b/docs/conf.py index f89b39fd..e86a7b9e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- # +import os + extensions = ['sphinx.ext.imgmath', 'sphinx.ext.viewcode'] templates_path = ['_templates'] source_suffix = '.rst' @@ -33,6 +35,10 @@ html_context = { 'display_github': True, 'github_user': 'mailu', 'github_repo': 'mailu', - 'github_version': 'master', + 'github_version': os.environ.get('VERSION', 'master'), + 'versions': [ + ('1.5', '/1.5/'), + ('master', '/master/') + ], 'conf_py_path': '/docs/' } diff --git a/docs/docker-compose.yml b/docs/docker-compose.yml new file mode 100644 index 00000000..0caaa7a4 --- /dev/null +++ b/docs/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' + + +services: + docs_master: + image: mailu/docs:master + labels: + - traefik.enable=true + - traefik.port=80 + - traefik.main.frontend.rule=Host:${hostname};PathPrefix:/master/ + + docs_15: + image: mailu/docs:1.5 + labels: + - traefik.enable=true + - traefik.port=80 + - traefik.root.frontend.redirect.regex=.* + - traefik.root.frontend.redirect.replacement=/1.5/ + - traefik.root.frontend.rule=Host:${hostname};PathPrefix:/ + - traefik.main.frontend.rule=Host:${hostname};PathPrefix:/1.5/ + From 13e3862ca701f5ec5d5e2f3cbb1fff196f0220ca Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 16 Oct 2018 22:55:05 +0200 Subject: [PATCH 24/51] Simplify the docs configuration --- docs/_templates/layout.html | 2 -- docs/conf.py | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html deleted file mode 100644 index 3f424a0a..00000000 --- a/docs/_templates/layout.html +++ /dev/null @@ -1,2 +0,0 @@ -{% set version=github_version %} -{% extends "!layout.html" %} diff --git a/docs/conf.py b/docs/conf.py index e86a7b9e..4ac9de3d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,9 +11,9 @@ master_doc = 'index' project = 'Mailu' copyright = '2018, Mailu authors' author = 'Mailu authors' -version = release = 'latest' +version = release = os.environ.get('VERSION', 'master') language = None -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'Dockerfile', 'docker-compose.yml'] pygments_style = 'sphinx' todo_include_todos = False html_theme = 'sphinx_rtd_theme' @@ -35,7 +35,7 @@ html_context = { 'display_github': True, 'github_user': 'mailu', 'github_repo': 'mailu', - 'github_version': os.environ.get('VERSION', 'master'), + 'github_version': version, 'versions': [ ('1.5', '/1.5/'), ('master', '/master/') From 1fc40bf93220e1c1b66343b1ea254d49585b59df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Tue, 16 Oct 2018 22:57:25 +0200 Subject: [PATCH 25/51] add healthcheck for postfix --- core/postfix/Dockerfile | 3 +++ core/postfix/health.sh | 8 ++++++++ 2 files changed, 11 insertions(+) create mode 100755 core/postfix/health.sh diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index ea58ce1d..4208714a 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -7,8 +7,11 @@ RUN apk add --no-cache postfix postfix-pcre rsyslog \ COPY conf /conf COPY start.py /start.py +COPY health.sh /health.sh EXPOSE 25/tcp 10025/tcp VOLUME ["/data"] CMD /start.py + +HEALTHCHECK CMD /health.sh diff --git a/core/postfix/health.sh b/core/postfix/health.sh new file mode 100755 index 00000000..192eac25 --- /dev/null +++ b/core/postfix/health.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +if [ "$(echo QUIT|nc localhost 25|tail -n1|cut -f1 -d ' ')" = "221" ]; then + echo "ping successful" +else + echo "ping failed" + exit 1 +fi From 0bc901a722f957faab0af81a3d71aff6510cf442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Tue, 16 Oct 2018 23:11:20 +0200 Subject: [PATCH 26/51] add healthcheck for dovecot --- core/dovecot/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index d8d4c55b..c0d3e3cd 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -13,3 +13,5 @@ EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp VOLUME ["/data", "/mail"] CMD /start.py + +HEALTHCHECK CMD echo QUIT|nc localhost 110|grep "Dovecot ready." From a412951a308c7c1e9a15d7790835f164d0e04814 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Tue, 16 Oct 2018 23:12:02 +0200 Subject: [PATCH 27/51] simpler healthcheck for postfix --- core/postfix/Dockerfile | 3 +-- core/postfix/health.sh | 8 -------- 2 files changed, 1 insertion(+), 10 deletions(-) delete mode 100755 core/postfix/health.sh diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 4208714a..5533499e 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -7,11 +7,10 @@ RUN apk add --no-cache postfix postfix-pcre rsyslog \ COPY conf /conf COPY start.py /start.py -COPY health.sh /health.sh EXPOSE 25/tcp 10025/tcp VOLUME ["/data"] CMD /start.py -HEALTHCHECK CMD /health.sh +HEALTHCHECK CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" diff --git a/core/postfix/health.sh b/core/postfix/health.sh deleted file mode 100755 index 192eac25..00000000 --- a/core/postfix/health.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -if [ "$(echo QUIT|nc localhost 25|tail -n1|cut -f1 -d ' ')" = "221" ]; then - echo "ping successful" -else - echo "ping failed" - exit 1 -fi From 53b9c031c99ccb6c2530cacc8c54a57f92f82a58 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 16 Oct 2018 23:19:48 +0200 Subject: [PATCH 28/51] Add a warning banner when not reading the stable docs --- docs/_templates/layout.html | 9 +++++++++ docs/conf.py | 1 + 2 files changed, 10 insertions(+) create mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 00000000..63db07e2 --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,9 @@ +{% extends "!layout.html" %} +{% block document %} +{% if version != stable_version %} +
+

You are currently browsing documentation for the {{ version }} branch. Documentation for the stable {{ stable_version }} branch can be found here.

+
+{% endif %} +{{ super() }} +{% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 4ac9de3d..64997eb1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -36,6 +36,7 @@ html_context = { 'github_user': 'mailu', 'github_repo': 'mailu', 'github_version': version, + 'stable_version': '1.5', 'versions': [ ('1.5', '/1.5/'), ('master', '/master/') From 4ea12deae792081d86372698dba8a4288bd68302 Mon Sep 17 00:00:00 2001 From: hacor Date: Wed, 17 Oct 2018 07:22:55 +0200 Subject: [PATCH 29/51] Added kubernetes to Mailu --- core/dovecot/Dockerfile | 4 +- core/nginx/Dockerfile | 4 +- core/nginx/conf/nginx.conf | 3 + core/postfix/Dockerfile | 2 +- core/postfix/conf/main.cf | 2 +- docs/compose/.env | 3 + docs/index.rst | 2 +- docs/kubernetes/index.rst | 26 -- docs/kubernetes/kubernetes-mailu.yaml | 419 ------------------ .../kubernetes-nginx-ingress-controller.yaml | 84 ---- docs/kubernetes/mailu/admin-ingress.yaml | 86 ++++ docs/kubernetes/mailu/admin.yaml | 63 +++ docs/kubernetes/mailu/configmap.yaml | 172 +++++++ docs/kubernetes/mailu/fetchmail.yaml | 39 ++ docs/kubernetes/mailu/front.yaml | 135 ++++++ docs/kubernetes/mailu/imap.yaml | 80 ++++ docs/kubernetes/mailu/index.rst | 193 ++++++++ docs/kubernetes/mailu/pvc.yaml | 27 ++ docs/kubernetes/mailu/rbac.yaml | 4 + docs/kubernetes/mailu/redis.yaml | 56 +++ docs/kubernetes/mailu/security-ingress.yaml | 30 ++ docs/kubernetes/mailu/security.yaml | 116 +++++ docs/kubernetes/mailu/smtp.yaml | 80 ++++ docs/kubernetes/mailu/webdav-ingress.yaml | 46 ++ docs/kubernetes/mailu/webdav.yaml | 63 +++ docs/kubernetes/mailu/webmail-ingress.yaml | 31 ++ docs/kubernetes/mailu/webmail.yaml | 59 +++ .../nginx/default-http-backend.yaml | 55 +++ docs/kubernetes/nginx/nginx-ingress.yaml | 127 ++++++ docs/kubernetes/nginx/rbac.yaml | 129 ++++++ services/rspamd/conf/worker-controller.inc | 4 + webmails/roundcube/Dockerfile | 6 +- 32 files changed, 1611 insertions(+), 539 deletions(-) delete mode 100644 docs/kubernetes/index.rst delete mode 100644 docs/kubernetes/kubernetes-mailu.yaml delete mode 100644 docs/kubernetes/kubernetes-nginx-ingress-controller.yaml create mode 100644 docs/kubernetes/mailu/admin-ingress.yaml create mode 100644 docs/kubernetes/mailu/admin.yaml create mode 100644 docs/kubernetes/mailu/configmap.yaml create mode 100644 docs/kubernetes/mailu/fetchmail.yaml create mode 100644 docs/kubernetes/mailu/front.yaml create mode 100644 docs/kubernetes/mailu/imap.yaml create mode 100644 docs/kubernetes/mailu/index.rst create mode 100644 docs/kubernetes/mailu/pvc.yaml create mode 100644 docs/kubernetes/mailu/rbac.yaml create mode 100644 docs/kubernetes/mailu/redis.yaml create mode 100644 docs/kubernetes/mailu/security-ingress.yaml create mode 100644 docs/kubernetes/mailu/security.yaml create mode 100644 docs/kubernetes/mailu/smtp.yaml create mode 100644 docs/kubernetes/mailu/webdav-ingress.yaml create mode 100644 docs/kubernetes/mailu/webdav.yaml create mode 100644 docs/kubernetes/mailu/webmail-ingress.yaml create mode 100644 docs/kubernetes/mailu/webmail.yaml create mode 100644 docs/kubernetes/nginx/default-http-backend.yaml create mode 100644 docs/kubernetes/nginx/nginx-ingress.yaml create mode 100644 docs/kubernetes/nginx/rbac.yaml diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index cacfe354..29957921 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:edge +FROM alpine:3.7 RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ && apk add --no-cache \ @@ -11,4 +11,4 @@ COPY start.py /start.py EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp -CMD /start.py +CMD /start.py \ No newline at end of file diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 3be4b50f..49627fb7 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:edge +FROM alpine:3.7 RUN apk add --no-cache nginx nginx-mod-mail python py-jinja2 certbot openssl @@ -7,4 +7,4 @@ COPY *.py / EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp -CMD /start.py +CMD /start.py \ No newline at end of file diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 8fcda1c3..b6e0fbd3 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -34,6 +34,8 @@ http { '' $scheme; } + # Disable the main http server when on kubernetes (port 80 and 443) + {% if KUBERNETES_INGRESS != 'true' %} # Main HTTP server server { # Variables for proxifying @@ -147,6 +149,7 @@ http { proxy_set_header Content-Length ""; } } + {% endif %} # Forwarding authentication server server { diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index bb5831a2..168f3c60 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine +FROM alpine:3.7 RUN apk add --no-cache postfix postfix-sqlite postfix-pcre rsyslog python py-jinja2 diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index 2f2c6990..c1d5e4b2 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -32,7 +32,7 @@ relayhost = {{ RELAYHOST }} recipient_delimiter = {{ RECIPIENT_DELIMITER }} # Only the front server is allowed to perform xclient -smtpd_authorized_xclient_hosts={{ FRONT_ADDRESS }} +smtpd_authorized_xclient_hosts={{ RELAYNETS }} ############### # TLS diff --git a/docs/compose/.env b/docs/compose/.env index 06038bc8..e7a1f55d 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -55,6 +55,9 @@ WEBDAV=none # Antivirus solution (value: clamav, none) ANTIVIRUS=none +# The password for the rspamd web interface +RSPAMD_PASSWORD=Secr3tPassWord + ################################### # Mail settings ################################### diff --git a/docs/index.rst b/docs/index.rst index 0920bb96..ebfbd7a7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -55,7 +55,7 @@ the version of Mailu that you are running. configuration compose/requirements compose/setup - kubernetes/index + kubernetes/mailu/index dns reverse diff --git a/docs/kubernetes/index.rst b/docs/kubernetes/index.rst deleted file mode 100644 index efd1ab7c..00000000 --- a/docs/kubernetes/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -Kubernetes setup -================ - -Please note that Kubernetes setup is not yet well supported or documented, all -tests currently run on Docker Compose. The configuration has not yet been updated -to work properly with ngin authentication proxy. - -Prepare the environment ------------------------ - -The resource configurations in this folder assume that you have `Kubernetes Ingress`_ -set up for your cluster. If you are not using the `NGINX Ingress Controller for Kubernetes`_, -please ensure that the configuration specified in the file matches your set up. - -.. _`Kubernetes Ingress`: https://kubernetes.io/docs/concepts/services-networking/ingress/ -.. _`NGINX Ingress Controller for Kubernetes`: https://github.com/kubernetes/ingress/tree/master/controllers/nginx - -Setup the Kubernetes service ----------------------------- - -Using the resource configurations is simple: - -1. ``kubectl apply -f kubernetes-nginx-ingress-controller.yaml`` to configure an ingress controller with the proper settings. (If you have one set up already you may need to port the configuration to your own ingress). -2. ``kubectl apply -f kubernetes-mailu.yaml`` to create the resources required to run Mailu. - -Based on the configuration, your Mailu instance should be available at ``mail..tld/admin`` (note that visiting just ``mail..tld`` will likely result in a 404 error). diff --git a/docs/kubernetes/kubernetes-mailu.yaml b/docs/kubernetes/kubernetes-mailu.yaml deleted file mode 100644 index a7bafccd..00000000 --- a/docs/kubernetes/kubernetes-mailu.yaml +++ /dev/null @@ -1,419 +0,0 @@ ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: mailu-admin-ing - labels: - app: mailu - role: mail - tier: backend -spec: - tls: - - hosts: - - "mail.example.com" - secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt - rules: - - host: "mail.example.com" - http: - paths: - - path: "/admin" - backend: - serviceName: mailu-admin - servicePort: 80 - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-redis -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-redis - role: mail - tier: backend - spec: - containers: - - name: redis - image: redis:4.0-alpine - imagePullPolicy: Always - volumeMounts: - - mountPath: /data - name: redisdata - ports: - - containerPort: 6379 - name: redis - protocol: TCP - volumes: - - name: redisdata - hostPath: - path: /var/data/mailu/redisdata - ---- - -apiVersion: v1 -kind: Service -metadata: - name: redis - labels: - app: mailu-redis - role: mail - tier: backend -spec: - selector: - app: mailu - role: mail - tier: backend - ports: - - name: redis - port: 6379 - protocol: TCP - ---- - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-imap -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-imap - role: mail - tier: backend - spec: - containers: - - name: imap - image: mailu/dovecot:stable - imagePullPolicy: Always - env: - - name : DOMAIN - value : example.com - - name : HOSTNAME - value : mail.example.com - - name : POSTMASTER - value : admin - volumeMounts: - - mountPath: /data - name: maildata - - mountPath: /mail - name: mailstate - - mountPath: /overrides - name: overrides - - mountPath: /certs - name: certs - readOnly: true - ports: - - containerPort: 2102 - - containerPort: 2525 - - containerPort: 143 - - containerPort: 993 - - containerPort: 4190 - volumes: - - name: maildata - hostPath: - path: /var/data/mailu/maildata - - name: mailstate - hostPath: - path: /var/data/mailu/mailstate - - name: overrides - hostPath: - path: /var/data/mailu/overrides - - name: certs - secret: - items: - - key: tls.crt - path: cert.pem - - key: tls.key - path: key.pem - secretName: letsencrypt-certs-all - ---- - -apiVersion: v1 -kind: Service -metadata: - name: imap - labels: - app: mailu - role: mail - tier: backend -spec: - selector: - app: mailu-imap - role: mail - tier: backend - ports: - ports: - - name: imap-auth - port: 2102 - protocol: TCP - - name: imap-transport - port: 2525 - protocol: TCP - - name: imap-default - port: 143 - protocol: TCP - - name: imap-ssl - port: 993 - protocol: TCP - - name: sieve - port: 4190 - protocol: TCP - ---- - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-smtp -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-smtp - role: mail - tier: backend - spec: - containers: - - name: smtp - image: mailu/postfix:stable - imagePullPolicy: Always - env: - - name : DOMAIN - value : example.com - - name : HOSTNAME - value : mail.example.com - - name : MESSAGE_SIZE_LIMIT - value : "50000000" - - name : RELAYHOST - value : "" - volumeMounts: - - mountPath: /data - name: maildata - - mountPath: /overrides - name: overrides - - mountPath: /certs - name: certs - readOnly: true - ports: - - name: smtp - containerPort: 25 - protocol: TCP - - name: smtp-ssl - containerPort: 465 - protocol: TCP - - name: smtp-starttls - containerPort: 587 - protocol: TCP - volumes: - - name: maildata - hostPath: - path: /var/data/mailu/maildata - - name: overrides - hostPath: - path: /var/data/mailu/overrides - - name: certs - secret: - items: - - key: tls.crt - path: cert.pem - - key: tls.key - path: key.pem - secretName: letsencrypt-certs-all - ---- - -apiVersion: v1 -kind: Service -metadata: - name: smtp - labels: - app: mailu - role: mail - tier: backend -spec: - selector: - app: mailu-smtp - role: mail - tier: backend - ports: - - name: smtp - port: 25 - protocol: TCP - - name: smtp-ssl - port: 465 - protocol: TCP - - name: smtp-starttls - port: 587 - protocol: TCP - ---- - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-security -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-security - role: mail - tier: backend - spec: - containers: - - name: antispam - image: mailu/rspamd:stable - imagePullPolicy: Always - ports: - - name: antispam - containerPort: 11333 - protocol: TCP - volumeMounts: - - name: filter - mountPath: /var/lib/rspamd - - name: antivirus - image: mailu/clamav:stable - imagePullPolicy: Always - ports: - - name: antivirus - containerPort: 3310 - protocol: TCP - volumeMounts: - - name: filter - mountPath: /data - volumes: - - name: filter - hostPath: - path: /var/data/mailu/filter - ---- - -apiVersion: v1 -kind: Service -metadata: - name: antispam - labels: - app: mailu-antispam - role: mail - tier: backend -spec: - selector: - app: mailu-security - role: mail - tier: backend - ports: - - name: antispam - port: 11333 - protocol: TCP - ---- - -apiVersion: v1 -kind: Service -metadata: - name: antivirus - labels: - app: mailu-antivirus - role: mail - tier: backend -spec: - selector: - app: mailu-security - role: mail - tier: backend - ports: - - name: antivirus - port: 3310 - protocol: TCP - ---- - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-admin -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-admin - role: mail - tier: backend - spec: - containers: - - name: admin - image: mailu/admin:stable - imagePullPolicy: Always - env: - - name : DOMAIN - value : example.com - - name : HOSTNAME - value : mail.example.com - - name : POSTMASTER - value : core - - name : SECRET_KEY - value : pleasereplacethiswithabetterkey - - name : DEBUG - value : "True" - volumeMounts: - - name: maildata - mountPath: /data - - name: dkim - mountPath: /dkim - - name: certs - mountPath: /certs - readOnly: true - # - name: docker - # mountPath: /var/run/docker.sock - # readOnly: true - ports: - - name: http - containerPort: 80 - protocol: TCP - volumes: - - name: maildata - hostPath: - path: /var/data/mailu/maildata - - name: dkim - hostPath: - path: /var/data/mailu/dkim - - name: certs - secret: - items: - - key: tls.crt - path: cert.pem - - key: tls.key - path: key.pem - secretName: letsencrypt-certs-all - # - name: docker - # hostPath: - # path: /var/run/docker.sock - ---- - -apiVersion: v1 -kind: Service -metadata: - name: mailu-admin - labels: - app: mailu-admin - role: mail - tier: backend -spec: - selector: - app: mailu-admin - role: mail - tier: backend - ports: - - name: http - port: 80 - protocol: TCP diff --git a/docs/kubernetes/kubernetes-nginx-ingress-controller.yaml b/docs/kubernetes/kubernetes-nginx-ingress-controller.yaml deleted file mode 100644 index 5ea9790a..00000000 --- a/docs/kubernetes/kubernetes-nginx-ingress-controller.yaml +++ /dev/null @@ -1,84 +0,0 @@ ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: nginx-configuration - namespace: ingress-nginx - labels: - app: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - namespace: ingress-nginx - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - namespace: ingress-nginx -data: - 25: "mailu/smtp:25" - 465: "mailu/smtp:465" - 587: "mailu/smtp:587" - 143: "mailu/imap:143" - 993: "mailu/imap:993" - ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: nginx-ingress-controller - namespace: kube-system - labels: - k8s-app: nginx-ingress-controller -spec: - replicas: 1 - template: - metadata: - labels: - k8s-app: nginx-ingress-controller - annotations: - prometheus.io/port: '10254' - prometheus.io/scrape: 'true' - spec: - # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration - # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host - # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used - # like with kubeadm - # hostNetwork: true - terminationGracePeriodSeconds: 60 - containers: - - image: gcr.io/google_containers/nginx-ingress-controller:0.11.0 - name: nginx-ingress-controller - args: - - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/default-http-backend - - --configmap=$(POD_NAMESPACE)/nginx-configuration - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --annotations-prefix=nginx.ingress.kubernetes.io - readinessProbe: - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - livenessProbe: - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - timeoutSeconds: 1 - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace diff --git a/docs/kubernetes/mailu/admin-ingress.yaml b/docs/kubernetes/mailu/admin-ingress.yaml new file mode 100644 index 00000000..72aafa68 --- /dev/null +++ b/docs/kubernetes/mailu/admin-ingress.yaml @@ -0,0 +1,86 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: mailu-admin-ingress + namespace: mailu-mailserver + annotations: + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + certmanager.k8s.io/cluster-issuer: letsencrypt-stage + ingress.kubernetes.io/permanent-redirect: "https://mail.example.com/admin/ui/" + ingress.kubernetes.io/follow-redirects: "true" + labels: + app: mailu + role: mail + tier: backend +spec: + tls: + - hosts: + - "mail.example.com" + secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt + rules: + - host: "mail.example.com" + http: + paths: + - path: "/admin" + backend: + serviceName: admin + servicePort: 80 +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: mailu-admin-ui-ingress + namespace: mailu-mailserver + annotations: + kubernetes.io/tls-acme: "true" + certmanager.k8s.io/cluster-issuer: letsencrypt-stage + ingress.kubernetes.io/rewrite-target: "/ui" + ingress.kubernetes.io/configuration-snippet: | + proxy_set_header X-Forwarded-Prefix /admin; + labels: + app: mailu + role: mail + tier: backend +spec: + tls: + - hosts: + - "mail.example.com" + secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt + rules: + - host: "mail.example.com" + http: + paths: + - path: "/admin/ui" + backend: + serviceName: admin + servicePort: 80 +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: mailu-admin-static-ingress + namespace: mailu-mailserver + annotations: + kubernetes.io/tls-acme: "true" + certmanager.k8s.io/cluster-issuer: letsencrypt-stage + ingress.kubernetes.io/rewrite-target: "/static" + ingress.kubernetes.io/configuration-snippet: | + proxy_set_header X-Forwarded-Prefix /admin; + labels: + app: mailu + role: mail + tier: backend +spec: + tls: + - hosts: + - "mail.example.com" + secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt + rules: + - host: "mail.example.com" + http: + paths: + - path: "/admin/static" + backend: + serviceName: admin + servicePort: 80 \ No newline at end of file diff --git a/docs/kubernetes/mailu/admin.yaml b/docs/kubernetes/mailu/admin.yaml new file mode 100644 index 00000000..435b7975 --- /dev/null +++ b/docs/kubernetes/mailu/admin.yaml @@ -0,0 +1,63 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mailu-admin + namespace: mailu-mailserver +spec: + replicas: 1 + template: + metadata: + labels: + app: mailu-admin + role: mail + tier: backend + spec: + containers: + - name: admin + image: mailu/admin:master + imagePullPolicy: Always + envFrom: + - configMapRef: + name: mailu-config + volumeMounts: + - name: maildata + mountPath: /data + subPath: maildata + - name: maildata + mountPath: /dkim + subPath: dkim + ports: + - name: http + containerPort: 80 + protocol: TCP + resources: + requests: + memory: 500Mi + cpu: 500m + limits: + memory: 500Mi + cpu: 500m + volumes: + - name: maildata + persistentVolumeClaim: + claimName: mail-storage +--- + +apiVersion: v1 +kind: Service +metadata: + name: admin + namespace: mailu-mailserver + labels: + app: mailu-admin + role: mail + tier: backend +spec: + selector: + app: mailu-admin + role: mail + tier: backend + ports: + - name: http + port: 80 + protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/mailu/configmap.yaml b/docs/kubernetes/mailu/configmap.yaml new file mode 100644 index 00000000..be39fcc9 --- /dev/null +++ b/docs/kubernetes/mailu/configmap.yaml @@ -0,0 +1,172 @@ + apiVersion: v1 + kind: ConfigMap + metadata: + name: mailu-config + namespace: mailu-mailserver + data: + # Mailu main configuration file + # + # Most configuration variables can be modified through the Web interface, + # these few settings must however be configured before starting the mail + # server and require a restart upon change. + + ################################### + # Common configuration variables + ################################### + + # Set this to the path where Mailu data and configuration is stored + ROOT: "/mailu" + + # Mailu version to run (1.0, 1.1, etc. or master) + VERSION: "master" + + # Set to a randomly generated 16 bytes string + SECRET_KEY: "MySup3rS3cr3tPas" + + # Address where listening ports should bind + BIND_ADDRESS4: "127.0.0.1" + #BIND_ADDRESS6: "::1" + + # Main mail domain + DOMAIN: "example.com" + + # Hostnames for this server, separated with comas + HOSTNAMES: "mail.example.com" + + # Postmaster local part (will append the main mail domain) + POSTMASTER: "admin" + + # Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt) + TLS_FLAVOR: "cert" + + # Authentication rate limit (per source IP address) + AUTH_RATELIMIT: "100/minute;10000/hour" + + # Opt-out of statistics, replace with "True" to opt out + DISABLE_STATISTICS: "False" + + ################################### + # Kubernetes configuration + ################################### + + # Use Kubernetes Ingress Controller to handle all actions on port 80 and 443 + # This way we can make use of the advantages of the cert-manager deployment + KUBERNETES_INGRESS: "true" + POD_ADDRESS_RANGE: "10.2.0.0/16" + + ################################### + # Optional features + ################################### + + # Expose the admin interface (value: true, false) + ADMIN: "true" + # Run the admin interface in debug mode + #DEBUG: "True" + + # Choose which webmail to run if any (values: roundcube, rainloop, none) + WEBMAIL: "roundcube" + + # Dav server implementation (value: radicale, none) + WEBDAV: "radicale" + + # Antivirus solution (value: clamav, none) + ANTIVIRUS: "clamav" + + # The password for the rspamd web interface + RSPAMD_PASSWORD: "Secr3tPassWord" + + ################################### + # Mail settings + ################################### + + # Message size limit in bytes + # Default: accept messages up to 50MB + MESSAGE_SIZE_LIMIT: "50000000" + + # Networks granted relay permissions, make sure that you include your Docker + # internal network (default to 172.17.0.0/16) + # For kubernetes this is the CIDR of the pod network + RELAYNETS: "10.2.0.0/16" + + # Will relay all outgoing mails if configured + #RELAYHOST= + + # This part is needed for the XCLIENT login for postfix. This should be the POD ADDRESS range + FRONT_ADDRESS: "front.mailu-mailserver.svc.cluster.local" + + # This value is needed by the webmail to find the correct imap backend + IMAP_ADDRESS: "imap.mailu-mailserver.svc.cluster.local" + + # This value is used by Dovecot to find the Redis server in the cluster + REDIS_ADDRESS: "redis.mailu-mailserver.svc.cluster.local" + + # Fetchmail delay + FETCHMAIL_DELAY: "600" + + # Recipient delimiter, character used to delimiter localpart from custom address part + # e.g. localpart+custom@domain;tld + RECIPIENT_DELIMITER: "+" + + # DMARC rua and ruf email + DMARC_RUA: "root" + DMARC_RUF: "root" + + # Welcome email, enable and set a topic and body if you wish to send welcome + # emails to all users. + WELCOME: "false" + WELCOME_SUBJECT: "Welcome to your new email account" + WELCOME_BODY: "Welcome to your new email account, if you can read this, then it is configured properly!" + + ################################### + # Web settings + ################################### + + # Path to the admin interface if enabled + # Kubernetes addition: You need to change ALL the ingresses, when you want this URL to be different!!! + WEB_ADMIN: "/admin" + + # Path to the webmail if enabled + # Currently, this is not used, because we intended to use a different subdomain: webmail.example.com + # This option can be added in a feature release + WEB_WEBMAIL: "/webmail" + + # Website name + SITENAME: "Mailu" + + # Linked Website URL + WEBSITE: "https://example.com" + + # Registration reCaptcha settings (warning, this has some privacy impact) + # RECAPTCHA_PUBLIC_KEY= + # RECAPTCHA_PRIVATE_KEY= + + # Domain registration, uncomment to enable + # DOMAIN_REGISTRATION=true + + ################################### + # Advanced settings + ################################### + + # Docker-compose project name, this will prepended to containers names. + COMPOSE_PROJECT_NAME: "mailu" + + # Default password scheme used for newly created accounts and changed passwords + # (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) + PASSWORD_SCHEME: "SHA512-CRYPT" + + # Header to take the real ip from + #REAL_IP_HEADER: + + # IPs for nginx set_real_ip_from (CIDR list separated by commas) + #REAL_IP_FROM: + + # Host settings + HOST_IMAP: "imap.mailu-mailserver.svc.cluster.local" + HOST_POP3: "imap.mailu-mailserver.svc.cluster.local" + HOST_SMTP: "smtp.mailu-mailserver.svc.cluster.local" + HOST_AUTHSMTP: "smtp.mailu-mailserver.svc.cluster.local" + HOST_WEBMAIL: "webmail.mailu-mailserver.svc.cluster.local" + HOST_ADMIN: "admin.mailu-mailserver.svc.cluster.local" + HOST_WEBDAV: "webdav.mailu-mailserver.svc.cluster.local:5232" + HOST_ANTISPAM: "antispam.mailu-mailserver.svc.cluster.local:11332" + HOST_REDIS: "redis.mailu-mailserver.svc.cluster.local" diff --git a/docs/kubernetes/mailu/fetchmail.yaml b/docs/kubernetes/mailu/fetchmail.yaml new file mode 100644 index 00000000..cf3271e7 --- /dev/null +++ b/docs/kubernetes/mailu/fetchmail.yaml @@ -0,0 +1,39 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mailu-fetchmail + namespace: mailu-mailserver +spec: + replicas: 1 + template: + metadata: + labels: + app: mailu-fetchmail + role: mail + tier: backend + spec: + containers: + - name: fetchmail + image: mailu/fetchmail:master + imagePullPolicy: Always + envFrom: + - configMapRef: + name: mailu-config + volumeMounts: + - name: maildata + mountPath: /data + subPath: maildata + ports: + - containerPort: 5232 + - containerPort: 80 + resources: + requests: + memory: 100Mi + cpu: 100m + limits: + memory: 100Mi + cpu: 100m + volumes: + - name: maildata + persistentVolumeClaim: + claimName: mail-storage \ No newline at end of file diff --git a/docs/kubernetes/mailu/front.yaml b/docs/kubernetes/mailu/front.yaml new file mode 100644 index 00000000..c13ecd9d --- /dev/null +++ b/docs/kubernetes/mailu/front.yaml @@ -0,0 +1,135 @@ +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + name: mailu-front + namespace: mailu-mailserver + labels: + k8s-app: mail-loadbalancer + component: ingress-controller + type: nginx +spec: + selector: + matchLabels: + k8s-app: mail-loadbalancer + component: ingress-controller + type: nginx + template: + metadata: + labels: + k8s-app: mail-loadbalancer + component: ingress-controller + type: nginx + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: DoesNotExist + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/node: "" + dnsPolicy: ClusterFirstWithHostNet + restartPolicy: Always + terminationGracePeriodSeconds: 60 + containers: + - name: front + image: mailu/nginx:master + imagePullPolicy: Always + envFrom: + - configMapRef: + name: mailu-config + volumeMounts: + - name: certs + mountPath: /certs + ports: + - name: pop3 + containerPort: 110 + protocol: TCP + - name: pop3s + containerPort: 995 + protocol: TCP + - name: imap + containerPort: 143 + protocol: TCP + - name: imaps + containerPort: 993 + protocol: TCP + - name: smtp + containerPort: 25 + protocol: TCP + - name: smtp-auth + containerPort: 10025 + protocol: TCP + - name: imap-auth + containerPort: 10143 + protocol: TCP + - name: smtps + containerPort: 465 + protocol: TCP + - name: smtpd + containerPort: 587 + protocol: TCP + - name: auth + containerPort: 8000 + protocol: TCP + resources: + requests: + memory: 100Mi + cpu: 100m + limits: + memory: 200Mi + cpu: 200m + volumes: + - name: certs + secret: + items: + - key: tls.crt + path: cert.pem + - key: tls.key + path: key.pem + secretName: letsencrypt-certs-all +--- +apiVersion: v1 +kind: Service +metadata: + name: front + namespace: mailu-mailserver + labels: + k8s-app: mail-loadbalancer + component: ingress-controller + type: nginx +spec: + selector: + k8s-app: mail-loadbalancer + component: ingress-controller + type: nginx + ports: + - name: pop3 + port: 110 + protocol: TCP + - name: pop3s + port: 995 + protocol: TCP + - name: imap + port: 143 + protocol: TCP + - name: imaps + port: 993 + protocol: TCP + - name: smtp + port: 25 + protocol: TCP + - name: smtps + port: 465 + protocol: TCP + - name: smtpd + port: 587 + protocol: TCP + - name: smtp-auth + port: 10025 + protocol: TCP + - name: imap-auth + port: 10143 + protocol: TCP diff --git a/docs/kubernetes/mailu/imap.yaml b/docs/kubernetes/mailu/imap.yaml new file mode 100644 index 00000000..37f4899e --- /dev/null +++ b/docs/kubernetes/mailu/imap.yaml @@ -0,0 +1,80 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mailu-imap + namespace: mailu-mailserver +spec: + replicas: 1 + template: + metadata: + labels: + app: mailu-imap + role: mail + tier: backend + spec: + containers: + - name: imap + image: mailu/dovecot:master + imagePullPolicy: Always + envFrom: + - configMapRef: + name: mailu-config + volumeMounts: + - mountPath: /data + name: maildata + subPath: maildata + - mountPath: /mail + name: maildata + subPath: mailstate + - mountPath: /overrides + name: maildata + subPath: overrides + ports: + - containerPort: 2102 + - containerPort: 2525 + - containerPort: 143 + - containerPort: 993 + - containerPort: 4190 + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 1Gi + cpu: 1000m + volumes: + - name: maildata + persistentVolumeClaim: + claimName: mail-storage +--- +apiVersion: v1 +kind: Service +metadata: + name: imap + namespace: mailu-mailserver + labels: + app: mailu + role: mail + tier: backend +spec: + selector: + app: mailu-imap + role: mail + tier: backend + ports: + ports: + - name: imap-auth + port: 2102 + protocol: TCP + - name: imap-transport + port: 2525 + protocol: TCP + - name: imap-default + port: 143 + protocol: TCP + - name: imap-ssl + port: 993 + protocol: TCP + - name: sieve + port: 4190 + protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/mailu/index.rst b/docs/kubernetes/mailu/index.rst new file mode 100644 index 00000000..ef12eb58 --- /dev/null +++ b/docs/kubernetes/mailu/index.rst @@ -0,0 +1,193 @@ +Install Mailu master on kubernetes +================================== + +Prequisites +----------- + +Structure +~~~~~~~~~ + +There’s chosen to have a double NGINX stack for Mailu, this way the main +ingress can still be used to access other websites/domains on your +cluster. This is the current structure: + +- ``NGINX Ingress controller``: Listens to the nodes ports 80 & 443. We have chosen to have a double NGINX stack for Mailu. +- ``Cert manager``: Creates automatic Lets Encrypt certificates based on an ``Ingress``-objects domain name. +- ``Mailu NGINX Front daemonset``: This daemonset runs in parallel with the Nginx Ingress Controller and only listens on all E-mail specific ports (25, 110, 143, 587,...) +- ``Mailu components``: All Mailu components (imap, smtp, security, webmail,...) are split into separate files to make them more handy to use, you can find the ``YAML`` files in this directory + +What you need +~~~~~~~~~~~~~ + +- A working Kubernetes cluster (tested with 1.10.5) +- A working `cert-manager`_ installation +- A working nginx-ingress controller needed for the lets-encrypt + certificates. You can find those files in the ``nginx`` subfolder + +Cert manager +^^^^^^^^^^^^ + +The ``Cert-manager`` is quite easy to deploy using Helm when reading the +`docs`_. After booting the ``Cert-manager`` you’ll need a +``ClusterIssuer`` which takes care of all required certificates through +``Ingress`` items. We chose to provide a ``clusterIssuer`` so you can provide SSL certificates +for other namespaces (different websites/services), if you don't need this option, you can easily change this by +changing ``clusterIssuer`` to ``Issuer`` and adding the ``namespace: mailu-mailserver`` to the metadata. +An example of a production and a staging ``clusterIssuer``: + +.. code:: yaml + + # This clusterIssuer example uses the staging environment for testing first + apiVersion: certmanager.k8s.io/v1alpha1 + kind: ClusterIssuer + metadata: + name: letsencrypt-stage + spec: + acme: + email: something@example.com + http01: {} + privateKeySecretRef: + name: letsencrypt-stage + server: https://acme-staging-v02.api.letsencrypt.org/directory + +.. code:: yaml + + # This clusterIssuer example uses the production environment + apiVersion: certmanager.k8s.io/v1alpha1 + kind: ClusterIssuer + metadata: + name: letsencrypt-prod + spec: + acme: + email: something@example.com + http01: {} + privateKeySecretRef: + name: letsencrypt-prod + server: https://acme-v02.api.letsencrypt.org/directory + +**IMPORTANT**: All ``*-ingress.yaml`` files use the ``letsencrypt-stage`` ``clusterIssuer``. If you are ready for production, +change this field in all ``*-ingress.yaml`` files to ``letsencrypt-prod`` or whatever name you chose for the production. +If you choose for ``Issuer`` instead of ``clusterIssuer`` you also need to change the annotation to ``certmanager.k8s.io/issuer`` instead of ``certmanager.k8s.io/cluster-issuer`` + +Deploying Mailu +--------------- + +All manifests can be found in the ``mailu`` subdirectory. All commands +below need to be run from this subdirectory + +Personalization +~~~~~~~~~~~~~~~ + +- All services run in the same namespace, currently ``mailu-mailserver``. So if you want to use a different one, change the ``namespace`` value in **every** file +- Check the ``storage-class`` field in the ``pvc.yaml`` file, you can also change the sizes to your liking. Note that you need ``RWX`` (read-write-many) and ``RWO`` (read-write-once) storageclasses. +- Check the ``configmap.yaml`` and adapt it to your needs. Be sure to check the kubernetes DNS values at the end (if you use a different namespace) +- Check the ``*-ingress.yaml`` files and change it to the domain you want (this is for the kubernetes ingress controller to handle the admin, webmail, webdav and auth connections) + +Installation +------------ + +Boot the Mailu components +~~~~~~~~~~~~~~~~~~~~~~~~~ + +To start Mailu, run the following commands from the ``docs/kubernetes/mailu`` directory + +.. code-block:: bash + + kubectl create -f rbac.yaml + kubectl create -f configmap.yaml + kubectl create -f pvc.yaml + kubectl create -f redis.yaml + kubectl create -f front.yaml + kubectl create -f webmail.yaml + kubectl create -f imap.yaml + kubectl create -f security.yaml + kubectl create -f smtp.yaml + kubectl create -f fetchmail.yaml + kubectl create -f admin.yaml + kubectl create -f webdav.yaml + kubectl create -f admin-ingress.yaml + kubectl create -f webdav-ingress.yaml + kubectl create -f security-ingress.yaml + kubectl create -f webmail-ingress.yaml + + +Create the first admin account +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When the cluster is online you need to create you master user to access https://mail.example.com/admin +Enter the main ``admin`` pod to create the root account: + +.. code-block:: bash + + kubectl -n mailu-mailserver get po + kubectl -n mailu-mailserver exec -it mailu-admin-.... /bin/sh + +And in the pod run the following command. The command uses following entries: + +.. code-block:: bash + + python manage.py admin root example.com password + +- ``admin`` Make it an admin user +- ``root`` The first part of the e-mail adres (ROOT@example.com) +- ``example.com`` the domain appendix +- ``password`` the chosen password for the user + + +Now you should be able to login on the mail account: https://mail.example.com/admin + +Adaptations +----------- + +Dovecot +~~~~~~~ + +- If you are using Dovecot on a shared file system (Glusterfs, NFS,...), you need to create a special override otherwise a lot of indexing errors will occur on your Dovecot pod. +- I also higher the number of max connections per IP. Now it's limited to 10. + +Enter the dovecot pod: + +.. code:: bash + + kubectl -n mailu-mailserver get po + kubectl -n mailu-mailserver exec -it mailu-imap-.... /bin/sh + +Create the file ``overrides/dovecot.conf`` + +.. code:: bash + + vi /overrides/dovecot.conf + +And enter following contents: + +.. code:: bash + + mail_nfs_index = yes + mail_nfs_storage = yes + mail_fsync = always + mmap_disable = yes + mail_max_userip_connections=100 + +Save and close the file and delete the imap pod to get it recreated. + +.. code:: bash + + kubectl -n mailu-mailserver delete po/mailu-imap-.... + +Wait for the pod to recreate and you're online! +Happy mailing! + +.. _here: https://github.com/hacor/Mailu/blob/master/core/postfix/conf/main.cf#L35 +.. _cert-manager: https://github.com/jetstack/cert-manager +.. _docs: https://cert-manager.readthedocs.io/en/latest/getting-started/2-installing.html + +Imap login fix +~~~~~~~~~~~~~~ + +If it seems you're not able to login using IMAP on your Mailu accounts, check the logs of the imap container to see whether it's a permissions problem on the database. +This problem can be easily fixed by running following commands: + +.. code:: bash + + kubectl -n mailu-mailserver exec -it maolu-imap-... /bin/sh + chmod 777 /data/main.db diff --git a/docs/kubernetes/mailu/pvc.yaml b/docs/kubernetes/mailu/pvc.yaml new file mode 100644 index 00000000..0ec2852f --- /dev/null +++ b/docs/kubernetes/mailu/pvc.yaml @@ -0,0 +1,27 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: redis-hdd + namespace: mailu-mailserver + annotations: + volume.beta.kubernetes.io/storage-class: "glusterblock-hdd" +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mail-storage + namespace: mailu-mailserver + annotations: + volume.beta.kubernetes.io/storage-class: "gluster-heketi-hdd" +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi diff --git a/docs/kubernetes/mailu/rbac.yaml b/docs/kubernetes/mailu/rbac.yaml new file mode 100644 index 00000000..33255130 --- /dev/null +++ b/docs/kubernetes/mailu/rbac.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: mailu-mailserver \ No newline at end of file diff --git a/docs/kubernetes/mailu/redis.yaml b/docs/kubernetes/mailu/redis.yaml new file mode 100644 index 00000000..d6bb1eb8 --- /dev/null +++ b/docs/kubernetes/mailu/redis.yaml @@ -0,0 +1,56 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mailu-redis + namespace: mailu-mailserver +spec: + replicas: 1 + template: + metadata: + labels: + app: mailu-redis + role: mail + tier: backend + spec: + containers: + - name: redis + image: redis:4.0-alpine + imagePullPolicy: Always + volumeMounts: + - mountPath: /data + name: redisdata + ports: + - containerPort: 6379 + name: redis + protocol: TCP + resources: + requests: + memory: 200Mi + cpu: 100m + limits: + memory: 300Mi + cpu: 200m + volumes: + - name: redisdata + persistentVolumeClaim: + claimName: redis-hdd +--- + +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: mailu-mailserver + labels: + app: mailu-redis + role: mail + tier: backend +spec: + selector: + app: mailu-redis + role: mail + tier: backend + ports: + - name: redis + port: 6379 + protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/mailu/security-ingress.yaml b/docs/kubernetes/mailu/security-ingress.yaml new file mode 100644 index 00000000..74ced47e --- /dev/null +++ b/docs/kubernetes/mailu/security-ingress.yaml @@ -0,0 +1,30 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: mailu-antispam-ingress + namespace: mailu-mailserver + annotations: + kubernetes.io/tls-acme: "true" + certmanager.k8s.io/cluster-issuer: letsencrypt-stage + ingress.kubernetes.io/configuration-snippet: | + rewrite ^/admin/antispam/(.*) /$1 break; + auth_request /internal/auth/admin; + proxy_set_header X-Real-IP ""; + proxy_set_header X-Forwarded-For ""; + labels: + app: mailu + role: mail + tier: frontend +spec: + tls: + - hosts: + - "mail.example.com" + secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt + rules: + - host: "mail.example.com" + http: + paths: + - path: "/admin/antispam" + backend: + serviceName: antispam + servicePort: 11334 \ No newline at end of file diff --git a/docs/kubernetes/mailu/security.yaml b/docs/kubernetes/mailu/security.yaml new file mode 100644 index 00000000..80fde812 --- /dev/null +++ b/docs/kubernetes/mailu/security.yaml @@ -0,0 +1,116 @@ + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mailu-security + namespace: mailu-mailserver +spec: + replicas: 1 + template: + metadata: + labels: + app: mailu-security + role: mail + tier: backend + spec: + containers: + - name: antispam + image: mailu/rspamd:master + imagePullPolicy: Always + envFrom: + - configMapRef: + name: mailu-config + resources: + requests: + memory: 100Mi + cpu: 100m + limits: + memory: 200Mi + cpu: 200m + ports: + - name: antispam + containerPort: 11332 + protocol: TCP + - name: antispam-http + containerPort: 11334 + protocol: TCP + volumeMounts: + - name: filter + subPath: filter + mountPath: /var/lib/rspamd + - name: filter + mountPath: /dkim + subPath: dkim + - name: filter + mountPath: /etc/rspamd/override.d + subPath: rspamd-overrides + - name: antivirus + image: mailu/clamav:master + imagePullPolicy: Always + resources: + requests: + memory: 1Gi + cpu: 1000m + limits: + memory: 2Gi + cpu: 1000m + envFrom: + - configMapRef: + name: mailu-config + ports: + - name: antivirus + containerPort: 3310 + protocol: TCP + volumeMounts: + - name: filter + subPath: filter + mountPath: /data + volumes: + - name: filter + persistentVolumeClaim: + claimName: mail-storage + +--- + +apiVersion: v1 +kind: Service +metadata: + name: antispam + namespace: mailu-mailserver + labels: + app: mailu-antispam + role: mail + tier: backend +spec: + selector: + app: mailu-security + role: mail + tier: backend + ports: + - name: antispam + port: 11332 + protocol: TCP + - name: antispam-http + protocol: TCP + port: 11334 + +--- + +apiVersion: v1 +kind: Service +metadata: + name: antivirus + namespace: mailu-mailserver + labels: + app: mailu-antivirus + role: mail + tier: backend +spec: + selector: + app: mailu-security + role: mail + tier: backend + ports: + - name: antivirus + port: 3310 + protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/mailu/smtp.yaml b/docs/kubernetes/mailu/smtp.yaml new file mode 100644 index 00000000..926a2b7c --- /dev/null +++ b/docs/kubernetes/mailu/smtp.yaml @@ -0,0 +1,80 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mailu-smtp + namespace: mailu-mailserver +spec: + replicas: 1 + template: + metadata: + labels: + app: mailu-smtp + role: mail + tier: backend + spec: + containers: + - name: smtp + image: mailu/postfix:master + imagePullPolicy: Always + envFrom: + - configMapRef: + name: mailu-config + resources: + requests: + memory: 2Gi + cpu: 500m + limits: + memory: 2Gi + cpu: 500m + volumeMounts: + - mountPath: /data + name: maildata + subPath: maildata + - mountPath: /overrides + name: maildata + subPath: overrides + ports: + - name: smtp + containerPort: 25 + protocol: TCP + - name: smtp-ssl + containerPort: 465 + protocol: TCP + - name: smtp-starttls + containerPort: 587 + protocol: TCP + - name: smtp-auth + containerPort: 10025 + protocol: TCP + volumes: + - name: maildata + persistentVolumeClaim: + claimName: mail-storage +--- +apiVersion: v1 +kind: Service +metadata: + name: smtp + namespace: mailu-mailserver + labels: + app: mailu + role: mail + tier: backend +spec: + selector: + app: mailu-smtp + role: mail + tier: backend + ports: + - name: smtp + port: 25 + protocol: TCP + - name: smtp-ssl + port: 465 + protocol: TCP + - name: smtp-starttls + port: 587 + protocol: TCP + - name: smtp-auth + port: 10025 + protocol: TCP diff --git a/docs/kubernetes/mailu/webdav-ingress.yaml b/docs/kubernetes/mailu/webdav-ingress.yaml new file mode 100644 index 00000000..3498eb02 --- /dev/null +++ b/docs/kubernetes/mailu/webdav-ingress.yaml @@ -0,0 +1,46 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: mailu-webdav-ingress + namespace: mailu-mailserver + annotations: + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + certmanager.k8s.io/cluster-issuer: letsencrypt-stage + #ingress.kubernetes.io/auth-url: http://admin.mailu-mailserver.svc.cluster.local/internal/auth/basic + ingress.kubernetes.io/configuration-snippet: | + rewrite ^/webdav/(.*) /$1 break; + auth_request /internal/auth/basic; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + auth_request_set $user $upstream_http_x_user; + proxy_set_header X-Remote-User $user; + proxy_set_header X-Script-Name /webdav; + ingress.kubernetes.io/server-snippet: | + location /internal { + internal; + + proxy_set_header Authorization $http_authorization; + proxy_pass_header Authorization; + proxy_pass http://admin.mailu-mailserver.svc.cluster.local; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + } + labels: + app: mailu + role: mail + tier: frontend +spec: + tls: + - hosts: + - "mail.example.com" + secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt + rules: + - host: "mail.example.com" + http: + paths: + - path: "/webdav" + backend: + serviceName: webdav + servicePort: 5232 \ No newline at end of file diff --git a/docs/kubernetes/mailu/webdav.yaml b/docs/kubernetes/mailu/webdav.yaml new file mode 100644 index 00000000..07b7733c --- /dev/null +++ b/docs/kubernetes/mailu/webdav.yaml @@ -0,0 +1,63 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mailu-webdav + namespace: mailu-mailserver +spec: + replicas: 1 + template: + metadata: + labels: + app: mailu-webdav + role: mail + tier: backend + spec: + containers: + - name: radicale + image: mailu/radicale:master + imagePullPolicy: Always + envFrom: + - configMapRef: + name: mailu-config + volumeMounts: + - mountPath: /data + name: maildata + subPath: dav + ports: + - containerPort: 5232 + - containerPort: 80 + resources: + requests: + memory: 100Mi + cpu: 100m + limits: + memory: 100Mi + cpu: 100m + volumes: + - name: maildata + persistentVolumeClaim: + claimName: mail-storage +--- + +apiVersion: v1 +kind: Service +metadata: + name: webdav + namespace: mailu-mailserver + labels: + app: mailu-webdav + role: mail + tier: backend +spec: + selector: + app: mailu-webdav + role: mail + tier: backend + ports: + ports: + - name: http + port: 80 + protocol: TCP + - name: http-ui + port: 5232 + protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/mailu/webmail-ingress.yaml b/docs/kubernetes/mailu/webmail-ingress.yaml new file mode 100644 index 00000000..40655ca2 --- /dev/null +++ b/docs/kubernetes/mailu/webmail-ingress.yaml @@ -0,0 +1,31 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: mailu-webmail-ingress + namespace: mailu-mailserver + annotations: + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + certmanager.k8s.io/cluster-issuer: letsencrypt-stage + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + labels: + app: mailu + role: mail + tier: backend +spec: + tls: + - hosts: + - "webmail.example.com" + secretName: letsencrypt-webmail # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt + rules: + - host: "webmail.example.com" + http: + paths: + - path: "/" + backend: + serviceName: webmail + servicePort: 80 \ No newline at end of file diff --git a/docs/kubernetes/mailu/webmail.yaml b/docs/kubernetes/mailu/webmail.yaml new file mode 100644 index 00000000..bbbeb09d --- /dev/null +++ b/docs/kubernetes/mailu/webmail.yaml @@ -0,0 +1,59 @@ + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: mailu-roundcube + namespace: mailu-mailserver +spec: + replicas: 1 + template: + metadata: + labels: + app: mailu-roundcube + role: mail + tier: frontend + spec: + containers: + - name: roundcube + image: mailu/roundcube:master + imagePullPolicy: Always + envFrom: + - configMapRef: + name: mailu-config + resources: + requests: + memory: 100Mi + cpu: 100m + limits: + memory: 200Mi + cpu: 200m + volumeMounts: + - mountPath: /data + name: maildata + subPath: webmail + ports: + - containerPort: 80 + volumes: + - name: maildata + persistentVolumeClaim: + claimName: mail-storage +--- +apiVersion: v1 +kind: Service +metadata: + name: webmail + namespace: mailu-mailserver + labels: + app: mailu-roundcube + role: mail + tier: frontend +spec: + selector: + app: mailu-roundcube + role: mail + tier: frontend + ports: + ports: + - name: http + port: 80 + protocol: TCP diff --git a/docs/kubernetes/nginx/default-http-backend.yaml b/docs/kubernetes/nginx/default-http-backend.yaml new file mode 100644 index 00000000..097fe7c5 --- /dev/null +++ b/docs/kubernetes/nginx/default-http-backend.yaml @@ -0,0 +1,55 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: default-http-backend + labels: + app: default-http-backend + namespace: kube-ingress +spec: + replicas: 1 + selector: + matchLabels: + app: default-http-backend + template: + metadata: + labels: + app: default-http-backend + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: default-http-backend + # Any image is permissible as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.4 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi +--- + +apiVersion: v1 +kind: Service +metadata: + name: default-http-backend + namespace: kube-ingress + labels: + app: default-http-backend +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + app: default-http-backend \ No newline at end of file diff --git a/docs/kubernetes/nginx/nginx-ingress.yaml b/docs/kubernetes/nginx/nginx-ingress.yaml new file mode 100644 index 00000000..d8b71e21 --- /dev/null +++ b/docs/kubernetes/nginx/nginx-ingress.yaml @@ -0,0 +1,127 @@ +apiVersion: v1 +kind: Service +metadata: + # keep it under 24 chars + name: ingress-lb + namespace: kube-ingress + labels: + k8s-app: ingress-lb + component: ingress-controller +spec: + type: ClusterIP + selector: + k8s-app: ingress-lb + component: ingress-controller + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 80 + - name: https + protocol: TCP + port: 443 + targetPort: 443 +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: udp-services + namespace: kube-ingress + +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: tcp-services + namespace: kube-ingress +data: + +--- +apiVersion: v1 +data: + enable-vts-status: "true" +kind: ConfigMap +metadata: + name: nginx-ingress-lb-conf + namespace: kube-ingress +--- +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + name: ingress-controller + namespace: kube-ingress + annotations: + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + k8s-app: ingress-lb + component: ingress-controller + type: nginx +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + k8s-app: ingress-lb + component: ingress-controller + type: nginx + template: + metadata: + labels: + k8s-app: ingress-lb + component: ingress-controller + type: nginx + spec: + serviceAccount: kube-nginx-ingress + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: DoesNotExist + containers: + - name: nginx-ingress-lb + image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.2 + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --annotations-prefix=ingress.kubernetes.io + - --enable-ssl-passthrough + # use downward API + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: http + containerPort: 80 + - name: https + containerPort: 443 + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + initialDelaySeconds: 10 + timeoutSeconds: 1 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/node: "" + dnsPolicy: ClusterFirstWithHostNet + restartPolicy: Always + terminationGracePeriodSeconds: 60 diff --git a/docs/kubernetes/nginx/rbac.yaml b/docs/kubernetes/nginx/rbac.yaml new file mode 100644 index 00000000..d3c01384 --- /dev/null +++ b/docs/kubernetes/nginx/rbac.yaml @@ -0,0 +1,129 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-ingress +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-nginx-ingress + namespace: kube-ingress +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: kube-nginx-ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "extensions" + resources: + - ingresses/status + verbs: + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: kube-nginx-ingress + namespace: kube-ingress +rules: + - apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - namespaces + verbs: + - get + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "ingress-controller-leader-nginx" + verbs: + - get + - update + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - create + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: kube-nginx-ingress + namespace: kube-ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kube-nginx-ingress +subjects: + - kind: ServiceAccount + name: kube-nginx-ingress + namespace: kube-ingress +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kube-nginx-ingress +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-nginx-ingress +subjects: + - kind: ServiceAccount + name: kube-nginx-ingress + namespace: kube-ingress \ No newline at end of file diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index 6a020672..d3ab55f4 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,3 +1,7 @@ bind_socket = "*:11334"; +{% if RSPAMD_PASSWORD %} +password = "{{ RSPAMD_PASSWORD }}"; +{% else %} password = "mailu"; +{% endif %} secure_ip = "{{ FRONT_ADDRESS }}"; diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index c779e71a..e8a1aeaf 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -4,10 +4,10 @@ RUN apt-get update && apt-get install -y \ libfreetype6-dev \ libjpeg62-turbo-dev \ libmcrypt-dev \ - libpng12-dev \ + libpng-dev \ && docker-php-ext-install pdo_mysql mcrypt zip -ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.6/roundcubemail-1.3.6-complete.tar.gz +ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.7/roundcubemail-1.3.7-complete.tar.gz RUN echo date.timezone=UTC > /usr/local/etc/php/conf.d/timezone.ini @@ -28,4 +28,4 @@ COPY config.inc.php /var/www/html/config/ COPY start.sh /start.sh -CMD ["/start.sh"] +CMD ["/start.sh"] \ No newline at end of file From e67a0d464b16fb339710e61b90c91798db7e30d8 Mon Sep 17 00:00:00 2001 From: Hans Cornelis Date: Wed, 17 Oct 2018 07:44:21 +0200 Subject: [PATCH 30/51] Deleted old folder --- core/dovecot/Dockerfile_BACKUP_7007 | 19 +++ core/dovecot/Dockerfile_BASE_7007 | 14 ++ core/dovecot/Dockerfile_LOCAL_7007 | 14 ++ core/dovecot/Dockerfile_REMOTE_7007 | 15 ++ docs/kubernetes/1.6/README.md | 157 ------------------ docs/kubernetes/1.6/mailu/admin.yaml | 64 ------- docs/kubernetes/1.6/mailu/configmap.yaml | 153 ----------------- docs/kubernetes/1.6/mailu/fetchmail.yaml | 39 ----- docs/kubernetes/1.6/mailu/front.yaml | 129 -------------- docs/kubernetes/1.6/mailu/imap.yaml | 80 --------- docs/kubernetes/1.6/mailu/ingress-ssl.yaml | 32 ---- docs/kubernetes/1.6/mailu/pvc.yaml | 27 --- docs/kubernetes/1.6/mailu/rbac.yaml | 4 - docs/kubernetes/1.6/mailu/redis.yaml | 56 ------- docs/kubernetes/1.6/mailu/security.yaml | 110 ------------ docs/kubernetes/1.6/mailu/smtp.yaml | 80 --------- docs/kubernetes/1.6/mailu/static-ips.yaml | 0 docs/kubernetes/1.6/mailu/webdav.yaml | 63 ------- docs/kubernetes/1.6/mailu/webmail.yaml | 59 ------- .../1.6/nginx/default-http-backend.yaml | 55 ------ docs/kubernetes/1.6/nginx/nginx-ingress.yaml | 139 ---------------- docs/kubernetes/1.6/nginx/rbac.yaml | 129 -------------- 22 files changed, 62 insertions(+), 1376 deletions(-) create mode 100644 core/dovecot/Dockerfile_BACKUP_7007 create mode 100644 core/dovecot/Dockerfile_BASE_7007 create mode 100644 core/dovecot/Dockerfile_LOCAL_7007 create mode 100644 core/dovecot/Dockerfile_REMOTE_7007 delete mode 100644 docs/kubernetes/1.6/README.md delete mode 100644 docs/kubernetes/1.6/mailu/admin.yaml delete mode 100644 docs/kubernetes/1.6/mailu/configmap.yaml delete mode 100644 docs/kubernetes/1.6/mailu/fetchmail.yaml delete mode 100644 docs/kubernetes/1.6/mailu/front.yaml delete mode 100644 docs/kubernetes/1.6/mailu/imap.yaml delete mode 100644 docs/kubernetes/1.6/mailu/ingress-ssl.yaml delete mode 100644 docs/kubernetes/1.6/mailu/pvc.yaml delete mode 100644 docs/kubernetes/1.6/mailu/rbac.yaml delete mode 100644 docs/kubernetes/1.6/mailu/redis.yaml delete mode 100644 docs/kubernetes/1.6/mailu/security.yaml delete mode 100644 docs/kubernetes/1.6/mailu/smtp.yaml delete mode 100644 docs/kubernetes/1.6/mailu/static-ips.yaml delete mode 100644 docs/kubernetes/1.6/mailu/webdav.yaml delete mode 100644 docs/kubernetes/1.6/mailu/webmail.yaml delete mode 100644 docs/kubernetes/1.6/nginx/default-http-backend.yaml delete mode 100644 docs/kubernetes/1.6/nginx/nginx-ingress.yaml delete mode 100644 docs/kubernetes/1.6/nginx/rbac.yaml diff --git a/core/dovecot/Dockerfile_BACKUP_7007 b/core/dovecot/Dockerfile_BACKUP_7007 new file mode 100644 index 00000000..b5590a73 --- /dev/null +++ b/core/dovecot/Dockerfile_BACKUP_7007 @@ -0,0 +1,19 @@ +<<<<<<< HEAD +FROM alpine:3.7 +======= +FROM alpine:3.8 +>>>>>>> upstream/master + +RUN apk add --no-cache \ + dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ + python3 py3-pip \ + && pip3 install --upgrade pip \ + && pip3 install jinja2 podop tenacity + +COPY conf /conf +COPY start.py /start.py + +EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp +VOLUME ["/data", "/mail"] + +CMD /start.py \ No newline at end of file diff --git a/core/dovecot/Dockerfile_BASE_7007 b/core/dovecot/Dockerfile_BASE_7007 new file mode 100644 index 00000000..cacfe354 --- /dev/null +++ b/core/dovecot/Dockerfile_BASE_7007 @@ -0,0 +1,14 @@ +FROM alpine:edge + +RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ + && apk add --no-cache \ + dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \ + rspamd-client@testing python py-jinja2 + +COPY conf /conf +COPY sieve /var/lib/dovecot +COPY start.py /start.py + +EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp + +CMD /start.py diff --git a/core/dovecot/Dockerfile_LOCAL_7007 b/core/dovecot/Dockerfile_LOCAL_7007 new file mode 100644 index 00000000..29957921 --- /dev/null +++ b/core/dovecot/Dockerfile_LOCAL_7007 @@ -0,0 +1,14 @@ +FROM alpine:3.7 + +RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ + && apk add --no-cache \ + dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \ + rspamd-client@testing python py-jinja2 + +COPY conf /conf +COPY sieve /var/lib/dovecot +COPY start.py /start.py + +EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp + +CMD /start.py \ No newline at end of file diff --git a/core/dovecot/Dockerfile_REMOTE_7007 b/core/dovecot/Dockerfile_REMOTE_7007 new file mode 100644 index 00000000..d8d4c55b --- /dev/null +++ b/core/dovecot/Dockerfile_REMOTE_7007 @@ -0,0 +1,15 @@ +FROM alpine:3.8 + +RUN apk add --no-cache \ + dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ + python3 py3-pip \ + && pip3 install --upgrade pip \ + && pip3 install jinja2 podop tenacity + +COPY conf /conf +COPY start.py /start.py + +EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp +VOLUME ["/data", "/mail"] + +CMD /start.py diff --git a/docs/kubernetes/1.6/README.md b/docs/kubernetes/1.6/README.md deleted file mode 100644 index c0dd935b..00000000 --- a/docs/kubernetes/1.6/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# Install Mailu master on kubernetes - -## Prequisites - -### Structure - -There's chosen to have a double NGINX stack for Mailu, this way the main ingress can still be used to access other websites/domains on your cluster. This is the current structure: - -- `NGINX Ingress controller`: Listens to the nodes ports 80 & 443 and directly forwards all TCP traffic on the E-amail ports (993,143,25,587,...). This is because this `DaemonSet` already consumes ports 80 & 443 and uses `hostNetwork: true` -- `Cert manager`: Creates automatic Lets Encrypt certificates based on an `Ingress`-objects domain name. -- `Mailu NGINX Front container`: This container receives all the mail traffic forwarded from the ingress controller. The web traffic is also forwarded based on an ingress -- `Mailu components`: All Mailu components are split into separate files to make them more - -### What you need -- A working Kubernetes cluster (tested with 1.10.5) -- A working [cert-manager](https://github.com/jetstack/cert-manager) installation -- A working nginx-ingress controller needed for the lets-encrypt certificates. You can find those files in the `nginx` subfolder - -#### Cert manager - -The `Cert-manager` is quite easy to deploy using Helm when reading the [docs](https://cert-manager.readthedocs.io/en/latest/getting-started/2-installing.html). -After booting the `Cert-manager` you'll need a `ClusterIssuer` which takes care of all required certificates through `Ingress` items. An example: - -```yaml -apiVersion: certmanager.k8s.io/v1alpha1 -kind: ClusterIssuer -metadata: - name: letsencrypt-prod -spec: - acme: - email: something@example.com - http01: {} - privateKeySecretRef: - key: "" - name: letsencrypt-stage - server: https://acme-v02.api.letsencrypt.org/directory -``` - -## Deploying Mailu - -All manifests can be found in the `mailu` subdirectory. All commands below need to be run from this subdirectory - -### Personalization -- All services run in the same namespace, currently `mailu-mailserver`. So if you want to use a different one, change the `namespace` value in **every** file -- Check the `storage-class` field in the `pvc.yaml` file, you can also change the sizes to your liking. Note that you need `RWX` (read-write-many) and `RWO` (read-write-once) storageclasses. -- Check the `configmap.yaml` and adapt it to your needs. Be sure to check the kubernetes DNS values at the end (if you use a different namespace) -- Check the `ingress-ssl.yaml` and change it to the domain you want (this is for the kubernetes ingress controller, it will forward to `mailu/nginx` a.k.a. the `front` pod) - -## Installation -First run the command to start Mailu: - -```bash -kubectl create -f rbac.yaml -kubectl create -f configmap.yaml -kubectl create -f pvc.yaml -kubectl create -f ingress-ssl.yaml -kubectl create -f redis.yaml -kubectl create -f front.yaml -kubectl create -f webmail.yaml -kubectl create -f imap.yaml -kubectl create -f security.yaml -kubectl create -f smtp.yaml -kubectl create -f fetchmail.yaml -kubectl create -f admin.yaml -kubectl create -f webdav.yaml -``` - -## Create the first admin account - -When the cluster is online you need to create you master user to access `https://mail.example.com/admin`. -Enter the main `admin` pod to create the root account: - -```bash -kubectl -n mailu-mailserver get po -kubectl -n mailu-mailserver exec -it mailu-admin-.... /bin/sh -``` - -And in the pod run the following command. The command uses following entries: -- `admin` Make it an admin user -- `root` The first part of the e-mail adres (ROOT@example.com) -- `example.com` the domain appendix -- `password` the chosen password for the user - -```bash -python manage.py admin root example.com password -``` - -Now you should be able to login on the mail account: `https://mail.example.com/admin` - -## Adaptations - -### Postfix -I noticed you need an override for the `postfix` server in order to be able to send mail. I noticed Google wasn't able to deliver mail to my account and it had to do with the `smtpd_authorized_xclient_hosts` value in the config file. The config can be read [here](https://github.com/hacor/Mailu/blob/master/core/postfix/conf/main.cf#L35) and is pointing to a single IP of the service. But the requests come from the host IPs (the NGINX Ingress proxy) and they don't use the service specific IP. - -Enter the `postfix` pod: - -```bash -kubectl -n mailu-mailserver get po -kubectl -n mailu-mailserver exec -it mailu-smtp-.... /bin/sh -``` - -Now you're in the pod, create an override file like so: - -```bash -vi /overrides/postfix.cf -``` - -And give it the following contents, off course replacing `10.2.0.0/16` with the CIDR of your pod range. This way the NGINX pods can also restart and your mail server will still operate - -```bash -not_needed = true -smtpd_authorized_xclient_hosts = 10.2.0.0/16 -``` - -The first line seems stupid, but is needed because its pasted after a #, so from the second line we're really in action. -Save and close the file and exit. Now you need to delete the pod in order to recreate the config file. - -```bash -kubectl -n mailu-mailserver delete po/mailu-smtp-.... -``` - -### Dovecot -- If you are using Dovecot on a shared file system (Glusterfs, NFS,...), you need to create a special override otherwise a lot of indexing errors will occur on your Dovecot pod. -- I also higher the number of max connections per IP. Now it's limited to 10. -Enter the dovecot pod: - -```bash -kubectl -n mailu-mailserver get po -kubectl -n mailu-mailserver exec -it mailu-imap-.... /bin/sh -``` - -Create the file `/overrides/dovecot.conf` - -```bash -vi /overrides/dovecot.conf -``` - -And enter following contents: -```bash -mail_nfs_index = yes -mail_nfs_storage = yes -mail_fsync = always -mmap_disable = yes -mail_max_userip_connections=100 -``` - -Save and close the file and delete the imap pod to get it recreated. - -```bash -kubectl -n mailu-mailserver delete po/mailu-imap-.... -``` - -Wait for the pod to recreate and you're online! -Happy mailing! - -Wait for the pod to recreate and you're online! -Happy mailing! \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/admin.yaml b/docs/kubernetes/1.6/mailu/admin.yaml deleted file mode 100644 index b36760a2..00000000 --- a/docs/kubernetes/1.6/mailu/admin.yaml +++ /dev/null @@ -1,64 +0,0 @@ - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-admin - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-admin - role: mail - tier: backend - spec: - containers: - - name: admin - image: mailu/admin:master - imagePullPolicy: Always - envFrom: - - configMapRef: - name: mailu-config - volumeMounts: - - name: maildata - mountPath: /data - subPath: maildata - - name: maildata - mountPath: /dkim - subPath: dkim - ports: - - name: http - containerPort: 80 - protocol: TCP - resources: - requests: - memory: 500Mi - cpu: 500m - limits: - memory: 500Mi - cpu: 500m - volumes: - - name: maildata - persistentVolumeClaim: - claimName: mail-storage ---- - -apiVersion: v1 -kind: Service -metadata: - name: admin - namespace: mailu-mailserver - labels: - app: mailu-admin - role: mail - tier: backend -spec: - selector: - app: mailu-admin - role: mail - tier: backend - ports: - - name: http - port: 80 - protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/configmap.yaml b/docs/kubernetes/1.6/mailu/configmap.yaml deleted file mode 100644 index 9ebce8b1..00000000 --- a/docs/kubernetes/1.6/mailu/configmap.yaml +++ /dev/null @@ -1,153 +0,0 @@ - apiVersion: v1 - kind: ConfigMap - metadata: - name: mailu-config - namespace: mailu-mailserver - data: - # Mailu main configuration file - # - # Most configuration variables can be modified through the Web interface, - # these few settings must however be configured before starting the mail - # server and require a restart upon change. - - ################################### - # Common configuration variables - ################################### - - # Set this to the path where Mailu data and configuration is stored - ROOT: "/mailu" - - # Mailu version to run (1.0, 1.1, etc. or master) - VERSION: "master" - - # Set to a randomly generated 16 bytes string - SECRET_KEY: "YourKeyHere" - - # Address where listening ports should bind - BIND_ADDRESS4: "127.0.0.1" - #BIND_ADDRESS6: "::1" - - # Main mail domain - DOMAIN: "example.com" - - # Hostnames for this server, separated with comas - HOSTNAMES: "mail.example.com" - - # Postmaster local part (will append the main mail domain) - POSTMASTER: "admin" - - # Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt) - TLS_FLAVOR: "cert" - - # Authentication rate limit (per source IP address) - AUTH_RATELIMIT: "10/minute;1000/hour" - - # Opt-out of statistics, replace with "True" to opt out - DISABLE_STATISTICS: "False" - - ################################### - # Optional features - ################################### - - # Expose the admin interface (value: true, false) - ADMIN: "true" - # Run the admin interface in debug mode - #DEBUG: "True" - - # Choose which webmail to run if any (values: roundcube, rainloop, none) - WEBMAIL: "roundcube" - - # Dav server implementation (value: radicale, none) - WEBDAV: "radicale" - - # Antivirus solution (value: clamav, none) - ANTIVIRUS: "clamav" - - ################################### - # Mail settings - ################################### - - # Message size limit in bytes - # Default: accept messages up to 50MB - MESSAGE_SIZE_LIMIT: "50000000" - - # Networks granted relay permissions, make sure that you include your Docker - # internal network (default to 172.17.0.0/16) - # For kubernetes this is the CIDR of the pod network - RELAYNETS: "10.2.0.0/16" - POD_ADDRESS_RANGE: "10.2.0.0/16" - - - # Will relay all outgoing mails if configured - #RELAYHOST= - - # This part is needed for the XCLIENT login for postfix. This should be the POD ADDRESS range - FRONT_ADDRESS: "front.mailu-mailserver.svc.cluster.local" - - # Fetchmail delay - FETCHMAIL_DELAY: "600" - - # Recipient delimiter, character used to delimiter localpart from custom address part - # e.g. localpart+custom@domain;tld - RECIPIENT_DELIMITER: "+" - - # DMARC rua and ruf email - DMARC_RUA: "root" - DMARC_RUF: "root" - - # Welcome email, enable and set a topic and body if you wish to send welcome - # emails to all users. - WELCOME: "false" - WELCOME_SUBJECT: "Welcome to your new email account" - WELCOME_BODY: "Welcome to your new email account, if you can read this, then it is configured properly!" - - ################################### - # Web settings - ################################### - - # Path to the admin interface if enabled - WEB_ADMIN: "/admin" - - # Path to the webmail if enabled - WEB_WEBMAIL: "/webmail" - - # Website name - SITENAME: "AppSynth" - - # Linked Website URL - WEBSITE: "https://example.com" - - # Registration reCaptcha settings (warning, this has some privacy impact) - # RECAPTCHA_PUBLIC_KEY= - # RECAPTCHA_PRIVATE_KEY= - - # Domain registration, uncomment to enable - # DOMAIN_REGISTRATION=true - - ################################### - # Advanced settings - ################################### - - # Docker-compose project name, this will prepended to containers names. - COMPOSE_PROJECT_NAME: "mailu" - - # Default password scheme used for newly created accounts and changed passwords - # (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) - PASSWORD_SCHEME: "SHA512-CRYPT" - - # Header to take the real ip from - #REAL_IP_HEADER: - - # IPs for nginx set_real_ip_from (CIDR list separated by commas) - #REAL_IP_FROM: - - # Host settings - HOST_IMAP: "imap.mailu-mailserver.svc.cluster.local" - HOST_POP3: "imap.mailu-mailserver.svc.cluster.local" - HOST_SMTP: "smtp.mailu-mailserver.svc.cluster.local" - HOST_AUTHSMTP: "smtp.mailu-mailserver.svc.cluster.local" - HOST_WEBMAIL: "webmail.mailu-mailserver.svc.cluster.local" - HOST_ADMIN: "admin.mailu-mailserver.svc.cluster.local" - HOST_WEBDAV: "webdav.mailu-mailserver.svc.cluster.local:5232" - HOST_ANTISPAM: "antispam.mailu-mailserver.svc.cluster.local:11332" - HOST_REDIS: "redis.mailu-mailserver.svc.cluster.local" diff --git a/docs/kubernetes/1.6/mailu/fetchmail.yaml b/docs/kubernetes/1.6/mailu/fetchmail.yaml deleted file mode 100644 index cf3271e7..00000000 --- a/docs/kubernetes/1.6/mailu/fetchmail.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-fetchmail - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-fetchmail - role: mail - tier: backend - spec: - containers: - - name: fetchmail - image: mailu/fetchmail:master - imagePullPolicy: Always - envFrom: - - configMapRef: - name: mailu-config - volumeMounts: - - name: maildata - mountPath: /data - subPath: maildata - ports: - - containerPort: 5232 - - containerPort: 80 - resources: - requests: - memory: 100Mi - cpu: 100m - limits: - memory: 100Mi - cpu: 100m - volumes: - - name: maildata - persistentVolumeClaim: - claimName: mail-storage \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/front.yaml b/docs/kubernetes/1.6/mailu/front.yaml deleted file mode 100644 index e25ac828..00000000 --- a/docs/kubernetes/1.6/mailu/front.yaml +++ /dev/null @@ -1,129 +0,0 @@ - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-front - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-front - role: mail - tier: backend - spec: - restartPolicy: Always - terminationGracePeriodSeconds: 60 - containers: - - name: front - image: mailu/nginx:latest - imagePullPolicy: Always - envFrom: - - configMapRef: - name: mailu-config - volumeMounts: - - name: certs - mountPath: /certs - ports: - - name: http - containerPort: 80 - protocol: TCP - - name: https - containerPort: 443 - protocol: TCP - - name: pop3 - containerPort: 110 - protocol: TCP - - name: pop3s - containerPort: 995 - protocol: TCP - - name: imap - containerPort: 143 - protocol: TCP - - name: imaps - containerPort: 993 - protocol: TCP - - name: smtp - containerPort: 25 - protocol: TCP - - name: smtp-auth - containerPort: 10025 - protocol: TCP - - name: imap-auth - containerPort: 10143 - protocol: TCP - - name: smtps - containerPort: 465 - protocol: TCP - - name: smtpd - containerPort: 587 - protocol: TCP - - name: auth - containerPort: 8000 - protocol: TCP - resources: - requests: - memory: 100Mi - cpu: 100m - limits: - memory: 200Mi - cpu: 200m - volumes: - - name: certs - secret: - items: - - key: tls.crt - path: cert.pem - - key: tls.key - path: key.pem - secretName: letsencrypt-certs-all ---- -apiVersion: v1 -kind: Service -metadata: - name: front - namespace: mailu-mailserver - labels: - app: mailu-admin - role: mail - tier: backend -spec: - selector: - app: mailu-front - role: mail - tier: backend - ports: - - name: http - port: 80 - protocol: TCP - - name: https - port: 443 - protocol: TCP - - name: pop3 - port: 110 - protocol: TCP - - name: pop3s - port: 995 - protocol: TCP - - name: imap - port: 143 - protocol: TCP - - name: imaps - port: 993 - protocol: TCP - - name: smtp - port: 25 - protocol: TCP - - name: smtps - port: 465 - protocol: TCP - - name: smtpd - port: 587 - protocol: TCP - - name: smtp-auth - port: 10025 - protocol: TCP - - name: imap-auth - port: 10143 - protocol: TCP diff --git a/docs/kubernetes/1.6/mailu/imap.yaml b/docs/kubernetes/1.6/mailu/imap.yaml deleted file mode 100644 index 069b7730..00000000 --- a/docs/kubernetes/1.6/mailu/imap.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-imap - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-imap - role: mail - tier: backend - spec: - containers: - - name: imap - image: mailu/dovecot:master - imagePullPolicy: Always - envFrom: - - configMapRef: - name: mailu-config - volumeMounts: - - mountPath: /data - name: maildata - subPath: maildata - - mountPath: /mail - name: maildata - subPath: mailstate - - mountPath: /overrides - name: maildata - subPath: overrides - ports: - - containerPort: 2102 - - containerPort: 2525 - - containerPort: 143 - - containerPort: 993 - - containerPort: 4190 - resources: - requests: - memory: 500Mi - cpu: 500m - limits: - memory: 1Gi - cpu: 1000m - volumes: - - name: maildata - persistentVolumeClaim: - claimName: mail-storage ---- -apiVersion: v1 -kind: Service -metadata: - name: imap - namespace: mailu-mailserver - labels: - app: mailu - role: mail - tier: backend -spec: - selector: - app: mailu-imap - role: mail - tier: backend - ports: - ports: - - name: imap-auth - port: 2102 - protocol: TCP - - name: imap-transport - port: 2525 - protocol: TCP - - name: imap-default - port: 143 - protocol: TCP - - name: imap-ssl - port: 993 - protocol: TCP - - name: sieve - port: 4190 - protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/ingress-ssl.yaml b/docs/kubernetes/1.6/mailu/ingress-ssl.yaml deleted file mode 100644 index 61ae3cf7..00000000 --- a/docs/kubernetes/1.6/mailu/ingress-ssl.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: mailu-ssl-ingress - namespace: mailu-mailserver - annotations: - kubernetes.io/ingress.class: tectonic - kubernetes.io/tls-acme: "true" - nginx.ingress.kubernetes.io/proxy-body-size: "0" - ingress.kubernetes.io/ssl-redirect: "true" - # Replace letsencrypt-prod with the name of the certificate issuer - certmanager.k8s.io/cluster-issuer: letsencrypt-prod - #ingress.kubernetes.io/rewrite-target: "/" - #ingress.kubernetes.io/app-root: "/ui" - #ingress.kubernetes.io/follow-redirects: "true" - labels: - app: mailu - role: mail - tier: backend -spec: - tls: - - hosts: - - "mail.example.com" - secretName: letsencrypt-certs-all # If unsure how to generate these, check out https://github.com/ployst/docker-letsencrypt - rules: - - host: "mail.example.com" - http: - paths: - - path: "/" - backend: - serviceName: front - servicePort: 80 \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/pvc.yaml b/docs/kubernetes/1.6/mailu/pvc.yaml deleted file mode 100644 index 0ec2852f..00000000 --- a/docs/kubernetes/1.6/mailu/pvc.yaml +++ /dev/null @@ -1,27 +0,0 @@ -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: redis-hdd - namespace: mailu-mailserver - annotations: - volume.beta.kubernetes.io/storage-class: "glusterblock-hdd" -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: mail-storage - namespace: mailu-mailserver - annotations: - volume.beta.kubernetes.io/storage-class: "gluster-heketi-hdd" -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 100Gi diff --git a/docs/kubernetes/1.6/mailu/rbac.yaml b/docs/kubernetes/1.6/mailu/rbac.yaml deleted file mode 100644 index 33255130..00000000 --- a/docs/kubernetes/1.6/mailu/rbac.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: mailu-mailserver \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/redis.yaml b/docs/kubernetes/1.6/mailu/redis.yaml deleted file mode 100644 index d6bb1eb8..00000000 --- a/docs/kubernetes/1.6/mailu/redis.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-redis - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-redis - role: mail - tier: backend - spec: - containers: - - name: redis - image: redis:4.0-alpine - imagePullPolicy: Always - volumeMounts: - - mountPath: /data - name: redisdata - ports: - - containerPort: 6379 - name: redis - protocol: TCP - resources: - requests: - memory: 200Mi - cpu: 100m - limits: - memory: 300Mi - cpu: 200m - volumes: - - name: redisdata - persistentVolumeClaim: - claimName: redis-hdd ---- - -apiVersion: v1 -kind: Service -metadata: - name: redis - namespace: mailu-mailserver - labels: - app: mailu-redis - role: mail - tier: backend -spec: - selector: - app: mailu-redis - role: mail - tier: backend - ports: - - name: redis - port: 6379 - protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/security.yaml b/docs/kubernetes/1.6/mailu/security.yaml deleted file mode 100644 index c1c1ac0b..00000000 --- a/docs/kubernetes/1.6/mailu/security.yaml +++ /dev/null @@ -1,110 +0,0 @@ - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-security - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-security - role: mail - tier: backend - spec: - containers: - - name: antispam - image: mailu/rspamd:master - imagePullPolicy: Always - envFrom: - - configMapRef: - name: mailu-config - resources: - requests: - memory: 100Mi - cpu: 100m - limits: - memory: 200Mi - cpu: 200m - ports: - - name: antispam - containerPort: 11332 - protocol: TCP - volumeMounts: - - name: filter - subPath: filter - mountPath: /var/lib/rspamd - - name: filter - mountPath: /dkim - subPath: dkim - - name: filter - mountPath: /etc/rspamd/override.d - subPath: rspamd-overrides - - name: antivirus - image: mailu/clamav:master - imagePullPolicy: Always - resources: - requests: - memory: 1Gi - cpu: 1000m - limits: - memory: 2Gi - cpu: 1000m - envFrom: - - configMapRef: - name: mailu-config - ports: - - name: antivirus - containerPort: 3310 - protocol: TCP - volumeMounts: - - name: filter - subPath: filter - mountPath: /data - volumes: - - name: filter - persistentVolumeClaim: - claimName: mail-storage - ---- - -apiVersion: v1 -kind: Service -metadata: - name: antispam - namespace: mailu-mailserver - labels: - app: mailu-antispam - role: mail - tier: backend -spec: - selector: - app: mailu-security - role: mail - tier: backend - ports: - - name: antispam - port: 11332 - protocol: TCP - ---- - -apiVersion: v1 -kind: Service -metadata: - name: antivirus - namespace: mailu-mailserver - labels: - app: mailu-antivirus - role: mail - tier: backend -spec: - selector: - app: mailu-security - role: mail - tier: backend - ports: - - name: antivirus - port: 3310 - protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/smtp.yaml b/docs/kubernetes/1.6/mailu/smtp.yaml deleted file mode 100644 index 454b8ed7..00000000 --- a/docs/kubernetes/1.6/mailu/smtp.yaml +++ /dev/null @@ -1,80 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-smtp - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-smtp - role: mail - tier: backend - spec: - containers: - - name: smtp - image: mailu/postfix:master - imagePullPolicy: Always - envFrom: - - configMapRef: - name: mailu-config - resources: - requests: - memory: 500Mi - cpu: 200m - limits: - memory: 1Gi - cpu: 500m - volumeMounts: - - mountPath: /data - name: maildata - subPath: maildata - - mountPath: /overrides - name: maildata - subPath: overrides - ports: - - name: smtp - containerPort: 25 - protocol: TCP - - name: smtp-ssl - containerPort: 465 - protocol: TCP - - name: smtp-starttls - containerPort: 587 - protocol: TCP - - name: smtp-auth - containerPort: 10025 - protocol: TCP - volumes: - - name: maildata - persistentVolumeClaim: - claimName: mail-storage ---- -apiVersion: v1 -kind: Service -metadata: - name: smtp - namespace: mailu-mailserver - labels: - app: mailu - role: mail - tier: backend -spec: - selector: - app: mailu-smtp - role: mail - tier: backend - ports: - - name: smtp - port: 25 - protocol: TCP - - name: smtp-ssl - port: 465 - protocol: TCP - - name: smtp-starttls - port: 587 - protocol: TCP - - name: smtp-auth - port: 10025 - protocol: TCP diff --git a/docs/kubernetes/1.6/mailu/static-ips.yaml b/docs/kubernetes/1.6/mailu/static-ips.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/kubernetes/1.6/mailu/webdav.yaml b/docs/kubernetes/1.6/mailu/webdav.yaml deleted file mode 100644 index 07b7733c..00000000 --- a/docs/kubernetes/1.6/mailu/webdav.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-webdav - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-webdav - role: mail - tier: backend - spec: - containers: - - name: radicale - image: mailu/radicale:master - imagePullPolicy: Always - envFrom: - - configMapRef: - name: mailu-config - volumeMounts: - - mountPath: /data - name: maildata - subPath: dav - ports: - - containerPort: 5232 - - containerPort: 80 - resources: - requests: - memory: 100Mi - cpu: 100m - limits: - memory: 100Mi - cpu: 100m - volumes: - - name: maildata - persistentVolumeClaim: - claimName: mail-storage ---- - -apiVersion: v1 -kind: Service -metadata: - name: webdav - namespace: mailu-mailserver - labels: - app: mailu-webdav - role: mail - tier: backend -spec: - selector: - app: mailu-webdav - role: mail - tier: backend - ports: - ports: - - name: http - port: 80 - protocol: TCP - - name: http-ui - port: 5232 - protocol: TCP \ No newline at end of file diff --git a/docs/kubernetes/1.6/mailu/webmail.yaml b/docs/kubernetes/1.6/mailu/webmail.yaml deleted file mode 100644 index 81798782..00000000 --- a/docs/kubernetes/1.6/mailu/webmail.yaml +++ /dev/null @@ -1,59 +0,0 @@ - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: mailu-roundcube - namespace: mailu-mailserver -spec: - replicas: 1 - template: - metadata: - labels: - app: mailu-roundcube - role: mail - tier: frontend - spec: - containers: - - name: roundcube - image: mailu/roundcube:1.5 - imagePullPolicy: Always - envFrom: - - configMapRef: - name: mailu-config - resources: - requests: - memory: 100Mi - cpu: 100m - limits: - memory: 200Mi - cpu: 200m - volumeMounts: - - mountPath: /data - name: maildata - subPath: webmail - ports: - - containerPort: 80 - volumes: - - name: maildata - persistentVolumeClaim: - claimName: mail-storage ---- -apiVersion: v1 -kind: Service -metadata: - name: webmail - namespace: mailu-mailserver - labels: - app: mailu-roundcube - role: mail - tier: frontend -spec: - selector: - app: mailu-roundcube - role: mail - tier: frontend - ports: - ports: - - name: http - port: 80 - protocol: TCP diff --git a/docs/kubernetes/1.6/nginx/default-http-backend.yaml b/docs/kubernetes/1.6/nginx/default-http-backend.yaml deleted file mode 100644 index 097fe7c5..00000000 --- a/docs/kubernetes/1.6/nginx/default-http-backend.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: default-http-backend - labels: - app: default-http-backend - namespace: kube-ingress -spec: - replicas: 1 - selector: - matchLabels: - app: default-http-backend - template: - metadata: - labels: - app: default-http-backend - spec: - terminationGracePeriodSeconds: 60 - containers: - - name: default-http-backend - # Any image is permissible as long as: - # 1. It serves a 404 page at / - # 2. It serves 200 on a /healthz endpoint - image: gcr.io/google_containers/defaultbackend:1.4 - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - ports: - - containerPort: 8080 - resources: - limits: - cpu: 10m - memory: 20Mi - requests: - cpu: 10m - memory: 20Mi ---- - -apiVersion: v1 -kind: Service -metadata: - name: default-http-backend - namespace: kube-ingress - labels: - app: default-http-backend -spec: - ports: - - port: 80 - targetPort: 8080 - selector: - app: default-http-backend \ No newline at end of file diff --git a/docs/kubernetes/1.6/nginx/nginx-ingress.yaml b/docs/kubernetes/1.6/nginx/nginx-ingress.yaml deleted file mode 100644 index 90b24f24..00000000 --- a/docs/kubernetes/1.6/nginx/nginx-ingress.yaml +++ /dev/null @@ -1,139 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - # keep it under 24 chars - name: appsynth-lb - namespace: kube-ingress - labels: - k8s-app: appsynth-lb - component: ingress-controller -spec: - type: ClusterIP - selector: - k8s-app: appsynth-lb - component: ingress-controller - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 80 - - name: https - protocol: TCP - port: 443 - targetPort: 443 ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: udp-services - namespace: kube-ingress - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: tcp-services - namespace: kube-ingress -data: - 25: "mailu-mailserver/front:25" - 110: "mailu-mailserver/front:110" - 465: "mailu-mailserver/front:465" - 587: "mailu-mailserver/front:587" - 143: "mailu-mailserver/front:143" - 993: "mailu-mailserver/front:993" - 995: "mailu-mailserver/front:995" - ---- -apiVersion: v1 -data: - enable-vts-status: "true" -kind: ConfigMap -metadata: - name: nginx-ingress-lb-conf - namespace: kube-ingress ---- -apiVersion: apps/v1beta2 -kind: DaemonSet -metadata: - name: ingress-controller - namespace: kube-ingress - annotations: - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - k8s-app: appsynth-lb - component: ingress-controller - type: nginx -spec: - updateStrategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate - selector: - matchLabels: - k8s-app: appsynth-lb - component: ingress-controller - type: nginx - template: - metadata: - labels: - k8s-app: appsynth-lb - component: ingress-controller - type: nginx - spec: - serviceAccount: kube-nginx-ingress - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: DoesNotExist - containers: - - name: nginx-ingress-lb - image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.2 - args: - - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/tectonic-custom-error - - --default-backend-service=$(POD_NAMESPACE)/default-http-backend - #- --default-ssl-certificate=tectonic-system/tectonic-ingress-tls-secret - - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - - --annotations-prefix=ingress.kubernetes.io - - --enable-ssl-passthrough - - --ingress-class=tectonic - # use downward API - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - name: http - containerPort: 80 - hostPort: 80 - - name: https - containerPort: 443 - hostPort: 443 - readinessProbe: - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - livenessProbe: - initialDelaySeconds: 10 - timeoutSeconds: 1 - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - hostNetwork: true - nodeSelector: - node-role.kubernetes.io/node: "" - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 60 diff --git a/docs/kubernetes/1.6/nginx/rbac.yaml b/docs/kubernetes/1.6/nginx/rbac.yaml deleted file mode 100644 index d3c01384..00000000 --- a/docs/kubernetes/1.6/nginx/rbac.yaml +++ /dev/null @@ -1,129 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kube-ingress ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kube-nginx-ingress - namespace: kube-ingress ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: kube-nginx-ingress -rules: - - apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - apiGroups: - - "extensions" - resources: - - ingresses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - apiGroups: - - "extensions" - resources: - - ingresses/status - verbs: - - update ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: Role -metadata: - name: kube-nginx-ingress - namespace: kube-ingress -rules: - - apiGroups: - - "" - resources: - - configmaps - - pods - - secrets - - namespaces - verbs: - - get - - apiGroups: - - "" - resources: - - configmaps - resourceNames: - - "ingress-controller-leader-nginx" - verbs: - - get - - update - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - create - - update ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: RoleBinding -metadata: - name: kube-nginx-ingress - namespace: kube-ingress -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kube-nginx-ingress -subjects: - - kind: ServiceAccount - name: kube-nginx-ingress - namespace: kube-ingress ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: kube-nginx-ingress -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kube-nginx-ingress -subjects: - - kind: ServiceAccount - name: kube-nginx-ingress - namespace: kube-ingress \ No newline at end of file From ef55ca525c1286006c5090f4c653d133557851bb Mon Sep 17 00:00:00 2001 From: Hans Cornelis Date: Wed, 17 Oct 2018 07:48:52 +0200 Subject: [PATCH 31/51] Deleted conflicting merge files Signed-off-by: Hans Cornelis --- core/dovecot/Dockerfile_BACKUP_7007 | 19 ------------------- core/dovecot/Dockerfile_BASE_7007 | 14 -------------- core/dovecot/Dockerfile_LOCAL_7007 | 14 -------------- core/dovecot/Dockerfile_REMOTE_7007 | 15 --------------- 4 files changed, 62 deletions(-) delete mode 100644 core/dovecot/Dockerfile_BACKUP_7007 delete mode 100644 core/dovecot/Dockerfile_BASE_7007 delete mode 100644 core/dovecot/Dockerfile_LOCAL_7007 delete mode 100644 core/dovecot/Dockerfile_REMOTE_7007 diff --git a/core/dovecot/Dockerfile_BACKUP_7007 b/core/dovecot/Dockerfile_BACKUP_7007 deleted file mode 100644 index b5590a73..00000000 --- a/core/dovecot/Dockerfile_BACKUP_7007 +++ /dev/null @@ -1,19 +0,0 @@ -<<<<<<< HEAD -FROM alpine:3.7 -======= -FROM alpine:3.8 ->>>>>>> upstream/master - -RUN apk add --no-cache \ - dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ - python3 py3-pip \ - && pip3 install --upgrade pip \ - && pip3 install jinja2 podop tenacity - -COPY conf /conf -COPY start.py /start.py - -EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp -VOLUME ["/data", "/mail"] - -CMD /start.py \ No newline at end of file diff --git a/core/dovecot/Dockerfile_BASE_7007 b/core/dovecot/Dockerfile_BASE_7007 deleted file mode 100644 index cacfe354..00000000 --- a/core/dovecot/Dockerfile_BASE_7007 +++ /dev/null @@ -1,14 +0,0 @@ -FROM alpine:edge - -RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ - && apk add --no-cache \ - dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \ - rspamd-client@testing python py-jinja2 - -COPY conf /conf -COPY sieve /var/lib/dovecot -COPY start.py /start.py - -EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp - -CMD /start.py diff --git a/core/dovecot/Dockerfile_LOCAL_7007 b/core/dovecot/Dockerfile_LOCAL_7007 deleted file mode 100644 index 29957921..00000000 --- a/core/dovecot/Dockerfile_LOCAL_7007 +++ /dev/null @@ -1,14 +0,0 @@ -FROM alpine:3.7 - -RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ - && apk add --no-cache \ - dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \ - rspamd-client@testing python py-jinja2 - -COPY conf /conf -COPY sieve /var/lib/dovecot -COPY start.py /start.py - -EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp - -CMD /start.py \ No newline at end of file diff --git a/core/dovecot/Dockerfile_REMOTE_7007 b/core/dovecot/Dockerfile_REMOTE_7007 deleted file mode 100644 index d8d4c55b..00000000 --- a/core/dovecot/Dockerfile_REMOTE_7007 +++ /dev/null @@ -1,15 +0,0 @@ -FROM alpine:3.8 - -RUN apk add --no-cache \ - dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ - python3 py3-pip \ - && pip3 install --upgrade pip \ - && pip3 install jinja2 podop tenacity - -COPY conf /conf -COPY start.py /start.py - -EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp -VOLUME ["/data", "/mail"] - -CMD /start.py From 0a5dbf6230d9749dbf1a1129b606abe6f4dad0fc Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 17:41:21 +0200 Subject: [PATCH 32/51] Re-enable local dovecot sieve scripts --- core/dovecot/conf/dovecot.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index d6c4eb21..a9ec2676 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -136,7 +136,8 @@ service managesieve { } plugin { - sieve = dict:proxy:/tmp/podop.socket:sieve + sieve = file:~/sieve;active=~/.dovecot.sieve + sieve_before = dict:proxy:/tmp/podop.socket:sieve sieve_plugins = sieve_imapsieve sieve_extprograms sieve_extensions = +spamtest +spamtestplus +editheader sieve_global_extensions = +vnd.dovecot.execute From b9b4a8cd77c4a483d7dbf70314db777f021f0443 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:47:55 +0200 Subject: [PATCH 33/51] Explicitely specify the fuzzy worker listen address --- services/rspamd/conf/worker-fuzzy.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc index a0021a03..0f71ba32 100644 --- a/services/rspamd/conf/worker-fuzzy.inc +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -1,4 +1,5 @@ type = "fuzzy"; +bind_socket = "*:11335"; count = 1; backend = "redis"; expire = 90d; From ce0bf3366d2193b01e34c04ede5f870c94d250c3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:48:28 +0200 Subject: [PATCH 34/51] Learn fuzzy hashes automatically --- core/dovecot/Dockerfile | 2 +- core/dovecot/conf/bin/ham | 4 ++++ core/dovecot/conf/bin/mailtrain | 3 --- core/dovecot/conf/bin/spam | 4 ++++ core/dovecot/conf/report-ham.sieve | 2 +- core/dovecot/conf/report-spam.sieve | 2 +- 6 files changed, 11 insertions(+), 6 deletions(-) create mode 100755 core/dovecot/conf/bin/ham delete mode 100755 core/dovecot/conf/bin/mailtrain create mode 100755 core/dovecot/conf/bin/spam diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index d8d4c55b..41437e23 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.8 RUN apk add --no-cache \ dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ - python3 py3-pip \ + bash python3 py3-pip \ && pip3 install --upgrade pip \ && pip3 install jinja2 podop tenacity diff --git a/core/dovecot/conf/bin/ham b/core/dovecot/conf/bin/ham new file mode 100755 index 00000000..c74a97bd --- /dev/null +++ b/core/dovecot/conf/bin/ham @@ -0,0 +1,4 @@ +#!/bin/bash + +tee >(rspamc -h antispam:11334 -P mailu learn_ham /dev/stdin) \ + | rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /dev/stdin \ No newline at end of file diff --git a/core/dovecot/conf/bin/mailtrain b/core/dovecot/conf/bin/mailtrain deleted file mode 100755 index cfa36398..00000000 --- a/core/dovecot/conf/bin/mailtrain +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -rspamc -h antispam:11334 -P mailu "learn_$1" /dev/stdin <&0 diff --git a/core/dovecot/conf/bin/spam b/core/dovecot/conf/bin/spam new file mode 100755 index 00000000..e6a66f89 --- /dev/null +++ b/core/dovecot/conf/bin/spam @@ -0,0 +1,4 @@ +#!/bin/bash + +tee >(rspamc -h antispam:11334 -P mailu learn_spam /dev/stdin) \ + >(rspamc -h antispam:11334 -P mailu -f 11 fuzzy_add /dev/stdin) \ No newline at end of file diff --git a/core/dovecot/conf/report-ham.sieve b/core/dovecot/conf/report-ham.sieve index 1ad8abdf..0c69d67b 100644 --- a/core/dovecot/conf/report-ham.sieve +++ b/core/dovecot/conf/report-ham.sieve @@ -8,4 +8,4 @@ if string "${mailbox}" "Trash" { stop; } -execute :pipe "mailtrain" "ham"; +execute :pipe "ham"; diff --git a/core/dovecot/conf/report-spam.sieve b/core/dovecot/conf/report-spam.sieve index b2a544a6..108d6210 100644 --- a/core/dovecot/conf/report-spam.sieve +++ b/core/dovecot/conf/report-spam.sieve @@ -1,3 +1,3 @@ require "vnd.dovecot.execute"; -execute :pipe "mailtrain" "spam"; +execute :pipe "spam"; From d5162328eca5e2344c4ca1e432ec8f31bc5b7e1f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:48:59 +0200 Subject: [PATCH 35/51] Allow dovecot to write the source configuration directory for compiling sieve scripts --- core/dovecot/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index afd0513e..b65b4db9 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -36,5 +36,5 @@ for dovecot_file in glob.glob("/conf/*.conf"): # Run Podop, then postfix multiprocessing.Process(target=start_podop).start() -os.system("chown -R mail:mail /mail /var/lib/dovecot") +os.system("chown -R mail:mail /mail /var/lib/dovecot /conf") os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"]) From 86bdce840774f7c7170d4eccfc7dc9e9d3d9ea5e Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Wed, 17 Oct 2018 18:49:19 +0000 Subject: [PATCH 36/51] Explicitely specify the fuzzy worker listen address --- services/rspamd/conf/worker-fuzzy.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc index a0021a03..0f71ba32 100644 --- a/services/rspamd/conf/worker-fuzzy.inc +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -1,4 +1,5 @@ type = "fuzzy"; +bind_socket = "*:11335"; count = 1; backend = "redis"; expire = 90d; From dba8f1810d3d9271ff9ca9dcc21521d3bef7327c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 21:21:00 +0200 Subject: [PATCH 37/51] Do not check the password another time in Dovecot --- core/admin/mailu/internal/views/dovecot.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index c2f53794..036140f0 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -1,14 +1,24 @@ -from mailu import db, models +from mailu import db, models, app from mailu.internal import internal import flask +import socket @internal.route("/dovecot/passdb/") def dovecot_passdb_dict(user_email): user = models.User.query.get(user_email) or flask.abort(404) + allow_nets = [] + allow_nets.append( + app.config.get("POD_ADDRESS_RANGE") or + socket.gethostbyname(app.config["HOST_FRONT"]) + ) + allow_nets.append(socket.gethostbyname(app.config["HOST_WEBMAIL"])) + print(allow_nets) return flask.jsonify({ - "password": user.password, + "password": None, + "nopassword": "Y", + "allow_nets": ",".join(allow_nets) }) From 988e09e65eecab6daae5f090556826b0310a434a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 21:21:24 +0200 Subject: [PATCH 38/51] Add a profiler in debug mode for improving performance --- core/admin/mailu/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 167f04ae..a73e6ab9 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -12,7 +12,7 @@ import docker import socket import uuid -from werkzeug.contrib import fixers +from werkzeug.contrib import fixers, profiler # Create application app = flask.Flask(__name__) @@ -62,7 +62,10 @@ default_config = { 'HOST_IMAP': 'imap', 'HOST_POP3': 'imap', 'HOST_SMTP': 'smtp', + 'HOST_WEBMAIL': 'webmail', + 'HOST_FRONT': 'front', 'HOST_AUTHSMTP': os.environ.get('HOST_SMTP', 'smtp'), + 'POD_ADDRESS_RANGE': None } # Load configuration from the environment if available @@ -80,6 +83,10 @@ if app.config.get("DEBUG"): import flask_debugtoolbar toolbar = flask_debugtoolbar.DebugToolbarExtension(app) +# Profiler +if app.config.get("DEBUG"): + app.wsgi_app = profiler.ProfilerMiddleware(app.wsgi_app, restrictions=[30]) + # Manager commnad manager = flask_script.Manager(app) manager.add_command('db', flask_migrate.MigrateCommand) @@ -129,4 +136,5 @@ class PrefixMiddleware(object): environ['SCRIPT_NAME'] = prefix return self.app(environ, start_response) + app.wsgi_app = PrefixMiddleware(fixers.ProxyFix(app.wsgi_app)) From 01fa1797674e2badeaa42292172807107338c41a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 21:21:35 +0200 Subject: [PATCH 39/51] Update the user password in database when needed --- core/admin/mailu/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 0c80fd4f..51d07f24 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -276,7 +276,8 @@ class User(Base, Email): else: return self.email - scheme_dict = {'BLF-CRYPT': "bcrypt", + scheme_dict = {'PBKDF2': "pbkdf2_sha512", + 'BLF-CRYPT': "bcrypt", 'SHA512-CRYPT': "sha512_crypt", 'SHA256-CRYPT': "sha256_crypt", 'MD5-CRYPT': "md5_crypt", @@ -287,8 +288,14 @@ class User(Base, Email): ) def check_password(self, password): + context = User.pw_context reference = re.match('({[^}]+})?(.*)', self.password).group(2) - return User.pw_context.verify(password, reference) + result = context.verify(password, reference) + if result and context.identify(reference) != context.default_scheme(): + self.set_password(password) + db.session.add(self) + db.session.commit() + return result def set_password(self, password, hash_scheme=app.config['PASSWORD_SCHEME'], raw=False): """Set password for user with specified encryption scheme From 828d96f8f01da044611950b3df57137ef1797c11 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 21:26:44 +0200 Subject: [PATCH 40/51] Switch the default password scheme to PBKDF2 --- docs/compose/.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/compose/.env b/docs/compose/.env index 7823bc3e..ce8e9eb9 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -130,8 +130,8 @@ LOG_DRIVER=json-file COMPOSE_PROJECT_NAME=mailu # Default password scheme used for newly created accounts and changed passwords -# (value: BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) -PASSWORD_SCHEME=BLF-CRYPT +# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) +PASSWORD_SCHEME=PBKDF2 # Header to take the real ip from REAL_IP_HEADER= From d4f32c3e7d1d4150cc25a9e930ad7737521a65d3 Mon Sep 17 00:00:00 2001 From: hoellen Date: Thu, 18 Oct 2018 14:27:28 +0200 Subject: [PATCH 41/51] remove rewrite if webmail is on root --- core/nginx/conf/nginx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 8fcda1c3..7eaba003 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -91,8 +91,10 @@ http { {% endif %} location {{ WEB_WEBMAIL }} { + {% if WEB_WEBMAIL != '/' %} rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent; rewrite ^{{ WEB_WEBMAIL }}/(.*) /$1 break; + {% endif %} include /etc/nginx/proxy.conf; client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; proxy_pass http://$webmail; From 9c639eebd48cfc8b49f73345a13bcb33303054d6 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 18 Oct 2018 18:46:59 +0000 Subject: [PATCH 42/51] Made the instructions more simple (moved the nfs-volume example to another file) Removed he part around variable substitution as we can use a "trick" --- docs/swarm/master/README.md | 192 +++---------- docs/swarm/master/README_nfs_example.md | 357 ++++++++++++++++++++++++ 2 files changed, 394 insertions(+), 155 deletions(-) create mode 100644 docs/swarm/master/README_nfs_example.md diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index 2a2a021a..c6120228 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -29,34 +29,7 @@ mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active ### Volume definition For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. -Hereafter we will use a NFS share: -```bash -core@coreos-01 ~ $ showmount -e 192.168.0.30 -Export list for 192.168.0.30: -/mnt/Pool1/pv 192.168.0.0 -``` - -on the nfs server, I am using the following /etc/exports -```bash -$more /etc/exports -/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 -``` -on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up) -```bash -$mkdir /mnt/Pool1/pv/mailu -``` - -On your manager node, mount the nfs share to check that the share is available: -```bash -core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/ -``` -If this is ok, you can umount it: -```bash -core@coreos-01 ~ $ sudo umount /mnt/local/ -``` - - -## Networking mode +Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs". (GlusterFS and nfs shares have been successfully used) On this example, we are using: - the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. - the default ingress mode. @@ -94,10 +67,13 @@ As a side effect of this ingress mode "feature", make sure that the ingress subn - redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) ## Variable substitution and docker-compose.yml -The docker stack deploy command doesn't support variable substitution in the .yml file itself (but we still can use .env file to pass variables to the services). As a consequence we need to adjust the docker-compose file in order to : -- remove all variables : $VERSION , $BIND_ADDRESS4 , $BIND_ADDRESS6 , $ANTIVIRUS , $WEBMAIL , etc +The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around: +``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` + +We need also to: - change the way we define the volumes (nfs share in our case) - add a deploy section for every service +- the way the ports are defined for the front service ## Docker compose An example of docker-compose-stack.yml file is available here: @@ -109,7 +85,7 @@ version: '3.2' services: front: - image: mailu/nginx:master + image: mailu/nginx:$VERSION restart: always env_file: .env ports: @@ -132,10 +108,7 @@ services: - target: 587 published: 587 volumes: -# - "$ROOT/certs:/certs" - - type: volume - source: mailu_certs - target: /certs + - "$ROOT/certs:/certs" deploy: replicas: 2 @@ -143,118 +116,77 @@ services: image: redis:alpine restart: always volumes: -# - "$ROOT/redis:/data" - - type: volume - source: mailu_redis - target: /data + - "$ROOT/redis:/data" deploy: replicas: 1 imap: - image: mailu/dovecot:master + image: mailu/dovecot:$VERSION restart: always env_file: .env environment: - POD_ADDRESS_RANGE=10.0.1.0/24 volumes: -# - "$ROOT/data:/data" - - type: volume - source: mailu_data - target: /data -# - "$ROOT/mail:/mail" - - type: volume - source: mailu_mail - target: /mail -# - "$ROOT/overrides:/overrides" - - type: volume - source: mailu_overrides - target: /overrides + - "$ROOT/mail:/mail" + - "$ROOT/overrides:/overrides" depends_on: - front deploy: replicas: 2 smtp: - image: mailu/postfix:master + image: mailu/postfix:$VERSION restart: always env_file: .env environment: - POD_ADDRESS_RANGE=10.0.1.0/24 volumes: -# - "$ROOT/data:/data" - - type: volume - source: mailu_data - target: /data -# - "$ROOT/overrides:/overrides" - - type: volume - source: mailu_overrides - target: /overrides + - "$ROOT/overrides:/overrides" depends_on: - front deploy: replicas: 2 antispam: - image: mailu/rspamd:master + image: mailu/rspamd:$VERSION restart: always env_file: .env environment: - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: + - "$ROOT/filter:/var/lib/rspamd" + - "$ROOT/dkim:/dkim" + - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" depends_on: - front - volumes: -# - "$ROOT/filter:/var/lib/rspamd" - - type: volume - source: mailu_filter - target: /var/lib/rspamd -# - "$ROOT/dkim:/dkim" - - type: volume - source: mailu_dkim - target: /dkim -# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" - - type: volume - source: mailu_overrides_rspamd - target: /etc/rspamd/override.d deploy: replicas: 1 antivirus: - image: mailu/none:master + image: mailu/none:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/filter:/data" - - type: volume - source: mailu_filter - target: /data + - "$ROOT/filter:/data" deploy: replicas: 1 webdav: - image: mailu/none:master + image: mailu/none:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/dav:/data" - - type: volume - source: mailu_dav - target: /data + - "$ROOT/dav:/data" deploy: replicas: 1 admin: - image: mailu/admin:master + image: mailu/admin:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/data:/data" - - type: volume - source: mailu_data - target: /data -# - "$ROOT/dkim:/dkim" - - type: volume - source: mailu_dkim - target: /dkim + - "$ROOT/data:/data" + - "$ROOT/dkim:/dkim" - /var/run/docker.sock:/var/run/docker.sock:ro depends_on: - redis @@ -262,28 +194,21 @@ services: replicas: 1 webmail: - image: "mailu/roundcube:master" + image: mailu/roundcube:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/webmail:/data" - - type: volume - source: mailu_data - target: /data + - "$ROOT/webmail:/data" depends_on: - imap deploy: replicas: 2 fetchmail: - image: mailu/fetchmail:master + image: mailu/fetchmail:$VERSION restart: always env_file: .env volumes: -# - "$ROOT/data:/data" - - type: volume - source: mailu_data - target: /data deploy: replicas: 1 @@ -291,72 +216,29 @@ networks: default: external: name: mailu_default - -volumes: - mailu_filter: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/filter" - mailu_dkim: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/dkim" - mailu_overrides_rspamd: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/overrides/rspamd" - mailu_data: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/data" - mailu_mail: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/mail" - mailu_overrides: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/overrides" - mailu_dav: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/dav" - mailu_certs: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/certs" - mailu_redis: - driver_opts: - type: "nfs" - o: "addr=192.168.0.30,nolock,soft,rw" - device: ":/mnt/Pool1/pv/mailu/redis" ``` ## Deploy Mailu on the docker swarm Run the following command: ```bash -docker stack deploy -c docker-compose-stack.yml mailu +echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` See how the services are being deployed: ```bash core@coreos-01 ~ $ docker service ls ID NAME MODE REPLICAS IMAGE PORTS -ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:1.5 -pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:1.5 +ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:master +pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:master ``` check a specific service: ```bash core@coreos-01 ~ $ docker service ps mailu_fetchmail ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS -tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:1.5 coreos-01 Running Running 11 days ago +tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:master coreos-01 Running Running 11 days ago +``` +You might also have a look on the logs: +```bash +core@coreos-01 ~ $ docker service logs -f mailu_fetchmail ``` ## Remove the stack diff --git a/docs/swarm/master/README_nfs_example.md b/docs/swarm/master/README_nfs_example.md new file mode 100644 index 00000000..5cfd0a73 --- /dev/null +++ b/docs/swarm/master/README_nfs_example.md @@ -0,0 +1,357 @@ +# Install Mailu on a docker swarm + +## Prequisites + +### Swarm + +In order to deploy Mailu on a swarm, you will first need to initialize the swarm: + +The main command will be: +```bash +docker swarm init --advertise-addr +``` +See https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/ + +If you want to add other managers or workers, please use: +```bash +docker swarm join --token xxxxx +``` +See https://docs.docker.com/engine/swarm/join-nodes/ + +You have now a working swarm, and you can check its status with: +```bash +core@coreos-01 ~/git/Mailu/docs/swarm/1.5 $ docker node ls +ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION +xhgeekkrlttpmtgmapt5hyxrb black-pearl Ready Active 18.06.0-ce +sczlqjgfhehsfdjhfhhph1nvb * coreos-01 Ready Active Leader 18.03.1-ce +mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active 18.06.0-ce +``` + +### Volume definition +For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. +Hereafter we will use a NFS share: +```bash +core@coreos-01 ~ $ showmount -e 192.168.0.30 +Export list for 192.168.0.30: +/mnt/Pool1/pv 192.168.0.0 +``` + +on the nfs server, I am using the following /etc/exports +```bash +$more /etc/exports +/mnt/Pool1/pv -alldirs -mapall=root -network 192.168.0.0 -mask 255.255.255.0 +``` +on the nfs server, I created the Mailu directory (in fact I copied a working Mailu set-up) +```bash +$mkdir /mnt/Pool1/pv/mailu +``` + +On your manager node, mount the nfs share to check that the share is available: +```bash +core@coreos-01 ~ $ sudo mount -t nfs 192.168.0.30:/mnt/Pool1/pv/mailu /mnt/local/ +``` +If this is ok, you can umount it: +```bash +core@coreos-01 ~ $ sudo umount /mnt/local/ +``` + + +## Networking mode +On this example, we are using: +- the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. +- the default ingress mode. + +### Allow authentification with the mesh routing +In order to allow every (front & webmail) container to access the other services, we will use the variable POD_ADDRESS_RANGE. + +Let's create the mailu_default network: +```bash +core@coreos-01 ~ $ docker network create -d overlay --attachable mailu_default +core@coreos-01 ~ $ docker network inspect mailu_default | grep Subnet + "Subnet": "10.0.1.0/24", +``` +In the docker-compose.yml file, we will then use POD_ADDRESS_RANGE = 10.0.1.0/24 +In fact, imap & smtp logs doesn't show the IPs from the front(s) container(s), but the IP of "mailu_default-endpoint". So it is sufficient to set POD_ADDRESS_RANGE to this specific ip (which can be found by inspecting mailu_default network). The issue is that this endpoint is created while the stack is created, I did'nt figure a way to determine this IP before the stack creation... + +### Limitation with the ingress mode +With the default ingress mode, the front(s) container(s) will see origin IP(s) all being 10.255.0.x (which is the ingress-endpoint, can be found by inspecting the ingress network) + +This issue is known and discussed here: + +https://github.com/moby/moby/issues/25526 + +A workaround (using network host mode and global deployment) is discussed here: + +https://github.com/moby/moby/issues/25526#issuecomment-336363408 + +### Don't create an open relay ! +As a side effect of this ingress mode "feature", make sure that the ingress subnet is not in your RELAYHOST, otherwise you would create an smtp open relay :-( + + +## Scalability +- smtp and imap are scalable +- front and webmail are scalable (pending POD_ADDRESS_RANGE is used), although the let's encrypt magic might not like it (race condidtion ? or risk to be banned by let's encrypt server if too many front containers attemps to renew the certs at the same time) +- redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) + +## Variable substitution and docker-compose.yml +The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around: +``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` + +We need also to: +- change the way we define the volumes (nfs share in our case) +- add a deploy section for every service +- the way the ports are defined for the front service + +## Docker compose +An example of docker-compose-stack.yml file is available here: + +```yaml + +version: '3.2' + +services: + + front: + image: mailu/nginx:$VERSION + restart: always + env_file: .env + ports: + - target: 80 + published: 80 + - target: 443 + published: 443 + - target: 110 + published: 110 + - target: 143 + published: 143 + - target: 993 + published: 993 + - target: 995 + published: 995 + - target: 25 + published: 25 + - target: 465 + published: 465 + - target: 587 + published: 587 + volumes: +# - "$ROOT/certs:/certs" + - type: volume + source: mailu_certs + target: /certs + deploy: + replicas: 2 + + redis: + image: redis:alpine + restart: always + volumes: +# - "$ROOT/redis:/data" + - type: volume + source: mailu_redis + target: /data + deploy: + replicas: 1 + + imap: + image: mailu/dovecot:$VERSION + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: +# - "$ROOT/mail:/mail" + - type: volume + source: mailu_mail + target: /mail +# - "$ROOT/overrides:/overrides" + - type: volume + source: mailu_overrides + target: /overrides + depends_on: + - front + deploy: + replicas: 2 + + smtp: + image: mailu/postfix:$VERSION + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + volumes: +# - "$ROOT/overrides:/overrides" + - type: volume + source: mailu_overrides + target: /overrides + depends_on: + - front + deploy: + replicas: 2 + + antispam: + image: mailu/rspamd:$VERSION + restart: always + env_file: .env + environment: + - POD_ADDRESS_RANGE=10.0.1.0/24 + depends_on: + - front + volumes: +# - "$ROOT/filter:/var/lib/rspamd" + - type: volume + source: mailu_filter + target: /var/lib/rspamd +# - "$ROOT/dkim:/dkim" + - type: volume + source: mailu_dkim + target: /dkim +# - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" + - type: volume + source: mailu_overrides_rspamd + target: /etc/rspamd/override.d + deploy: + replicas: 1 + + antivirus: + image: mailu/none:$VERSION + restart: always + env_file: .env + volumes: +# - "$ROOT/filter:/data" + - type: volume + source: mailu_filter + target: /data + deploy: + replicas: 1 + + webdav: + image: mailu/none:$VERSION + restart: always + env_file: .env + volumes: +# - "$ROOT/dav:/data" + - type: volume + source: mailu_dav + target: /data + deploy: + replicas: 1 + + admin: + image: mailu/admin:$VERSION + restart: always + env_file: .env + volumes: +# - "$ROOT/data:/data" + - type: volume + source: mailu_data + target: /data +# - "$ROOT/dkim:/dkim" + - type: volume + source: mailu_dkim + target: /dkim + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - redis + deploy: + replicas: 1 + + webmail: + image: mailu/roundcube:$VERSION + restart: always + env_file: .env + volumes: +# - "$ROOT/webmail:/data" + - type: volume + source: mailu_data + target: /data + depends_on: + - imap + deploy: + replicas: 2 + + fetchmail: + image: mailu/fetchmail:$VERSION + restart: always + env_file: .env + volumes: + deploy: + replicas: 1 + +networks: + default: + external: + name: mailu_default + +volumes: + mailu_filter: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/filter" + mailu_dkim: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/dkim" + mailu_overrides_rspamd: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/overrides/rspamd" + mailu_data: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/data" + mailu_mail: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/mail" + mailu_overrides: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/overrides" + mailu_dav: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/dav" + mailu_certs: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/certs" + mailu_redis: + driver_opts: + type: "nfs" + o: "addr=192.168.0.30,soft,rw" + device: ":/mnt/Pool1/pv/mailu/redis" +``` + +## Deploy Mailu on the docker swarm +Run the following command: +```bash +echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu +``` +See how the services are being deployed: +```bash +core@coreos-01 ~ $ docker service ls +ID NAME MODE REPLICAS IMAGE PORTS +ywnsetmtkb1l mailu_antivirus replicated 1/1 mailu/none:master +pqokiaz0q128 mailu_fetchmail replicated 1/1 mailu/fetchmail:master +``` +check a specific service: +```bash +core@coreos-01 ~ $ docker service ps mailu_fetchmail +ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS +tbu8ppgsdffj mailu_fetchmail.1 mailu/fetchmail:master coreos-01 Running Running 11 days ago +``` + +## Remove the stack +Run the follwoing command: +```bash +core@coreos-01 ~ $ docker stack rm mailu +``` From 5b8deed06ba8e67ab592a20d104033a9ee63445e Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 18 Oct 2018 18:59:22 +0000 Subject: [PATCH 43/51] Made the instructions more simple Updated the volume defintion to the latest master status Removed the part around .yml variable substitution as we can use a "trick" --- docs/swarm/master/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index c6120228..ba5e0011 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -29,7 +29,9 @@ mzrm9nbdggsfz4sgq6dhs5i6n flying-dutchman Ready Active ### Volume definition For data persistance (the Mailu services might be launched/relaunched on any of the swarm nodes), we need to have Mailu data stored in a manner accessible by every manager or worker in the swarm. -Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs". (GlusterFS and nfs shares have been successfully used) + +Hereafter we will assume that "Mailu Data" is available on every node at "$ROOT/certs:/certs" (GlusterFS and nfs shares have been successfully used). + On this example, we are using: - the mesh routing mode (default mode). With this mode, each service is given a virtual IP adress and docker manages the routing between this virtual IP and the container(s) providing this service. - the default ingress mode. @@ -67,13 +69,16 @@ As a side effect of this ingress mode "feature", make sure that the ingress subn - redis, antispam, antivirus, fetchmail, admin, webdav have not been tested (hence replicas=1 in the following docker-compose.yml file) ## Variable substitution and docker-compose.yml -The docker stack deploy command doesn't support variable substitution in the .yml file itself. As a consequence, we need to use the following work-around: +The docker stack deploy command doesn't support variable substitution in the .yml file itself. +As a consequence, we cannot simply use ``` docker stack deploy -c docker.compose.yml mailu ``` +Instead, we will use the following work-around: ``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` We need also to: - change the way we define the volumes (nfs share in our case) - add a deploy section for every service - the way the ports are defined for the front service +- add the POD_ADDRESS_RANGE definition imap, smtp and antispam services ## Docker compose An example of docker-compose-stack.yml file is available here: From 7a3922c2e733a9d4c9a58910e86a7dcf032e6788 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 18 Oct 2018 19:14:53 +0000 Subject: [PATCH 44/51] Fixes few typos --- docs/swarm/master/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/swarm/master/README.md b/docs/swarm/master/README.md index ba5e0011..44c19dc7 100644 --- a/docs/swarm/master/README.md +++ b/docs/swarm/master/README.md @@ -75,10 +75,9 @@ Instead, we will use the following work-around: ``` echo "$(docker-compose -f /mnt/docker/apps/mailu/docker-compose.yml config 2>/dev/null)" | docker stack deploy -c- mailu ``` We need also to: -- change the way we define the volumes (nfs share in our case) - add a deploy section for every service -- the way the ports are defined for the front service -- add the POD_ADDRESS_RANGE definition imap, smtp and antispam services +- modify the way the ports are defined for the front service +- add the POD_ADDRESS_RANGE definition for imap, smtp and antispam services ## Docker compose An example of docker-compose-stack.yml file is available here: From 4ccefd6d5e013ace4d8a27b6af4a26074d15b672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 19 Oct 2018 13:43:09 +0300 Subject: [PATCH 45/51] Documentation update on local docs container --- docs/contributors/environment.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index a1cce193..af610c27 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -101,7 +101,8 @@ Documentation is maintained in the ``docs`` directory and are maintained as `reS docker build -t docs docs docker run -p 127.0.0.1:8080:80 docs -You can now read the local documentation by navigating to http://localhost:8080. +In a local build Docker always assumes the version to be master. +You can read the local documentation by navigating to http://localhost:8080/master. .. note:: After modifying the documentation, the image needs to be rebuild and the container restarted for the changes to become visible. From 6fb0b932405b143e53ef5a9dbd66dc13baf9fc61 Mon Sep 17 00:00:00 2001 From: Hans Cornelis Date: Fri, 19 Oct 2018 13:39:32 +0200 Subject: [PATCH 46/51] - Removed RSPAMD Password feature - Updated roles on the Front DS - Reverted the Auth limits Signed-off-by: Hans Cornelis --- docs/compose/.env | 3 --- docs/kubernetes/mailu/configmap.yaml | 11 +---------- docs/kubernetes/mailu/front.yaml | 4 ++-- services/rspamd/conf/worker-controller.inc | 4 ---- 4 files changed, 3 insertions(+), 19 deletions(-) diff --git a/docs/compose/.env b/docs/compose/.env index 64981db2..7823bc3e 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -55,9 +55,6 @@ WEBDAV=none # Antivirus solution (value: clamav, none) ANTIVIRUS=none -# The password for the rspamd web interface -RSPAMD_PASSWORD=Secr3tPassWord - ################################### # Mail settings ################################### diff --git a/docs/kubernetes/mailu/configmap.yaml b/docs/kubernetes/mailu/configmap.yaml index be39fcc9..4f8dad81 100644 --- a/docs/kubernetes/mailu/configmap.yaml +++ b/docs/kubernetes/mailu/configmap.yaml @@ -40,7 +40,7 @@ TLS_FLAVOR: "cert" # Authentication rate limit (per source IP address) - AUTH_RATELIMIT: "100/minute;10000/hour" + AUTH_RATELIMIT: "10/minute;1000/hour" # Opt-out of statistics, replace with "True" to opt out DISABLE_STATISTICS: "False" @@ -52,7 +52,6 @@ # Use Kubernetes Ingress Controller to handle all actions on port 80 and 443 # This way we can make use of the advantages of the cert-manager deployment KUBERNETES_INGRESS: "true" - POD_ADDRESS_RANGE: "10.2.0.0/16" ################################### # Optional features @@ -72,9 +71,6 @@ # Antivirus solution (value: clamav, none) ANTIVIRUS: "clamav" - # The password for the rspamd web interface - RSPAMD_PASSWORD: "Secr3tPassWord" - ################################### # Mail settings ################################### @@ -83,11 +79,6 @@ # Default: accept messages up to 50MB MESSAGE_SIZE_LIMIT: "50000000" - # Networks granted relay permissions, make sure that you include your Docker - # internal network (default to 172.17.0.0/16) - # For kubernetes this is the CIDR of the pod network - RELAYNETS: "10.2.0.0/16" - # Will relay all outgoing mails if configured #RELAYHOST= diff --git a/docs/kubernetes/mailu/front.yaml b/docs/kubernetes/mailu/front.yaml index c13ecd9d..9951f30c 100644 --- a/docs/kubernetes/mailu/front.yaml +++ b/docs/kubernetes/mailu/front.yaml @@ -25,8 +25,8 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: node-role.kubernetes.io/master - operator: DoesNotExist + - key: node-role.kubernetes.io/node + operator: Exists hostNetwork: true nodeSelector: node-role.kubernetes.io/node: "" diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index d3ab55f4..6a020672 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,7 +1,3 @@ bind_socket = "*:11334"; -{% if RSPAMD_PASSWORD %} -password = "{{ RSPAMD_PASSWORD }}"; -{% else %} password = "mailu"; -{% endif %} secure_ip = "{{ FRONT_ADDRESS }}"; From 771e0ee6a2d21393a83b7eabd343355c03882897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 19 Oct 2018 16:29:47 +0300 Subject: [PATCH 47/51] Remove old crypt settings from .env As per conversation in PR --- docs/compose/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compose/.env b/docs/compose/.env index ce8e9eb9..2100e27a 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -130,7 +130,7 @@ LOG_DRIVER=json-file COMPOSE_PROJECT_NAME=mailu # Default password scheme used for newly created accounts and changed passwords -# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) +# (value: PBKDF2, BLF-CRYPT, SHA512-CRYPT, SHA256-CRYPT) PASSWORD_SCHEME=PBKDF2 # Header to take the real ip from From 72d4fa2bc95fe4ce74b2b69137c0d3f69c95b7d6 Mon Sep 17 00:00:00 2001 From: hoellen Date: Fri, 19 Oct 2018 22:13:38 +0200 Subject: [PATCH 48/51] remove empty line from merge conflict --- core/admin/mailu/models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 08ce8e8c..3653011f 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -278,7 +278,6 @@ class User(Base, Email): else: return self.email - @property def reply_active(self): now = date.today() From 78bd5aea1c66422f5e1ff816dfd0e71174d8db7f Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Fri, 19 Oct 2018 11:51:33 -0600 Subject: [PATCH 49/51] enable http2, because it's that easy --- core/nginx/conf/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 7eaba003..3f2e9a42 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -48,8 +48,8 @@ http { # Only enable HTTPS if TLS is enabled with no error {% if TLS and not TLS_ERROR %} - listen 443 ssl; - listen [::]:443 ssl; + listen 443 ssl http2; + listen [::]:443 ssl http2; include /etc/nginx/tls.conf; ssl_session_cache shared:SSLHTTP:50m; From c3e89967fb8ce7fc1fd431a2e618b82cc856cb4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sun, 21 Oct 2018 20:45:41 +0300 Subject: [PATCH 50/51] Fix front health checking - Specified seperated /health path in order to allow for healthcheck even if webmail and admin are not seletectd. This also allows healthchecking fom external services like DNS load balancers; - Make curl not to fail on TLS because localhost is not included in the certificates. --- core/nginx/Dockerfile | 2 +- core/nginx/conf/nginx.conf | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 87951c03..00ecf84e 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -13,4 +13,4 @@ VOLUME ["/certs"] CMD /start.py -HEALTHCHECK CMD curl -f -L http://localhost/ || exit 1 +HEALTHCHECK CMD curl -k -f -L http://localhost/health || exit 1 diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index 8fcda1c3..d2cbc7fe 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -146,6 +146,10 @@ http { proxy_pass_request_body off; proxy_set_header Content-Length ""; } + + location /health { + return 204; + } } # Forwarding authentication server From a2fea36c79367ce967e4b96a028aa5ff86c869dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sun, 21 Oct 2018 20:49:01 +0300 Subject: [PATCH 51/51] Increase HEALTHCHECK start time for services that need to wait for host resolving during startup. In Docker Swarm mode the services listed below can get stuck in their start script, while they are waiting for other services become available. Now, with HEALTHCHECK enabled, docker does not resolve names of services that not pass HEALTHCHECK yet. Meaning that if one of the depenend services is not yet available, it will create a chain of failing services. The services below retry to resolve 100 time, with an average of 3.5 seconds. Hence, the --start-time flag is now set at 350 seconds. - dovecot (imap) - postfix (smtp) - rspamd (antispam) --- core/dovecot/Dockerfile | 2 +- core/postfix/Dockerfile | 2 +- services/rspamd/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index c0d3e3cd..e19631ee 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -14,4 +14,4 @@ VOLUME ["/data", "/mail"] CMD /start.py -HEALTHCHECK CMD echo QUIT|nc localhost 110|grep "Dovecot ready." +HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 110|grep "Dovecot ready." diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 5533499e..e0529e01 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -13,4 +13,4 @@ VOLUME ["/data"] CMD /start.py -HEALTHCHECK CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" +HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index d87a64f5..4337fb2e 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -18,4 +18,4 @@ VOLUME ["/var/lib/rspamd"] CMD /start.py -HEALTHCHECK CMD curl -f -L http://localhost:11334/ || exit 1 +HEALTHCHECK --start-period=350s CMD curl -f -L http://localhost:11334/ || exit 1