Merge branch 'master' into feat-abstract-db

.mergify.yml

@@ -0,0 +1,10 @@
+rules:
+  default: null
+  branches:
+    master:
+      protection:
+        required_status_checks:
+          contexts:
+            - continuous-integration/travis-ci 
+        required_pull_request_reviews:
+          required_approving_review_count: 2

.travis.yml

@@ -8,4 +8,15 @@ env:
   - VERSION=$TRAVIS_BRANCH
 
 script:
-- docker-compose -f tests/build.yml -p Mailu build
+  # Default to mailu for DOCKER_ORG
+  - if [ -z "$DOCKER_ORG" ]; then export DOCKER_ORG="mailu"; fi
+  - docker-compose -f tests/build.yml build
+  - tests/compose/test-script.sh
+
+deploy:
+  provider: script
+  script: bash tests/deploy.sh
+  on:
+    all_branches: true
+    condition: -n $DOCKER_UN
+

core/admin/Dockerfile

@@ -17,5 +17,6 @@ COPY start.sh /start.sh
 RUN pybabel compile -d mailu/translations
 
 EXPOSE 80/tcp
+VOLUME ["/data"]
 
 CMD ["/start.sh"]

core/dovecot/Dockerfile

@@ -3,11 +3,13 @@ FROM alpine:3.8
 RUN apk add --no-cache \
      dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \
      python3 py3-pip \
- && pip3 install jinja2 podop
+ && pip3 install --upgrade pip \
+ && pip3 install jinja2 podop tenacity
 
 COPY conf /conf
 COPY start.py /start.py
 
 EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp
+VOLUME ["/data", "/mail"]
 
 CMD /start.py

core/dovecot/start.py

@@ -5,7 +5,9 @@ import os
 import socket
 import glob
 import multiprocessing
+import tenacity
 
+from tenacity import retry
 from podop import run_server
 
 
@@ -19,8 +21,15 @@ def start_podop():
 
 convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
-# Actual startup script
+@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
+def resolve():
 	os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front"))
+	os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis"))
+	if os.environ["WEBMAIL"] != "none":
+		os.environ["WEBMAIL_ADDRESS"] = socket.gethostbyname(os.environ.get("WEBMAIL_ADDRESS", "webmail"))
+
+# Actual startup script
+resolve()
 
 for dovecot_file in glob.glob("/conf/*.conf"):
     convert(dovecot_file, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))

core/nginx/Dockerfile

@@ -6,5 +6,6 @@ COPY conf /conf
 COPY *.py /
 
 EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp
+VOLUME ["/certs"]
 
 CMD /start.py

core/postfix/Dockerfile

@@ -2,11 +2,13 @@ FROM alpine:3.8
 
 RUN apk add --no-cache postfix postfix-pcre rsyslog \
     python3 py3-pip \
- && pip3 install jinja2 podop
+ && pip3 install --upgrade pip
+ && pip3 install jinja2 podop tenacity
 
 COPY conf /conf
 COPY start.py /start.py
 
 EXPOSE 25/tcp 10025/tcp
+VOLUME ["/data"]
 
 CMD /start.py

core/postfix/conf/master.cf

@@ -8,6 +8,7 @@ smtp      inet  n       -       n       -       -       smtpd
 10025     inet  n       -       n       -       -       smtpd
   -o smtpd_sasl_auth_enable=yes
   -o smtpd_client_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
+  -o smtpd_reject_unlisted_recipient={% if REJECT_UNLISTED_RECIPIENT %}{{ REJECT_UNLISTED_RECIPIENT }}{% else %}no{% endif %}
   -o cleanup_service_name=outclean
 outclean  unix n       -       n       -       0       cleanup
   -o header_checks=pcre:/etc/postfix/outclean_header_filter.cf

core/postfix/start.py

@@ -5,8 +5,10 @@ import os
 import socket
 import glob
 import shutil
+import tenacity
 import multiprocessing
 
+from tenacity import retry
 from podop import run_server
 
 
@@ -22,8 +24,12 @@ def start_podop():
 
 convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
-# Actual startup script
+@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
+def resolve():
 	os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front"))
+
+# Actual startup script
+resolve()
 os.environ["HOST_ANTISPAM"] = os.environ.get("HOST_ANTISPAM", "antispam:11332")
 os.environ["HOST_LMTP"] = os.environ.get("HOST_LMTP", "imap:2525")
 

docs/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3-alpine
+
+COPY requirements.txt /requirements.txt
+
+RUN pip install -r /requirements.txt \
+ && apk add --no-cache nginx \
+ && mkdir /run/nginx
+
+COPY ./nginx.conf /etc/nginx/conf.d/default.conf
+COPY . /docs
+
+RUN sphinx-build /docs /build
+
+CMD nginx -g "daemon off;"

docs/compose/.env

@@ -132,3 +132,6 @@ REAL_IP_HEADER=
 
 # IPs for nginx set_real_ip_from (CIDR list separated by commas)
 REAL_IP_FROM=
+
+# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
+REJECT_UNLISTED_RECIPIENT=

docs/conf.py

@@ -7,7 +7,7 @@ templates_path = ['_templates']
 source_suffix = '.rst'
 master_doc = 'index'
 project = 'Mailu'
-copyright = '2017, Mailu authors'
+copyright = '2018, Mailu authors'
 author = 'Mailu authors'
 version = release = 'latest'
 language = None
@@ -23,7 +23,7 @@ htmlhelp_basename = 'Mailudoc'
 # to template names.
 html_sidebars = {
     '**': [
-        'relations.html',  # needs 'show_related': True theme option to display
+        'relations.html', 
         'searchbox.html',
     ]
 }
@@ -36,24 +36,3 @@ html_context = {
     'github_version': 'master',
     'conf_py_path': '/docs/'
 }
-
-
-# Upload function when the script is called directly
-if __name__ == "__main__":
-    import os, sys, paramiko
-    build_dir, hostname, username, password, dest_dir = sys.argv[1:]
-    transport = paramiko.Transport((hostname, 22))
-    transport.connect(username=username, password=password)
-    sftp = paramiko.SFTPClient.from_transport(transport)
-    os.chdir(build_dir)
-    for dirpath, dirnames, filenames in os.walk("."):
-        remote_path = os.path.join(dest_dir, dirpath)
-        try:
-            sftp.mkdir(remote_path)
-        except:
-            pass
-        for filename in filenames:
-            sftp.put(
-                os.path.join(dirpath, filename),
-                os.path.join(remote_path, filename)
-            )

docs/contributors/environment.rst

@@ -89,3 +89,20 @@ Any change to the files will automatically restart the Web server and reload the
 
 When using the development environment, a debugging toolbar is displayed on the right side
 of the screen, that you can open to access query details, internal variables, etc.
+
+Documentation
+-------------
+
+Documentation is maintained in the ``docs`` directory and are maintained as `reStructuredText`_ files. It is possible to run a local documentation server for reviewing purposes, using Docker:
+
+.. code-block:: bash
+
+  cd <Mailu repo>
+  docker build -t docs docs
+  docker run -p 127.0.0.1:8080:80 docs
+
+You can now read the local documentation by navigating to http://localhost:8080.
+
+.. note:: After modifying the documentation, the image needs to be rebuild and the container restarted for the changes to become visible.
+
+.. _`reStructuredText`: http://docutils.sourceforge.net/rst.html

docs/nginx.conf

@@ -0,0 +1,5 @@
+server {
+    listen 80;
+    listen [::]:80;
+    root /build;
+}

docs/requirements.txt

@@ -2,5 +2,3 @@ recommonmark
 Sphinx
 sphinx-autobuild
 sphinx-rtd-theme
-sphinxcontrib-versioning
-paramiko

optional/clamav/Dockerfile

@@ -6,5 +6,6 @@ COPY conf /etc/clamav
 COPY start.sh /start.sh
 
 EXPOSE 3310/tcp
+VOLUME ["/data"]
 
 CMD ["/start.sh"]

optional/radicale/Dockerfile

@@ -6,5 +6,6 @@ RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/re
 COPY radicale.conf /radicale.conf
 
 EXPOSE 5232/tcp
+VOLUME ["/data"]
 
 CMD radicale -f -S -C /radicale.conf

services/rspamd/Dockerfile

@@ -1,6 +1,8 @@
 FROM alpine:edge
 
-RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates
+RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \
+ && pip install --upgrade pip \
+ && pip install tenacity
 
 RUN mkdir /run/rspamd
 
@@ -12,4 +14,6 @@ RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
 
 EXPOSE 11332/tcp 11334/tcp
 
+VOLUME ["/var/lib/rspamd"]
+
 CMD /start.py

services/rspamd/conf/arc.conf

@@ -0,0 +1,4 @@
+try_fallback = true;
+path = "/dkim/$domain.$selector.key";
+selector = "dkim"
+use_esld = false;

services/rspamd/start.py

@@ -4,11 +4,17 @@ import jinja2
 import os
 import socket
 import glob
+import tenacity
+from tenacity import retry
 
 convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
-# Actual startup script
+@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
+def resolve():
 	os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front"))
+
+# Actual startup script
+resolve()
 if "HOST_REDIS" not in os.environ: os.environ["HOST_REDIS"] = "redis"
 
 for rspamd_file in glob.glob("/conf/*"):

tests/build.yml

@@ -3,45 +3,54 @@ version: '3'
 services:
 
   front:
-    image: mailu/nginx:$VERSION
+    image: $DOCKER_ORG/nginx:$VERSION
     build: ../core/nginx
 
   imap:
-    image: mailu/dovecot:$VERSION
+    image: $DOCKER_ORG/dovecot:$VERSION
     build: ../core/dovecot
 
   smtp:
-    image: mailu/postfix:$VERSION
+    image: $DOCKER_ORG/postfix:$VERSION
     build: ../core/postfix
 
   antispam:
-    image: mailu/rspamd:$VERSION
+    image: $DOCKER_ORG/rspamd:$VERSION
     build: ../services/rspamd
 
   antivirus:
-    image: mailu/clamav:$VERSION
+    image: $DOCKER_ORG/clamav:$VERSION
     build: ../optional/clamav
 
   webdav:
-    image: mailu/radicale:$VERSION
+    image: $DOCKER_ORG/radicale:$VERSION
     build: ../optional/radicale
 
   admin:
-    image: mailu/admin:$VERSION
+    image: $DOCKER_ORG/admin:$VERSION
     build: ../core/admin
 
   roundcube:
-    image: mailu/roundcube:$VERSION
+    image: $DOCKER_ORG/roundcube:$VERSION
     build: ../webmails/roundcube
 
   rainloop:
-    image: mailu/rainloop:$VERSION
+    image: $DOCKER_ORG/rainloop:$VERSION
     build: ../webmails/rainloop
 
   fetchmail:
-    image: mailu/fetchmail:$VERSION
+    image: $DOCKER_ORG/fetchmail:$VERSION
     build: ../services/fetchmail
 
   none:
-    image: mailu/none:$VERSION
+    image: $DOCKER_ORG/none:$VERSION
     build: ../core/none
+
+  docs:
+    image: $DOCKER_ORG/docs:$VERSION
+    build: ../docs
+
+  setup:
+    image: $DOCKER_ORG/setup:$VERSION
+    build: ../setup
+

tests/compose/core.env

@@ -0,0 +1,134 @@
+# Mailu main configuration file
+#
+# Most configuration variables can be modified through the Web interface,
+# these few settings must however be configured before starting the mail
+# server and require a restart upon change.
+
+###################################
+# Common configuration variables
+###################################
+
+# Set this to the path where Mailu data and configuration is stored
+ROOT=/mailu
+
+# Mailu version to run (1.0, 1.1, etc. or master)
+#VERSION=master
+
+# Set to a randomly generated 16 bytes string
+SECRET_KEY=ChangeMeChangeMe
+
+# Address where listening ports should bind
+BIND_ADDRESS4=127.0.0.1
+#BIND_ADDRESS6=::1
+
+# Main mail domain
+DOMAIN=mailu.io
+
+# Hostnames for this server, separated with comas
+HOSTNAMES=mail.mailu.io,alternative.mailu.io,yetanother.mailu.io
+
+# Postmaster local part (will append the main mail domain)
+POSTMASTER=admin
+
+# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
+TLS_FLAVOR=cert
+
+# Authentication rate limit (per source IP address)
+AUTH_RATELIMIT=10/minute;1000/hour
+
+# Opt-out of statistics, replace with "True" to opt out
+DISABLE_STATISTICS=False
+
+###################################
+# Optional features
+###################################
+
+# Expose the admin interface (value: true, false)
+ADMIN=false
+
+# Choose which webmail to run if any (values: roundcube, rainloop, none)
+WEBMAIL=none
+
+# Dav server implementation (value: radicale, none)
+WEBDAV=none
+
+# Antivirus solution (value: clamav, none)
+ANTIVIRUS=none
+
+###################################
+# Mail settings
+###################################
+
+# Message size limit in bytes
+# Default: accept messages up to 50MB
+MESSAGE_SIZE_LIMIT=50000000
+
+# Networks granted relay permissions, make sure that you include your Docker
+# internal network (default to 172.17.0.0/16)
+RELAYNETS=172.16.0.0/12
+
+# Will relay all outgoing mails if configured
+RELAYHOST=
+
+# Fetchmail delay
+FETCHMAIL_DELAY=600
+
+# Recipient delimiter, character used to delimiter localpart from custom address part
+# e.g. localpart+custom@domain;tld
+RECIPIENT_DELIMITER=+
+
+# DMARC rua and ruf email
+DMARC_RUA=admin
+DMARC_RUF=admin
+
+# Welcome email, enable and set a topic and body if you wish to send welcome
+# emails to all users.
+WELCOME=false
+WELCOME_SUBJECT=Welcome to your new email account
+WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly!
+
+# Maildir Compression
+# choose compression-method, default: none (value: bz2, gz)
+COMPRESSION=
+# change compression-level, default: 6 (value: 1-9)
+COMPRESSION_LEVEL=
+
+###################################
+# Web settings
+###################################
+
+# Path to the admin interface if enabled
+WEB_ADMIN=/admin
+
+# Path to the webmail if enabled
+WEB_WEBMAIL=/webmail
+
+# Website name
+SITENAME=Mailu
+
+# Linked Website URL
+WEBSITE=https://mailu.io
+
+# Registration reCaptcha settings (warning, this has some privacy impact)
+# RECAPTCHA_PUBLIC_KEY=
+# RECAPTCHA_PRIVATE_KEY=
+
+# Domain registration, uncomment to enable
+# DOMAIN_REGISTRATION=true
+
+###################################
+# Advanced settings
+###################################
+
+# Docker-compose project name, this will prepended to containers names.
+#COMPOSE_PROJECT_NAME=mailu
+
+# Default password scheme used for newly created accounts and changed passwords
+# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
+PASSWORD_SCHEME=SHA512-CRYPT
+
+# Header to take the real ip from
+REAL_IP_HEADER=
+
+# IPs for nginx set_real_ip_from (CIDR list separated by commas)
+REAL_IP_FROM=

tests/compose/run.yml

@@ -0,0 +1,99 @@
+version: '2'
+
+services:
+
+  front:
+    image: $DOCKER_ORG/nginx:$VERSION
+    restart: 'no'
+    env_file: $PWD/.env
+    ports:
+    - "$BIND_ADDRESS4:80:80"
+    - "$BIND_ADDRESS4:443:443"
+    - "$BIND_ADDRESS4:110:110"
+    - "$BIND_ADDRESS4:143:143"
+    - "$BIND_ADDRESS4:993:993"
+    - "$BIND_ADDRESS4:995:995"
+    - "$BIND_ADDRESS4:25:25"
+    - "$BIND_ADDRESS4:465:465"
+    - "$BIND_ADDRESS4:587:587"
+    volumes:
+      - "$ROOT/certs:/certs"
+
+  redis:
+    image: redis:alpine
+    restart: 'no'
+    volumes:
+      - "$ROOT/redis:/data"
+
+  imap:
+    image: $DOCKER_ORG/dovecot:$VERSION
+    restart: 'no'
+    env_file: $PWD/.env
+    volumes:
+      - "$ROOT/data:/data"
+      - "$ROOT/mail:/mail"
+      - "$ROOT/overrides:/overrides"
+    depends_on:
+      - front
+
+  smtp:
+    image: $DOCKER_ORG/postfix:$VERSION
+    restart: 'no'
+    env_file: $PWD/.env
+    volumes:
+      - "$ROOT/data:/data"
+      - "$ROOT/overrides:/overrides"
+    depends_on:
+      - front
+
+  antispam:
+    image: $DOCKER_ORG/rspamd:$VERSION
+    restart: 'no'
+    env_file: $PWD/.env
+    volumes:
+      - "$ROOT/filter:/var/lib/rspamd"
+      - "$ROOT/dkim:/dkim"
+      - "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
+    depends_on:
+      - front
+
+  antivirus:
+    image: $DOCKER_ORG/$ANTIVIRUS:$VERSION
+    restart: 'no'
+    env_file: $PWD/.env
+    volumes:
+      - "$ROOT/filter:/data"
+
+  webdav:
+    image: $DOCKER_ORG/$WEBDAV:$VERSION
+    restart: 'no'
+    env_file: $PWD/.env
+    volumes:
+      - "$ROOT/dav:/data"
+
+  admin:
+    image: $DOCKER_ORG/admin:$VERSION
+    restart: 'no'
+    env_file: $PWD/.env
+    volumes:
+      - "$ROOT/data:/data"
+      - "$ROOT/dkim:/dkim"
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    depends_on:
+      - redis
+
+  webmail:
+    image: "$DOCKER_ORG/$WEBMAIL:$VERSION"
+    restart: 'no'
+    env_file: $PWD/.env
+    volumes:
+      - "$ROOT/webmail:/data"
+    depends_on:
+      - imap
+
+  fetchmail:
+    image: $DOCKER_ORG/fetchmail:$VERSION
+    restart: 'no'
+    env_file: $PWD/.env
+    volumes:
+      - "$ROOT/data:/data"

tests/compose/test-script.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+containers=(
+	webmail
+	imap
+	smtp
+	antispam
+	admin
+	redis
+	antivirus
+	webdav
+#	fetchmail
+	front
+)
+
+# Time to sleep in minutes after starting the containers
+WAIT=1
+
+containers_check() {
+	status=0
+	for container in "${containers[@]}"; do
+		name="${DOCKER_ORG}_${container}_1"
+		echo "Checking $name"
+		docker inspect "$name" | grep '"Status": "running"' || status=1
+	done
+	docker ps -a
+	return $status
+}
+
+container_logs() {
+	for container in "${containers[@]}"; do
+		name="${DOCKER_ORG}_${container}_1"
+                echo "Showing logs for $name"
+                docker container logs "$name"
+        done
+}
+
+clean() {
+	docker-compose -f tests/compose/run.yml -p $DOCKER_ORG down || exit 1
+	rm -fv .env
+}
+
+# Cleanup before callig exit
+die() {
+	clean
+	exit $1
+}
+
+for file in tests/compose/*.env ; do
+	cp $file .env
+	docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d
+	echo -e "\nSleeping for ${WAIT} minutes" # Clean terminal distortion from docker-compose in travis
+	travis_wait sleep ${WAIT}m || sleep ${WAIT}m #Fallback sleep for local run
+	container_logs
+	containers_check || die 1
+	clean
+done
+

tests/deploy.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+docker login -u $DOCKER_UN -p $DOCKER_PW
+docker-compose -f tests/build.yml push

webmails/rainloop/Dockerfile

@@ -24,4 +24,7 @@ COPY default.ini /default.ini
 
 COPY start.py /start.py
 
+EXPOSE 80/tcp
+VOLUME ["/data"]
+
 CMD /start.py

webmails/rainloop/start.py

@@ -18,4 +18,7 @@ os.makedirs(base + "configs", exist_ok=True)
 convert("/default.ini", "/data/_data_/_default_/domains/default.ini")
 convert("/config.ini", "/data/_data_/_default_/configs/config.ini")
 
+os.system("chown -R www-data:www-data /data")
+
 os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"])
+

webmails/roundcube/Dockerfile

@@ -25,4 +25,7 @@ COPY config.inc.php /var/www/html/config/
 
 COPY start.sh /start.sh
 
+EXPOSE 80/tcp
+VOLUME ["/data"]
+
 CMD ["/start.sh"]