From cc8e15748bebd090f8abcc6ca06147843236864d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Wed, 8 Aug 2018 17:54:15 +0300 Subject: [PATCH 01/55] Retry 10 times when resolving fails in start.py scripts --- core/dovecot/start.py | 20 ++++++++++++++++---- core/postfix/start.py | 16 ++++++++++++++-- services/rspamd/start.py | 14 +++++++++++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 83f91fab..2de114fd 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -4,14 +4,26 @@ import jinja2 import os import socket import glob +import time convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) # Actual startup script -os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) -os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis")) -if os.environ["WEBMAIL"] != "none": - os.environ["WEBMAIL_ADDRESS"] = socket.gethostbyname(os.environ.get("WEBMAIL_ADDRESS", "webmail")) +i = 0 +t = 10 +while True: + i += 1 + try: + os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) + os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis")) + if os.environ["WEBMAIL"] != "none": + os.environ["WEBMAIL_ADDRESS"] = socket.gethostbyname(os.environ.get("WEBMAIL_ADDRESS", "webmail")) + except socket.gaierror as err: + if i >= t: + raise + time.sleep(10) + continue + break for dovecot_file in glob.glob("/conf/*"): convert(dovecot_file, os.path.join("/etc/dovecot", os.path.basename(dovecot_file))) diff --git a/core/postfix/start.py b/core/postfix/start.py index 4dbf2206..f3c6aaca 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -5,11 +5,23 @@ import os import socket import glob import shutil - +import time + convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) # Actual startup script -os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) +i = 0 +t = 10 +while True: + i += 1 + try: + os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) + except socket.gaierror as err: + if i >= t: + raise + time.sleep(10) + continue + break os.environ["HOST_ANTISPAM"] = os.environ.get("HOST_ANTISPAM", "antispam:11332") os.environ["HOST_LMTP"] = os.environ.get("HOST_LMTP", "imap:2525") diff --git a/services/rspamd/start.py b/services/rspamd/start.py index 87309cee..08301a0d 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -4,11 +4,23 @@ import jinja2 import os import socket import glob +import time convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) # Actual startup script -os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) +i = 0 +t = 10 +while True: + i += 1 + try: + os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) + except socket.gaierror as err: + if i >= t: + raise + time.sleep(10) + continue + break if "HOST_REDIS" not in os.environ: os.environ["HOST_REDIS"] = "redis" for rspamd_file in glob.glob("/conf/*"): From 5341ee4472e0f9067b5391d918dda38f77e99c46 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 25 Sep 2018 21:04:30 +0200 Subject: [PATCH 02/55] Add a Dockerfile for buliding the docs --- docs/Dockerfile | 13 +++++++++++++ docs/requirements.txt | 1 - 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 docs/Dockerfile diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 00000000..a850a664 --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3-alpine + +RUN apk add --no-cache git + +COPY docs/requirements.txt requirements.txt +RUN pip install -r /requirements.txt \ + && mkdir /src + +WORKDIR /src +COPY .git /src/.git + +RUN sphinx-versioning build -b -B 1.5 -r 1.5 -w '^[0-9.]*$' -w master -W '^$' /src /build + diff --git a/docs/requirements.txt b/docs/requirements.txt index 2572817f..98338e63 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,4 +3,3 @@ Sphinx sphinx-autobuild sphinx-rtd-theme sphinxcontrib-versioning -paramiko From 72cfadd5e8d722b0a4b7e039c6f095c10241fb69 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 25 Sep 2018 21:08:04 +0200 Subject: [PATCH 03/55] Build the docs during tests --- tests/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/build.yml b/tests/build.yml index 674abf8c..308b821a 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -45,3 +45,9 @@ services: none: image: mailu/none:$VERSION build: ../core/none + + docs: + image: mailu/docs:$VERSION + build: + context: ../ + dockerfile: ../docs/Dockerfile From 69c19dca55f321e95d66fa5d6f9aaa1a4ad3fa42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Thu, 27 Sep 2018 21:45:06 +0300 Subject: [PATCH 04/55] Attempt to fix the docs build context --- tests/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build.yml b/tests/build.yml index 308b821a..ff08de5d 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -50,4 +50,4 @@ services: image: mailu/docs:$VERSION build: context: ../ - dockerfile: ../docs/Dockerfile + dockerfile: docs/Dockerfile From 339b3c1b24c7821c91f7090fb3a714efb328947f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 28 Sep 2018 10:41:17 +0200 Subject: [PATCH 05/55] Build the documentation as a Docker image --- docs/Dockerfile | 13 +++++++------ docs/conf.py | 25 ++----------------------- docs/nginx.conf | 5 +++++ docs/requirements.txt | 1 - tests/build.yml | 4 +--- 5 files changed, 15 insertions(+), 33 deletions(-) create mode 100644 docs/nginx.conf diff --git a/docs/Dockerfile b/docs/Dockerfile index a850a664..af481a27 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,13 +1,14 @@ FROM python:3-alpine -RUN apk add --no-cache git +COPY requirements.txt /requirements.txt -COPY docs/requirements.txt requirements.txt RUN pip install -r /requirements.txt \ - && mkdir /src + && apk add --no-cache nginx \ + && mkdir /run/nginx -WORKDIR /src -COPY .git /src/.git +COPY ./nginx.conf /etc/nginx/conf.d/default.conf +COPY . /docs -RUN sphinx-versioning build -b -B 1.5 -r 1.5 -w '^[0-9.]*$' -w master -W '^$' /src /build +RUN sphinx-build /docs /build +CMD nginx -g "daemon off;" \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 7a378132..f89b39fd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,7 +7,7 @@ templates_path = ['_templates'] source_suffix = '.rst' master_doc = 'index' project = 'Mailu' -copyright = '2017, Mailu authors' +copyright = '2018, Mailu authors' author = 'Mailu authors' version = release = 'latest' language = None @@ -23,7 +23,7 @@ htmlhelp_basename = 'Mailudoc' # to template names. html_sidebars = { '**': [ - 'relations.html', # needs 'show_related': True theme option to display + 'relations.html', 'searchbox.html', ] } @@ -36,24 +36,3 @@ html_context = { 'github_version': 'master', 'conf_py_path': '/docs/' } - - -# Upload function when the script is called directly -if __name__ == "__main__": - import os, sys, paramiko - build_dir, hostname, username, password, dest_dir = sys.argv[1:] - transport = paramiko.Transport((hostname, 22)) - transport.connect(username=username, password=password) - sftp = paramiko.SFTPClient.from_transport(transport) - os.chdir(build_dir) - for dirpath, dirnames, filenames in os.walk("."): - remote_path = os.path.join(dest_dir, dirpath) - try: - sftp.mkdir(remote_path) - except: - pass - for filename in filenames: - sftp.put( - os.path.join(dirpath, filename), - os.path.join(remote_path, filename) - ) diff --git a/docs/nginx.conf b/docs/nginx.conf new file mode 100644 index 00000000..75b5be50 --- /dev/null +++ b/docs/nginx.conf @@ -0,0 +1,5 @@ +server { + listen 80; + listen [::]:80; + root /build; +} diff --git a/docs/requirements.txt b/docs/requirements.txt index 98338e63..4afd9bb6 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,4 +2,3 @@ recommonmark Sphinx sphinx-autobuild sphinx-rtd-theme -sphinxcontrib-versioning diff --git a/tests/build.yml b/tests/build.yml index ff08de5d..c39b0af4 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -48,6 +48,4 @@ services: docs: image: mailu/docs:$VERSION - build: - context: ../ - dockerfile: docs/Dockerfile + build: ../docs From f97d0d9e437bceaf7dfd54d37cac23c46f1fd7b7 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 25 Sep 2018 21:04:30 +0200 Subject: [PATCH 06/55] Add a Dockerfile for buliding the docs --- docs/Dockerfile | 13 +++++++++++++ docs/requirements.txt | 1 - 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 docs/Dockerfile diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 00000000..a850a664 --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3-alpine + +RUN apk add --no-cache git + +COPY docs/requirements.txt requirements.txt +RUN pip install -r /requirements.txt \ + && mkdir /src + +WORKDIR /src +COPY .git /src/.git + +RUN sphinx-versioning build -b -B 1.5 -r 1.5 -w '^[0-9.]*$' -w master -W '^$' /src /build + diff --git a/docs/requirements.txt b/docs/requirements.txt index 2572817f..98338e63 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,4 +3,3 @@ Sphinx sphinx-autobuild sphinx-rtd-theme sphinxcontrib-versioning -paramiko From b287a85124f0485ee97e7deb963d4cb7d53c6ae2 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 25 Sep 2018 21:08:04 +0200 Subject: [PATCH 07/55] Build the docs during tests --- tests/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/build.yml b/tests/build.yml index 674abf8c..308b821a 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -45,3 +45,9 @@ services: none: image: mailu/none:$VERSION build: ../core/none + + docs: + image: mailu/docs:$VERSION + build: + context: ../ + dockerfile: ../docs/Dockerfile From 11bcae4c5744a8c93929e3c6d26030813dff477c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Thu, 27 Sep 2018 21:45:06 +0300 Subject: [PATCH 08/55] Attempt to fix the docs build context --- tests/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build.yml b/tests/build.yml index 308b821a..ff08de5d 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -50,4 +50,4 @@ services: image: mailu/docs:$VERSION build: context: ../ - dockerfile: ../docs/Dockerfile + dockerfile: docs/Dockerfile From 2cba045013acab4a14abc338fb0e2650f07df4a0 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 28 Sep 2018 17:12:50 +0200 Subject: [PATCH 09/55] Explicitely declare required volumes, fixes #568 --- core/dovecot/Dockerfile | 1 + core/nginx/Dockerfile | 1 + core/postfix/Dockerfile | 1 + optional/clamav/Dockerfile | 1 + optional/radicale/Dockerfile | 1 + services/rspamd/Dockerfile | 2 ++ webmails/rainloop/Dockerfile | 3 +++ webmails/roundcube/Dockerfile | 3 +++ 8 files changed, 13 insertions(+) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 80e3539a..363a7244 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -10,5 +10,6 @@ COPY sieve /var/lib/dovecot COPY start.py /start.py EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp +VOLUME ["/data", "/mail"] CMD /start.py diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 8a6536eb..adb785d8 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -6,5 +6,6 @@ COPY conf /conf COPY *.py / EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp +VOLUME ["/certs"] CMD /start.py diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 168f3c60..d853c9f9 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -6,5 +6,6 @@ COPY conf /conf COPY start.py /start.py EXPOSE 25/tcp 10025/tcp +VOLUME ["/data"] CMD /start.py diff --git a/optional/clamav/Dockerfile b/optional/clamav/Dockerfile index 92309c45..1c83d9c7 100644 --- a/optional/clamav/Dockerfile +++ b/optional/clamav/Dockerfile @@ -6,5 +6,6 @@ COPY conf /etc/clamav COPY start.sh /start.sh EXPOSE 3310/tcp +VOLUME ["/data"] CMD ["/start.sh"] diff --git a/optional/radicale/Dockerfile b/optional/radicale/Dockerfile index b1e63d7b..b82a0804 100644 --- a/optional/radicale/Dockerfile +++ b/optional/radicale/Dockerfile @@ -6,5 +6,6 @@ RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/re COPY radicale.conf /radicale.conf EXPOSE 5232/tcp +VOLUME ["/data"] CMD radicale -f -S -C /radicale.conf diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index c6c2afdd..d5e93db7 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -12,4 +12,6 @@ RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf EXPOSE 11332/tcp 11334/tcp +VOLUME ["/var/lib/rspamd"] + CMD /start.py diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile index dfc6c83e..f4571944 100644 --- a/webmails/rainloop/Dockerfile +++ b/webmails/rainloop/Dockerfile @@ -24,4 +24,7 @@ COPY default.ini /default.ini COPY start.py /start.py +EXPOSE 80/tcp +VOLUME ["/data"] + CMD /start.py diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index 99567502..ad198236 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -25,4 +25,7 @@ COPY config.inc.php /var/www/html/config/ COPY start.sh /start.sh +EXPOSE 80/tcp +VOLUME ["/data"] + CMD ["/start.sh"] From 4d70a8737e17ac049f9bc030b27cad41c8912d31 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 28 Sep 2018 17:42:10 +0200 Subject: [PATCH 10/55] Expose the data volume for admin container --- core/admin/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 0adc626c..08de0e88 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -17,5 +17,6 @@ COPY start.sh /start.sh RUN pybabel compile -d mailu/translations EXPOSE 80/tcp +VOLUME ["/data"] CMD ["/start.sh"] From 76923d80d89447dbaa08ef05e5ee3a372b3d2ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Sat, 29 Sep 2018 13:03:33 +0200 Subject: [PATCH 11/55] implement support for ARC --- services/rspamd/conf/arc.conf | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 services/rspamd/conf/arc.conf diff --git a/services/rspamd/conf/arc.conf b/services/rspamd/conf/arc.conf new file mode 100644 index 00000000..205d4284 --- /dev/null +++ b/services/rspamd/conf/arc.conf @@ -0,0 +1,4 @@ +try_fallback = true; +path = "/dkim/$domain.$selector.key"; +selector = "dkim" +use_esld = false; From 73add1b428450d78d68852369c670b8f199b4271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Mon, 1 Oct 2018 01:47:40 +0300 Subject: [PATCH 12/55] Documentation on running a local docs container --- docs/contributors/environment.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index 0aac71f4..a1cce193 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -89,3 +89,20 @@ Any change to the files will automatically restart the Web server and reload the When using the development environment, a debugging toolbar is displayed on the right side of the screen, that you can open to access query details, internal variables, etc. + +Documentation +------------- + +Documentation is maintained in the ``docs`` directory and are maintained as `reStructuredText`_ files. It is possible to run a local documentation server for reviewing purposes, using Docker: + +.. code-block:: bash + + cd + docker build -t docs docs + docker run -p 127.0.0.1:8080:80 docs + +You can now read the local documentation by navigating to http://localhost:8080. + +.. note:: After modifying the documentation, the image needs to be rebuild and the container restarted for the changes to become visible. + +.. _`reStructuredText`: http://docutils.sourceforge.net/rst.html From 6490a43492550fa1a9b42e175668aea0d9995c4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Mon, 1 Oct 2018 12:06:18 +0300 Subject: [PATCH 13/55] Revert "Attempt to fix the docs build context" This reverts commit 11bcae4c5744a8c93929e3c6d26030813dff477c. --- tests/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build.yml b/tests/build.yml index ff08de5d..308b821a 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -50,4 +50,4 @@ services: image: mailu/docs:$VERSION build: context: ../ - dockerfile: docs/Dockerfile + dockerfile: ../docs/Dockerfile From 07af9978e254162d14b303ca63b72f7dbaeb969e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Mon, 1 Oct 2018 12:06:56 +0300 Subject: [PATCH 14/55] Revert "Build the docs during tests" This reverts commit b287a85124f0485ee97e7deb963d4cb7d53c6ae2. --- tests/build.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/build.yml b/tests/build.yml index 308b821a..674abf8c 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -45,9 +45,3 @@ services: none: image: mailu/none:$VERSION build: ../core/none - - docs: - image: mailu/docs:$VERSION - build: - context: ../ - dockerfile: ../docs/Dockerfile From 6479f5177b350769ebfa9c8635bc8d4d81802a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Mon, 1 Oct 2018 12:07:49 +0300 Subject: [PATCH 15/55] Revert "Add a Dockerfile for buliding the docs" This reverts commit f97d0d9e437bceaf7dfd54d37cac23c46f1fd7b7. --- docs/Dockerfile | 13 ------------- docs/requirements.txt | 1 + 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 docs/Dockerfile diff --git a/docs/Dockerfile b/docs/Dockerfile deleted file mode 100644 index a850a664..00000000 --- a/docs/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM python:3-alpine - -RUN apk add --no-cache git - -COPY docs/requirements.txt requirements.txt -RUN pip install -r /requirements.txt \ - && mkdir /src - -WORKDIR /src -COPY .git /src/.git - -RUN sphinx-versioning build -b -B 1.5 -r 1.5 -w '^[0-9.]*$' -w master -W '^$' /src /build - diff --git a/docs/requirements.txt b/docs/requirements.txt index 98338e63..2572817f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,3 +3,4 @@ Sphinx sphinx-autobuild sphinx-rtd-theme sphinxcontrib-versioning +paramiko From 5cc5b7c40a63105c9a01737de32e8724ddabd006 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Thu, 4 Oct 2018 11:48:56 +0300 Subject: [PATCH 16/55] Copied docs/compose/docker-compose.yml to tests/compose/run.yml Edit for test environment --- tests/compose/run.yml | 108 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/compose/run.yml diff --git a/tests/compose/run.yml b/tests/compose/run.yml new file mode 100644 index 00000000..c1667db3 --- /dev/null +++ b/tests/compose/run.yml @@ -0,0 +1,108 @@ +version: '2' + +services: + + front: + image: mailu/nginx:$VERSION + restart: no + env_file: $ENV_FILE + ports: + - "$BIND_ADDRESS4:80:80" + - "$BIND_ADDRESS4:443:443" + - "$BIND_ADDRESS4:110:110" + - "$BIND_ADDRESS4:143:143" + - "$BIND_ADDRESS4:993:993" + - "$BIND_ADDRESS4:995:995" + - "$BIND_ADDRESS4:25:25" + - "$BIND_ADDRESS4:465:465" + - "$BIND_ADDRESS4:587:587" + - "$BIND_ADDRESS6:80:80" + - "$BIND_ADDRESS6:443:443" + - "$BIND_ADDRESS6:110:110" + - "$BIND_ADDRESS6:143:143" + - "$BIND_ADDRESS6:993:993" + - "$BIND_ADDRESS6:995:995" + - "$BIND_ADDRESS6:25:25" + - "$BIND_ADDRESS6:465:465" + - "$BIND_ADDRESS6:587:587" + volumes: + - "$ROOT/certs:/certs" + + redis: + image: redis:alpine + restart: no + volumes: + - "$ROOT/redis:/data" + + imap: + image: mailu/dovecot:$VERSION + restart: no + env_file: $ENV_FILE + volumes: + - "$ROOT/data:/data" + - "$ROOT/mail:/mail" + - "$ROOT/overrides:/overrides" + depends_on: + - front + + smtp: + image: mailu/postfix:$VERSION + restart: no + env_file: $ENV_FILE + volumes: + - "$ROOT/data:/data" + - "$ROOT/overrides:/overrides" + depends_on: + - front + + antispam: + image: mailu/rspamd:$VERSION + restart: no + env_file: $ENV_FILE + volumes: + - "$ROOT/filter:/var/lib/rspamd" + - "$ROOT/dkim:/dkim" + - "$ROOT/overrides/rspamd:/etc/rspamd/override.d" + depends_on: + - front + + antivirus: + image: mailu/$ANTIVIRUS:$VERSION + restart: no + env_file: $ENV_FILE + volumes: + - "$ROOT/filter:/data" + + webdav: + image: mailu/$WEBDAV:$VERSION + restart: no + env_file: $ENV_FILE + volumes: + - "$ROOT/dav:/data" + + admin: + image: mailu/admin:$VERSION + restart: no + env_file: $ENV_FILE + volumes: + - "$ROOT/data:/data" + - "$ROOT/dkim:/dkim" + - /var/run/docker.sock:/var/run/docker.sock:ro + depends_on: + - redis + + webmail: + image: "mailu/$WEBMAIL:$VERSION" + restart: no + env_file: $ENV_FILE + volumes: + - "$ROOT/webmail:/data" + depends_on: + - imap + + fetchmail: + image: mailu/fetchmail:$VERSION + restart: no + env_file: $ENV_FILE + volumes: + - "$ROOT/data:/data" From 0067d97f26094c6a79b7a51c9fa204578d28d2e3 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Thu, 4 Oct 2018 11:50:32 +0300 Subject: [PATCH 17/55] Copied docs/compose/.env to tests/compose/core.env --- tests/compose/core.env | 134 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 tests/compose/core.env diff --git a/tests/compose/core.env b/tests/compose/core.env new file mode 100644 index 00000000..9477448a --- /dev/null +++ b/tests/compose/core.env @@ -0,0 +1,134 @@ +# Mailu main configuration file +# +# Most configuration variables can be modified through the Web interface, +# these few settings must however be configured before starting the mail +# server and require a restart upon change. + +################################### +# Common configuration variables +################################### + +# Set this to the path where Mailu data and configuration is stored +ROOT=/mailu + +# Mailu version to run (1.0, 1.1, etc. or master) +VERSION=master + +# Set to a randomly generated 16 bytes string +SECRET_KEY=ChangeMeChangeMe + +# Address where listening ports should bind +BIND_ADDRESS4=127.0.0.1 +BIND_ADDRESS6=::1 + +# Main mail domain +DOMAIN=mailu.io + +# Hostnames for this server, separated with comas +HOSTNAMES=mail.mailu.io,alternative.mailu.io,yetanother.mailu.io + +# Postmaster local part (will append the main mail domain) +POSTMASTER=admin + +# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt) +TLS_FLAVOR=cert + +# Authentication rate limit (per source IP address) +AUTH_RATELIMIT=10/minute;1000/hour + +# Opt-out of statistics, replace with "True" to opt out +DISABLE_STATISTICS=False + +################################### +# Optional features +################################### + +# Expose the admin interface (value: true, false) +ADMIN=false + +# Choose which webmail to run if any (values: roundcube, rainloop, none) +WEBMAIL=none + +# Dav server implementation (value: radicale, none) +WEBDAV=none + +# Antivirus solution (value: clamav, none) +ANTIVIRUS=none + +################################### +# Mail settings +################################### + +# Message size limit in bytes +# Default: accept messages up to 50MB +MESSAGE_SIZE_LIMIT=50000000 + +# Networks granted relay permissions, make sure that you include your Docker +# internal network (default to 172.17.0.0/16) +RELAYNETS=172.16.0.0/12 + +# Will relay all outgoing mails if configured +RELAYHOST= + +# Fetchmail delay +FETCHMAIL_DELAY=600 + +# Recipient delimiter, character used to delimiter localpart from custom address part +# e.g. localpart+custom@domain;tld +RECIPIENT_DELIMITER=+ + +# DMARC rua and ruf email +DMARC_RUA=admin +DMARC_RUF=admin + +# Welcome email, enable and set a topic and body if you wish to send welcome +# emails to all users. +WELCOME=false +WELCOME_SUBJECT=Welcome to your new email account +WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly! + +# Maildir Compression +# choose compression-method, default: none (value: bz2, gz) +COMPRESSION= +# change compression-level, default: 6 (value: 1-9) +COMPRESSION_LEVEL= + +################################### +# Web settings +################################### + +# Path to the admin interface if enabled +WEB_ADMIN=/admin + +# Path to the webmail if enabled +WEB_WEBMAIL=/webmail + +# Website name +SITENAME=Mailu + +# Linked Website URL +WEBSITE=https://mailu.io + +# Registration reCaptcha settings (warning, this has some privacy impact) +# RECAPTCHA_PUBLIC_KEY= +# RECAPTCHA_PRIVATE_KEY= + +# Domain registration, uncomment to enable +# DOMAIN_REGISTRATION=true + +################################### +# Advanced settings +################################### + +# Docker-compose project name, this will prepended to containers names. +COMPOSE_PROJECT_NAME=mailu + +# Default password scheme used for newly created accounts and changed passwords +# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) +PASSWORD_SCHEME=SHA512-CRYPT + +# Header to take the real ip from +REAL_IP_HEADER= + +# IPs for nginx set_real_ip_from (CIDR list separated by commas) +REAL_IP_FROM= From ebe90fc44704be34d5a40c6a87e0e5a7c70e5a6d Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Thu, 4 Oct 2018 14:42:34 +0300 Subject: [PATCH 18/55] Added run script for travis --- .travis.yml | 4 +++- tests/compose/core.env | 2 +- tests/compose/run.yml | 38 ++++++++++++++++++------------------ tests/compose/test-script.sh | 5 +++++ 4 files changed, 28 insertions(+), 21 deletions(-) create mode 100755 tests/compose/test-script.sh diff --git a/.travis.yml b/.travis.yml index d5114c0d..f1d36f3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,4 +8,6 @@ env: - VERSION=$TRAVIS_BRANCH script: -- docker-compose -f tests/build.yml -p Mailu build + - docker-compose -f tests/build.yml -p Mailu build + - tests/compose/test-script.sh + diff --git a/tests/compose/core.env b/tests/compose/core.env index 9477448a..c92c977b 100644 --- a/tests/compose/core.env +++ b/tests/compose/core.env @@ -12,7 +12,7 @@ ROOT=/mailu # Mailu version to run (1.0, 1.1, etc. or master) -VERSION=master +#VERSION=master # Set to a randomly generated 16 bytes string SECRET_KEY=ChangeMeChangeMe diff --git a/tests/compose/run.yml b/tests/compose/run.yml index c1667db3..ceee3a3a 100644 --- a/tests/compose/run.yml +++ b/tests/compose/run.yml @@ -4,8 +4,8 @@ services: front: image: mailu/nginx:$VERSION - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env ports: - "$BIND_ADDRESS4:80:80" - "$BIND_ADDRESS4:443:443" @@ -30,14 +30,14 @@ services: redis: image: redis:alpine - restart: no + restart: 'no' volumes: - "$ROOT/redis:/data" imap: image: mailu/dovecot:$VERSION - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env volumes: - "$ROOT/data:/data" - "$ROOT/mail:/mail" @@ -47,8 +47,8 @@ services: smtp: image: mailu/postfix:$VERSION - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env volumes: - "$ROOT/data:/data" - "$ROOT/overrides:/overrides" @@ -57,8 +57,8 @@ services: antispam: image: mailu/rspamd:$VERSION - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env volumes: - "$ROOT/filter:/var/lib/rspamd" - "$ROOT/dkim:/dkim" @@ -68,22 +68,22 @@ services: antivirus: image: mailu/$ANTIVIRUS:$VERSION - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env volumes: - "$ROOT/filter:/data" webdav: image: mailu/$WEBDAV:$VERSION - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env volumes: - "$ROOT/dav:/data" admin: image: mailu/admin:$VERSION - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env volumes: - "$ROOT/data:/data" - "$ROOT/dkim:/dkim" @@ -93,8 +93,8 @@ services: webmail: image: "mailu/$WEBMAIL:$VERSION" - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env volumes: - "$ROOT/webmail:/data" depends_on: @@ -102,7 +102,7 @@ services: fetchmail: image: mailu/fetchmail:$VERSION - restart: no - env_file: $ENV_FILE + restart: 'no' + env_file: $PWD/.env volumes: - "$ROOT/data:/data" diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh new file mode 100755 index 00000000..2f33f12f --- /dev/null +++ b/tests/compose/test-script.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for file in ../*.env ; do + cp $file .env + docker-compose -f ../run.yml up -d +done From 28ebcd500775124a64aecdb4397cd371f470abaf Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Thu, 4 Oct 2018 15:19:40 +0300 Subject: [PATCH 19/55] Fixed paths to files --- tests/compose/test-script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 2f33f12f..35dba8c3 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -1,5 +1,5 @@ #!/bin/bash -for file in ../*.env ; do +for file in tests/compose/*.env ; do cp $file .env - docker-compose -f ../run.yml up -d + docker-compose -f tests/compose/run.yml up -d done From 645a7f3c96cbc2be5f8ba36dbcdd79b8dce2df21 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Thu, 4 Oct 2018 15:41:44 +0300 Subject: [PATCH 20/55] Removed BIND_ADDRESS6 --- tests/compose/core.env | 2 +- tests/compose/run.yml | 9 --------- tests/compose/test-script.sh | 1 + 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/compose/core.env b/tests/compose/core.env index c92c977b..67b746f8 100644 --- a/tests/compose/core.env +++ b/tests/compose/core.env @@ -19,7 +19,7 @@ SECRET_KEY=ChangeMeChangeMe # Address where listening ports should bind BIND_ADDRESS4=127.0.0.1 -BIND_ADDRESS6=::1 +#BIND_ADDRESS6=::1 # Main mail domain DOMAIN=mailu.io diff --git a/tests/compose/run.yml b/tests/compose/run.yml index ceee3a3a..631aaa83 100644 --- a/tests/compose/run.yml +++ b/tests/compose/run.yml @@ -16,15 +16,6 @@ services: - "$BIND_ADDRESS4:25:25" - "$BIND_ADDRESS4:465:465" - "$BIND_ADDRESS4:587:587" - - "$BIND_ADDRESS6:80:80" - - "$BIND_ADDRESS6:443:443" - - "$BIND_ADDRESS6:110:110" - - "$BIND_ADDRESS6:143:143" - - "$BIND_ADDRESS6:993:993" - - "$BIND_ADDRESS6:995:995" - - "$BIND_ADDRESS6:25:25" - - "$BIND_ADDRESS6:465:465" - - "$BIND_ADDRESS6:587:587" volumes: - "$ROOT/certs:/certs" diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 35dba8c3..a0bb3c3b 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -3,3 +3,4 @@ for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml up -d done +sleep 2m From bc70be35c22007aadf72d3f6c3ddca71e2665993 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Thu, 4 Oct 2018 17:18:48 +0300 Subject: [PATCH 21/55] Added function to check docker containers --- tests/compose/test-script.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index a0bb3c3b..ffc36373 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -1,6 +1,14 @@ #!/bin/bash +containers_check() { + for container in mailu_webmail_1 mailu_imap_1 mailu_smtp_1 mailu_antispam_1 mailu_admin_1 mailu_redis_1 mailu_antivirus_1 mailu_webdav_1 mailu_fetchmail_1 mailu_front_1; do + docker inspect $container | grep '"Status": "running"' || exit 1 + done +} + for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml up -d + sleep 1m + containers_check done -sleep 2m + From 027b8f36dfb5a2a8b0a479324af7a00294a85e11 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Thu, 4 Oct 2018 17:37:09 +0300 Subject: [PATCH 22/55] Added a check for container status --- tests/compose/test-script.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index ffc36373..27d78585 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -1,7 +1,8 @@ #!/bin/bash containers_check() { for container in mailu_webmail_1 mailu_imap_1 mailu_smtp_1 mailu_antispam_1 mailu_admin_1 mailu_redis_1 mailu_antivirus_1 mailu_webdav_1 mailu_fetchmail_1 mailu_front_1; do - docker inspect $container | grep '"Status": "running"' || exit 1 + echo "Checking $container" + docker inspect $container | grep '"Status": "running"' || exit 1 done } From 71eed72e212689e28a4dd366aba4df5b15219fe1 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Thu, 4 Oct 2018 17:51:17 +0300 Subject: [PATCH 23/55] Changed exit condition --- tests/compose/test-script.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 27d78585..47b1fc8c 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -1,15 +1,17 @@ #!/bin/bash containers_check() { + STATUS=0 for container in mailu_webmail_1 mailu_imap_1 mailu_smtp_1 mailu_antispam_1 mailu_admin_1 mailu_redis_1 mailu_antivirus_1 mailu_webdav_1 mailu_fetchmail_1 mailu_front_1; do echo "Checking $container" - docker inspect $container | grep '"Status": "running"' || exit 1 + docker inspect $container | grep '"Status": "running"' || STATUS=1 done + return $STATUS } for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml up -d sleep 1m - containers_check + containers_check || exit 1 done From cc17962c86fdbd9e16fae4b5f5e613021bccc626 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 4 Oct 2018 17:25:22 +0000 Subject: [PATCH 24/55] fixes #583 --- core/postfix/conf/master.cf | 1 + 1 file changed, 1 insertion(+) diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index cbcc5e56..aa1e967a 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -8,6 +8,7 @@ smtp inet n - n - - smtpd 10025 inet n - n - - smtpd -o smtpd_sasl_auth_enable=yes -o smtpd_recipient_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit + -o smtpd_reject_unlisted_recipient={{ REJECT_UNLISTED_RECIPIENT }} -o cleanup_service_name=outclean outclean unix n - n - 0 cleanup -o header_checks=pcre:/etc/postfix/outclean_header_filter.cf From 09d77bc2de8b639b1c655b18c290314297836345 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 4 Oct 2018 18:31:04 +0000 Subject: [PATCH 25/55] Handle the case where the variable REJECT_UNLISTED_RECIPIENT is not set --- core/postfix/conf/master.cf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/postfix/conf/master.cf b/core/postfix/conf/master.cf index aa1e967a..15fd62dc 100644 --- a/core/postfix/conf/master.cf +++ b/core/postfix/conf/master.cf @@ -8,7 +8,7 @@ smtp inet n - n - - smtpd 10025 inet n - n - - smtpd -o smtpd_sasl_auth_enable=yes -o smtpd_recipient_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit - -o smtpd_reject_unlisted_recipient={{ REJECT_UNLISTED_RECIPIENT }} + -o smtpd_reject_unlisted_recipient={% if REJECT_UNLISTED_RECIPIENT %}{{ REJECT_UNLISTED_RECIPIENT }}{% else %}no{% endif %} -o cleanup_service_name=outclean outclean unix n - n - 0 cleanup -o header_checks=pcre:/etc/postfix/outclean_header_filter.cf From 58a83a93e6599c1b0c090bb2bc237f37d2b10f1d Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Thu, 4 Oct 2018 18:44:27 +0000 Subject: [PATCH 26/55] Add REJECT_UNLISTED_RECIPIENT variable in .env file --- docs/compose/.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/compose/.env b/docs/compose/.env index 9477448a..721aaf22 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -132,3 +132,6 @@ REAL_IP_HEADER= # IPs for nginx set_real_ip_from (CIDR list separated by commas) REAL_IP_FROM= + +# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no) +REJECT_UNLISTED_RECIPIENT= From 20f37607bbbcec0a61d1f1d7938a3f3cbb8e5ae6 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Fri, 5 Oct 2018 11:55:01 +0300 Subject: [PATCH 27/55] Added docker ps and logs --- tests/compose/test-script.sh | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 47b1fc8c..711fd985 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -1,17 +1,39 @@ #!/bin/bash +containers=( + mailu_webmail_1 + mailu_imap_1 + mailu_smtp_1 + mailu_antispam_1 + mailu_admin_1 + mailu_redis_1 + mailu_antivirus_1 + mailu_webdav_1 +# mailu_fetchmail_1 + mailu_front_1 +) + containers_check() { STATUS=0 - for container in mailu_webmail_1 mailu_imap_1 mailu_smtp_1 mailu_antispam_1 mailu_admin_1 mailu_redis_1 mailu_antivirus_1 mailu_webdav_1 mailu_fetchmail_1 mailu_front_1; do + for container in "${containers[@]}"; do echo "Checking $container" docker inspect $container | grep '"Status": "running"' || STATUS=1 done return $STATUS } +container_logs() { + for container in "${containers[@]}"; do + echo "Showing logs for $container" + docker container logs $container + done +} + for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml up -d sleep 1m + docker ps + container_logs containers_check || exit 1 done From 59877036318607a1a7c9ae1c52719d7358078174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 5 Oct 2018 11:51:35 +0300 Subject: [PATCH 28/55] Tavis-ci to upload images to the Docker hub, after succesfull tests. For this to work, the following variables will need to be set at Docker hub: - DOCKER_ORG first part of docker repo name (example: mailu) - DOCKER_UN Docker-hub login username (private) - DOCKER_PW Docker-hub login password (private) Note that also Docker hub autobuilds will have to be disabled. --- .travis.yml | 4 ++++ tests/build.yml | 22 +++++++++++----------- tests/deploy.sh | 4 ++++ 3 files changed, 19 insertions(+), 11 deletions(-) create mode 100755 tests/deploy.sh diff --git a/.travis.yml b/.travis.yml index f1d36f3f..9fc1d02f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,3 +11,7 @@ script: - docker-compose -f tests/build.yml -p Mailu build - tests/compose/test-script.sh +deploy: + provider: script + script: bash tests/deploy.sh + diff --git a/tests/build.yml b/tests/build.yml index 674abf8c..a195fdf7 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -3,45 +3,45 @@ version: '3' services: front: - image: mailu/nginx:$VERSION + image: $DOCKER_ORG/nginx:$VERSION build: ../core/nginx imap: - image: mailu/dovecot:$VERSION + image: $DOCKER_ORG/dovecot:$VERSION build: ../core/dovecot smtp: - image: mailu/postfix:$VERSION + image: $DOCKER_ORG/postfix:$VERSION build: ../core/postfix antispam: - image: mailu/rspamd:$VERSION + image: $DOCKER_ORG/rspamd:$VERSION build: ../services/rspamd antivirus: - image: mailu/clamav:$VERSION + image: $DOCKER_ORG/clamav:$VERSION build: ../optional/clamav webdav: - image: mailu/radicale:$VERSION + image: $DOCKER_ORG/radicale:$VERSION build: ../optional/radicale admin: - image: mailu/admin:$VERSION + image: $DOCKER_ORG/admin:$VERSION build: ../core/admin roundcube: - image: mailu/roundcube:$VERSION + image: $DOCKER_ORG/roundcube:$VERSION build: ../webmails/roundcube rainloop: - image: mailu/rainloop:$VERSION + image: $DOCKER_ORG/rainloop:$VERSION build: ../webmails/rainloop fetchmail: - image: mailu/fetchmail:$VERSION + image: $DOCKER_ORG/fetchmail:$VERSION build: ../services/fetchmail none: - image: mailu/none:$VERSION + image: $DOCKER_ORG/none:$VERSION build: ../core/none diff --git a/tests/deploy.sh b/tests/deploy.sh new file mode 100755 index 00000000..0526a217 --- /dev/null +++ b/tests/deploy.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +docker login -u $DOCKER_UN -p $DOCKER_PW +docker-compose -f tests/build.yml push From 3224a8ecad8a5e2af50ab77c513af4040003f08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 5 Oct 2018 12:52:19 +0300 Subject: [PATCH 29/55] Further introduction of the DOCKER_ORG variable --- tests/compose/core.env | 2 +- tests/compose/run.yml | 18 +++++++++--------- tests/compose/test-script.sh | 30 +++++++++++++++--------------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/compose/core.env b/tests/compose/core.env index 67b746f8..89120d4f 100644 --- a/tests/compose/core.env +++ b/tests/compose/core.env @@ -121,7 +121,7 @@ WEBSITE=https://mailu.io ################################### # Docker-compose project name, this will prepended to containers names. -COMPOSE_PROJECT_NAME=mailu +#COMPOSE_PROJECT_NAME=mailu # Default password scheme used for newly created accounts and changed passwords # (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT) diff --git a/tests/compose/run.yml b/tests/compose/run.yml index 631aaa83..56ea1627 100644 --- a/tests/compose/run.yml +++ b/tests/compose/run.yml @@ -3,7 +3,7 @@ version: '2' services: front: - image: mailu/nginx:$VERSION + image: $DOCKER_ORG/nginx:$VERSION restart: 'no' env_file: $PWD/.env ports: @@ -26,7 +26,7 @@ services: - "$ROOT/redis:/data" imap: - image: mailu/dovecot:$VERSION + image: $DOCKER_ORG/dovecot:$VERSION restart: 'no' env_file: $PWD/.env volumes: @@ -37,7 +37,7 @@ services: - front smtp: - image: mailu/postfix:$VERSION + image: $DOCKER_ORG/postfix:$VERSION restart: 'no' env_file: $PWD/.env volumes: @@ -47,7 +47,7 @@ services: - front antispam: - image: mailu/rspamd:$VERSION + image: $DOCKER_ORG/rspamd:$VERSION restart: 'no' env_file: $PWD/.env volumes: @@ -58,21 +58,21 @@ services: - front antivirus: - image: mailu/$ANTIVIRUS:$VERSION + image: $DOCKER_ORG/$ANTIVIRUS:$VERSION restart: 'no' env_file: $PWD/.env volumes: - "$ROOT/filter:/data" webdav: - image: mailu/$WEBDAV:$VERSION + image: $DOCKER_ORG/$WEBDAV:$VERSION restart: 'no' env_file: $PWD/.env volumes: - "$ROOT/dav:/data" admin: - image: mailu/admin:$VERSION + image: $DOCKER_ORG/admin:$VERSION restart: 'no' env_file: $PWD/.env volumes: @@ -83,7 +83,7 @@ services: - redis webmail: - image: "mailu/$WEBMAIL:$VERSION" + image: "$DOCKER_ORG/$WEBMAIL:$VERSION" restart: 'no' env_file: $PWD/.env volumes: @@ -92,7 +92,7 @@ services: - imap fetchmail: - image: mailu/fetchmail:$VERSION + image: $DOCKER_ORG/fetchmail:$VERSION restart: 'no' env_file: $PWD/.env volumes: diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 711fd985..d7d5b23c 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -1,36 +1,36 @@ #!/bin/bash containers=( - mailu_webmail_1 - mailu_imap_1 - mailu_smtp_1 - mailu_antispam_1 - mailu_admin_1 - mailu_redis_1 - mailu_antivirus_1 - mailu_webdav_1 -# mailu_fetchmail_1 - mailu_front_1 + webmail_1 + imap_1 + smtp_1 + antispam_1 + admin_1 + redis_1 + antivirus_1 + webdav_1 +# fetchmail_1 + front_1 ) containers_check() { STATUS=0 for container in "${containers[@]}"; do - echo "Checking $container" - docker inspect $container | grep '"Status": "running"' || STATUS=1 + echo "Checking ${DOCKER_ORG}_${container}" + docker inspect "${DOCKER_ORG}_${container}" | grep '"Status": "running"' || STATUS=1 done return $STATUS } container_logs() { for container in "${containers[@]}"; do - echo "Showing logs for $container" - docker container logs $container + echo "Showing logs for ${DOCKER_ORG}_${container}" + docker container logs "${DOCKER_ORG}_${container}" done } for file in tests/compose/*.env ; do cp $file .env - docker-compose -f tests/compose/run.yml up -d + docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d sleep 1m docker ps container_logs From 0c73be7c6e00ad8561642cff156f3d1157bd1513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Fri, 5 Oct 2018 13:02:10 +0300 Subject: [PATCH 30/55] Deploy on all branches --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9fc1d02f..2baa1183 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,4 +14,6 @@ script: deploy: provider: script script: bash tests/deploy.sh + on: + all_branches: true From 38db35f1d6a52ab6e7a0551b73a6722e42bc37e0 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Fri, 5 Oct 2018 15:04:04 +0300 Subject: [PATCH 31/55] Make docker ps more verbose --- tests/compose/test-script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index d7d5b23c..05285c2c 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -8,7 +8,7 @@ containers=( redis_1 antivirus_1 webdav_1 -# fetchmail_1 + fetchmail_1 front_1 ) @@ -32,7 +32,7 @@ for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d sleep 1m - docker ps + docker ps -a container_logs containers_check || exit 1 done From 4133bab280ac08d7285bf57d076cce28bb8bf404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 15:52:39 +0300 Subject: [PATCH 32/55] Default to mailu for DOCKER_ORG --- tests/compose/test-script.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 05285c2c..8b049cd6 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -12,6 +12,11 @@ containers=( front_1 ) +# Default to mailu for DOCKER_ORG +if [ -z "$DOCKER_ORG" ]; then + export DOCKER_ORG="mailu" +fi + containers_check() { STATUS=0 for container in "${containers[@]}"; do From 42f557ff382a2750752adfd9abd1610cff68ad0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 16:03:07 +0300 Subject: [PATCH 33/55] Cleanup containers --- tests/compose/test-script.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 8b049cd6..5fdddf04 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -33,12 +33,24 @@ container_logs() { done } +clean() { + docker-compose -f tests/compose/run.yml -p $DOCKER_ORG down || exit 1 + rm -fv .env +} + +# Cleanup before callig exit +die() { + clean + exit $1 +} + for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d sleep 1m docker ps -a container_logs - containers_check || exit 1 + containers_check || die 1 + clean done From 59eb4a5a00fbb4c283d244de4fef448152fc2380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 16:12:05 +0300 Subject: [PATCH 34/55] Minor script flow restructure --- tests/compose/test-script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 5fdddf04..7e3ab266 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -18,11 +18,13 @@ if [ -z "$DOCKER_ORG" ]; then fi containers_check() { + sleep 1m STATUS=0 for container in "${containers[@]}"; do echo "Checking ${DOCKER_ORG}_${container}" docker inspect "${DOCKER_ORG}_${container}" | grep '"Status": "running"' || STATUS=1 done + docker ps -a return $STATUS } @@ -47,8 +49,6 @@ die() { for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d - sleep 1m - docker ps -a container_logs containers_check || die 1 clean From 508796eaabcf1934c3dca3c7c5d2f56504b45228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 16:25:12 +0300 Subject: [PATCH 35/55] Verbose sleep --- tests/compose/test-script.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 7e3ab266..f287b4a0 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -17,8 +17,19 @@ if [ -z "$DOCKER_ORG" ]; then export DOCKER_ORG="mailu" fi +# Verbose sleep, to prevent Travis to cancel the build +# First argument is desired sleep time in minutes +v_sleep() { + COUNT=$1 + until [ $COUNT -eq 0 ]; do + echo "Sleep for $COUNT more minutes" + sleep 1m + ((COUNT--)) + done; +} + containers_check() { - sleep 1m + v_sleep 1 STATUS=0 for container in "${containers[@]}"; do echo "Checking ${DOCKER_ORG}_${container}" From 39c159bae9d74b5e56051c8e63e5f5d00bbc4064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 16:34:57 +0300 Subject: [PATCH 36/55] Make the code a bit more DRY. Put sleep back into main loop --- tests/compose/test-script.sh | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index f287b4a0..4ad8ad28 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -1,15 +1,15 @@ #!/bin/bash containers=( - webmail_1 - imap_1 - smtp_1 - antispam_1 - admin_1 - redis_1 - antivirus_1 - webdav_1 - fetchmail_1 - front_1 + webmail + imap + smtp + antispam + admin + redis + antivirus + webdav + fetchmail + front ) # Default to mailu for DOCKER_ORG @@ -29,20 +29,21 @@ v_sleep() { } containers_check() { - v_sleep 1 - STATUS=0 + status=0 for container in "${containers[@]}"; do - echo "Checking ${DOCKER_ORG}_${container}" - docker inspect "${DOCKER_ORG}_${container}" | grep '"Status": "running"' || STATUS=1 + name="${DOCKER_ORG}_${container}_1" + echo "Checking $name" + docker inspect "$name" | grep '"Status": "running"' || status=1 done docker ps -a - return $STATUS + return $status } container_logs() { for container in "${containers[@]}"; do - echo "Showing logs for ${DOCKER_ORG}_${container}" - docker container logs "${DOCKER_ORG}_${container}" + name="${DOCKER_ORG}_${container}_1" + echo "Showing logs for $name" + docker container logs "$name" done } @@ -60,6 +61,7 @@ die() { for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d + v_sleep 1 container_logs containers_check || die 1 clean From bc85dff27e349749d763579d0781cc9b3792830b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 16:43:04 +0300 Subject: [PATCH 37/55] Don't try deploy is DOCKER_UN is not set --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2baa1183..66c8f405 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,4 +16,5 @@ deploy: script: bash tests/deploy.sh on: all_branches: true + condition: -z $DOCKER_UN From bb6b984610bc3ab8956a3728d7e83e42e6121a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 16:46:01 +0300 Subject: [PATCH 38/55] Exclude fetchmail for the time being. See Mailu/Mailu#412 --- tests/compose/test-script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 4ad8ad28..59b8c8a4 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -8,7 +8,7 @@ containers=( redis antivirus webdav - fetchmail +# fetchmail front ) From 7915f2631fdbf46f1fc8ec0d141ea9955aaf1d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 17:04:14 +0300 Subject: [PATCH 39/55] DOCKER_UN needs to be non-zero for deploy --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 66c8f405..1b07da5f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,5 +16,5 @@ deploy: script: bash tests/deploy.sh on: all_branches: true - condition: -z $DOCKER_UN + condition: -n $DOCKER_UN From a19e11d5524b395e059724f6f816ecf025458d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 17:06:23 +0300 Subject: [PATCH 40/55] Clean terminal distortion from docker-compose in travis --- tests/compose/test-script.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 59b8c8a4..30240bd0 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -61,6 +61,7 @@ die() { for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d + echo "\n" # Clean terminal distortion from docker-compose in travis v_sleep 1 container_logs containers_check || die 1 From f7d8f20c87ff1b412bcbebf359da31e229ddad80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 17:24:42 +0300 Subject: [PATCH 41/55] Move DOCKER_ORG default to travis.yml --- .travis.yml | 2 ++ tests/compose/test-script.sh | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b07da5f..a8fe1709 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,8 @@ env: - VERSION=$TRAVIS_BRANCH script: + # Default to mailu for DOCKER_ORG + - if [ -z "$DOCKER_ORG" ]; then; export DOCKER_ORG="mailu"; fi - docker-compose -f tests/build.yml -p Mailu build - tests/compose/test-script.sh diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index 30240bd0..a1ae2cf4 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -12,11 +12,6 @@ containers=( front ) -# Default to mailu for DOCKER_ORG -if [ -z "$DOCKER_ORG" ]; then - export DOCKER_ORG="mailu" -fi - # Verbose sleep, to prevent Travis to cancel the build # First argument is desired sleep time in minutes v_sleep() { From 940b88a641c70e5d849e66215aa9c73883f9a3b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 17:31:01 +0300 Subject: [PATCH 42/55] Fix syntax error --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a8fe1709..8dbc2206 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ env: script: # Default to mailu for DOCKER_ORG - - if [ -z "$DOCKER_ORG" ]; then; export DOCKER_ORG="mailu"; fi + - if [ -z "$DOCKER_ORG" ]; then export DOCKER_ORG="mailu"; fi - docker-compose -f tests/build.yml -p Mailu build - tests/compose/test-script.sh From c09ac713a37f04538dc1c3911f38008307cd2aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 17:36:59 +0300 Subject: [PATCH 43/55] Project name can be ommitted in docker-compose build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8dbc2206..c3a19529 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ env: script: # Default to mailu for DOCKER_ORG - if [ -z "$DOCKER_ORG" ]; then export DOCKER_ORG="mailu"; fi - - docker-compose -f tests/build.yml -p Mailu build + - docker-compose -f tests/build.yml build - tests/compose/test-script.sh deploy: From c135b37b076c148597eb59c1dcb3b5d38fb144ee Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 6 Oct 2018 18:53:49 +0200 Subject: [PATCH 44/55] Enable mergify --- .mergify.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .mergify.yml diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 00000000..27d34606 --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,11 @@ +rules: + default: null + branches: + master: + protection: + required_status_checks: + strict: true + contexts: + - continuous-integration/travis-ci + required_pull_request_reviews: + required_approving_review_count: 2 From 7a7854bf3fcdaecc8a23193f6707bf5dab2a879b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 6 Oct 2018 19:09:59 +0200 Subject: [PATCH 45/55] Disable strict checking --- .mergify.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.mergify.yml b/.mergify.yml index 27d34606..7195e58e 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,7 +4,6 @@ rules: master: protection: required_status_checks: - strict: true contexts: - continuous-integration/travis-ci required_pull_request_reviews: From 5716ca933e26c4976124f7ae593b0768b9f75d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 20:24:54 +0300 Subject: [PATCH 46/55] Fix autodeploy after merge with master --- tests/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build.yml b/tests/build.yml index 88040160..256e3f73 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -47,5 +47,5 @@ services: build: ../core/none docs: - image: mailu/docs:$VERSION + image: $DOCKER_ORG/docs:$VERSION build: ../docs From 1bbb86eab5aa60c9045b614edeec42934625c0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sat, 6 Oct 2018 20:26:39 +0300 Subject: [PATCH 47/55] Add autobuild for 'setup' --- tests/build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/build.yml b/tests/build.yml index 256e3f73..0b6858a0 100644 --- a/tests/build.yml +++ b/tests/build.yml @@ -49,3 +49,8 @@ services: docs: image: $DOCKER_ORG/docs:$VERSION build: ../docs + + setup: + image: $DOCKER_ORG/setup:$VERSION + build: ../setup + From c457ccfa6060229faf39cfa5aceedd0fd047d1d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sun, 7 Oct 2018 00:32:05 +0300 Subject: [PATCH 48/55] Use tenacity for resolver retries --- core/dovecot/Dockerfile | 3 ++- core/dovecot/start.py | 26 +++++++++----------------- core/postfix/Dockerfile | 3 ++- core/postfix/start.py | 19 ++++++------------- services/rspamd/Dockerfile | 3 ++- services/rspamd/start.py | 19 ++++++------------- 6 files changed, 27 insertions(+), 46 deletions(-) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 363a7244..3bfd67fc 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -3,7 +3,8 @@ FROM alpine:3.7 RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \ && apk add --no-cache \ dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \ - dovecot-fts-lucene rspamd-client@testing python py-jinja2 + dovecot-fts-lucene rspamd-client@testing python py-jinja2 py-pip \ + && pip install tenacity COPY conf /conf COPY sieve /var/lib/dovecot diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 2de114fd..bab5f1ee 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -4,27 +4,19 @@ import jinja2 import os import socket import glob -import time +from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -# Actual startup script -i = 0 -t = 10 -while True: - i += 1 - try: - os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) - os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis")) - if os.environ["WEBMAIL"] != "none": - os.environ["WEBMAIL_ADDRESS"] = socket.gethostbyname(os.environ.get("WEBMAIL_ADDRESS", "webmail")) - except socket.gaierror as err: - if i >= t: - raise - time.sleep(10) - continue - break +@retry(stop=stop_after_attempt(10), wait=wait_random(min=2, max=5)) +def resolve(): + os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) + os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis")) + if os.environ["WEBMAIL"] != "none": + os.environ["WEBMAIL_ADDRESS"] = socket.gethostbyname(os.environ.get("WEBMAIL_ADDRESS", "webmail")) +# Actual startup script +resolve() for dovecot_file in glob.glob("/conf/*"): convert(dovecot_file, os.path.join("/etc/dovecot", os.path.basename(dovecot_file))) diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index d853c9f9..61b8f1ea 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,6 +1,7 @@ FROM alpine:3.7 -RUN apk add --no-cache postfix postfix-sqlite postfix-pcre rsyslog python py-jinja2 +RUN apk add --no-cache postfix postfix-sqlite postfix-pcre rsyslog python py-jinja2 py-pip \ + && pip install tenacity COPY conf /conf COPY start.py /start.py diff --git a/core/postfix/start.py b/core/postfix/start.py index f3c6aaca..30167da5 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -5,23 +5,16 @@ import os import socket import glob import shutil -import time +from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +@retry(stop=stop_after_attempt(10), wait=wait_random(min=2, max=5)) +def resolve(): + os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) + # Actual startup script -i = 0 -t = 10 -while True: - i += 1 - try: - os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) - except socket.gaierror as err: - if i >= t: - raise - time.sleep(10) - continue - break +resolve() os.environ["HOST_ANTISPAM"] = os.environ.get("HOST_ANTISPAM", "antispam:11332") os.environ["HOST_LMTP"] = os.environ.get("HOST_LMTP", "imap:2525") diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index d5e93db7..e8eb49f9 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -1,6 +1,7 @@ FROM alpine:edge -RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates +RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \ + && pip install tenacity RUN mkdir /run/rspamd diff --git a/services/rspamd/start.py b/services/rspamd/start.py index 08301a0d..cb9a1788 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -4,23 +4,16 @@ import jinja2 import os import socket import glob -import time +from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +@retry(stop=stop_after_attempt(10), wait=wait_random(min=2, max=5)) +def resolve(): + os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) + # Actual startup script -i = 0 -t = 10 -while True: - i += 1 - try: - os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) - except socket.gaierror as err: - if i >= t: - raise - time.sleep(10) - continue - break +resolve() if "HOST_REDIS" not in os.environ: os.environ["HOST_REDIS"] = "redis" for rspamd_file in glob.glob("/conf/*"): From 1bae5968ade8a945873c63c5e5f7bbe92564fcf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sun, 7 Oct 2018 01:39:02 +0300 Subject: [PATCH 49/55] Import tenacy and fix syntax errors --- core/dovecot/start.py | 3 ++- core/postfix/start.py | 3 ++- services/rspamd/start.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index bab5f1ee..40e1edf0 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -4,11 +4,12 @@ import jinja2 import os import socket import glob +import tenacity from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -@retry(stop=stop_after_attempt(10), wait=wait_random(min=2, max=5)) +@retry(stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_random(min=2, max=5)) def resolve(): os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis")) diff --git a/core/postfix/start.py b/core/postfix/start.py index 30167da5..bfa13da2 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -5,11 +5,12 @@ import os import socket import glob import shutil +import tenacity from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -@retry(stop=stop_after_attempt(10), wait=wait_random(min=2, max=5)) +@retry(stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_random(min=2, max=5)) def resolve(): os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) diff --git a/services/rspamd/start.py b/services/rspamd/start.py index cb9a1788..3ef309b2 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -4,11 +4,12 @@ import jinja2 import os import socket import glob +import tenacity from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -@retry(stop=stop_after_attempt(10), wait=wait_random(min=2, max=5)) +@retry(stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_random(min=2, max=5)) def resolve(): os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) From 16469d72823c36718a30691cb210cb74fcd2fc70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sun, 7 Oct 2018 01:40:22 +0300 Subject: [PATCH 50/55] Upgrade to newer pip version --- core/dovecot/Dockerfile | 1 + core/postfix/Dockerfile | 1 + services/rspamd/Dockerfile | 1 + 3 files changed, 3 insertions(+) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 3bfd67fc..a9a3f854 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -4,6 +4,7 @@ RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/re && apk add --no-cache \ dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \ dovecot-fts-lucene rspamd-client@testing python py-jinja2 py-pip \ + && pip install --upgrade pip && pip install tenacity COPY conf /conf diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index 61b8f1ea..bb7acb9b 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -1,6 +1,7 @@ FROM alpine:3.7 RUN apk add --no-cache postfix postfix-sqlite postfix-pcre rsyslog python py-jinja2 py-pip \ + && pip install --upgrade pip \ && pip install tenacity COPY conf /conf diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index e8eb49f9..987e5ab0 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -1,6 +1,7 @@ FROM alpine:edge RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \ + && pip install --upgrade pip \ && pip install tenacity RUN mkdir /run/rspamd From 716ed16f34056e717fd92a851c047bc5b3f77e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sun, 7 Oct 2018 01:52:52 +0300 Subject: [PATCH 51/55] Fix typo --- core/dovecot/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index a9a3f854..36effc6a 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -4,7 +4,7 @@ RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/re && apk add --no-cache \ dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \ dovecot-fts-lucene rspamd-client@testing python py-jinja2 py-pip \ - && pip install --upgrade pip + && pip install --upgrade pip \ && pip install tenacity COPY conf /conf From 081762986990bd26ddf7d2cd891e60c57e6ec50b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sun, 7 Oct 2018 02:10:13 +0300 Subject: [PATCH 52/55] Increase attempts as it failed on fresh Swarm host --- core/dovecot/start.py | 2 +- core/postfix/start.py | 2 +- services/rspamd/start.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 40e1edf0..0aa7a365 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -9,7 +9,7 @@ from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -@retry(stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_random(min=2, max=5)) +@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) def resolve(): os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis")) diff --git a/core/postfix/start.py b/core/postfix/start.py index bfa13da2..e3c13110 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -10,7 +10,7 @@ from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -@retry(stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_random(min=2, max=5)) +@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) def resolve(): os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) diff --git a/services/rspamd/start.py b/services/rspamd/start.py index 3ef309b2..b979517e 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -9,7 +9,7 @@ from tenacity import retry convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -@retry(stop=tenacity.stop_after_attempt(10), wait=tenacity.wait_random(min=2, max=5)) +@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) def resolve(): os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front")) From c316c040c5bed10e8480bfb61bede0603970bb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Sun, 7 Oct 2018 14:08:45 +0300 Subject: [PATCH 53/55] Use travis_wait for sleep --- tests/compose/test-script.sh | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/compose/test-script.sh b/tests/compose/test-script.sh index a1ae2cf4..0a3c2237 100755 --- a/tests/compose/test-script.sh +++ b/tests/compose/test-script.sh @@ -12,16 +12,8 @@ containers=( front ) -# Verbose sleep, to prevent Travis to cancel the build -# First argument is desired sleep time in minutes -v_sleep() { - COUNT=$1 - until [ $COUNT -eq 0 ]; do - echo "Sleep for $COUNT more minutes" - sleep 1m - ((COUNT--)) - done; -} +# Time to sleep in minutes after starting the containers +WAIT=1 containers_check() { status=0 @@ -56,8 +48,8 @@ die() { for file in tests/compose/*.env ; do cp $file .env docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d - echo "\n" # Clean terminal distortion from docker-compose in travis - v_sleep 1 + echo -e "\nSleeping for ${WAIT} minutes" # Clean terminal distortion from docker-compose in travis + travis_wait sleep ${WAIT}m || sleep ${WAIT}m #Fallback sleep for local run container_logs containers_check || die 1 clean From d59d6b0d0d59855f8701e8e43f3fc4557f7d8095 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Mon, 8 Oct 2018 16:35:09 +0300 Subject: [PATCH 54/55] Added chwon command --- webmails/rainloop/start.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/webmails/rainloop/start.py b/webmails/rainloop/start.py index a9f5da32..26d849f7 100755 --- a/webmails/rainloop/start.py +++ b/webmails/rainloop/start.py @@ -19,3 +19,6 @@ convert("/default.ini", "/data/_data_/_default_/domains/default.ini") convert("/config.ini", "/data/_data_/_default_/configs/config.ini") os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"]) + +os.system("chown -R www-data:www-data /data") + From 29e4ca0768c8992c02453c2da067ae5da18e3d30 Mon Sep 17 00:00:00 2001 From: Ionut Filip Date: Mon, 8 Oct 2018 16:47:10 +0300 Subject: [PATCH 55/55] Changed command placement --- webmails/rainloop/start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webmails/rainloop/start.py b/webmails/rainloop/start.py index 26d849f7..9e8465a2 100755 --- a/webmails/rainloop/start.py +++ b/webmails/rainloop/start.py @@ -18,7 +18,7 @@ os.makedirs(base + "configs", exist_ok=True) convert("/default.ini", "/data/_data_/_default_/domains/default.ini") convert("/config.ini", "/data/_data_/_default_/configs/config.ini") -os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"]) - os.system("chown -R www-data:www-data /data") +os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"]) +