Merge #1082

1082: Use socrate instead of Mailustart r=mergify[bot] a=hoellen

## What type of PR?

enhancement

## What does this PR do?
This PR updates the `Dockerfile`, `setup.py` and `config.py` of each image to support the new [Mailu/socrate](https://github.com/Mailu/socrate) python package. So [MailuStart](https://github.com/Mailu/MailuStart) is not used anymore for resolving DNS and configuration processing. 

## Prerequistes
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [x] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/guide.html#changelog) entry file.


Co-authored-by: hoellen <dev@hoellen.eu>

core/dovecot/Dockerfile

@@ -3,10 +3,13 @@ FROM alpine:3.10
 RUN apk add --no-cache \
     python3 py3-pip git bash \
   && pip3 install --upgrade pip
-# Shared layer between rspamd, postfix, dovecot, unbound, rainloop, roundcube and nginx
-RUN pip3 install git+https://github.com/Mailu/MailuStart.git#egg=mailustart
+
+# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
+RUN pip3 install socrate
+
 # Shared layer between dovecot and postfix
 RUN pip3 install "podop>0.2.5"
+
 # Image specific layers under this line
 RUN apk add --no-cache \
   dovecot dovecot-lmtpd dovecot-pop3d dovecot-submissiond dovecot-pigeonhole-plugin rspamd-client \

core/dovecot/start.py

@@ -5,9 +5,9 @@ import glob
 import multiprocessing
 import logging as log
 import sys
-from mailustart import resolve, convert
 
-from podop import run_server
+from podop   import run_server
+from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
@@ -21,14 +21,14 @@ def start_podop():
     ])
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
-os.environ["REDIS_ADDRESS"] = resolve(os.environ.get("REDIS_ADDRESS", "redis"))
-os.environ["ADMIN_ADDRESS"] = resolve(os.environ.get("ADMIN_ADDRESS", "admin"))
+os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("FRONT_ADDRESS", "front"))
+os.environ["REDIS_ADDRESS"] = system.resolve_address(os.environ.get("REDIS_ADDRESS", "redis"))
+os.environ["ADMIN_ADDRESS"] = system.resolve_address(os.environ.get("ADMIN_ADDRESS", "admin"))
 if os.environ["WEBMAIL"] != "none":
-    os.environ["WEBMAIL_ADDRESS"] = resolve(os.environ.get("WEBMAIL_ADDRESS", "webmail"))
+    os.environ["WEBMAIL_ADDRESS"] = system.resolve_address(os.environ.get("WEBMAIL_ADDRESS", "webmail"))
 
 for dovecot_file in glob.glob("/conf/*.conf"):
-    convert(dovecot_file, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
+    conf.jinja(dovecot_file, os.environ, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
 
 # Run Podop, then postfix
 multiprocessing.Process(target=start_podop).start()

core/nginx/Dockerfile

@@ -3,11 +3,13 @@ FROM alpine:3.10
 RUN apk add --no-cache \
     python3 py3-pip git bash \
   && pip3 install --upgrade pip
-# Shared layer between rspamd, postfix, dovecot, unbound, rainloop, roundcube and nginx
-RUN pip3 install git+https://github.com/Mailu/MailuStart.git#egg=mailustart
+
+# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
+RUN pip3 install socrate
+
 # Image specific layers under this line
 RUN apk add --no-cache certbot nginx nginx-mod-mail openssl curl \
- && pip3 install idna requests watchdog
+ && pip3 install watchdog
 
 COPY conf /conf
 COPY static /static

core/nginx/config.py

@@ -3,7 +3,7 @@
 import os
 import logging as log
 import sys
-from mailustart import resolve, convert
+from socrate import system, conf
 
 args = os.environ.copy()
 
@@ -14,14 +14,14 @@ with open("/etc/resolv.conf") as handle:
     content = handle.read().split()
     args["RESOLVER"] = content[content.index("nameserver") + 1]
 
-args["HOST_ADMIN"] = resolve(args.get("HOST_ADMIN", "admin"))
-args["HOST_ANTISPAM"] = resolve(args.get("HOST_ANTISPAM", "antispam:11334"))
+args["HOST_ADMIN"] = system.resolve_address(args.get("HOST_ADMIN", "admin"))
+args["HOST_ANTISPAM"] = system.resolve_address(args.get("HOST_ANTISPAM", "antispam:11334"))
 args["HOST_WEBMAIL"] = args.get("HOST_WEBMAIL", "webmail")
 if args["WEBMAIL"] != "none":
-    args["HOST_WEBMAIL"] = resolve(args.get("HOST_WEBMAIL"))
+    args["HOST_WEBMAIL"] = system.resolve_address(args.get("HOST_WEBMAIL"))
 args["HOST_WEBDAV"] = args.get("HOST_WEBDAV", "webdav:5232")
 if args["WEBDAV"] != "none":
-    args["HOST_WEBDAV"] = resolve(args.get("HOST_WEBDAV"))
+    args["HOST_WEBDAV"] = system.resolve_address(args.get("HOST_WEBDAV"))
 
 # TLS configuration
 cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem")
@@ -41,8 +41,8 @@ if args["TLS"] and not all(os.path.exists(file_path) for file_path in args["TLS"
     args["TLS_ERROR"] = "yes"
 
 # Build final configuration paths
-convert("/conf/tls.conf", "/etc/nginx/tls.conf", args)
-convert("/conf/proxy.conf", "/etc/nginx/proxy.conf", args)
-convert("/conf/nginx.conf", "/etc/nginx/nginx.conf", args)
+conf.jinja("/conf/tls.conf", args, "/etc/nginx/tls.conf")
+conf.jinja("/conf/proxy.conf", args, "/etc/nginx/proxy.conf")
+conf.jinja("/conf/nginx.conf", args, "/etc/nginx/nginx.conf")
 if os.path.exists("/var/run/nginx.pid"):
     os.system("nginx -s reload")

core/postfix/Dockerfile

@@ -3,12 +3,14 @@ FROM alpine:3.10
 RUN apk add --no-cache \
     python3 py3-pip git bash \
   && pip3 install --upgrade pip
-# Shared layer between rspamd, postfix, dovecot, unbound, rainloop, roundcube and nginx
-RUN pip3 install git+https://github.com/Mailu/MailuStart.git#egg=mailustart
+
+# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
+RUN pip3 install socrate
+
 # Shared layer between dovecot and postfix
 RUN pip3 install "podop>0.2.5"
-# Image specific layers under this line
 
+# Image specific layers under this line
 RUN apk add --no-cache postfix postfix-pcre cyrus-sasl-plain
 
 COPY conf /conf

core/postfix/start.py

@@ -6,9 +6,9 @@ import shutil
 import multiprocessing
 import logging as log
 import sys
-from mailustart import resolve, convert
 
-from podop import run_server
+from podop   import run_server
+from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
@@ -26,13 +26,13 @@ def start_podop():
     ])
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
-os.environ["ADMIN_ADDRESS"] = resolve(os.environ.get("ADMIN_ADDRESS", "admin"))
-os.environ["HOST_ANTISPAM"] = resolve(os.environ.get("HOST_ANTISPAM", "antispam:11332"))
-os.environ["HOST_LMTP"] = resolve(os.environ.get("HOST_LMTP", "imap:2525"))
+os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("FRONT_ADDRESS", "front"))
+os.environ["ADMIN_ADDRESS"] = system.resolve_address(os.environ.get("ADMIN_ADDRESS", "admin"))
+os.environ["HOST_ANTISPAM"] = system.resolve_address(os.environ.get("HOST_ANTISPAM", "antispam:11332"))
+os.environ["HOST_LMTP"] = system.resolve_address(os.environ.get("HOST_LMTP", "imap:2525"))
 
 for postfix_file in glob.glob("/conf/*.cf"):
-    convert(postfix_file, os.path.join("/etc/postfix", os.path.basename(postfix_file)))
+    conf.jinja(postfix_file, os.environ, os.path.join("/etc/postfix", os.path.basename(postfix_file)))
 
 if os.path.exists("/overrides/postfix.cf"):
     for line in open("/overrides/postfix.cf").read().strip().split("\n"):
@@ -50,7 +50,7 @@ for map_file in glob.glob("/overrides/*.map"):
 
 if "RELAYUSER" in os.environ:
     path = "/etc/postfix/sasl_passwd"
-    convert("/conf/sasl_passwd", path)
+    conf.jinja("/conf/sasl_passwd", os.environ, path)
     os.system("postmap {}".format(path))
 
 # Run Podop and Postfix

optional/postgresql/Dockerfile

@@ -3,8 +3,10 @@ FROM alpine:3.10
 RUN apk add --no-cache \
     python3 py3-pip bash \
   && pip3 install --upgrade pip
-# Shared layer between rspamd, postfix, dovecot, unbound, rainloop, roundcube and nginx
-RUN pip3 install jinja2
+
+# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
+RUN pip3 install socrate
+
 # Image specific layers under this line
 RUN apk add --no-cache \
     postgresql postgresql-libs busybox-suid sudo tar \

optional/postgresql/start.py

@@ -6,6 +6,7 @@ import jinja2
 import glob
 import os
 import subprocess
+from socrate import conf
 
 def setup():
     conn =  psycopg2.connect(user='postgres')
@@ -47,9 +48,8 @@ os.system("mkdir -p /backup/wal_archive")
 os.system("chown -R postgres:postgres /backup")
 
 # Render config files
-convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 for pg_file in glob.glob("/conf/*.conf"):
-    convert(pg_file, os.path.join("/data", os.path.basename(pg_file)))
+    conf.jinja(pg_file, os.environ, os.path.join("/data", os.path.basename(pg_file)))
 
 # (Re)start postgresql locally for DB and user creation
 os.system("sudo -u postgres pg_ctl start -D /data -o '-h \"''\" '")

services/fetchmail/Dockerfile

@@ -12,7 +12,6 @@ RUN cd fetchmail-7.0.0-alpha6 && \
 
 FROM alpine:3.10
 
-
 # python3 shared with most images
 RUN apk add --no-cache \
     python3 py3-pip bash \

services/rspamd/Dockerfile

@@ -3,8 +3,10 @@ FROM alpine:3.10
 RUN apk add --no-cache \
     python3 py3-pip git bash \
   && pip3 install --upgrade pip
-# Shared layer between rspamd, postfix, dovecot, unbound, rainloop, roundcube and nginx
-RUN pip3 install git+https://github.com/Mailu/MailuStart.git#egg=mailustart
+
+# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
+RUN pip3 install socrate
+
 # Image specific layers under this line
 RUN apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl
 

services/rspamd/start.py

@@ -4,17 +4,17 @@ import os
 import glob
 import logging as log
 import sys
-from mailustart import resolve, convert
+from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
+os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("FRONT_ADDRESS", "front"))
 
 if "HOST_REDIS" not in os.environ: os.environ["HOST_REDIS"] = "redis"
 
 for rspamd_file in glob.glob("/conf/*"):
-    convert(rspamd_file, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
+    conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
 
 # Run rspamd
 os.execv("/usr/sbin/rspamd", ["rspamd", "-i", "-f"])

services/unbound/Dockerfile

@@ -3,8 +3,10 @@ FROM alpine:3.10
 RUN apk add --no-cache \
     python3 py3-pip git bash \
   && pip3 install --upgrade pip
-# Shared layer between rspamd, postfix, dovecot, unbound, rainloop, roundcube and nginx
-RUN pip3 install git+https://github.com/Mailu/MailuStart.git#egg=mailustart
+
+# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
+RUN pip3 install socrate
+
 # Image specific layers under this line
 RUN apk add --no-cache unbound curl bind-tools \
   && curl -o /etc/unbound/root.hints https://www.internic.net/domain/named.cache \

services/unbound/start.py

@@ -3,10 +3,10 @@
 import os
 import logging as log
 import sys
-from mailustart import convert
+from socrate import conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
-convert("/unbound.conf", "/etc/unbound/unbound.conf")
+conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf")
 
 os.execv("/usr/sbin/unbound", ["-c /etc/unbound/unbound.conf"])

towncrier/newsfragments/1082.feature

@@ -0,0 +1 @@
+Use python package socrate instead of Mailustart

webmails/rainloop/Dockerfile

@@ -5,6 +5,9 @@ RUN apt-get update && apt-get install -y \
   && rm -rf /var/lib/apt/lists \
   && echo "ServerSignature Off" >> /etc/apache2/apache2.conf
 
+# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
+RUN pip3 install socrate
+
 ENV RAINLOOP_URL https://github.com/RainLoop/rainloop-webmail/releases/download/v1.13.0/rainloop-community-1.13.0.zip
 
 RUN apt-get update && apt-get install -y \
@@ -22,9 +25,6 @@ RUN apt-get update && apt-get install -y \
  && apt-get purge -y unzip \
  && rm -rf /var/lib/apt/lists
 
-# Shared layer between rspamd, postfix, dovecot, unbound, rainloop, roundcube and nginx
-RUN pip3 install git+https://github.com/Mailu/MailuStart.git#egg=mailustart
-
 COPY include.php /var/www/html/include.php
 COPY php.ini /php.ini
 

webmails/rainloop/start.py

@@ -4,13 +4,13 @@ import os
 import shutil
 import logging as log
 import sys
-from mailustart import resolve, convert
+from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
-os.environ["IMAP_ADDRESS"] = resolve(os.environ.get("IMAP_ADDRESS", "imap"))
+os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("FRONT_ADDRESS", "front"))
+os.environ["IMAP_ADDRESS"] = system.resolve_address(os.environ.get("IMAP_ADDRESS", "imap"))
 
 os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576))
 
@@ -19,9 +19,9 @@ shutil.rmtree(base + "domains/", ignore_errors=True)
 os.makedirs(base + "domains", exist_ok=True)
 os.makedirs(base + "configs", exist_ok=True)
 
-convert("/default.ini", "/data/_data_/_default_/domains/default.ini")
-convert("/application.ini", "/data/_data_/_default_/configs/application.ini")
-convert("/php.ini", "/usr/local/etc/php/conf.d/rainloop.ini")
+conf.jinja("/default.ini", os.environ, "/data/_data_/_default_/domains/default.ini")
+conf.jinja("/application.ini", os.environ, "/data/_data_/_default_/configs/application.ini")
+conf.jinja("/php.ini", os.environ, "/usr/local/etc/php/conf.d/rainloop.ini")
 
 os.system("chown -R www-data:www-data /data")
 

webmails/roundcube/Dockerfile

@@ -5,6 +5,9 @@ RUN apt-get update && apt-get install -y \
   && rm -rf /var/lib/apt/lists \
   && echo "ServerSignature Off" >> /etc/apache2/apache2.conf
 
+# Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
+RUN pip3 install socrate
+
 ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.9/roundcubemail-1.3.9-complete.tar.gz
 
 RUN apt-get update && apt-get install -y \
@@ -24,9 +27,6 @@ RUN apt-get update && apt-get install -y \
  && chown -R www-data: logs temp \
  && rm -rf /var/lib/apt/lists
 
-# Shared layer between rspamd, postfix, dovecot, unbound, rainloop, roundcube and nginx
-RUN pip3 install git+https://github.com/Mailu/MailuStart.git#egg=mailustart
-
 COPY php.ini /php.ini
 COPY config.inc.php /var/www/html/config/
 COPY start.py /start.py

webmails/roundcube/start.py

@@ -3,13 +3,13 @@
 import os
 import logging as log
 import sys
-from mailustart import convert
+from socrate import conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576))
 
-convert("/php.ini", "/usr/local/etc/php/conf.d/roundcube.ini")
+conf.jinja("/php.ini", os.environ, "/usr/local/etc/php/conf.d/roundcube.ini")
 
 # Fix some permissions
 os.system("mkdir -p /data/gpg")