Enable dynamic resolution of hostnames

Get rid of all HOST_* variables, sanitize the environment in socrates

core/admin/mailu/configuration.py

@@ -1,7 +1,6 @@
 import os
 
 from datetime import timedelta
-from socrate import system
 import ipaddress
 
 DEFAULT_CONFIG = {
@@ -83,17 +82,6 @@ DEFAULT_CONFIG = {
     'PROXY_AUTH_WHITELIST': '',
     'PROXY_AUTH_HEADER': 'X-Auth-Email',
     'PROXY_AUTH_CREATE': False,
-    # Host settings
-    'HOST_IMAP': 'imap',
-    'HOST_LMTP': 'imap:2525',
-    'HOST_POP3': 'imap',
-    'HOST_SMTP': 'smtp',
-    'HOST_AUTHSMTP': 'smtp',
-    'HOST_ADMIN': 'admin',
-    'HOST_WEBMAIL': 'webmail',
-    'HOST_WEBDAV': 'webdav:5232',
-    'HOST_REDIS': 'redis',
-    'HOST_FRONT': 'front',
     'SUBNET': '192.168.203.0/24',
     'SUBNET6': None
 }
@@ -111,18 +99,9 @@ class ConfigManager:
     def __init__(self):
         self.config = dict()
 
-    def get_host_address(self, name):
-        # if MYSERVICE_ADDRESS is defined, use this
-        if f'{name}_ADDRESS' in os.environ:
-            return os.environ.get(f'{name}_ADDRESS')
-        # otherwise use the host name and resolve it
-        return system.resolve_address(self.config[f'HOST_{name}'])
-
     def resolve_hosts(self):
-        for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']:
+        for key in ['ADMIN', 'FRONT', 'SMTP', 'IMAP', 'REDIS', 'ANTIVIRUS:', 'ANTISPAM', 'WEBMAIL', 'WEBDAV']:
-            self.config[f'{key}_ADDRESS'] = self.get_host_address(key)
+            self.config[f'{key}_ADDRESS'] = os.environ.get(f'{key}_ADDRESS')
-        if self.config['WEBMAIL'] != 'none':
-            self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL')
 
     def __get_env(self, key, value):
         key_file = key + "_FILE"
@@ -148,6 +127,7 @@ class ConfigManager:
             key: self.__coerce_value(self.__get_env(key, value))
             for key, value in DEFAULT_CONFIG.items()
         })
+
         self.resolve_hosts()
 
         # automatically set the sqlalchemy string

core/admin/mailu/internal/nginx.py

@@ -2,7 +2,6 @@ from mailu import models, utils
 from flask import current_app as app
 from socrate import system
 
-import re
 import urllib
 import ipaddress
 import sqlalchemy.exc
@@ -126,20 +125,16 @@ def get_status(protocol, status):
     status, codes = STATUSES[status]
     return status, codes[protocol]
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
 def get_server(protocol, authenticated=False):
     if protocol == "imap":
-        hostname, port = extract_host_port(app.config['IMAP_ADDRESS'], 143)
+        hostname, port = app.config['IMAP_ADDRESS'], 143
     elif protocol == "pop3":
-        hostname, port = extract_host_port(app.config['POP3_ADDRESS'], 110)
+        hostname, port = app.config['POP3_ADDRESS'], 110
     elif protocol == "smtp":
         if authenticated:
-            hostname, port = extract_host_port(app.config['AUTHSMTP_ADDRESS'], 10025)
+            hostname, port = app.config['SMTP_ADDRESS'], 10025
         else:
-            hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25)
+            hostname, port = app.config['SMTP_ADDRESS'], 25
     try:
         # test if hostname is already resolved to an ip address
         ipaddress.ip_address(hostname)

core/admin/mailu/models.py

@@ -421,8 +421,7 @@ class Email(object):
         """ send an email to the address """
         try:
             f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}'
-            ip, port = app.config['HOST_LMTP'].rsplit(':')
+            with smtplib.LMTP(ip=app.config['IMAP_ADDRESS'], port=2525) as lmtp:
-            with smtplib.LMTP(ip, port=port) as lmtp:
                 to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}'
                 msg = text.MIMEText(body)
                 msg['Subject'] = subject

core/admin/start.py

@@ -4,6 +4,7 @@ import os
 import logging as log
 from pwd import getpwnam
 import sys
+from socrate import system
 
 os.system("chown mailu:mailu -R /dkim")
 os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
@@ -12,6 +13,7 @@ os.setgid(mailu_id.pw_gid)
 os.setuid(mailu_id.pw_uid)
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
+system.set_env(['SECRET'])
 
 os.system("flask mailu advertise")
 os.system("flask db upgrade")

core/base/libs/socrate/socrate/system.py

@@ -1,7 +1,8 @@
+import hmac
+import logging as log
+import os
 import socket
 import tenacity
-from os import environ
-import logging as log
 
 @tenacity.retry(stop=tenacity.stop_after_attempt(100),
                 wait=tenacity.wait_random(min=2, max=5))
@@ -15,24 +16,29 @@ def resolve_hostname(hostname):
         log.warn("Unable to lookup '%s': %s",hostname,e)
         raise e
 
+SERVICE = {
+    "ADMIN": "admin",
+    "FRONT": "front",
+    "SMTP": "smtp",
+    "IMAP": "imap",
+    "REDIS": "redis",
+    "ANTIVIRUS:": "antivirus",
+    "ANTISPAM": "antispam",
+    "WEBMAIL": "webmail",
+    "WEBDAV": "webdav",
+}
+def set_env(required_secrets=[]):
+    """ This will set all the environment variables and retains only the secrets we need """
+    for service in SERVICE:
+        if not os.environ.get(f'{service}_ADDRESS'):
+            os.environ[f'{service}_ADDRESS'] = f'{SERVICE[service]}'
 
-def resolve_address(address):
+    secret_key = os.environ.get('SECRET_KEY')
-    """ This function is identical to ``resolve_hostname`` but also supports
+    clean_env()
-    resolving an address, i.e. including a port.
+    # derive the keys we need
-    """
+    for secret in required_secrets:
-    hostname, *rest = address.rsplit(":", 1)
+        os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest()
-    ip_address = resolve_hostname(hostname)
-    if ":" in ip_address:
-        ip_address = "[{}]".format(ip_address)
-    return ip_address + "".join(":" + port for port in rest)
 
-
+def clean_env():
-def get_host_address_from_environment(name, default):
+    """ remove all secret keys """
-    """ This function looks up an envionment variable ``{{ name }}_ADDRESS``.
+    [os.environ.pop(key, None) for key in os.environ.keys() if key.endswith("_KEY")]
-    If it's defined, it is returned unmodified. If it's undefined, an environment
-    variable ``HOST_{{ name }}`` is looked up and resolved to an ip address.
-    If this is also not defined, the default is resolved to an ip address.
-    """
-    if "{}_ADDRESS".format(name) in environ:
-        return environ.get("{}_ADDRESS".format(name))
-    return resolve_address(environ.get("HOST_{}".format(name), default))

core/dovecot/conf/ham.script

@@ -1,9 +1,8 @@
 #!/bin/bash
-{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %}
+RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\  -f1):11334"
-RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\  -f1):{{ port }}"
 if [[ $? -ne 0 ]]
 then
-	echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2
+	echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2
 	exit 1
 fi
 

core/dovecot/conf/spam.script

@@ -1,9 +1,8 @@
 #!/bin/bash
-{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %}
+RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\  -f1):11334"
-RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\  -f1):{{ port }}"
 if [[ $? -ne 0 ]]
 then
-	echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2
+	echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2
 	exit 1
 fi
 

core/dovecot/start.py

@@ -10,6 +10,7 @@ from podop import run_server
 from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 def start_podop():
     os.setuid(8)
@@ -21,10 +22,6 @@ def start_podop():
     ])
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-os.environ["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334")
-
 for dovecot_file in glob.glob("/conf/*.conf"):
     conf.jinja(dovecot_file, os.environ, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
 

core/nginx/conf/nginx.conf

@@ -77,12 +77,12 @@ http {
       root /static;
       # Variables for proxifying
       set $admin {{ ADMIN_ADDRESS }};
-      set $antispam {{ ANTISPAM_WEBUI_ADDRESS }};
+      set $antispam {{ ANTISPAM_ADDRESS }}:11334;
       {% if WEBMAIL_ADDRESS %}
       set $webmail {{ WEBMAIL_ADDRESS }};
       {% endif %}
       {% if WEBDAV_ADDRESS %}
-      set $webdav {{ WEBDAV_ADDRESS }};
+      set $webdav {{ WEBDAV_ADDRESS }}:5232;
       {% endif %}
       client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }};
 

core/nginx/config.py

@@ -5,8 +5,8 @@ import logging as log
 import sys
 from socrate import system, conf
 
+system.set_env()
 args = os.environ.copy()
-
 log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING"))
 
 args['TLS_PERMISSIVE'] = str(args.get('TLS_PERMISSIVE')).lower() not in ('false', 'no')
@@ -17,13 +17,6 @@ with open("/etc/resolv.conf") as handle:
     resolver = content[content.index("nameserver") + 1]
     args["RESOLVER"] = f"[{resolver}]" if ":" in resolver else resolver
 
-args["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-args["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334")
-if args["WEBMAIL"] != "none":
-    args["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail")
-if args["WEBDAV"] != "none":
-    args["WEBDAV_ADDRESS"] = system.get_host_address_from_environment("WEBDAV", "webdav:5232")
-
 # TLS configuration
 cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem")
 keypair_name = os.getenv("TLS_KEYPAIR_FILENAME", default="key.pem")

core/postfix/conf/main.cf

@@ -81,7 +81,7 @@ virtual_mailbox_maps = ${podop}mailbox
 # Mails are transported if required, then forwarded to Dovecot for delivery
 relay_domains = ${podop}transport
 transport_maps = lmdb:/etc/postfix/transport.map, ${podop}transport
-virtual_transport = lmtp:inet:{{ LMTP_ADDRESS }}
+virtual_transport = lmtp:inet:{{ IMAP_ADDRESS }}:2525
 
 # Sender and recipient canonical maps, mostly for SRS
 sender_canonical_maps = ${podop}sendermap
@@ -126,7 +126,7 @@ unverified_recipient_reject_reason = Address lookup failure
 # Milter
 ###############
 
-smtpd_milters = inet:{{ ANTISPAM_MILTER_ADDRESS }}
+smtpd_milters = inet:{{ ANTISPAM_ADDRESS }}:11332
 milter_protocol = 6
 milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen}
 milter_default_action = tempfail

core/postfix/start.py

@@ -13,6 +13,7 @@ from pwd import getpwnam
 from socrate import system, conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 def start_podop():
     os.setuid(getpwnam('postfix').pw_uid)
@@ -43,10 +44,6 @@ def is_valid_postconf_line(line):
 
 # Actual startup script
 os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True'
-os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_MILTER", "antispam:11332")
-os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
 os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local")
 os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "")
 

core/rspamd/conf/antivirus.conf

@@ -3,7 +3,7 @@ clamav {
   scan_mime_parts = true;
   symbol = "CLAM_VIRUS";
   type = "clamav";
-  servers = "{{ ANTIVIRUS_ADDRESS }}";
+  servers = "{{ ANTIVIRUS_ADDRESS }}:3310";
   {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %}
   action = "reject"
   {% endif %}

core/rspamd/start.py

@@ -6,18 +6,13 @@ import logging as log
 import requests
 import sys
 import time
-from socrate import system, conf
+from socrate import system,conf
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 # Actual startup script
 
-os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis")
-os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
-
-if os.environ.get("ANTIVIRUS") == 'clamav':
-    os.environ["ANTIVIRUS_ADDRESS"] = system.get_host_address_from_environment("ANTIVIRUS", "antivirus:3310")
-
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
 

docs/configuration.rst

@@ -249,32 +249,23 @@ virus mails during SMTP dialogue, so the sender will receive a reject message.
 Infrastructure settings
 -----------------------
 
-Various environment variables ``HOST_*`` can be used to run Mailu containers
+Various environment variables ``*_ADDRESS`` can be used to run Mailu containers
 separately from a supported orchestrator. It is used by the various components
 to find the location of the other containers it depends on. They can contain an
 optional port number. Those variables are:
 
-- ``HOST_IMAP``: the container that is running the IMAP server (default: ``imap``, port 143)
+- ``ADMIN_ADDRESS``
-- ``HOST_LMTP``: the container that is running the LMTP server (default: ``imap:2525``)
+- ``ANTISPAM_ADDRESS``
-- ``HOST_HOSTIMAP``: the container that is running the IMAP server for the webmail (default: ``imap``, port 10143)
+- ``ANTIVIRUS_ADDRESS``
-- ``HOST_POP3``: the container that is running the POP3 server (default: ``imap``, port 110)
+- ``FRONT_ADDRESS``
-- ``HOST_SMTP``: the container that is running the SMTP server (default: ``smtp``, port 25)
+- ``IMAP_ADDRESS``
-- ``HOST_AUTHSMTP``: the container that is running the authenticated SMTP server for the webnmail (default: ``smtp``, port 10025)
+- ``REDIS_ADDRESS``
-- ``HOST_ADMIN``: the container that is running the admin interface (default: ``admin``)
+- ``SMTP_ADDRESS``
-- ``HOST_ANTISPAM_MILTER``: the container that is running the antispam milter service (default: ``antispam:11332``)
+- ``WEBDAV_ADDRESS``
-- ``HOST_ANTISPAM_WEBUI``: the container that is running the antispam webui service (default: ``antispam:11334``)
+- ``WEBMAIL_ADDRESS``
-- ``HOST_ANTIVIRUS``: the container that is running the antivirus service (default: ``antivirus:3310``)
-- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
-- ``HOST_WEBDAV``: the container that is running the webdav server (default: ``webdav:5232``)
-- ``HOST_REDIS``: the container that is running the redis daemon (default: ``redis``)
-- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
 
-The startup scripts will resolve ``HOST_*`` to their IP addresses and store the result in ``*_ADDRESS`` for further use.
+This can be used to rely on DNS based service discovery with changing services IP addresses.
-
+When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames (without port numbers).
-Alternatively, ``*_ADDRESS`` can directly be set. In this case, the values of ``*_ADDRESS`` is kept and not
-resolved. This can be used to rely on DNS based service discovery with changing services IP addresses.
-When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to
-resolve the hostnames.
 
 .. _db_settings:
 

optional/fetchmail/fetchmail.py

@@ -34,11 +34,6 @@ poll "{host}" proto {protocol}  port {port}
 """
 
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
-
 def escape_rc_string(arg):
     return "".join("\\x%2x" % ord(char) for char in arg)
 
@@ -54,20 +49,7 @@ def fetchmail(fetchmailrc):
 
 def run(debug):
     try:
-        os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
-        os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
         fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
-        smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
-        if smtpport is None:
-            smtphostport = smtphost
-        else:
-            smtphostport = "%s/%d" % (smtphost, smtpport)
-        os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
-        lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
-        if lmtpport is None:
-            lmtphostport = lmtphost
-        else:
-            lmtphostport = "%s/%d" % (lmtphost, lmtpport)
         for fetch in fetches:
             fetchmailrc = ""
             options = "options antispam 501, 504, 550, 553, 554"
@@ -79,7 +61,7 @@ def run(debug):
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=smtphostport if fetch['scan'] else lmtphostport,
+                smtphost=f'{os.environ["FRONT_ADDRESS"]}:25' if fetch['scan'] else f'{os.environ["IMAP_ADDRESS"]}:2525',
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
                 options=options,
@@ -118,8 +100,9 @@ if __name__ == "__main__":
     os.chmod("/data/fetchids", 0o700)
     os.setgid(id_fetchmail.pw_gid)
     os.setuid(id_fetchmail.pw_uid)
+    system.set_env()
     while True:
-        delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
+        delay = int(os.environ.get("FETCHMAIL_DELAY", 900))
         print("Sleeping for {} seconds".format(delay))
         time.sleep(delay)
 

optional/unbound/start.py

@@ -3,9 +3,10 @@
 import os
 import logging as log
 import sys
-from socrate import conf
+from socrate import conf, system
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+system.set_env()
 
 conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf")
 

towncrier/newsfragments/1341.misc

@@ -0,0 +1,2 @@
+Remove HOST_* variables, use *_ADDRESS everywhere instead. Please note that those should only contain a FQDN (no port number).
+Derive a different key for admin/SECRET_KEY; this will invalidate existing sessions

webmails/start.py

@@ -13,14 +13,13 @@ from socrate import conf, system
 env = os.environ
 
 logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING"))
+system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS'])
 
 # jinja context
 context = {}
 context.update(env)
 
 context["MAX_FILESIZE"] = str(int(int(env.get("MESSAGE_SIZE_LIMIT", "50000000")) * 0.66 / 1048576))
-context["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
-context["IMAP_ADDRESS"] = system.get_host_address_from_environment("IMAP", "imap")
 
 db_flavor = env.get("ROUNDCUBE_DB_FLAVOR", "sqlite")
 if db_flavor == "sqlite":
@@ -43,17 +42,6 @@ else:
     print(f"Unknown ROUNDCUBE_DB_FLAVOR: {db_flavor}", file=sys.stderr)
     exit(1)
 
-# derive roundcube secret key
-secret_key = env.get("SECRET_KEY")
-if not secret_key:
-    try:
-        secret_key = open(env.get("SECRET_KEY_FILE"), "r").read().strip()
-    except Exception as exc:
-        print(f"Can't read SECRET_KEY from file: {exc}", file=sys.stderr)
-        exit(2)
-
-context['ROUNDCUBE_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest()
-context['SNUFFLEUPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEUPAGUS_KEY', 'utf-8'), 'sha256').hexdigest()
 conf.jinja("/etc/snuffleupagus.rules.tpl", context, "/etc/snuffleupagus.rules")
 
 # roundcube plugins
@@ -127,8 +115,7 @@ conf.jinja("/conf/nginx-webmail.conf", context, "/etc/nginx/http.d/webmail.conf"
 if os.path.exists("/var/run/nginx.pid"):
     os.system("nginx -s reload")
 
-# clean env
+system.clean_env()
-[env.pop(key, None) for key in env.keys() if key == "SECRET_KEY" or key.endswith("_KEY")]
 
 # run nginx
 os.system("php-fpm81")