Merge #1114

1114: Resolve HOST to ADDRESS only if ADDRESS is not already set r=mergify[bot] a=micw

## What type of PR?

bug-fix

## What does this PR do?

~Makes the rsolving from hosts to ips at startup configurable~

I rewrote the pull request after #940 was merged. Now it resolves HOSTs to ADDRESSes only of ADDRESSes are not already set. So on kubernetes we can jsut set the address and have working service discovery.

### Related issue(s)
- closes #1113

## Prerequistes

~Minor change, backward compatible~
Changelog will be added

Co-authored-by: Michael Wyraz <michael@wyraz.de>

core/admin/mailu/configuration.py

@@ -60,7 +60,9 @@ DEFAULT_CONFIG = {
     'HOST_SMTP': 'smtp',
     'HOST_AUTHSMTP': 'smtp',
     'HOST_ADMIN': 'admin',
+    'ANTISPAM': 'none',
     'HOST_ANTISPAM': 'antispam:11334',
+    'WEBMAIL': 'none',
     'HOST_WEBMAIL': 'webmail',
     'HOST_WEBDAV': 'webdav:5232',
     'HOST_REDIS': 'redis',
@@ -79,18 +81,26 @@ class ConfigManager(dict):
         'mysql': 'mysql://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}'
     }
 
-    HOSTS = ('IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS')
-    OPTIONAL_HOSTS = ('WEBMAIL', 'ANTISPAM')
-
     def __init__(self):
         self.config = dict()
 
-    def resolve_host(self):
-        optional = [item for item in self.OPTIONAL_HOSTS if item in self.config and self.config[item] != "none"]
-        for item in list(self.HOSTS) + optional:
-            host = 'HOST_' + item
-            address = item + '_ADDRESS'
-            self.config[address] = system.resolve_address(self.config[host])
+    def get_host_address(self, name):
+        # if MYSERVICE_ADDRESS is defined, use this
+        if '{}_ADDRESS'.format(name) in os.environ:
+            return os.environ.get('{}_ADDRESS'.format(name))
+        # otherwise use the host name and resolve it
+        return system.resolve_address(self.config['HOST_{}'.format(name)])
+
+    def resolve_hosts(self):
+        self.config["IMAP_ADDRESS"] = self.get_host_address("IMAP")
+        self.config["POP3_ADDRESS"] = self.get_host_address("POP3")
+        self.config["AUTHSMTP_ADDRESS"] = self.get_host_address("AUTHSMTP")
+        self.config["SMTP_ADDRESS"] = self.get_host_address("SMTP")
+        self.config["REDIS_ADDRESS"] = self.get_host_address("REDIS")
+        if self.config["WEBMAIL"] != "none":
+            self.config["WEBMAIL_ADDRESS"] = self.get_host_address("WEBMAIL")
+        if self.config["ANTISPAM"] != "none":
+            self.config["ANTISPAM_ADDRESS"] = self.get_host_address("ANTISPAM")
 
     def __coerce_value(self, value):
         if isinstance(value, str) and value.lower() in ('true','yes'):
@@ -106,7 +116,7 @@ class ConfigManager(dict):
             key: self.__coerce_value(os.environ.get(key, value))
             for key, value in DEFAULT_CONFIG.items()
         })
-        self.resolve_host()
+        self.resolve_hosts()
 
         # automatically set the sqlalchemy string
         if self.config['DB_FLAVOR']:

core/admin/mailu/internal/nginx.py

@@ -3,6 +3,9 @@ from flask import current_app as app
 
 import re
 import urllib
+import ipaddress
+import socket
+import tenacity
 
 
 SUPPORTED_AUTH_METHODS = ["none", "plain"]
@@ -88,4 +91,18 @@ def get_server(protocol, authenticated=False):
             hostname, port = extract_host_port(app.config['AUTHSMTP_ADDRESS'], 10025)
         else:
             hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25)
+    try:
+        # test if hostname is already resolved to an ip adddress
+        ipaddress.ip_address(hostname)
+    except:
+        # hostname is not an ip address - so we need to resolve it
+        hostname = resolve_hostname(hostname)
     return hostname, port
+
+@tenacity.retry(stop=tenacity.stop_after_attempt(100),
+                wait=tenacity.wait_random(min=2, max=5))
+def resolve_hostname(hostname):
+    """ This function uses system DNS to resolve a hostname.
+    It is capable of retrying in case the host is not immediately available
+    """
+    return socket.gethostbyname(hostname)

core/dovecot/Dockerfile

@@ -5,7 +5,7 @@ RUN apk add --no-cache \
   && pip3 install --upgrade pip
 
 # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
-RUN pip3 install socrate
+RUN pip3 install socrate==0.2.0
 
 # Shared layer between dovecot and postfix
 RUN pip3 install "podop>0.2.5"

core/dovecot/start.py

@@ -21,12 +21,13 @@ def start_podop():
     ])
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front"))
-os.environ["REDIS_ADDRESS"] = system.resolve_address(os.environ.get("HOST_REDIS", "redis"))
-os.environ["ADMIN_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ADMIN", "admin"))
-os.environ["ANTISPAM_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ANTISPAM", "antispam:11334"))
+
+os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
+os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis")
+os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
+os.environ["ANTISPAM_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM", "antispam:11334")
 if os.environ["WEBMAIL"] != "none":
-    os.environ["WEBMAIL_ADDRESS"] = system.resolve_address(os.environ.get("HOST_WEBMAIL", "webmail"))
+    os.environ["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail")
 
 for dovecot_file in glob.glob("/conf/*.conf"):
     conf.jinja(dovecot_file, os.environ, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))

core/nginx/Dockerfile

@@ -5,7 +5,7 @@ RUN apk add --no-cache \
   && pip3 install --upgrade pip
 
 # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
-RUN pip3 install socrate
+RUN pip3 install socrate==0.2.0
 
 # Image specific layers under this line
 RUN apk add --no-cache certbot nginx nginx-mod-mail openssl curl \

core/nginx/config.py

@@ -14,12 +14,12 @@ with open("/etc/resolv.conf") as handle:
     content = handle.read().split()
     args["RESOLVER"] = content[content.index("nameserver") + 1]
 
-args["ADMIN_ADDRESS"] = system.resolve_address(args.get("HOST_ADMIN", "admin"))
-args["ANTISPAM_ADDRESS"] = system.resolve_address(args.get("HOST_ANTISPAM", "antispam:11334"))
+args["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
+args["ANTISPAM_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM", "antispam:11334")
 if args["WEBMAIL"] != "none":
-    args["WEBMAIL_ADDRESS"] = system.resolve_address(args.get("HOST_WEBMAIL", "webmail"))
+    args["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail")
 if args["WEBDAV"] != "none":
-    args["WEBDAV_ADDRESS"] = system.resolve_address(args.get("HOST_WEBDAV", "webdav:5232"))
+    args["WEBDAV_ADDRESS"] = system.get_host_address_from_environment("WEBDAV", "webdav:5232")
 
 # TLS configuration
 cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem")

core/postfix/Dockerfile

@@ -5,7 +5,7 @@ RUN apk add --no-cache \
   && pip3 install --upgrade pip
 
 # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
-RUN pip3 install socrate
+RUN pip3 install socrate==0.2.0
 
 # Shared layer between dovecot and postfix
 RUN pip3 install "podop>0.2.5"

core/postfix/start.py

@@ -26,10 +26,10 @@ def start_podop():
     ])
 
 # Actual startup script
-os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front"))
-os.environ["ADMIN_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ADMIN", "admin"))
-os.environ["ANTISPAM_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ANTISPAM", "antispam:11332"))
-os.environ["LMTP_ADDRESS"] = system.resolve_address(os.environ.get("HOST_LMTP", "imap:2525"))
+os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
+os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
+os.environ["ANTISPAM_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM", "antispam:11332")
+os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
 
 for postfix_file in glob.glob("/conf/*.cf"):
     conf.jinja(postfix_file, os.environ, os.path.join("/etc/postfix", os.path.basename(postfix_file)))

docs/configuration.rst

@@ -151,11 +151,13 @@ optional port number. Those variables are:
 - ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
 - ``HOST_WEBDAV``: the container that is running the webdav server (default: ``webdav:5232``)
 - ``HOST_REDIS``: the container that is running the redis daemon (default: ``redis``)
+- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``)
+
+The startup scripts will resolve HOST_* to their IP addresses and store the result in *_ADDRESS for further use.
+
+Alternatively, *_ADDRESS can directly be set. In this case, the values of *_ADDRESS is kept and not
+resolved. This can be used to rely on DNS based service discovery with changing services IP addresses.
+When using *_ADDRESS, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to
+resolve the hostnames.
 
-Additional variables are used to locate other containers without dialing a
-specific port number. It is used to either whitelist connection from these
-addresses or connect to containers on the docker network:
 
-- ``FRONT_ADDRESS``: the nginx container address (default: ``front``)
-- ``WEBMAIL_ADDRESS``: the webmail container address (default: ``webmail``)
-- ``IMAP_ADDRESS``: the webmail container address (default: ``webmail``)

services/rspamd/Dockerfile

@@ -5,7 +5,7 @@ RUN apk add --no-cache \
   && pip3 install --upgrade pip
 
 # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube
-RUN pip3 install socrate
+RUN pip3 install socrate==0.2.0
 
 # Image specific layers under this line
 RUN apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl

services/rspamd/start.py

@@ -10,11 +10,11 @@ log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 # Actual startup script
 
-os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front"))
-os.environ["REDIS_ADDRESS"] = system.resolve_address(os.environ.get("HOST_REDIS", "redis"))
+os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front")
+os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis")
 
 if os.environ.get("ANTIVIRUS") == 'clamav':
-    os.environ["ANTIVIRUS_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ANTIVIRUS", "antivirus:3310"))
+    os.environ["ANTIVIRUS_ADDRESS"] = system.get_host_address_from_environment("ANTIVIRUS", "antivirus:3310")
 
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))

towncrier/newsfragments/1113.feature

@@ -0,0 +1 @@
+Resolve hosts to IPs if only HOST_* is set. If *_ADDRESS is set, leave it unresolved.