From de2f166bd1f7a6912db04819798a1e984537c0ce Mon Sep 17 00:00:00 2001 From: Michael Wyraz Date: Fri, 23 Aug 2019 08:43:59 +0200 Subject: [PATCH] Resolve HOST_* to *_ADDRESS only if *_ADDRESS is not already set --- core/admin/mailu/configuration.py | 30 ++++++++++++++++++---------- core/admin/mailu/internal/nginx.py | 17 ++++++++++++++++ core/dovecot/Dockerfile | 2 +- core/dovecot/start.py | 11 +++++----- core/nginx/Dockerfile | 2 +- core/nginx/config.py | 8 ++++---- core/postfix/Dockerfile | 2 +- core/postfix/start.py | 8 ++++---- docs/configuration.rst | 14 +++++++------ services/rspamd/Dockerfile | 2 +- services/rspamd/start.py | 6 +++--- towncrier/newsfragments/1113.feature | 1 + 12 files changed, 67 insertions(+), 36 deletions(-) create mode 100644 towncrier/newsfragments/1113.feature diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 39539b54..f01e1bb9 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -60,7 +60,9 @@ DEFAULT_CONFIG = { 'HOST_SMTP': 'smtp', 'HOST_AUTHSMTP': 'smtp', 'HOST_ADMIN': 'admin', + 'ANTISPAM': 'none', 'HOST_ANTISPAM': 'antispam:11334', + 'WEBMAIL': 'none', 'HOST_WEBMAIL': 'webmail', 'HOST_WEBDAV': 'webdav:5232', 'HOST_REDIS': 'redis', @@ -79,18 +81,26 @@ class ConfigManager(dict): 'mysql': 'mysql://{DB_USER}:{DB_PW}@{DB_HOST}/{DB_NAME}' } - HOSTS = ('IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS') - OPTIONAL_HOSTS = ('WEBMAIL', 'ANTISPAM') - def __init__(self): self.config = dict() - def resolve_host(self): - optional = [item for item in self.OPTIONAL_HOSTS if item in self.config and self.config[item] != "none"] - for item in list(self.HOSTS) + optional: - host = 'HOST_' + item - address = item + '_ADDRESS' - self.config[address] = system.resolve_address(self.config[host]) + def get_host_address(self, name): + # if MYSERVICE_ADDRESS is defined, use this + if '{}_ADDRESS'.format(name) in os.environ: + return os.environ.get('{}_ADDRESS'.format(name)) + # otherwise use the host name and resolve it + return system.resolve_address(self.config['HOST_{}'.format(name)]) + + def resolve_hosts(self): + self.config["IMAP_ADDRESS"] = self.get_host_address("IMAP") + self.config["POP3_ADDRESS"] = self.get_host_address("POP3") + self.config["AUTHSMTP_ADDRESS"] = self.get_host_address("AUTHSMTP") + self.config["SMTP_ADDRESS"] = self.get_host_address("SMTP") + self.config["REDIS_ADDRESS"] = self.get_host_address("REDIS") + if self.config["WEBMAIL"] != "none": + self.config["WEBMAIL_ADDRESS"] = self.get_host_address("WEBMAIL") + if self.config["ANTISPAM"] != "none": + self.config["ANTISPAM_ADDRESS"] = self.get_host_address("ANTISPAM") def __coerce_value(self, value): if isinstance(value, str) and value.lower() in ('true','yes'): @@ -106,7 +116,7 @@ class ConfigManager(dict): key: self.__coerce_value(os.environ.get(key, value)) for key, value in DEFAULT_CONFIG.items() }) - self.resolve_host() + self.resolve_hosts() # automatically set the sqlalchemy string if self.config['DB_FLAVOR']: diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index d5f2d697..e9e3c21a 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -3,6 +3,9 @@ from flask import current_app as app import re import urllib +import ipaddress +import socket +import tenacity SUPPORTED_AUTH_METHODS = ["none", "plain"] @@ -88,4 +91,18 @@ def get_server(protocol, authenticated=False): hostname, port = extract_host_port(app.config['AUTHSMTP_ADDRESS'], 10025) else: hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25) + try: + # test if hostname is already resolved to an ip adddress + ipaddress.ip_address(hostname) + except: + # hostname is not an ip address - so we need to resolve it + hostname = resolve_hostname(hostname) return hostname, port + +@tenacity.retry(stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5)) +def resolve_hostname(hostname): + """ This function uses system DNS to resolve a hostname. + It is capable of retrying in case the host is not immediately available + """ + return socket.gethostbyname(hostname) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 4c25bfcb..75545011 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -5,7 +5,7 @@ RUN apk add --no-cache \ && pip3 install --upgrade pip # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube -RUN pip3 install socrate +RUN pip3 install socrate==0.2.0 # Shared layer between dovecot and postfix RUN pip3 install "podop>0.2.5" diff --git a/core/dovecot/start.py b/core/dovecot/start.py index ae429748..f522cf1e 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -21,12 +21,13 @@ def start_podop(): ]) # Actual startup script -os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front")) -os.environ["REDIS_ADDRESS"] = system.resolve_address(os.environ.get("HOST_REDIS", "redis")) -os.environ["ADMIN_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ADMIN", "admin")) -os.environ["ANTISPAM_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ANTISPAM", "antispam:11334")) + +os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") +os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis") +os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") +os.environ["ANTISPAM_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM", "antispam:11334") if os.environ["WEBMAIL"] != "none": - os.environ["WEBMAIL_ADDRESS"] = system.resolve_address(os.environ.get("HOST_WEBMAIL", "webmail")) + os.environ["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail") for dovecot_file in glob.glob("/conf/*.conf"): conf.jinja(dovecot_file, os.environ, os.path.join("/etc/dovecot", os.path.basename(dovecot_file))) diff --git a/core/nginx/Dockerfile b/core/nginx/Dockerfile index 2b6da845..6b54cbf9 100644 --- a/core/nginx/Dockerfile +++ b/core/nginx/Dockerfile @@ -5,7 +5,7 @@ RUN apk add --no-cache \ && pip3 install --upgrade pip # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube -RUN pip3 install socrate +RUN pip3 install socrate==0.2.0 # Image specific layers under this line RUN apk add --no-cache certbot nginx nginx-mod-mail openssl curl \ diff --git a/core/nginx/config.py b/core/nginx/config.py index cafe2f26..65beeb6a 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -14,12 +14,12 @@ with open("/etc/resolv.conf") as handle: content = handle.read().split() args["RESOLVER"] = content[content.index("nameserver") + 1] -args["ADMIN_ADDRESS"] = system.resolve_address(args.get("HOST_ADMIN", "admin")) -args["ANTISPAM_ADDRESS"] = system.resolve_address(args.get("HOST_ANTISPAM", "antispam:11334")) +args["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") +args["ANTISPAM_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM", "antispam:11334") if args["WEBMAIL"] != "none": - args["WEBMAIL_ADDRESS"] = system.resolve_address(args.get("HOST_WEBMAIL", "webmail")) + args["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail") if args["WEBDAV"] != "none": - args["WEBDAV_ADDRESS"] = system.resolve_address(args.get("HOST_WEBDAV", "webdav:5232")) + args["WEBDAV_ADDRESS"] = system.get_host_address_from_environment("WEBDAV", "webdav:5232") # TLS configuration cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem") diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index ef23f9f4..f3b1b24a 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -5,7 +5,7 @@ RUN apk add --no-cache \ && pip3 install --upgrade pip # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube -RUN pip3 install socrate +RUN pip3 install socrate==0.2.0 # Shared layer between dovecot and postfix RUN pip3 install "podop>0.2.5" diff --git a/core/postfix/start.py b/core/postfix/start.py index 28b31468..4df5a3d4 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -26,10 +26,10 @@ def start_podop(): ]) # Actual startup script -os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front")) -os.environ["ADMIN_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ADMIN", "admin")) -os.environ["ANTISPAM_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ANTISPAM", "antispam:11332")) -os.environ["LMTP_ADDRESS"] = system.resolve_address(os.environ.get("HOST_LMTP", "imap:2525")) +os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") +os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") +os.environ["ANTISPAM_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM", "antispam:11332") +os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525") for postfix_file in glob.glob("/conf/*.cf"): conf.jinja(postfix_file, os.environ, os.path.join("/etc/postfix", os.path.basename(postfix_file))) diff --git a/docs/configuration.rst b/docs/configuration.rst index 18e8b75b..02931d35 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -151,11 +151,13 @@ optional port number. Those variables are: - ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``) - ``HOST_WEBDAV``: the container that is running the webdav server (default: ``webdav:5232``) - ``HOST_REDIS``: the container that is running the redis daemon (default: ``redis``) +- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``) + +The startup scripts will resolve HOST_* to their IP addresses and store the result in *_ADDRESS for further use. + +Alternatively, *_ADDRESS can directly be set. In this case, the values of *_ADDRESS is kept and not +resolved. This can be used to rely on DNS based service discovery with changing services IP addresses. +When using *_ADDRESS, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to +resolve the hostnames. -Additional variables are used to locate other containers without dialing a -specific port number. It is used to either whitelist connection from these -addresses or connect to containers on the docker network: -- ``FRONT_ADDRESS``: the nginx container address (default: ``front``) -- ``WEBMAIL_ADDRESS``: the webmail container address (default: ``webmail``) -- ``IMAP_ADDRESS``: the webmail container address (default: ``webmail``) diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index 1646330d..b7ee8653 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -5,7 +5,7 @@ RUN apk add --no-cache \ && pip3 install --upgrade pip # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube -RUN pip3 install socrate +RUN pip3 install socrate==0.2.0 # Image specific layers under this line RUN apk add --no-cache rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates curl diff --git a/services/rspamd/start.py b/services/rspamd/start.py index ee3cffdc..bde708f2 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -10,11 +10,11 @@ log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) # Actual startup script -os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front")) -os.environ["REDIS_ADDRESS"] = system.resolve_address(os.environ.get("HOST_REDIS", "redis")) +os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") +os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis") if os.environ.get("ANTIVIRUS") == 'clamav': - os.environ["ANTIVIRUS_ADDRESS"] = system.resolve_address(os.environ.get("HOST_ANTIVIRUS", "antivirus:3310")) + os.environ["ANTIVIRUS_ADDRESS"] = system.get_host_address_from_environment("ANTIVIRUS", "antivirus:3310") for rspamd_file in glob.glob("/conf/*"): conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file))) diff --git a/towncrier/newsfragments/1113.feature b/towncrier/newsfragments/1113.feature new file mode 100644 index 00000000..156a89df --- /dev/null +++ b/towncrier/newsfragments/1113.feature @@ -0,0 +1 @@ +Resolve hosts to IPs if only HOST_* is set. If *_ADDRESS is set, leave it unresolved.