From 4e3874b0c147e2c0dbca6a9a85f09d8ea54d734a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 8 Dec 2022 12:46:31 +0100 Subject: [PATCH] Enable dynamic resolution of hostnames --- core/admin/mailu/configuration.py | 30 +++------------- core/admin/mailu/internal/nginx.py | 13 +++---- core/admin/mailu/models.py | 3 +- core/admin/mailu/ui/templates/client.html | 4 +-- core/admin/run_dev.sh | 11 +++--- core/admin/start.py | 2 ++ core/base/Dockerfile | 20 ++++++++--- core/base/libs/socrate/socrate/system.py | 42 ++++++++++------------- core/dovecot/conf/ham.script | 5 ++- core/dovecot/conf/spam.script | 5 ++- core/dovecot/start.py | 5 +-- core/nginx/conf/nginx.conf | 4 +-- core/nginx/config.py | 9 +---- core/postfix/conf/main.cf | 4 +-- core/postfix/start.py | 5 +-- core/rspamd/conf/antivirus.conf | 2 +- core/rspamd/start.py | 9 ++--- docs/configuration.rst | 36 +++++++------------ optional/fetchmail/fetchmail.py | 22 ++---------- optional/unbound/start.py | 3 +- setup/flavors/compose/docker-compose.yml | 3 ++ towncrier/newsfragments/1341.misc | 4 +++ webmails/start.py | 17 ++------- 23 files changed, 94 insertions(+), 164 deletions(-) create mode 100644 towncrier/newsfragments/1341.misc diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index d447e570..89469ea8 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -1,7 +1,6 @@ import os from datetime import timedelta -from socrate import system import ipaddress DEFAULT_CONFIG = { @@ -83,17 +82,6 @@ DEFAULT_CONFIG = { 'PROXY_AUTH_WHITELIST': '', 'PROXY_AUTH_HEADER': 'X-Auth-Email', 'PROXY_AUTH_CREATE': False, - # Host settings - 'HOST_IMAP': 'imap', - 'HOST_LMTP': 'imap:2525', - 'HOST_POP3': 'imap', - 'HOST_SMTP': 'smtp', - 'HOST_AUTHSMTP': 'smtp', - 'HOST_ADMIN': 'admin', - 'HOST_WEBMAIL': 'webmail', - 'HOST_WEBDAV': 'webdav:5232', - 'HOST_REDIS': 'redis', - 'HOST_FRONT': 'front', 'SUBNET': '192.168.203.0/24', 'SUBNET6': None } @@ -111,19 +99,6 @@ class ConfigManager: def __init__(self): self.config = dict() - def get_host_address(self, name): - # if MYSERVICE_ADDRESS is defined, use this - if f'{name}_ADDRESS' in os.environ: - return os.environ.get(f'{name}_ADDRESS') - # otherwise use the host name and resolve it - return system.resolve_address(self.config[f'HOST_{name}']) - - def resolve_hosts(self): - for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']: - self.config[f'{key}_ADDRESS'] = self.get_host_address(key) - if self.config['WEBMAIL'] != 'none': - self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL') - def __get_env(self, key, value): key_file = key + "_FILE" if key_file in os.environ: @@ -144,11 +119,14 @@ class ConfigManager: # get current app config self.config.update(app.config) # get environment variables + for key in os.environ: + if key.endswith('_ADDRESS'): + self.config[key] = os.environ[key] + self.config.update({ key: self.__coerce_value(self.__get_env(key, value)) for key, value in DEFAULT_CONFIG.items() }) - self.resolve_hosts() # automatically set the sqlalchemy string if self.config['DB_FLAVOR']: diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 5b321ad3..577e5a44 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -2,7 +2,6 @@ from mailu import models, utils from flask import current_app as app from socrate import system -import re import urllib import ipaddress import sqlalchemy.exc @@ -128,20 +127,16 @@ def get_status(protocol, status): status, codes = STATUSES[status] return status, codes[protocol] -def extract_host_port(host_and_port, default_port): - host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups() - return host, int(port) if port else default_port - def get_server(protocol, authenticated=False): if protocol == "imap": - hostname, port = extract_host_port(app.config['IMAP_ADDRESS'], 143) + hostname, port = app.config['IMAP_ADDRESS'], 143 elif protocol == "pop3": - hostname, port = extract_host_port(app.config['POP3_ADDRESS'], 110) + hostname, port = app.config['IMAP_ADDRESS'], 110 elif protocol == "smtp": if authenticated: - hostname, port = extract_host_port(app.config['AUTHSMTP_ADDRESS'], 10025) + hostname, port = app.config['SMTP_ADDRESS'], 10025 else: - hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25) + hostname, port = app.config['SMTP_ADDRESS'], 25 try: # test if hostname is already resolved to an ip address ipaddress.ip_address(hostname) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index b33a0776..a7d0d006 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -421,8 +421,7 @@ class Email(object): """ send an email to the address """ try: f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' - ip, port = app.config['HOST_LMTP'].rsplit(':') - with smtplib.LMTP(ip, port=port) as lmtp: + with smtplib.LMTP(ip=app.config['IMAP_ADDRESS'], port=2525) as lmtp: to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' msg = text.MIMEText(body) msg['Subject'] = subject diff --git a/core/admin/mailu/ui/templates/client.html b/core/admin/mailu/ui/templates/client.html index fddbe0d2..593fd258 100644 --- a/core/admin/mailu/ui/templates/client.html +++ b/core/admin/mailu/ui/templates/client.html @@ -21,7 +21,7 @@ {% trans %}Server name{% endtrans %} - 
{{ config["HOSTNAMES"] }}
+ 
{{ config["HOSTNAME"] }}
{% trans %}Username{% endtrans %} @@ -46,7 +46,7 @@ {% trans %}Server name{% endtrans %} - 
{{ config["HOSTNAMES"] }}
+ 
{{ config["HOSTNAME"] }}
{% trans %}Username{% endtrans %} diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index cf05fba3..947ad873 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -75,12 +75,15 @@ ENV \ DEBUG_ASSETS="/app/static" \ DEBUG_TB_INTERCEPT_REDIRECTS=False \ \ - IMAP_ADDRESS="127.0.0.1" \ - POP3_ADDRESS="127.0.0.1" \ - AUTHSMTP_ADDRESS="127.0.0.1" \ + ADMIN_ADDRESS="127.0.0.1" \ + FRONT_ADDRESS="127.0.0.1" \ SMTP_ADDRESS="127.0.0.1" \ + IMAP_ADDRESS="127.0.0.1" \ REDIS_ADDRESS="127.0.0.1" \ - WEBMAIL_ADDRESS="127.0.0.1" + ANTIVIRUS_ADDRESS="127.0.0.1" \ + ANTISPAM_ADDRESS="127.0.0.1" \ + WEBMAIL_ADDRESS="127.0.0.1" \ + WEBDAV_ADDRESS="127.0.0.1" CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"] EOF diff --git a/core/admin/start.py b/core/admin/start.py index e2163398..6aa0d1a4 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -4,6 +4,7 @@ import os import logging as log from pwd import getpwnam import sys +from socrate import system os.system("chown mailu:mailu -R /dkim") os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu") @@ -12,6 +13,7 @@ os.setgid(mailu_id.pw_gid) os.setuid(mailu_id.pw_uid) log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO")) +system.set_env(['SECRET']) os.system("flask mailu advertise") os.system("flask db upgrade") diff --git a/core/base/Dockerfile b/core/base/Dockerfile index e3e53dc6..c4ba6b3f 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -17,11 +17,21 @@ RUN set -euxo pipefail \ ; ! [[ "${machine}" == x86_64 ]] \ || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0 -ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so -ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" -ENV CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" -ENV CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" -ENV LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" +ENV \ + LD_PRELOAD="/usr/lib/libhardened_malloc.so" \ + CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \ + CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" \ + CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" \ + LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" \ + ADMIN_ADDRESS="admin" \ + FRONT_ADDRESS="front" \ + SMTP_ADDRESS="smtp" \ + IMAP_ADDRESS="imap" \ + REDIS_ADDRESS="redis" \ + ANTIVIRUS_ADDRESS="antivirus" \ + ANTISPAM_ADDRESS="antispam" \ + WEBMAIL_ADDRESS="webmail" \ + WEBDAV_ADDRESS="webdav" WORKDIR /app diff --git a/core/base/libs/socrate/socrate/system.py b/core/base/libs/socrate/socrate/system.py index d4e3802a..319da0a9 100644 --- a/core/base/libs/socrate/socrate/system.py +++ b/core/base/libs/socrate/socrate/system.py @@ -1,7 +1,8 @@ +import hmac +import logging as log +import os import socket import tenacity -from os import environ -import logging as log @tenacity.retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) @@ -14,25 +15,20 @@ def resolve_hostname(hostname): except Exception as e: log.warn("Unable to lookup '%s': %s",hostname,e) raise e +def set_env(required_secrets=[]): + """ This will set all the environment variables and retains only the secrets we need """ + secret_key = os.environ.get('SECRET_KEY') + if not secret_key: + try: + secret_key = open(env.get("SECRET_KEY_FILE"), "r").read().strip() + except Exception as exc: + log.error(f"Can't read SECRET_KEY from file: {exc}") + raise exc + clean_env() + # derive the keys we need + for secret in required_secrets: + os.environ[f'{secret}_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray(secret, 'utf-8'), 'sha256').hexdigest() - -def resolve_address(address): - """ This function is identical to ``resolve_hostname`` but also supports - resolving an address, i.e. including a port. - """ - hostname, *rest = address.rsplit(":", 1) - ip_address = resolve_hostname(hostname) - if ":" in ip_address: - ip_address = "[{}]".format(ip_address) - return ip_address + "".join(":" + port for port in rest) - - -def get_host_address_from_environment(name, default): - """ This function looks up an envionment variable ``{{ name }}_ADDRESS``. - If it's defined, it is returned unmodified. If it's undefined, an environment - variable ``HOST_{{ name }}`` is looked up and resolved to an ip address. - If this is also not defined, the default is resolved to an ip address. - """ - if "{}_ADDRESS".format(name) in environ: - return environ.get("{}_ADDRESS".format(name)) - return resolve_address(environ.get("HOST_{}".format(name), default)) +def clean_env(): + """ remove all secret keys """ + [os.environ.pop(key, None) for key in os.environ.keys() if key.endswith("_KEY")] diff --git a/core/dovecot/conf/ham.script b/core/dovecot/conf/ham.script index 57112747..7066d170 100755 --- a/core/dovecot/conf/ham.script +++ b/core/dovecot/conf/ham.script @@ -1,9 +1,8 @@ #!/bin/bash -{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %} -RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\ -f1):{{ port }}" +RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\ -f1):11334" if [[ $? -ne 0 ]] then - echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2 + echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2 exit 1 fi diff --git a/core/dovecot/conf/spam.script b/core/dovecot/conf/spam.script index 2e3872b0..94d664ae 100755 --- a/core/dovecot/conf/spam.script +++ b/core/dovecot/conf/spam.script @@ -1,9 +1,8 @@ #!/bin/bash -{% set hostname,port = ANTISPAM_WEBUI_ADDRESS.split(':') %} -RSPAMD_HOST="$(getent hosts {{ hostname }}|cut -d\ -f1):{{ port }}" +RSPAMD_HOST="$(getent hosts {{ ANTISPAM_ADDRESS }}|cut -d\ -f1):11334" if [[ $? -ne 0 ]] then - echo "Failed to lookup {{ ANTISPAM_WEBUI_ADDRESS }}" >&2 + echo "Failed to lookup {{ ANTISPAM_ADDRESS }}" >&2 exit 1 fi diff --git a/core/dovecot/start.py b/core/dovecot/start.py index cfa477bc..4da7a09c 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -11,6 +11,7 @@ from podop import run_server from socrate import system, conf log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) +system.set_env() def start_podop(): id_mail = getpwnam('mail') @@ -24,10 +25,6 @@ def start_podop(): ]) # Actual startup script -os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") -os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") -os.environ["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334") - for dovecot_file in glob.glob("/conf/*.conf"): conf.jinja(dovecot_file, os.environ, os.path.join("/etc/dovecot", os.path.basename(dovecot_file))) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index f9278f38..b373fb13 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -77,12 +77,12 @@ http { root /static; # Variables for proxifying set $admin {{ ADMIN_ADDRESS }}; - set $antispam {{ ANTISPAM_WEBUI_ADDRESS }}; + set $antispam {{ ANTISPAM_ADDRESS }}:11334; {% if WEBMAIL_ADDRESS %} set $webmail {{ WEBMAIL_ADDRESS }}; {% endif %} {% if WEBDAV_ADDRESS %} - set $webdav {{ WEBDAV_ADDRESS }}; + set $webdav {{ WEBDAV_ADDRESS }}:5232; {% endif %} client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; diff --git a/core/nginx/config.py b/core/nginx/config.py index 7930ff12..cee8bce4 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -5,8 +5,8 @@ import logging as log import sys from socrate import system, conf +system.set_env() args = os.environ.copy() - log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING")) args['TLS_PERMISSIVE'] = str(args.get('TLS_PERMISSIVE')).lower() not in ('false', 'no') @@ -17,13 +17,6 @@ with open("/etc/resolv.conf") as handle: resolver = content[content.index("nameserver") + 1] args["RESOLVER"] = f"[{resolver}]" if ":" in resolver else resolver -args["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") -args["ANTISPAM_WEBUI_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_WEBUI", "antispam:11334") -if args["WEBMAIL"] != "none": - args["WEBMAIL_ADDRESS"] = system.get_host_address_from_environment("WEBMAIL", "webmail") -if args["WEBDAV"] != "none": - args["WEBDAV_ADDRESS"] = system.get_host_address_from_environment("WEBDAV", "webdav:5232") - # TLS configuration cert_name = os.getenv("TLS_CERT_FILENAME", default="cert.pem") keypair_name = os.getenv("TLS_KEYPAIR_FILENAME", default="key.pem") diff --git a/core/postfix/conf/main.cf b/core/postfix/conf/main.cf index f3b789f9..2f0275b7 100644 --- a/core/postfix/conf/main.cf +++ b/core/postfix/conf/main.cf @@ -81,7 +81,7 @@ virtual_mailbox_maps = ${podop}mailbox # Mails are transported if required, then forwarded to Dovecot for delivery relay_domains = ${podop}transport transport_maps = lmdb:/etc/postfix/transport.map, ${podop}transport -virtual_transport = lmtp:inet:{{ LMTP_ADDRESS }} +virtual_transport = lmtp:inet:{{ IMAP_ADDRESS }}:2525 # Sender and recipient canonical maps, mostly for SRS sender_canonical_maps = ${podop}sendermap @@ -126,7 +126,7 @@ unverified_recipient_reject_reason = Address lookup failure # Milter ############### -smtpd_milters = inet:{{ ANTISPAM_MILTER_ADDRESS }} +smtpd_milters = inet:{{ ANTISPAM_ADDRESS }}:11332 milter_protocol = 6 milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen} milter_default_action = tempfail diff --git a/core/postfix/start.py b/core/postfix/start.py index 80c1c4bf..b9a058f7 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -13,6 +13,7 @@ from pwd import getpwnam from socrate import system, conf log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) +system.set_env() os.system("flock -n /queue/pid/master.pid rm /queue/pid/master.pid") @@ -45,10 +46,6 @@ def is_valid_postconf_line(line): # Actual startup script os.environ['DEFER_ON_TLS_ERROR'] = os.environ['DEFER_ON_TLS_ERROR'] if 'DEFER_ON_TLS_ERROR' in os.environ else 'True' -os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") -os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") -os.environ["ANTISPAM_MILTER_ADDRESS"] = system.get_host_address_from_environment("ANTISPAM_MILTER", "antispam:11332") -os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525") os.environ["POSTFIX_LOG_SYSLOG"] = os.environ.get("POSTFIX_LOG_SYSLOG","local") os.environ["POSTFIX_LOG_FILE"] = os.environ.get("POSTFIX_LOG_FILE", "") diff --git a/core/rspamd/conf/antivirus.conf b/core/rspamd/conf/antivirus.conf index 1d492850..53da0768 100644 --- a/core/rspamd/conf/antivirus.conf +++ b/core/rspamd/conf/antivirus.conf @@ -3,7 +3,7 @@ clamav { scan_mime_parts = true; symbol = "CLAM_VIRUS"; type = "clamav"; - servers = "{{ ANTIVIRUS_ADDRESS }}"; + servers = "{{ ANTIVIRUS_ADDRESS }}:3310"; {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %} action = "reject" {% endif %} diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 37de1df9..507da65d 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -6,18 +6,13 @@ import logging as log import requests import sys import time -from socrate import system, conf +from socrate import system,conf log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) +system.set_env() # Actual startup script -os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis") -os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") - -if os.environ.get("ANTIVIRUS") == 'clamav': - os.environ["ANTIVIRUS_ADDRESS"] = system.get_host_address_from_environment("ANTIVIRUS", "antivirus:3310") - for rspamd_file in glob.glob("/conf/*"): conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file))) diff --git a/docs/configuration.rst b/docs/configuration.rst index b5affad6..f4510626 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -249,32 +249,22 @@ virus mails during SMTP dialogue, so the sender will receive a reject message. Infrastructure settings ----------------------- -Various environment variables ``HOST_*`` can be used to run Mailu containers +Various environment variables ``*_ADDRESS`` can be used to run Mailu containers separately from a supported orchestrator. It is used by the various components -to find the location of the other containers it depends on. They can contain an -optional port number. Those variables are: +to find the location of the other containers it depends on. Those variables are: -- ``HOST_IMAP``: the container that is running the IMAP server (default: ``imap``, port 143) -- ``HOST_LMTP``: the container that is running the LMTP server (default: ``imap:2525``) -- ``HOST_HOSTIMAP``: the container that is running the IMAP server for the webmail (default: ``imap``, port 10143) -- ``HOST_POP3``: the container that is running the POP3 server (default: ``imap``, port 110) -- ``HOST_SMTP``: the container that is running the SMTP server (default: ``smtp``, port 25) -- ``HOST_AUTHSMTP``: the container that is running the authenticated SMTP server for the webnmail (default: ``smtp``, port 10025) -- ``HOST_ADMIN``: the container that is running the admin interface (default: ``admin``) -- ``HOST_ANTISPAM_MILTER``: the container that is running the antispam milter service (default: ``antispam:11332``) -- ``HOST_ANTISPAM_WEBUI``: the container that is running the antispam webui service (default: ``antispam:11334``) -- ``HOST_ANTIVIRUS``: the container that is running the antivirus service (default: ``antivirus:3310``) -- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``) -- ``HOST_WEBDAV``: the container that is running the webdav server (default: ``webdav:5232``) -- ``HOST_REDIS``: the container that is running the redis daemon (default: ``redis``) -- ``HOST_WEBMAIL``: the container that is running the webmail (default: ``webmail``) +- ``ADMIN_ADDRESS`` +- ``ANTISPAM_ADDRESS`` +- ``ANTIVIRUS_ADDRESS`` +- ``FRONT_ADDRESS`` +- ``IMAP_ADDRESS`` +- ``REDIS_ADDRESS`` +- ``SMTP_ADDRESS`` +- ``WEBDAV_ADDRESS`` +- ``WEBMAIL_ADDRESS`` -The startup scripts will resolve ``HOST_*`` to their IP addresses and store the result in ``*_ADDRESS`` for further use. - -Alternatively, ``*_ADDRESS`` can directly be set. In this case, the values of ``*_ADDRESS`` is kept and not -resolved. This can be used to rely on DNS based service discovery with changing services IP addresses. -When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to -resolve the hostnames. +These are used for DNS based service discovery with possibly changing services IP addresses. +``*_ADDRESS`` values must be fully qualified domain names without port numbers. .. _db_settings: diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 62bd7124..20af1e7f 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -7,7 +7,6 @@ from pwd import getpwnam import tempfile import shlex import subprocess -import re import requests from socrate import system import sys @@ -34,11 +33,6 @@ poll "{host}" proto {protocol} port {port} """ -def extract_host_port(host_and_port, default_port): - host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups() - return host, int(port) if port else default_port - - def escape_rc_string(arg): return "".join("\\x%2x" % ord(char) for char in arg) @@ -54,20 +48,7 @@ def fetchmail(fetchmailrc): def run(debug): try: - os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp") - os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json() - smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None) - if smtpport is None: - smtphostport = smtphost - else: - smtphostport = "%s/%d" % (smtphost, smtpport) - os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525") - lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None) - if lmtpport is None: - lmtphostport = lmtphost - else: - lmtphostport = "%s/%d" % (lmtphost, lmtpport) for fetch in fetches: fetchmailrc = "" options = "options antispam 501, 504, 550, 553, 554" @@ -79,7 +60,7 @@ def run(debug): protocol=fetch["protocol"], host=escape_rc_string(fetch["host"]), port=fetch["port"], - smtphost=smtphostport if fetch['scan'] else lmtphostport, + smtphost=f'{os.environ["SMTP_ADDRESS"]}' if fetch['scan'] else f'{os.environ["IMAP_ADDRESS"]}/2525', username=escape_rc_string(fetch["username"]), password=escape_rc_string(fetch["password"]), options=options, @@ -118,6 +99,7 @@ if __name__ == "__main__": os.chmod("/data/fetchids", 0o700) os.setgid(id_fetchmail.pw_gid) os.setuid(id_fetchmail.pw_uid) + system.set_env() while True: delay = int(os.environ.get("FETCHMAIL_DELAY", 60)) print("Sleeping for {} seconds".format(delay)) diff --git a/optional/unbound/start.py b/optional/unbound/start.py index f3a5bee7..df768092 100755 --- a/optional/unbound/start.py +++ b/optional/unbound/start.py @@ -3,9 +3,10 @@ import os import logging as log import sys -from socrate import conf +from socrate import conf, system log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) +system.set_env() conf.jinja("/unbound.conf", os.environ, "/etc/unbound/unbound.conf") diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index b6c99ca5..cd470098 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -113,6 +113,9 @@ services: - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d:ro" depends_on: - front + {% if antivirus_enabled %} + - antivirus + {% endif %} {% if resolver_enabled %} - resolver dns: diff --git a/towncrier/newsfragments/1341.misc b/towncrier/newsfragments/1341.misc new file mode 100644 index 00000000..53f8df91 --- /dev/null +++ b/towncrier/newsfragments/1341.misc @@ -0,0 +1,4 @@ +Remove HOST_* variables, use *_ADDRESS everywhere instead. Please note that those should only contain a FQDN (no port number). +Derive a different key for admin/SECRET_KEY; this will invalidate existing sessions +Ensure that rspamd starts after clamav +Only display a single HOSTNAME on the client configuration page diff --git a/webmails/start.py b/webmails/start.py index f6dd4d56..954c8407 100755 --- a/webmails/start.py +++ b/webmails/start.py @@ -13,14 +13,13 @@ from socrate import conf, system env = os.environ logging.basicConfig(stream=sys.stderr, level=env.get("LOG_LEVEL", "WARNING")) +system.set_env(['ROUNDCUBE','SNUFFLEUPAGUS']) # jinja context context = {} context.update(env) context["MAX_FILESIZE"] = str(int(int(env.get("MESSAGE_SIZE_LIMIT", "50000000")) * 0.66 / 1048576)) -context["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") -context["IMAP_ADDRESS"] = system.get_host_address_from_environment("IMAP", "imap") db_flavor = env.get("ROUNDCUBE_DB_FLAVOR", "sqlite") if db_flavor == "sqlite": @@ -43,17 +42,6 @@ else: print(f"Unknown ROUNDCUBE_DB_FLAVOR: {db_flavor}", file=sys.stderr) exit(1) -# derive roundcube secret key -secret_key = env.get("SECRET_KEY") -if not secret_key: - try: - secret_key = open(env.get("SECRET_KEY_FILE"), "r").read().strip() - except Exception as exc: - print(f"Can't read SECRET_KEY from file: {exc}", file=sys.stderr) - exit(2) - -context['ROUNDCUBE_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest() -context['SNUFFLEUPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEUPAGUS_KEY', 'utf-8'), 'sha256').hexdigest() conf.jinja("/etc/snuffleupagus.rules.tpl", context, "/etc/snuffleupagus.rules") # roundcube plugins @@ -127,8 +115,7 @@ conf.jinja("/conf/nginx-webmail.conf", context, "/etc/nginx/http.d/webmail.conf" if os.path.exists("/var/run/nginx.pid"): os.system("nginx -s reload") -# clean env -[env.pop(key, None) for key in env.keys() if key == "SECRET_KEY" or key.endswith("_KEY")] +system.clean_env() # run nginx os.system("php-fpm81")