From c2d45a47fe96b5093f5f2ee3f34ec41c8dfb93de Mon Sep 17 00:00:00 2001 From: Dario Ernst Date: Thu, 27 Dec 2018 15:36:24 +0100 Subject: [PATCH 01/15] Attempt stripping recipient delimiter from localpart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since postfix now asks us for the complete email over podop, which includes the recipient-delimiter-and-what-follows not stripped, we need to attempt to find both the verbatim localpart, as well as the localpart stripped of the delimited part …. Fixes #755 --- core/admin/mailu/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a3fc15a4..2b70e408 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -260,10 +260,19 @@ class Email(object): @classmethod def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False): + localpart_stripped = None + if os.environ.get('RECIPIENT_DELIMITER') in localpart: + localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0] + alias = Alias.resolve(localpart, domain_name) + if not alias and localpart_stripped: + alias = Alias.resolve(localpart_stripped, domain_name) if alias: return alias.destination + user = User.query.get('{}@{}'.format(localpart, domain_name)) + if not user and localpart_stripped: + user = User.query.get('{}@{}'.format(localpart_stripped, domain_name)) if user: if user.forward_enabled: destination = user.forward_destination From b9313488dd3cf40fc93553ff847699c30c00972e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Mon, 7 Jan 2019 23:49:10 +0200 Subject: [PATCH 02/15] Add logging for tenacity.retry In the process we found that the previous way of tenacity syntax caused it not to honor any args. In this commit we've refactored to use the @decorator syntax, in which tenacity seems to behave better. --- core/dovecot/start.py | 18 ++++++++++++++++-- core/postfix/start.py | 20 +++++++++++++++++--- services/rspamd/start.py | 19 +++++++++++++++++-- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 8bf66efd..2fdc7361 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -6,14 +6,17 @@ import socket import glob import multiprocessing import tenacity +import logging as log +import sys from tenacity import retry from podop import run_server +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") def start_podop(): os.setuid(8) - run_server(3 if "DEBUG" in os.environ else 0, "dovecot", "/tmp/podop.socket", [ + run_server(0, "dovecot", "/tmp/podop.socket", [ ("quota", "url", "http://admin/internal/dovecot/§"), ("auth", "url", "http://admin/internal/dovecot/§"), ("sieve", "url", "http://admin/internal/dovecot/§"), @@ -21,8 +24,19 @@ def start_podop(): convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +@retry( + stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5), + before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG), + before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO), + after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG) + ) +def resolve(hostname): + logger = log.getLogger("resolve()") + logger.info(hostname) + return socket.gethostbyname(hostname) + # Actual startup script -resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front")) os.environ["REDIS_ADDRESS"] = resolve(os.environ.get("REDIS_ADDRESS", "redis")) if os.environ["WEBMAIL"] != "none": diff --git a/core/postfix/start.py b/core/postfix/start.py index 86e9a827..c41a2f4c 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -7,14 +7,18 @@ import glob import shutil import tenacity import multiprocessing +import logging as log +import sys from tenacity import retry from podop import run_server +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") def start_podop(): os.setuid(100) - run_server(3 if "DEBUG" in os.environ else 0, "postfix", "/tmp/podop.socket", [ + # TODO: Remove verbosity setting from Podop? + run_server(0, "postfix", "/tmp/podop.socket", [ ("transport", "url", "http://admin/internal/postfix/transport/§"), ("alias", "url", "http://admin/internal/postfix/alias/§"), ("domain", "url", "http://admin/internal/postfix/domain/§"), @@ -25,9 +29,19 @@ def start_podop(): convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -# Actual startup script -resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) +@retry( + stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5), + before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG), + before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO), + after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG) + ) +def resolve(hostname): + logger = log.getLogger("resolve()") + logger.info(hostname) + return socket.gethostbyname(hostname) +# Actual startup script os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front")) os.environ["HOST_ANTISPAM"] = os.environ.get("HOST_ANTISPAM", "antispam:11332") os.environ["HOST_LMTP"] = os.environ.get("HOST_LMTP", "imap:2525") diff --git a/services/rspamd/start.py b/services/rspamd/start.py index 0b3c48a8..cc4efa85 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -5,13 +5,28 @@ import os import socket import glob import tenacity +import logging as log +import sys + from tenacity import retry +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") + convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -# Actual startup script -resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) +@retry( + stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5), + before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG), + before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO), + after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG) + ) +def resolve(hostname): + logger = log.getLogger("resolve()") + logger.info(hostname) + return socket.gethostbyname(hostname) +# Actual startup script os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front")) if "HOST_REDIS" not in os.environ: os.environ["HOST_REDIS"] = "redis" From e994e94512be8f78aa9528ad7549d11047119744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 8 Jan 2019 00:35:52 +0200 Subject: [PATCH 03/15] Implement logging in clamav start script --- optional/clamav/start.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/optional/clamav/start.py b/optional/clamav/start.py index d4701d2d..19f91bcf 100755 --- a/optional/clamav/start.py +++ b/optional/clamav/start.py @@ -1,12 +1,21 @@ #!/usr/bin/python3 import os +import logging as log +import sys + +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") +logger=log.getLogger(__name__) # Bootstrap the database if clamav is running for the first time -os.system("[ -f /data/main.cvd ] || freshclam") +if not os.path.isfile("/data/main.cvd"): + logger.info("Starting primary virus DB download") + os.system("freshclam") # Run the update daemon +logger.info("Starting the update daemon") os.system("freshclam -d -c 6") # Run clamav +logger.info("Starting clamav") os.system("clamd") From b04a9d1c2833511f653eaf479f2e44b1cee750dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 8 Jan 2019 00:38:06 +0200 Subject: [PATCH 04/15] Implement debug logging for template rendering --- core/dovecot/start.py | 5 ++++- core/nginx/config.py | 11 +++++++++-- core/postfix/start.py | 5 ++++- services/rspamd/start.py | 5 ++++- services/unbound/start.py | 10 +++++++++- webmails/rainloop/start.py | 9 ++++++++- webmails/roundcube/start.py | 11 +++++++++-- 7 files changed, 47 insertions(+), 9 deletions(-) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 2fdc7361..68043cd1 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -22,7 +22,10 @@ def start_podop(): ("sieve", "url", "http://admin/internal/dovecot/§"), ]) -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) @retry( stop=tenacity.stop_after_attempt(100), diff --git a/core/nginx/config.py b/core/nginx/config.py index 07b7ea32..719a6450 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -2,11 +2,18 @@ import jinja2 import os - -convert = lambda src, dst, args: open(dst, "w").write(jinja2.Template(open(src).read()).render(**args)) +import logging as log +import sys args = os.environ.copy() +log.basicConfig(stream=sys.stderr, level=args["LOG_LEVEL"] if "LOG_LEVEL" in args else "WARN") + +def convert(src, dst, args): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**args)) + # Get the first DNS server with open("/etc/resolv.conf") as handle: content = handle.read().split() diff --git a/core/postfix/start.py b/core/postfix/start.py index c41a2f4c..3214e077 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -27,7 +27,10 @@ def start_podop(): ("senderlogin", "url", "http://admin/internal/postfix/sender/login/§") ]) -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) @retry( stop=tenacity.stop_after_attempt(100), diff --git a/services/rspamd/start.py b/services/rspamd/start.py index cc4efa85..8819407f 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -12,7 +12,10 @@ from tenacity import retry log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) @retry( stop=tenacity.stop_after_attempt(100), diff --git a/services/unbound/start.py b/services/unbound/start.py index 6f494762..198934c6 100755 --- a/services/unbound/start.py +++ b/services/unbound/start.py @@ -2,8 +2,16 @@ import jinja2 import os +import logging as log +import sys + +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") + +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) convert("/unbound.conf", "/etc/unbound/unbound.conf") os.execv("/usr/sbin/unbound", ["-c /etc/unbound/unbound.conf"]) diff --git a/webmails/rainloop/start.py b/webmails/rainloop/start.py index 4c116e09..3b4ac3da 100755 --- a/webmails/rainloop/start.py +++ b/webmails/rainloop/start.py @@ -3,8 +3,15 @@ import jinja2 import os import shutil +import logging as log +import sys -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") + +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) # Actual startup script os.environ["FRONT_ADDRESS"] = os.environ.get("FRONT_ADDRESS", "front") diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 3a0bd0bc..ccb5faf5 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -2,8 +2,15 @@ import os import jinja2 +import logging as log +import sys -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") + +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576)) @@ -14,4 +21,4 @@ os.system("mkdir -p /data/gpg") os.system("chown -R www-data:www-data /data") # Run apache -os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"]) \ No newline at end of file +os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"]) From 7d01bb2a4d78a75a29b6cdb00cd917ae4ff0cfa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 8 Jan 2019 00:58:01 +0200 Subject: [PATCH 05/15] LOG_LEVEL docs and changelog entry --- CHANGELOG.md | 1 + core/dovecot/start.py | 2 +- core/nginx/config.py | 2 +- core/postfix/start.py | 2 +- docs/compose/.env | 3 +++ docs/configuration.rst | 7 +++++++ optional/clamav/start.py | 2 +- services/rspamd/start.py | 2 +- services/unbound/start.py | 2 +- setup/flavors/compose/mailu.env | 3 +++ webmails/rainloop/start.py | 2 +- webmails/roundcube/start.py | 2 +- 12 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93b51220..bca7c66b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ v1.6.0 - unreleased - Enhancement: Move Mailu Docker network to a fixed subnet ([#727](https://github.com/Mailu/Mailu/issues/727)) - Enhancement: Added regex validation for alias username ([#764](https://github.com/Mailu/Mailu/issues/764)) - Enhancement: Update documentation +- Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to controll verbosity ([#588](https://github.com/Mailu/Mailu/issues/588)) - Upstream: Update Roundcube - Upstream: Update Rainloop - Bug: Rainloop fails with "domain not allowed" ([#93](https://github.com/Mailu/Mailu/issues/93)) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 68043cd1..b4289b76 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -12,7 +12,7 @@ import sys from tenacity import retry from podop import run_server -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") def start_podop(): os.setuid(8) diff --git a/core/nginx/config.py b/core/nginx/config.py index 719a6450..528a3247 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -7,7 +7,7 @@ import sys args = os.environ.copy() -log.basicConfig(stream=sys.stderr, level=args["LOG_LEVEL"] if "LOG_LEVEL" in args else "WARN") +log.basicConfig(stream=sys.stderr, level=args["LOG_LEVEL"] if "LOG_LEVEL" in args else "WARNING") def convert(src, dst, args): logger = log.getLogger("convert()") diff --git a/core/postfix/start.py b/core/postfix/start.py index 3214e077..5a8c2968 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -13,7 +13,7 @@ import sys from tenacity import retry from podop import run_server -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") def start_podop(): os.setuid(100) diff --git a/docs/compose/.env b/docs/compose/.env index 836e9dbf..f65c2f01 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -151,3 +151,6 @@ REAL_IP_FROM= # choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no) REJECT_UNLISTED_RECIPIENT= + +# Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET) +LOG_LEVEL=WARNING diff --git a/docs/configuration.rst b/docs/configuration.rst index 2f44b293..ecb42fe5 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -83,6 +83,13 @@ The ``PASSWORD_SCHEME`` is the password encryption scheme. You should use the default value, unless you are importing password from a separate system and want to keep using the old password encryption scheme. +The ``LOG_LEVEL`` setting is used by the python start-up scripts as a logging threshold. +Log messages equal or higher than this priority will be printed. +Can be one of: CRITICAL, ERROR, WARNING, INFO, DEBUG or NOTSET. +See the `python docs`_ for more information. + +.. _`python docs`: https://docs.python.org/3.6/library/logging.html#logging-levels + Infrastructure settings ----------------------- diff --git a/optional/clamav/start.py b/optional/clamav/start.py index 19f91bcf..c08df509 100755 --- a/optional/clamav/start.py +++ b/optional/clamav/start.py @@ -4,7 +4,7 @@ import os import logging as log import sys -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") logger=log.getLogger(__name__) # Bootstrap the database if clamav is running for the first time diff --git a/services/rspamd/start.py b/services/rspamd/start.py index 8819407f..047bfeba 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -10,7 +10,7 @@ import sys from tenacity import retry -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") def convert(src, dst): logger = log.getLogger("convert()") diff --git a/services/unbound/start.py b/services/unbound/start.py index 198934c6..43a52df1 100755 --- a/services/unbound/start.py +++ b/services/unbound/start.py @@ -5,7 +5,7 @@ import os import logging as log import sys -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") def convert(src, dst): logger = log.getLogger("convert()") diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 6bdc5e21..2d2b8735 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -160,3 +160,6 @@ REAL_IP_FROM={{ real_ip_from }} # choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no) REJECT_UNLISTED_RECIPIENT={{ reject_unlisted_recipient }} + +# Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET) +LOG_LEVEL=WARNING diff --git a/webmails/rainloop/start.py b/webmails/rainloop/start.py index 3b4ac3da..defc5be6 100755 --- a/webmails/rainloop/start.py +++ b/webmails/rainloop/start.py @@ -6,7 +6,7 @@ import shutil import logging as log import sys -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") def convert(src, dst): logger = log.getLogger("convert()") diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index ccb5faf5..8d48d3b6 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -5,7 +5,7 @@ import jinja2 import logging as log import sys -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARN") +log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") def convert(src, dst): logger = log.getLogger("convert()") From 0ac3cf961783e1213691882ed97eecb1794580b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 8 Jan 2019 05:00:15 +0200 Subject: [PATCH 06/15] Don't recursivly chown on mailboxes. This fixes #776. Recursion is not needed, as the permissions will only need to be set on the first invocation. --- CHANGELOG.md | 1 + core/dovecot/Dockerfile | 3 ++- core/dovecot/start.py | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8277743d..05013478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,7 @@ v1.6.0 - unreleased - Bug: Error when trying to log in with an account without domain ([#585](https://github.com/Mailu/Mailu/issues/585)) - Bug: Fix rainloop permissions ([#637](https://github.com/Mailu/Mailu/issues/637)) - Bug: Fix broken webmail and logo url in admin ([#792](https://github.com/Mailu/Mailu/issues/792)) +- Bug: Don't recursivly chown on mailboxes ([#776](https://github.com/Mailu/Mailu/issues/776)) v1.5.1 - 2017-11-21 ------------------- diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 1d4f7b91..dec2b520 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -10,7 +10,8 @@ RUN pip3 install tenacity # Image specific layers under this line RUN apk add --no-cache \ dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client bash \ - && pip3 install podop + && pip3 install podop \ + && mkdir /var/lib/dovecot COPY conf /conf COPY start.py /start.py diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 8bf66efd..c97b8a05 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -33,5 +33,6 @@ for dovecot_file in glob.glob("/conf/*.conf"): # Run Podop, then postfix multiprocessing.Process(target=start_podop).start() -os.system("chown -R mail:mail /mail /var/lib/dovecot /conf") +os.system("chown mail:mail /mail") +os.system("chown -R mail:mail /var/lib/dovecot /conf") os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"]) From 049ca9941f256dc6c83e990e8181d383b1ef1142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 8 Jan 2019 05:16:05 +0200 Subject: [PATCH 07/15] Cleanup syntax and fix typo --- CHANGELOG.md | 2 +- core/dovecot/start.py | 2 +- core/nginx/config.py | 2 +- core/postfix/start.py | 2 +- services/rspamd/start.py | 2 +- services/unbound/start.py | 2 +- webmails/rainloop/start.py | 2 +- webmails/roundcube/start.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79aefd40..e69c37f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,8 +76,8 @@ v1.6.0 - unreleased - Enhancement: Move Mailu Docker network to a fixed subnet ([#727](https://github.com/Mailu/Mailu/issues/727)) - Enhancement: Added regex validation for alias username ([#764](https://github.com/Mailu/Mailu/issues/764)) - Enhancement: Update documentation -- Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to controll verbosity ([#588](https://github.com/Mailu/Mailu/issues/588)) - Enhancement: Include favicon package ([#801](https://github.com/Mailu/Mailu/issues/801), ([#802](https://github.com/Mailu/Mailu/issues/802)) +- Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to control verbosity ([#588] - Upstream: Update Roundcube - Upstream: Update Rainloop - Bug: Rainloop fails with "domain not allowed" ([#93](https://github.com/Mailu/Mailu/issues/93)) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index b4289b76..d9273464 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -12,7 +12,7 @@ import sys from tenacity import retry from podop import run_server -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def start_podop(): os.setuid(8) diff --git a/core/nginx/config.py b/core/nginx/config.py index 528a3247..79370508 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -7,7 +7,7 @@ import sys args = os.environ.copy() -log.basicConfig(stream=sys.stderr, level=args["LOG_LEVEL"] if "LOG_LEVEL" in args else "WARNING") +log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING")) def convert(src, dst, args): logger = log.getLogger("convert()") diff --git a/core/postfix/start.py b/core/postfix/start.py index 5a8c2968..a06b3833 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -13,7 +13,7 @@ import sys from tenacity import retry from podop import run_server -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def start_podop(): os.setuid(100) diff --git a/services/rspamd/start.py b/services/rspamd/start.py index 047bfeba..744d4a9c 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -10,7 +10,7 @@ import sys from tenacity import retry -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def convert(src, dst): logger = log.getLogger("convert()") diff --git a/services/unbound/start.py b/services/unbound/start.py index 43a52df1..4dd5f3be 100755 --- a/services/unbound/start.py +++ b/services/unbound/start.py @@ -5,7 +5,7 @@ import os import logging as log import sys -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def convert(src, dst): logger = log.getLogger("convert()") diff --git a/webmails/rainloop/start.py b/webmails/rainloop/start.py index defc5be6..e2b917bf 100755 --- a/webmails/rainloop/start.py +++ b/webmails/rainloop/start.py @@ -6,7 +6,7 @@ import shutil import logging as log import sys -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def convert(src, dst): logger = log.getLogger("convert()") diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 8d48d3b6..4effd965 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -5,7 +5,7 @@ import jinja2 import logging as log import sys -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def convert(src, dst): logger = log.getLogger("convert()") From 3e97e7e1c208bc86d8661e105b870ec0bebb3832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 8 Jan 2019 05:26:04 +0200 Subject: [PATCH 08/15] Missed a spot of cleaning syntax --- optional/clamav/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optional/clamav/start.py b/optional/clamav/start.py index c08df509..56e1bcfe 100755 --- a/optional/clamav/start.py +++ b/optional/clamav/start.py @@ -4,7 +4,7 @@ import os import logging as log import sys -log.basicConfig(stream=sys.stderr, level=os.environ["LOG_LEVEL"] if "LOG_LEVEL" in os.environ else "WARNING") +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) logger=log.getLogger(__name__) # Bootstrap the database if clamav is running for the first time From 77ef7317a92d56bb7c95a3d50c3667524db84d6b Mon Sep 17 00:00:00 2001 From: hoellen Date: Tue, 8 Jan 2019 14:06:19 +0200 Subject: [PATCH 09/15] Add link in changelog Co-Authored-By: muhlemmer --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69c37f4..f650ca45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,7 +77,7 @@ v1.6.0 - unreleased - Enhancement: Added regex validation for alias username ([#764](https://github.com/Mailu/Mailu/issues/764)) - Enhancement: Update documentation - Enhancement: Include favicon package ([#801](https://github.com/Mailu/Mailu/issues/801), ([#802](https://github.com/Mailu/Mailu/issues/802)) -- Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to control verbosity ([#588] +- Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to control verbosity ([#588](https://github.com/Mailu/Mailu/issues/588)) - Upstream: Update Roundcube - Upstream: Update Rainloop - Bug: Rainloop fails with "domain not allowed" ([#93](https://github.com/Mailu/Mailu/issues/93)) From 732b5fe161843389de061a066bce7137a36a5cf3 Mon Sep 17 00:00:00 2001 From: hoellen Date: Tue, 8 Jan 2019 19:29:34 +0100 Subject: [PATCH 10/15] change password field type in fetch creation/edit and add validators. --- CHANGELOG.md | 1 + core/admin/mailu/ui/forms.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac4c83af..34c976e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,7 @@ v1.6.0 - unreleased - Bug: Fix rainloop permissions ([#637](https://github.com/Mailu/Mailu/issues/637)) - Bug: Fix broken webmail and logo url in admin ([#792](https://github.com/Mailu/Mailu/issues/792)) - Bug: Don't recursivly chown on mailboxes ([#776](https://github.com/Mailu/Mailu/issues/776)) +- Bug: Fetched accounts: Password field is of type "text" ([#789](https://github.com/issues/789)) v1.5.1 - 2017-11-21 ------------------- diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index 5ee6da7d..9967fefc 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -165,11 +165,11 @@ class FetchForm(flask_wtf.FlaskForm): protocol = fields.SelectField(_('Protocol'), choices=[ ('imap', 'IMAP'), ('pop3', 'POP3') ]) - host = fields.StringField(_('Hostname or IP')) - port = fields.IntegerField(_('TCP port')) + host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()]) + port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)]) tls = fields.BooleanField(_('Enable TLS')) - username = fields.StringField(_('Username')) - password = fields.StringField(_('Password')) + username = fields.StringField(_('Username'), [validators.DataRequired()]) + password = fields.PasswordField(_('Password'), [validators.DataRequired()]) keep = fields.BooleanField(_('Keep emails on the server')) submit = fields.SubmitField(_('Submit')) From ff9558026d855302a08fe00e9903105667b49ed1 Mon Sep 17 00:00:00 2001 From: hoellen Date: Tue, 8 Jan 2019 19:49:16 +0100 Subject: [PATCH 11/15] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c976e3..d0ce37ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,7 +111,7 @@ v1.6.0 - unreleased - Bug: Fix rainloop permissions ([#637](https://github.com/Mailu/Mailu/issues/637)) - Bug: Fix broken webmail and logo url in admin ([#792](https://github.com/Mailu/Mailu/issues/792)) - Bug: Don't recursivly chown on mailboxes ([#776](https://github.com/Mailu/Mailu/issues/776)) -- Bug: Fetched accounts: Password field is of type "text" ([#789](https://github.com/issues/789)) +- Bug: Fetched accounts: Password field is of type "text" ([#789](https://github.com/Mailu/Mailu/issues/789)) v1.5.1 - 2017-11-21 ------------------- From f08491dc469f8620caea5f70b21be5021c21e2ea Mon Sep 17 00:00:00 2001 From: hoellen Date: Wed, 9 Jan 2019 12:03:47 +0100 Subject: [PATCH 12/15] fix forced password on user edit --- CHANGELOG.md | 1 + core/admin/mailu/ui/forms.py | 2 +- core/admin/mailu/ui/views/users.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac4c83af..1c6b4870 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,7 @@ v1.6.0 - unreleased - Bug: Fix rainloop permissions ([#637](https://github.com/Mailu/Mailu/issues/637)) - Bug: Fix broken webmail and logo url in admin ([#792](https://github.com/Mailu/Mailu/issues/792)) - Bug: Don't recursivly chown on mailboxes ([#776](https://github.com/Mailu/Mailu/issues/776)) +- Bug: Fix forced password input for user edit ([#745](https://github.com/Mailu/Mailu/issues/745)) v1.5.1 - 2017-11-21 ------------------- diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index 5ee6da7d..5312e7d9 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -84,7 +84,7 @@ class RelayForm(flask_wtf.FlaskForm): class UserForm(flask_wtf.FlaskForm): localpart = fields.StringField(_('E-mail'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)]) - pw = fields.PasswordField(_('Password'), [validators.DataRequired()]) + pw = fields.PasswordField(_('Password')) pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')]) quota_bytes = fields_.IntegerSliderField(_('Quota'), default=1000000000) enable_imap = fields.BooleanField(_('Allow IMAP access'), default=True) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index e3c03848..8bdb76b1 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -23,6 +23,7 @@ def user_create(domain_name): return flask.redirect( flask.url_for('.user_list', domain_name=domain.name)) form = forms.UserForm() + form.pw.validators = [wtforms.validators.DataRequired()] if domain.max_quota_bytes: form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=domain.max_quota_bytes)] @@ -54,7 +55,6 @@ def user_edit(user_email): # Create the form form = forms.UserForm(obj=user) wtforms_components.read_only(form.localpart) - form.pw.validators = [] form.localpart.validators = [] if max_quota_bytes: form.quota_bytes.validators = [ From a59d5dad23e1e4c6c3420441f33fcd324c3926df Mon Sep 17 00:00:00 2001 From: hoellen Date: Wed, 9 Jan 2019 12:52:05 +0100 Subject: [PATCH 13/15] fix edit of fetched acc without changing password --- core/admin/mailu/ui/forms.py | 2 +- core/admin/mailu/ui/views/fetches.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index 9967fefc..b6b6f34e 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -169,7 +169,7 @@ class FetchForm(flask_wtf.FlaskForm): port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)]) tls = fields.BooleanField(_('Enable TLS')) username = fields.StringField(_('Username'), [validators.DataRequired()]) - password = fields.PasswordField(_('Password'), [validators.DataRequired()]) + password = fields.PasswordField(_('Password')) keep = fields.BooleanField(_('Keep emails on the server')) submit = fields.SubmitField(_('Submit')) diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index d9f55404..f2049fe9 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -3,6 +3,7 @@ from mailu.ui import ui, forms, access import flask import flask_login +import wtforms @ui.route('/fetch/list', methods=['GET', 'POST'], defaults={'user_email': None}) @@ -21,6 +22,7 @@ def fetch_create(user_email): user_email = user_email or flask_login.current_user.email user = models.User.query.get(user_email) or flask.abort(404) form = forms.FetchForm() + form.pw.validators = [wtforms.validators.DataRequired()] if form.validate_on_submit(): fetch = models.Fetch(user=user) form.populate_obj(fetch) @@ -38,6 +40,8 @@ def fetch_edit(fetch_id): fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) form = forms.FetchForm(obj=fetch) if form.validate_on_submit(): + if not form.password.data: + form.password.data = fetch.password form.populate_obj(fetch) models.db.session.commit() flask.flash('Fetch configuration updated') From b65d70cf1eb9bdb98578f0b0399fd387af94c0d1 Mon Sep 17 00:00:00 2001 From: hoellen Date: Wed, 9 Jan 2019 19:53:52 +0100 Subject: [PATCH 14/15] mark spam as seen --- CHANGELOG.md | 1 + core/dovecot/conf/report-spam.sieve | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ce37ae..735dd75e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ v1.6.0 - unreleased - Enhancement: Update documentation - Enhancement: Include favicon package ([#801](https://github.com/Mailu/Mailu/issues/801), ([#802](https://github.com/Mailu/Mailu/issues/802)) - Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to control verbosity ([#588](https://github.com/Mailu/Mailu/issues/588)) +- Enhancement: Mark message as seen when reporting as spam - Upstream: Update Roundcube - Upstream: Update Rainloop - Bug: Rainloop fails with "domain not allowed" ([#93](https://github.com/Mailu/Mailu/issues/93)) diff --git a/core/dovecot/conf/report-spam.sieve b/core/dovecot/conf/report-spam.sieve index 108d6210..87fd515e 100644 --- a/core/dovecot/conf/report-spam.sieve +++ b/core/dovecot/conf/report-spam.sieve @@ -1,3 +1,5 @@ +require "imap4flags"; require "vnd.dovecot.execute"; +setflag "\\seen"; execute :pipe "spam"; From 492f3867d891577d941ca0325ef9d9a4e79bcb33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20S=C3=A4nger?= Date: Thu, 10 Jan 2019 12:07:42 +0100 Subject: [PATCH 15/15] remove (broken) FTS --- CHANGELOG.md | 1 - core/dovecot/Dockerfile | 2 +- core/dovecot/conf/dovecot.conf | 16 ---------------- docs/compose/.env | 4 ---- 4 files changed, 1 insertion(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3632f9ff..0458ab5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,6 @@ v1.6.0 - unreleased - Feature: Add posibilty to run webmail on root ([#501](https://github.com/Mailu/Mailu/issues/501)) - Feature: Upgrade docker-compose.yml to version 3 ([#539](https://github.com/Mailu/Mailu/issues/539)) - Feature: Documentation to deploy mailu on a docker swarm ([#551](https://github.com/Mailu/Mailu/issues/551)) -- Feature: Add full-text search support ([#552](https://github.com/Mailu/Mailu/issues/552)) - Feature: Add optional Maildir-Compression ([#553](https://github.com/Mailu/Mailu/issues/553)) - Feature: Preserve rspamd history on container restart ([#561](https://github.com/Mailu/Mailu/issues/561)) - Feature: FAQ ([#564](https://github.com/Mailu/Mailu/issues/564), [#677](https://github.com/Mailu/Mailu/issues/677)) diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index dec2b520..83d23b52 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -9,7 +9,7 @@ RUN pip3 install jinja2 RUN pip3 install tenacity # Image specific layers under this line RUN apk add --no-cache \ - dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client bash \ + dovecot dovecot-pigeonhole-plugin rspamd-client bash \ && pip3 install podop \ && mkdir /var/lib/dovecot diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 83c78f16..b7cca76c 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -7,22 +7,6 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }} hostname = {{ HOSTNAMES.split(",")[0] }} submission_host = {{ FRONT_ADDRESS }} -{% if DISABLE_FTS_LUCENE != 'true' %} -############### -# Full-text search -############### -mail_plugins = $mail_plugins fts fts_lucene - -plugin { - fts = lucene - - fts_autoindex = yes - fts_autoindex_exclude = \Junk - - fts_lucene = whitespace_chars=@. -} -{% endif %} - ############### # Mailboxes ############### diff --git a/docs/compose/.env b/docs/compose/.env index f65c2f01..cf906b58 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -3,10 +3,6 @@ # these few settings must however be configured before starting the mail # server and require a restart upon change. -# Set this to `true` to disable full text search by lucene (value: true, false) -# This is a workaround for the bug in issue #751 (indexer-worker crashes) -DISABLE_FTS_LUCENE=false - ################################### # Common configuration variables ###################################