diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a5c65d4..344fe29a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,6 @@ v1.6.0 - unreleased - Feature: Add posibilty to run webmail on root ([#501](https://github.com/Mailu/Mailu/issues/501)) - Feature: Upgrade docker-compose.yml to version 3 ([#539](https://github.com/Mailu/Mailu/issues/539)) - Feature: Documentation to deploy mailu on a docker swarm ([#551](https://github.com/Mailu/Mailu/issues/551)) -- Feature: Add full-text search support ([#552](https://github.com/Mailu/Mailu/issues/552)) - Feature: Add optional Maildir-Compression ([#553](https://github.com/Mailu/Mailu/issues/553)) - Feature: Preserve rspamd history on container restart ([#561](https://github.com/Mailu/Mailu/issues/561)) - Feature: FAQ ([#564](https://github.com/Mailu/Mailu/issues/564), [#677](https://github.com/Mailu/Mailu/issues/677)) @@ -78,6 +77,8 @@ v1.6.0 - unreleased - Enhancement: Added regex validation for alias username ([#764](https://github.com/Mailu/Mailu/issues/764)) - Enhancement: Update documentation - Enhancement: Include favicon package ([#801](https://github.com/Mailu/Mailu/issues/801), ([#802](https://github.com/Mailu/Mailu/issues/802)) +- Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to control verbosity ([#588](https://github.com/Mailu/Mailu/issues/588)) +- Enhancement: Mark message as seen when reporting as spam - Upstream: Update Roundcube - Upstream: Update Rainloop - Bug: Rainloop fails with "domain not allowed" ([#93](https://github.com/Mailu/Mailu/issues/93)) @@ -110,6 +111,9 @@ v1.6.0 - unreleased - Bug: Error when trying to log in with an account without domain ([#585](https://github.com/Mailu/Mailu/issues/585)) - Bug: Fix rainloop permissions ([#637](https://github.com/Mailu/Mailu/issues/637)) - Bug: Fix broken webmail and logo url in admin ([#792](https://github.com/Mailu/Mailu/issues/792)) +- Bug: Don't recursivly chown on mailboxes ([#776](https://github.com/Mailu/Mailu/issues/776)) +- Bug: Fix forced password input for user edit ([#745](https://github.com/Mailu/Mailu/issues/745)) +- Bug: Fetched accounts: Password field is of type "text" ([#789](https://github.com/Mailu/Mailu/issues/789)) v1.5.1 - 2017-11-21 ------------------- diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index d4594df9..42aecbf0 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -267,10 +267,19 @@ class Email(object): @classmethod def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False): + localpart_stripped = None + if os.environ.get('RECIPIENT_DELIMITER') in localpart: + localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0] + alias = Alias.resolve(localpart, domain_name) + if not alias and localpart_stripped: + alias = Alias.resolve(localpart_stripped, domain_name) if alias: return alias.destination + user = User.query.get('{}@{}'.format(localpart, domain_name)) + if not user and localpart_stripped: + user = User.query.get('{}@{}'.format(localpart_stripped, domain_name)) if user: if user.forward_enabled: destination = user.forward_destination diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index 5ee6da7d..783b7166 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -84,7 +84,7 @@ class RelayForm(flask_wtf.FlaskForm): class UserForm(flask_wtf.FlaskForm): localpart = fields.StringField(_('E-mail'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)]) - pw = fields.PasswordField(_('Password'), [validators.DataRequired()]) + pw = fields.PasswordField(_('Password')) pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')]) quota_bytes = fields_.IntegerSliderField(_('Quota'), default=1000000000) enable_imap = fields.BooleanField(_('Allow IMAP access'), default=True) @@ -165,11 +165,11 @@ class FetchForm(flask_wtf.FlaskForm): protocol = fields.SelectField(_('Protocol'), choices=[ ('imap', 'IMAP'), ('pop3', 'POP3') ]) - host = fields.StringField(_('Hostname or IP')) - port = fields.IntegerField(_('TCP port')) + host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()]) + port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)]) tls = fields.BooleanField(_('Enable TLS')) - username = fields.StringField(_('Username')) - password = fields.StringField(_('Password')) + username = fields.StringField(_('Username'), [validators.DataRequired()]) + password = fields.PasswordField(_('Password')) keep = fields.BooleanField(_('Keep emails on the server')) submit = fields.SubmitField(_('Submit')) diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index d9f55404..f2049fe9 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -3,6 +3,7 @@ from mailu.ui import ui, forms, access import flask import flask_login +import wtforms @ui.route('/fetch/list', methods=['GET', 'POST'], defaults={'user_email': None}) @@ -21,6 +22,7 @@ def fetch_create(user_email): user_email = user_email or flask_login.current_user.email user = models.User.query.get(user_email) or flask.abort(404) form = forms.FetchForm() + form.pw.validators = [wtforms.validators.DataRequired()] if form.validate_on_submit(): fetch = models.Fetch(user=user) form.populate_obj(fetch) @@ -38,6 +40,8 @@ def fetch_edit(fetch_id): fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) form = forms.FetchForm(obj=fetch) if form.validate_on_submit(): + if not form.password.data: + form.password.data = fetch.password form.populate_obj(fetch) models.db.session.commit() flask.flash('Fetch configuration updated') diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index e3c03848..8bdb76b1 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -23,6 +23,7 @@ def user_create(domain_name): return flask.redirect( flask.url_for('.user_list', domain_name=domain.name)) form = forms.UserForm() + form.pw.validators = [wtforms.validators.DataRequired()] if domain.max_quota_bytes: form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=domain.max_quota_bytes)] @@ -54,7 +55,6 @@ def user_edit(user_email): # Create the form form = forms.UserForm(obj=user) wtforms_components.read_only(form.localpart) - form.pw.validators = [] form.localpart.validators = [] if max_quota_bytes: form.quota_bytes.validators = [ diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index 1d4f7b91..83d23b52 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -9,8 +9,9 @@ RUN pip3 install jinja2 RUN pip3 install tenacity # Image specific layers under this line RUN apk add --no-cache \ - dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client bash \ - && pip3 install podop + dovecot dovecot-pigeonhole-plugin rspamd-client bash \ + && pip3 install podop \ + && mkdir /var/lib/dovecot COPY conf /conf COPY start.py /start.py diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 83c78f16..b7cca76c 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -7,22 +7,6 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }} hostname = {{ HOSTNAMES.split(",")[0] }} submission_host = {{ FRONT_ADDRESS }} -{% if DISABLE_FTS_LUCENE != 'true' %} -############### -# Full-text search -############### -mail_plugins = $mail_plugins fts fts_lucene - -plugin { - fts = lucene - - fts_autoindex = yes - fts_autoindex_exclude = \Junk - - fts_lucene = whitespace_chars=@. -} -{% endif %} - ############### # Mailboxes ############### diff --git a/core/dovecot/conf/report-spam.sieve b/core/dovecot/conf/report-spam.sieve index 108d6210..87fd515e 100644 --- a/core/dovecot/conf/report-spam.sieve +++ b/core/dovecot/conf/report-spam.sieve @@ -1,3 +1,5 @@ +require "imap4flags"; require "vnd.dovecot.execute"; +setflag "\\seen"; execute :pipe "spam"; diff --git a/core/dovecot/start.py b/core/dovecot/start.py index 8bf66efd..15e370de 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -6,23 +6,40 @@ import socket import glob import multiprocessing import tenacity +import logging as log +import sys from tenacity import retry from podop import run_server +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def start_podop(): os.setuid(8) - run_server(3 if "DEBUG" in os.environ else 0, "dovecot", "/tmp/podop.socket", [ + run_server(0, "dovecot", "/tmp/podop.socket", [ ("quota", "url", "http://admin/internal/dovecot/§"), ("auth", "url", "http://admin/internal/dovecot/§"), ("sieve", "url", "http://admin/internal/dovecot/§"), ]) -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) + +@retry( + stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5), + before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG), + before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO), + after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG) + ) +def resolve(hostname): + logger = log.getLogger("resolve()") + logger.info(hostname) + return socket.gethostbyname(hostname) # Actual startup script -resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front")) os.environ["REDIS_ADDRESS"] = resolve(os.environ.get("REDIS_ADDRESS", "redis")) if os.environ["WEBMAIL"] != "none": @@ -33,5 +50,6 @@ for dovecot_file in glob.glob("/conf/*.conf"): # Run Podop, then postfix multiprocessing.Process(target=start_podop).start() -os.system("chown -R mail:mail /mail /var/lib/dovecot /conf") +os.system("chown mail:mail /mail") +os.system("chown -R mail:mail /var/lib/dovecot /conf") os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"]) diff --git a/core/nginx/config.py b/core/nginx/config.py index 07b7ea32..79370508 100755 --- a/core/nginx/config.py +++ b/core/nginx/config.py @@ -2,11 +2,18 @@ import jinja2 import os - -convert = lambda src, dst, args: open(dst, "w").write(jinja2.Template(open(src).read()).render(**args)) +import logging as log +import sys args = os.environ.copy() +log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING")) + +def convert(src, dst, args): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**args)) + # Get the first DNS server with open("/etc/resolv.conf") as handle: content = handle.read().split() diff --git a/core/postfix/start.py b/core/postfix/start.py index 86e9a827..a06b3833 100755 --- a/core/postfix/start.py +++ b/core/postfix/start.py @@ -7,14 +7,18 @@ import glob import shutil import tenacity import multiprocessing +import logging as log +import sys from tenacity import retry from podop import run_server +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) def start_podop(): os.setuid(100) - run_server(3 if "DEBUG" in os.environ else 0, "postfix", "/tmp/podop.socket", [ + # TODO: Remove verbosity setting from Podop? + run_server(0, "postfix", "/tmp/podop.socket", [ ("transport", "url", "http://admin/internal/postfix/transport/§"), ("alias", "url", "http://admin/internal/postfix/alias/§"), ("domain", "url", "http://admin/internal/postfix/domain/§"), @@ -23,11 +27,24 @@ def start_podop(): ("senderlogin", "url", "http://admin/internal/postfix/sender/login/§") ]) -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) + +@retry( + stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5), + before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG), + before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO), + after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG) + ) +def resolve(hostname): + logger = log.getLogger("resolve()") + logger.info(hostname) + return socket.gethostbyname(hostname) # Actual startup script -resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) - os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front")) os.environ["HOST_ANTISPAM"] = os.environ.get("HOST_ANTISPAM", "antispam:11332") os.environ["HOST_LMTP"] = os.environ.get("HOST_LMTP", "imap:2525") diff --git a/docs/compose/.env b/docs/compose/.env index 836e9dbf..cf906b58 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -3,10 +3,6 @@ # these few settings must however be configured before starting the mail # server and require a restart upon change. -# Set this to `true` to disable full text search by lucene (value: true, false) -# This is a workaround for the bug in issue #751 (indexer-worker crashes) -DISABLE_FTS_LUCENE=false - ################################### # Common configuration variables ################################### @@ -151,3 +147,6 @@ REAL_IP_FROM= # choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no) REJECT_UNLISTED_RECIPIENT= + +# Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET) +LOG_LEVEL=WARNING diff --git a/docs/configuration.rst b/docs/configuration.rst index b8f2a90c..ec114c97 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -91,6 +91,13 @@ The ``PASSWORD_SCHEME`` is the password encryption scheme. You should use the default value, unless you are importing password from a separate system and want to keep using the old password encryption scheme. +The ``LOG_LEVEL`` setting is used by the python start-up scripts as a logging threshold. +Log messages equal or higher than this priority will be printed. +Can be one of: CRITICAL, ERROR, WARNING, INFO, DEBUG or NOTSET. +See the `python docs`_ for more information. + +.. _`python docs`: https://docs.python.org/3.6/library/logging.html#logging-levels + Infrastructure settings ----------------------- diff --git a/optional/clamav/start.py b/optional/clamav/start.py index d4701d2d..56e1bcfe 100755 --- a/optional/clamav/start.py +++ b/optional/clamav/start.py @@ -1,12 +1,21 @@ #!/usr/bin/python3 import os +import logging as log +import sys + +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) +logger=log.getLogger(__name__) # Bootstrap the database if clamav is running for the first time -os.system("[ -f /data/main.cvd ] || freshclam") +if not os.path.isfile("/data/main.cvd"): + logger.info("Starting primary virus DB download") + os.system("freshclam") # Run the update daemon +logger.info("Starting the update daemon") os.system("freshclam -d -c 6") # Run clamav +logger.info("Starting clamav") os.system("clamd") diff --git a/services/rspamd/start.py b/services/rspamd/start.py index 0b3c48a8..744d4a9c 100755 --- a/services/rspamd/start.py +++ b/services/rspamd/start.py @@ -5,13 +5,31 @@ import os import socket import glob import tenacity +import logging as log +import sys + from tenacity import retry -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) + +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) + +@retry( + stop=tenacity.stop_after_attempt(100), + wait=tenacity.wait_random(min=2, max=5), + before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG), + before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO), + after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG) + ) +def resolve(hostname): + logger = log.getLogger("resolve()") + logger.info(hostname) + return socket.gethostbyname(hostname) # Actual startup script -resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5)) - os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front")) if "HOST_REDIS" not in os.environ: os.environ["HOST_REDIS"] = "redis" diff --git a/services/unbound/start.py b/services/unbound/start.py index 6f494762..4dd5f3be 100755 --- a/services/unbound/start.py +++ b/services/unbound/start.py @@ -2,8 +2,16 @@ import jinja2 import os +import logging as log +import sys + +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) + +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) convert("/unbound.conf", "/etc/unbound/unbound.conf") os.execv("/usr/sbin/unbound", ["-c /etc/unbound/unbound.conf"]) diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index bb500073..7d160011 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -161,6 +161,9 @@ REAL_IP_FROM={{ real_ip_from }} # choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no) REJECT_UNLISTED_RECIPIENT={{ reject_unlisted_recipient }} +# Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET) +LOG_LEVEL=WARNING + ################################### # Database settings ################################### diff --git a/webmails/rainloop/start.py b/webmails/rainloop/start.py index 4c116e09..e2b917bf 100755 --- a/webmails/rainloop/start.py +++ b/webmails/rainloop/start.py @@ -3,8 +3,15 @@ import jinja2 import os import shutil +import logging as log +import sys -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) + +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) # Actual startup script os.environ["FRONT_ADDRESS"] = os.environ.get("FRONT_ADDRESS", "front") diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 3a0bd0bc..4effd965 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -2,8 +2,15 @@ import os import jinja2 +import logging as log +import sys -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) + +def convert(src, dst): + logger = log.getLogger("convert()") + logger.debug("Source: %s, Destination: %s", src, dst) + open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576)) @@ -14,4 +21,4 @@ os.system("mkdir -p /data/gpg") os.system("chown -R www-data:www-data /data") # Run apache -os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"]) \ No newline at end of file +os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"])