Merge remote-tracking branch 'upstream/master' into feat-psql-support

CHANGELOG.md

@@ -31,7 +31,6 @@ v1.6.0 - unreleased
 - Feature: Add posibilty to run webmail on root ([#501](https://github.com/Mailu/Mailu/issues/501))
 - Feature: Upgrade docker-compose.yml to version 3 ([#539](https://github.com/Mailu/Mailu/issues/539))
 - Feature: Documentation to deploy mailu on a docker swarm ([#551](https://github.com/Mailu/Mailu/issues/551))
-- Feature: Add full-text search support ([#552](https://github.com/Mailu/Mailu/issues/552))
 - Feature: Add optional Maildir-Compression ([#553](https://github.com/Mailu/Mailu/issues/553))
 - Feature: Preserve rspamd history on container restart ([#561](https://github.com/Mailu/Mailu/issues/561))
 - Feature: FAQ ([#564](https://github.com/Mailu/Mailu/issues/564), [#677](https://github.com/Mailu/Mailu/issues/677))
@@ -78,6 +77,8 @@ v1.6.0 - unreleased
 - Enhancement: Added regex validation for alias username ([#764](https://github.com/Mailu/Mailu/issues/764))
 - Enhancement: Update documentation
 - Enhancement: Include favicon package ([#801](https://github.com/Mailu/Mailu/issues/801), ([#802](https://github.com/Mailu/Mailu/issues/802))
+- Enhancement: Add logging at critical places in python start.py scripts. Implement LOG_LEVEL to control verbosity ([#588](https://github.com/Mailu/Mailu/issues/588))
+- Enhancement: Mark message as seen when reporting as spam
 - Upstream: Update Roundcube
 - Upstream: Update Rainloop
 - Bug: Rainloop fails with "domain not allowed" ([#93](https://github.com/Mailu/Mailu/issues/93))
@@ -110,6 +111,9 @@ v1.6.0 - unreleased
 - Bug: Error when trying to log in with an account without domain ([#585](https://github.com/Mailu/Mailu/issues/585))
 - Bug: Fix rainloop permissions ([#637](https://github.com/Mailu/Mailu/issues/637))
 - Bug: Fix broken webmail and logo url in admin ([#792](https://github.com/Mailu/Mailu/issues/792))
+- Bug: Don't recursivly chown on mailboxes ([#776](https://github.com/Mailu/Mailu/issues/776))
+- Bug: Fix forced password input for user edit ([#745](https://github.com/Mailu/Mailu/issues/745))
+- Bug: Fetched accounts: Password field is of type "text" ([#789](https://github.com/Mailu/Mailu/issues/789))
 
 v1.5.1 - 2017-11-21
 -------------------

core/admin/mailu/models.py

@@ -267,10 +267,19 @@ class Email(object):
 
     @classmethod
     def resolve_destination(cls, localpart, domain_name, ignore_forward_keep=False):
+        localpart_stripped = None
+        if os.environ.get('RECIPIENT_DELIMITER') in localpart:
+            localpart_stripped = localpart.rsplit(os.environ.get('RECIPIENT_DELIMITER'), 1)[0]
+
         alias = Alias.resolve(localpart, domain_name)
+        if not alias and localpart_stripped:
+            alias = Alias.resolve(localpart_stripped, domain_name)
         if alias:
             return alias.destination
+
         user = User.query.get('{}@{}'.format(localpart, domain_name))
+        if not user and localpart_stripped:
+            user = User.query.get('{}@{}'.format(localpart_stripped, domain_name))
         if user:
             if user.forward_enabled:
                 destination = user.forward_destination

core/admin/mailu/ui/forms.py

@@ -84,7 +84,7 @@ class RelayForm(flask_wtf.FlaskForm):
 
 class UserForm(flask_wtf.FlaskForm):
     localpart = fields.StringField(_('E-mail'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)])
-    pw = fields.PasswordField(_('Password'), [validators.DataRequired()])
+    pw = fields.PasswordField(_('Password'))
     pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')])
     quota_bytes = fields_.IntegerSliderField(_('Quota'), default=1000000000)
     enable_imap = fields.BooleanField(_('Allow IMAP access'), default=True)
@@ -165,11 +165,11 @@ class FetchForm(flask_wtf.FlaskForm):
     protocol = fields.SelectField(_('Protocol'), choices=[
         ('imap', 'IMAP'), ('pop3', 'POP3')
     ])
-    host = fields.StringField(_('Hostname or IP'))
+    host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()])
-    port = fields.IntegerField(_('TCP port'))
+    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)])
     tls = fields.BooleanField(_('Enable TLS'))
-    username = fields.StringField(_('Username'))
+    username = fields.StringField(_('Username'), [validators.DataRequired()])
-    password = fields.StringField(_('Password'))
+    password = fields.PasswordField(_('Password'))
     keep = fields.BooleanField(_('Keep emails on the server'))
     submit = fields.SubmitField(_('Submit'))
 

core/admin/mailu/ui/views/fetches.py

@@ -3,6 +3,7 @@ from mailu.ui import ui, forms, access
 
 import flask
 import flask_login
+import wtforms
 
 
 @ui.route('/fetch/list', methods=['GET', 'POST'], defaults={'user_email': None})
@@ -21,6 +22,7 @@ def fetch_create(user_email):
     user_email = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email) or flask.abort(404)
     form = forms.FetchForm()
+    form.pw.validators = [wtforms.validators.DataRequired()]
     if form.validate_on_submit():
         fetch = models.Fetch(user=user)
         form.populate_obj(fetch)
@@ -38,6 +40,8 @@ def fetch_edit(fetch_id):
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     form = forms.FetchForm(obj=fetch)
     if form.validate_on_submit():
+        if not form.password.data:
+            form.password.data = fetch.password
         form.populate_obj(fetch)
         models.db.session.commit()
         flask.flash('Fetch configuration updated')

core/admin/mailu/ui/views/users.py

@@ -23,6 +23,7 @@ def user_create(domain_name):
         return flask.redirect(
             flask.url_for('.user_list', domain_name=domain.name))
     form = forms.UserForm()
+    form.pw.validators = [wtforms.validators.DataRequired()]
     if domain.max_quota_bytes:
         form.quota_bytes.validators = [
             wtforms.validators.NumberRange(max=domain.max_quota_bytes)]
@@ -54,7 +55,6 @@ def user_edit(user_email):
     # Create the form
     form = forms.UserForm(obj=user)
     wtforms_components.read_only(form.localpart)
-    form.pw.validators = []
     form.localpart.validators = []
     if max_quota_bytes:
         form.quota_bytes.validators = [

core/dovecot/Dockerfile

@@ -9,8 +9,9 @@ RUN pip3 install jinja2
 RUN pip3 install tenacity
 # Image specific layers under this line
 RUN apk add --no-cache \
-  dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client bash \
+  dovecot dovecot-pigeonhole-plugin rspamd-client bash \
-  && pip3 install podop
+  && pip3 install podop \
+  && mkdir /var/lib/dovecot
 
 COPY conf /conf
 COPY start.py /start.py

core/dovecot/conf/dovecot.conf

@@ -7,22 +7,6 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
 hostname = {{ HOSTNAMES.split(",")[0] }}
 submission_host = {{ FRONT_ADDRESS }}
 
-{% if DISABLE_FTS_LUCENE != 'true' %}
-###############
-# Full-text search
-###############
-mail_plugins =  $mail_plugins fts fts_lucene
-
-plugin {
-  fts = lucene
-
-  fts_autoindex = yes
-  fts_autoindex_exclude = \Junk
-
-  fts_lucene = whitespace_chars=@.
-}
-{% endif %}
-
 ###############
 # Mailboxes
 ###############

core/dovecot/conf/report-spam.sieve

@@ -1,3 +1,5 @@
+require "imap4flags";
 require "vnd.dovecot.execute";
 
+setflag "\\seen";
 execute :pipe "spam";

core/dovecot/start.py

@@ -6,23 +6,40 @@ import socket
 import glob
 import multiprocessing
 import tenacity
+import logging as log
+import sys
 
 from tenacity import retry
 from podop import run_server
 
+log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 def start_podop():
     os.setuid(8)
-    run_server(3 if "DEBUG" in os.environ else 0, "dovecot", "/tmp/podop.socket", [
+    run_server(0, "dovecot", "/tmp/podop.socket", [
 		("quota", "url", "http://admin/internal/dovecot/§"),
 		("auth", "url", "http://admin/internal/dovecot/§"),
 		("sieve", "url", "http://admin/internal/dovecot/§"),
     ])
 
-convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+def convert(src, dst):
+    logger = log.getLogger("convert()")
+    logger.debug("Source: %s, Destination: %s", src, dst)
+    open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+
+@retry(
+    stop=tenacity.stop_after_attempt(100),
+    wait=tenacity.wait_random(min=2, max=5),
+    before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG),
+    before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO),
+    after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG)
+    )
+def resolve(hostname):
+    logger = log.getLogger("resolve()")
+    logger.info(hostname)
+    return socket.gethostbyname(hostname)
 
 # Actual startup script
-resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
 os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
 os.environ["REDIS_ADDRESS"] = resolve(os.environ.get("REDIS_ADDRESS", "redis"))
 if os.environ["WEBMAIL"] != "none":
@@ -33,5 +50,6 @@ for dovecot_file in glob.glob("/conf/*.conf"):
 
 # Run Podop, then postfix
 multiprocessing.Process(target=start_podop).start()
-os.system("chown -R mail:mail /mail /var/lib/dovecot /conf")
+os.system("chown mail:mail /mail")
+os.system("chown -R mail:mail /var/lib/dovecot /conf")
 os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

core/nginx/config.py

@@ -2,11 +2,18 @@
 
 import jinja2
 import os
-
+import logging as log
-convert = lambda src, dst, args: open(dst, "w").write(jinja2.Template(open(src).read()).render(**args))
+import sys
 
 args = os.environ.copy()
 
+log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING"))
+
+def convert(src, dst, args):
+    logger = log.getLogger("convert()")
+    logger.debug("Source: %s, Destination: %s", src, dst)
+    open(dst, "w").write(jinja2.Template(open(src).read()).render(**args))
+
 # Get the first DNS server
 with open("/etc/resolv.conf") as handle:
     content = handle.read().split()

core/postfix/start.py

@@ -7,14 +7,18 @@ import glob
 import shutil
 import tenacity
 import multiprocessing
+import logging as log
+import sys
 
 from tenacity import retry
 from podop import run_server
 
+log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
 
 def start_podop():
     os.setuid(100)
-    run_server(3 if "DEBUG" in os.environ else 0, "postfix", "/tmp/podop.socket", [
+    # TODO: Remove verbosity setting from Podop?
+    run_server(0, "postfix", "/tmp/podop.socket", [
 		("transport", "url", "http://admin/internal/postfix/transport/§"),
 		("alias", "url", "http://admin/internal/postfix/alias/§"),
 		("domain", "url", "http://admin/internal/postfix/domain/§"),
@@ -23,11 +27,24 @@ def start_podop():
         ("senderlogin", "url", "http://admin/internal/postfix/sender/login/§")
     ])
 
-convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+def convert(src, dst):
+    logger = log.getLogger("convert()")
+    logger.debug("Source: %s, Destination: %s", src, dst)
+    open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+
+@retry(
+    stop=tenacity.stop_after_attempt(100),
+    wait=tenacity.wait_random(min=2, max=5),
+    before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG),
+    before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO),
+    after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG)
+    )
+def resolve(hostname):
+    logger = log.getLogger("resolve()")
+    logger.info(hostname)
+    return socket.gethostbyname(hostname)
 
 # Actual startup script
-resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
-
 os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
 os.environ["HOST_ANTISPAM"] = os.environ.get("HOST_ANTISPAM", "antispam:11332")
 os.environ["HOST_LMTP"] = os.environ.get("HOST_LMTP", "imap:2525")

docs/compose/.env

@@ -3,10 +3,6 @@
 # these few settings must however be configured before starting the mail
 # server and require a restart upon change.
 
-# Set this to `true` to disable full text search by lucene (value: true, false)
-# This is a workaround for the bug in issue #751 (indexer-worker crashes)
-DISABLE_FTS_LUCENE=false
-
 ###################################
 # Common configuration variables
 ###################################
@@ -151,3 +147,6 @@ REAL_IP_FROM=
 
 # choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
 REJECT_UNLISTED_RECIPIENT=
+
+# Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET)
+LOG_LEVEL=WARNING

docs/configuration.rst

@@ -91,6 +91,13 @@ The ``PASSWORD_SCHEME`` is the password encryption scheme. You should use the
 default value, unless you are importing password from a separate system and
 want to keep using the old password encryption scheme.
 
+The ``LOG_LEVEL`` setting is used by the python start-up scripts as a logging threshold.
+Log messages equal or higher than this priority will be printed.
+Can be one of: CRITICAL, ERROR, WARNING, INFO, DEBUG or NOTSET.
+See the `python docs`_ for more information.
+
+.. _`python docs`: https://docs.python.org/3.6/library/logging.html#logging-levels
+
 Infrastructure settings
 -----------------------
 

optional/clamav/start.py

@@ -1,12 +1,21 @@
 #!/usr/bin/python3
 
 import os
+import logging as log
+import sys
+
+log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+logger=log.getLogger(__name__)
 
 # Bootstrap the database if clamav is running for the first time
-os.system("[ -f /data/main.cvd ] || freshclam")
+if not os.path.isfile("/data/main.cvd"):
+    logger.info("Starting primary virus DB download")
+    os.system("freshclam")
 
 # Run the update daemon
+logger.info("Starting the update daemon")
 os.system("freshclam -d -c 6")
 
 # Run clamav
+logger.info("Starting clamav")
 os.system("clamd")

services/rspamd/start.py

@@ -5,13 +5,31 @@ import os
 import socket
 import glob
 import tenacity
+import logging as log
+import sys
+
 from tenacity import retry
 
-convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+
+def convert(src, dst):
+    logger = log.getLogger("convert()")
+    logger.debug("Source: %s, Destination: %s", src, dst)
+    open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+
+@retry(
+    stop=tenacity.stop_after_attempt(100),
+    wait=tenacity.wait_random(min=2, max=5),
+    before=tenacity.before_log(log.getLogger("tenacity.retry"), log.DEBUG),
+    before_sleep=tenacity.before_sleep_log(log.getLogger("tenacity.retry"), log.INFO),
+    after=tenacity.after_log(log.getLogger("tenacity.retry"), log.DEBUG)
+    )
+def resolve(hostname):
+    logger = log.getLogger("resolve()")
+    logger.info(hostname)
+    return socket.gethostbyname(hostname)
 
 # Actual startup script
-resolve = retry(socket.gethostbyname, stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
-
 os.environ["FRONT_ADDRESS"] = resolve(os.environ.get("FRONT_ADDRESS", "front"))
 
 if "HOST_REDIS" not in os.environ: os.environ["HOST_REDIS"] = "redis"

services/unbound/start.py

@@ -2,8 +2,16 @@
 
 import jinja2
 import os
+import logging as log
+import sys
+
+log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+
+def convert(src, dst):
+    logger = log.getLogger("convert()")
+    logger.debug("Source: %s, Destination: %s", src, dst)
+    open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
-convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 convert("/unbound.conf", "/etc/unbound/unbound.conf")
 
 os.execv("/usr/sbin/unbound", ["-c /etc/unbound/unbound.conf"])

setup/flavors/compose/mailu.env

@@ -161,6 +161,9 @@ REAL_IP_FROM={{ real_ip_from }}
 # choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
 REJECT_UNLISTED_RECIPIENT={{ reject_unlisted_recipient }}
 
+# Log level threshold in start.py (value: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET)
+LOG_LEVEL=WARNING
+
 ###################################
 # Database settings
 ###################################

webmails/rainloop/start.py

@@ -3,8 +3,15 @@
 import jinja2
 import os
 import shutil
+import logging as log
+import sys
 
-convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+
+def convert(src, dst):
+    logger = log.getLogger("convert()")
+    logger.debug("Source: %s, Destination: %s", src, dst)
+    open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
 # Actual startup script
 os.environ["FRONT_ADDRESS"] = os.environ.get("FRONT_ADDRESS", "front")

webmails/roundcube/start.py

@@ -2,8 +2,15 @@
 
 import os
 import jinja2
+import logging as log
+import sys
 
-convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
+log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING"))
+
+def convert(src, dst):
+    logger = log.getLogger("convert()")
+    logger.debug("Source: %s, Destination: %s", src, dst)
+    open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
 
 os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576))
 
@@ -14,4 +21,4 @@ os.system("mkdir -p /data/gpg")
 os.system("chown -R www-data:www-data /data")
 
 # Run apache
-os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"])
+os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"])