Merge branch 'master' of https://github.com/Mailu/Mailu into webmail-hardening

.github/workflows/build_test_deploy.yml

@@ -126,7 +126,7 @@ jobs:
           password: ${{ secrets.Docker_Password }}
       - name: Helper to convert docker org to lowercase
         id: string
-        uses: ASzc/change-string-case-action@v2
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ github.repository_owner }}
       - name: Build all docker images
@@ -182,7 +182,7 @@ jobs:
           password: ${{ secrets.Docker_Password }}
       - name: Helper to convert docker org to lowercase
         id: string
-        uses: ASzc/change-string-case-action@v2
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ github.repository_owner }}
       - name: Build all docker images
@@ -244,7 +244,7 @@ jobs:
           password: ${{ secrets.Docker_Password }}
       - name: Helper to convert docker org to lowercase
         id: string
-        uses: ASzc/change-string-case-action@v2
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ github.repository_owner }}
       - name: Build all docker images
@@ -307,7 +307,7 @@ jobs:
           password: ${{ secrets.Docker_Password }}
       - name: Helper to convert docker org to lowercase
         id: string
-        uses: ASzc/change-string-case-action@v2
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ github.repository_owner }}
       - name: Build all docker images
@@ -370,7 +370,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Helper to convert docker org to lowercase
         id: string
-        uses: ASzc/change-string-case-action@v2
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ github.repository_owner }}
       - name: Install python packages
@@ -416,7 +416,7 @@ jobs:
           password: ${{ secrets.Docker_Password }}
       - name: Helper to convert docker org to lowercase
         id: string
-        uses: ASzc/change-string-case-action@v2
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ github.repository_owner }}
       - name: Push image to Docker
@@ -461,7 +461,7 @@ jobs:
           password: ${{ secrets.Docker_Password }}
       - name: Helper to convert docker org to lowercase
         id: string
-        uses: ASzc/change-string-case-action@v2
+        uses: ASzc/change-string-case-action@v5
         with:
           string: ${{ github.repository_owner }}
       - name: Push image to Docker

core/admin/assets/Dockerfile

@@ -5,7 +5,6 @@ FROM node:16-alpine3.16
 WORKDIR /work
 
 COPY package.json ./
-COPY webpack.config.js ./
 
 RUN set -euxo pipefail \
   ; npm config set update-notifier false \
@@ -17,6 +16,7 @@ RUN set -euxo pipefail \
     done
 
 COPY assets/ ./assets/
+COPY webpack.config.js ./
 
 RUN set -euxo pipefail \
   ; node_modules/.bin/webpack-cli --color

core/admin/assets/assets/app.js

@@ -1,8 +1,3 @@
-require('./app.css');
-
-import logo from './mailu.png';
-import modules from "./*.json";
-
 // Inspired from https://github.com/mehdibo/hibp-js/blob/master/hibp.js
 function sha1(string) {
     var buffer = new TextEncoder("utf-8").encode(string);

core/admin/assets/assets/vendor.js

@@ -1,5 +1,5 @@
 // AdminLTE
-import 'admin-lte/plugins/jquery/jquery.min.js';
+window.$ = window.jQuery = require('admin-lte/plugins/jquery/jquery.min.js');
 import 'admin-lte/plugins/bootstrap/js/bootstrap.bundle.min.js';
 import 'admin-lte/build/scss/adminlte.scss';
 import 'admin-lte/build/js/AdminLTE.js';
@@ -18,7 +18,7 @@ import 'admin-lte/plugins/datatables/jquery.dataTables.min.js';
 import 'admin-lte/plugins/datatables-bs4/js/dataTables.bootstrap4.min.js';
 import 'admin-lte/plugins/datatables-responsive/js/dataTables.responsive.min.js';
 import 'admin-lte/plugins/datatables-responsive/js/responsive.bootstrap4.min.js';
+import modules from "./*.json";
 
 // clipboard.js
-import 'clipboard/dist/clipboard.min.js';
+window.ClipboardJS = require('clipboard/dist/clipboard.min.js');
-

core/admin/assets/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     mode: 'production',
     entry: {
         app: {
-            import: './assets/app.js',
+            import: ['./assets/app.css', './assets/mailu.png', './assets/app.js'],
             dependOn: 'vendor',
         },
         vendor: './assets/vendor.js',

core/admin/mailu/configuration.py

@@ -13,17 +13,19 @@ DEFAULT_CONFIG = {
     'RATELIMIT_STORAGE_URL': '',
     'DEBUG': False,
     'DEBUG_PROFILER': False,
+    'DEBUG_TB_INTERCEPT_REDIRECTS': False,
     'DEBUG_ASSETS': '',
     'DOMAIN_REGISTRATION': False,
     'TEMPLATES_AUTO_RELOAD': True,
     'MEMORY_SESSIONS': False,
+    'FETCHMAIL_ENABLED': False,
     # Database settings
     'DB_FLAVOR': None,
     'DB_USER': 'mailu',
     'DB_PW': None,
     'DB_HOST': 'database',
     'DB_NAME': 'mailu',
-    'SQLITE_DATABASE_FILE':'data/main.db',
+    'SQLITE_DATABASE_FILE': 'data/main.db',
     'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db',
     'SQLALCHEMY_TRACK_MODIFICATIONS': False,
     # Statistics management
@@ -60,7 +62,7 @@ DEFAULT_CONFIG = {
     # Web settings
     'SITENAME': 'Mailu',
     'WEBSITE': 'https://mailu.io',
-    'ADMIN' : 'none',
+    'ADMIN': 'none',
     'WEB_ADMIN': '/admin',
     'WEB_WEBMAIL': '/webmail',
     'WEBMAIL': 'none',
@@ -73,7 +75,7 @@ DEFAULT_CONFIG = {
     'SESSION_KEY_BITS': 128,
     'SESSION_TIMEOUT': 3600,
     'PERMANENT_SESSION_LIFETIME': 30*24*3600,
-    'SESSION_COOKIE_SECURE': True,
+    'SESSION_COOKIE_SECURE': None,
     'CREDENTIAL_ROUNDS': 12,
     'TLS_PERMISSIVE': True,
     'TZ': 'Etc/UTC',
@@ -156,6 +158,8 @@ class ConfigManager:
         self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3'
         self.config['SESSION_COOKIE_SAMESITE'] = 'Strict'
         self.config['SESSION_COOKIE_HTTPONLY'] = True
+        if self.config['SESSION_COOKIE_SECURE'] is None:
+            self.config['SESSION_COOKIE_SECURE'] = self.config['TLS_FLAVOR'] != 'notls'
         self.config['SESSION_PERMANENT'] = True
         self.config['SESSION_TIMEOUT'] = int(self.config['SESSION_TIMEOUT'])
         self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME'])

core/admin/mailu/internal/views/fetch.py

@@ -12,10 +12,12 @@ def fetch_list():
             "id": fetch.id,
             "tls": fetch.tls,
             "keep": fetch.keep,
+            "scan": fetch.scan,
             "user_email": fetch.user_email,
             "protocol": fetch.protocol,
             "host": fetch.host,
             "port": fetch.port,
+            "folders": fetch.folders,
             "username": fetch.username,
             "password": fetch.password
         } for fetch in models.Fetch.query.all()

core/admin/mailu/models.py

@@ -771,6 +771,8 @@ class Fetch(Base):
     username = db.Column(db.String(255), nullable=False)
     password = db.Column(db.String(255), nullable=False)
     keep = db.Column(db.Boolean, nullable=False, default=False)
+    scan = db.Column(db.Boolean, nullable=False, default=False)
+    folders = db.Column(CommaSeparatedList, nullable=True, default=list)
     last_check = db.Column(db.DateTime, nullable=True)
     error = db.Column(db.String(1023), nullable=True)
 

core/admin/mailu/sso/views/languages.py

@@ -1,7 +1,7 @@
 from mailu.sso import sso
 import flask
 
-@sso.route('/language/<language>', methods=['POST'])
+@sso.route('/language/<language>', methods=['GET','POST'])
 def set_language(language=None):
     if language:
         flask.session['language'] = language

core/admin/mailu/ui/forms.py

@@ -41,6 +41,16 @@ class MultipleEmailAddressesVerify(object):
         if not pattern.match(field.data.replace(" ", "")):
             raise validators.ValidationError(self.message)
 
+class MultipleFoldersVerify(object):
+    """ Ensure that we have CSV formated data """
+    def __init__(self,message=_('Invalid list of folders.')):
+        self.message = message
+
+    def __call__(self, form, field):
+        pattern = re.compile(r'^\w+(\s*,\s*\w+)*$')
+        if not pattern.match(field.data.replace(" ", "")):
+            raise validators.ValidationError(self.message)
+
 class ConfirmationForm(flask_wtf.FlaskForm):
     submit = fields.SubmitField(_('Confirm'))
 
@@ -164,11 +174,13 @@ class FetchForm(flask_wtf.FlaskForm):
         ('imap', 'IMAP'), ('pop3', 'POP3')
     ])
     host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()])
-    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)])
+    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993)
-    tls = fields.BooleanField(_('Enable TLS'))
+    tls = fields.BooleanField(_('Enable TLS'), default=True)
     username = fields.StringField(_('Username'), [validators.DataRequired()])
     password = fields.PasswordField(_('Password'))
     keep = fields.BooleanField(_('Keep emails on the server'))
+    scan = fields.BooleanField(_('Rescan emails locally'))
+    folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk')
     submit = fields.SubmitField(_('Submit'))
 
 

core/admin/mailu/ui/templates/fetch/create.html

@@ -24,6 +24,8 @@
 
   {%- call macros.card(title="Settings") %}
   {{ macros.form_field(form.keep) }}
+  {{ macros.form_field(form.scan) }}
+  {{ macros.form_field(form.folders) }}
   {%- endcall %}
 
   {{ macros.form_field(form.submit) }}

core/admin/mailu/ui/templates/fetch/list.html

@@ -20,6 +20,8 @@
     <th>{% trans %}Endpoint{% endtrans %}</th>
     <th>{% trans %}Username{% endtrans %}</th>
     <th>{% trans %}Keep emails{% endtrans %}</th>
+    <th>{% trans %}Rescan emails{% endtrans %}</th>
+    <th>{% trans %}Folders{% endtrans %}</th>
     <th>{% trans %}Last check{% endtrans %}</th>
     <th>{% trans %}Status{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
@@ -36,6 +38,8 @@
     <td>{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }}</td>
     <td>{{ fetch.username }}</td>
     <td data-sort="{{ fetch.keep }}">{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td data-sort="{{ fetch.scan }}">{% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td>{{ fetch.folders | join(',') }}</td>
     <td>{{ fetch.last_check | format_datetime or '-' }}</td>
     <td>{{ fetch.error or '-' }}</td>
     <td data-sort="{{ fetch.created_at or '0000-00-00' }}">{{ fetch.created_at | format_date }}</td>

core/admin/mailu/ui/templates/sidebar.html

@@ -31,12 +31,14 @@
           <p>{% trans %}Auto-reply{% endtrans %}</p>
         </a>
       </li>
+      {%- if config["FETCHMAIL_ENABLED"] %}
       <li class="nav-item" role="none">
         <a href="{{ url_for('.fetch_list') }}" class="nav-link" role="menuitem">
           <i class="nav-icon fas fa-download"></i>
           <p>{% trans %}Fetched accounts{% endtrans %}</p>
         </a>
       </li>
+      {%- endif %}
       <li class="nav-item" role="none">
         <a href="{{ url_for('.token_list') }}" class="nav-link" role="menuitem">
           <i class="nav-icon fas fa-ticket-alt"></i>

core/admin/mailu/ui/views/fetches.py

@@ -1,5 +1,6 @@
-from mailu import models
+from mailu import models, utils
 from mailu.ui import ui, forms, access
+from flask import current_app as app
 
 import flask
 import flask_login
@@ -10,6 +11,8 @@ import wtforms
 @ui.route('/fetch/list/<path:user_email>', methods=['GET'])
 @access.owner(models.User, 'user_email')
 def fetch_list(user_email):
+    if not app.config['FETCHMAIL_ENABLED']:
+        flask.abort(404)
     user_email = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email) or flask.abort(404)
     return flask.render_template('fetch/list.html', user=user)
@@ -19,13 +22,18 @@ def fetch_list(user_email):
 @ui.route('/fetch/create/<path:user_email>', methods=['GET', 'POST'])
 @access.owner(models.User, 'user_email')
 def fetch_create(user_email):
+    if not app.config['FETCHMAIL_ENABLED']:
+        flask.abort(404)
     user_email = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email) or flask.abort(404)
     form = forms.FetchForm()
     form.password.validators = [wtforms.validators.DataRequired()]
+    utils.formatCSVField(form.folders)
     if form.validate_on_submit():
         fetch = models.Fetch(user=user)
         form.populate_obj(fetch)
+        if form.folders.data:
+            fetch.folders = form.folders.data.replace(' ','').split(',')
         models.db.session.add(fetch)
         models.db.session.commit()
         flask.flash('Fetch configuration created')
@@ -37,12 +45,17 @@ def fetch_create(user_email):
 @ui.route('/fetch/edit/<fetch_id>', methods=['GET', 'POST'])
 @access.owner(models.Fetch, 'fetch_id')
 def fetch_edit(fetch_id):
+    if not app.config['FETCHMAIL_ENABLED']:
+        flask.abort(404)
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     form = forms.FetchForm(obj=fetch)
+    utils.formatCSVField(form.folders)
     if form.validate_on_submit():
         if not form.password.data:
             form.password.data = fetch.password
         form.populate_obj(fetch)
+        if form.folders.data:
+            fetch.folders = form.folders.data.replace(' ','').split(',')
         models.db.session.commit()
         flask.flash('Fetch configuration updated')
         return flask.redirect(
@@ -55,6 +68,8 @@ def fetch_edit(fetch_id):
 @access.confirmation_required("delete a fetched account")
 @access.owner(models.Fetch, 'fetch_id')
 def fetch_delete(fetch_id):
+    if not app.config['FETCHMAIL_ENABLED']:
+        flask.abort(404)
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     user = fetch.user
     models.db.session.delete(fetch)

core/admin/mailu/ui/views/users.py

@@ -64,6 +64,7 @@ def user_edit(user_email):
         form.quota_bytes.validators = [
             wtforms.validators.NumberRange(max=max_quota_bytes)]
     if form.validate_on_submit():
+        if form.pw.data:
             if msg := utils.isBadOrPwned(form):
                 flask.flash(msg, "error")
                 return flask.render_template('user/edit.html', form=form, user=user,
@@ -99,11 +100,7 @@ def user_settings(user_email):
     user_email_or_current = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email_or_current) or flask.abort(404)
     form = forms.UserSettingsForm(obj=user)
-    if isinstance(form.forward_destination.data,str):
+    utils.formatCSVField(form.forward_destination)
-        data = form.forward_destination.data.replace(" ","").split(",")
-    else:
-        data = form.forward_destination.data
-    form.forward_destination.data = ", ".join(data)
     if form.validate_on_submit():
         form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",")
         form.populate_obj(user)

core/admin/mailu/utils.py

@@ -518,3 +518,10 @@ def isBadOrPwned(form):
     if breaches > 0:
         return f"This password appears in {breaches} data breaches! It is not unique; please change it."
     return None
+
+def formatCSVField(field):
+    if isinstance(field.data,str):
+        data = field.data.replace(" ","").split(",")
+    else:
+        data = field.data
+    field.data = ", ".join(data)

core/admin/migrations/versions/f4f0f89e0047_.py

@@ -0,0 +1,25 @@
+""" Add fetch.scan and fetch.folders
+
+Revision ID: f4f0f89e0047
+Revises: 8f9ea78776f4
+Create Date: 2022-11-13 16:29:01.246509
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f4f0f89e0047'
+down_revision = '8f9ea78776f4'
+
+from alembic import op
+import sqlalchemy as sa
+import mailu
+
+def upgrade():
+    with op.batch_alter_table('fetch') as batch:
+        batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
+        batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True))
+
+def downgrade():
+    with op.batch_alter_table('fetch') as batch:
+        batch.drop_column('fetch', 'folders')
+        batch.drop_column('fetch', 'scan')

core/admin/run_dev.sh

@@ -48,7 +48,7 @@ sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Docker
 
 # assets
 cp "${assets}/package.json" .
-cp -r "${assets}/assets/" .
+cp -r "${assets}/assets" ./assets
 awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js
 sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile
 
@@ -65,7 +65,7 @@ RUN set -euxo pipefail \
   ; ln -s /app/start.py /
 
 ENV \
-    FLASK_ENV="development" \
+    FLASK_DEBUG="true" \
     MEMORY_SESSIONS="true" \
     RATELIMIT_STORAGE_URL="memory://" \
     SESSION_COOKIE_SECURE="false" \
@@ -73,7 +73,7 @@ ENV \
     DEBUG="true" \
     DEBUG_PROFILER="${DEV_PROFILER}" \
     DEBUG_ASSETS="/app/static" \
-    DEBUG_TB_ENABLED="true" \
+    DEBUG_TB_INTERCEPT_REDIRECTS=False \
     \
     IMAP_ADDRESS="127.0.0.1" \
     POP3_ADDRESS="127.0.0.1" \
@@ -82,7 +82,7 @@ ENV \
     REDIS_ADDRESS="127.0.0.1" \
     WEBMAIL_ADDRESS="127.0.0.1"
 
-CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null && flask run --host=0.0.0.0 --port=8080"]
+CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"]
 EOF
 
 # build

core/admin/start.py

@@ -2,8 +2,15 @@
 
 import os
 import logging as log
+from pwd import getpwnam
 import sys
 
+os.system("chown mailu:mailu -R /dkim")
+os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
+mailu_id = getpwnam('mailu')
+os.setgid(mailu_id.pw_gid)
+os.setuid(mailu_id.pw_uid)
+
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
 
 os.system("flask mailu advertise")

core/base/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile-upstream:1.4.3
 
 # base system image (intermediate)
-ARG DISTRO=alpine:3.16.2
+ARG DISTRO=alpine:3.16.3
 FROM $DISTRO as system
 
 ENV TZ=Etc/UTC LANG=C.UTF-8
@@ -12,7 +12,16 @@ ARG MAILU_GID=1000
 RUN set -euxo pipefail \
   ; addgroup -Sg ${MAILU_GID} mailu \
   ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \
-  ; apk add --no-cache bash ca-certificates curl python3 tzdata
+  ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \
+  ; machine="$(uname -m)" \
+  ; ! [[ "${machine}" == x86_64 ]] \
+    || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc
+
+ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so
+ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
+ENV CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
+ENV CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
+ENV LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now"
 
 WORKDIR /app
 
@@ -78,6 +87,7 @@ FROM system
 
 COPY --from=build /app/venv/ /app/venv/
 COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/
+RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn
 
 ENV VIRTUAL_ENV=/app/venv
 ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

core/none/Dockerfile

@@ -10,5 +10,5 @@ RUN echo $VERSION >/version
 
 HEALTHCHECK CMD true
 
-USER app
+USER mailu
 CMD ["/bin/bash", "-c", "sleep infinity"]

core/rspamd/start.py

@@ -33,4 +33,7 @@ while True:
     log.warning("Admin is not up just yet, retrying in 1 second")
 
 # Run rspamd
-os.execv("/usr/sbin/rspamd", ["rspamd", "-i", "-f"])
+os.system("mkdir -m 755 -p /run/rspamd")
+os.system("chown rspamd:rspamd /run/rspamd")
+os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown rspamd:rspamd")
+os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"])

docs/configuration.rst

@@ -104,6 +104,9 @@ support or e.g. mismatching TLS versions to deliver emails to Mailu.
 
 .. _fetchmail:
 
+When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is enabled in the admin interface.
+The container itself still needs to be deployed manually. ``FETCHMAIL_ENABLED`` defaults to ``True``.
+
 The ``FETCHMAIL_DELAY`` is a delay (in seconds) for the fetchmail service to
 go and fetch new email if available. Do not use too short delays if you do not
 want to be blacklisted by external services, but not too long delays if you
@@ -287,6 +290,10 @@ The admin service stores configurations in a database.
 - ``DB_USER``: the database user for mailu admin service. (when not ``sqlite``)
 - ``DB_NAME``: the database name for mailu admin service. (when not ``sqlite``)
 
+Alternatively, if you need more control, you can use a `DB URL`_ : do not set any of the ``DB_`` settings and set ``SQLALCHEMY_DATABASE_URI`` instead.
+
+.. _`DB URL`: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls
+
 The roundcube service stores configurations in a database.
 
 - ``ROUNDCUBE_DB_FLAVOR``: the database type for roundcube service. (``sqlite``, ``postgresql``, ``mysql``)

docs/webadministration.rst

@@ -157,7 +157,11 @@ You can add a fetched account by clicking on the `Add an account` button on the
 
 * Keep emails on the server. When ticked, retains the email message in the email account after retrieving it.
 
-Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after 10 minutes.
+* Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...).
+
+* Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled.
+
+Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``.
 
 
 Authentication tokens

optional/fetchmail/fetchmail.py

@@ -2,11 +2,14 @@
 
 import time
 import os
+from pathlib import Path
+from pwd import getpwnam
 import tempfile
 import shlex
 import subprocess
 import re
 import requests
+from socrate import system
 import sys
 import traceback
 
@@ -14,6 +17,7 @@ import traceback
 FETCHMAIL = """
 fetchmail -N \
     --idfile /data/fetchids --uidl \
+    --pidfile /dev/shm/fetchmail.pid \
     --sslcertck --sslcertpath /etc/ssl/certs \
     -f {}
 """
@@ -24,7 +28,9 @@ poll "{host}" proto {protocol}  port {port}
     user "{username}" password "{password}"
     is "{user_email}"
     smtphost "{smtphost}"
+    {folders}
     {options}
+    {lmtp}
 """
 
 
@@ -48,26 +54,37 @@ def fetchmail(fetchmailrc):
 
 def run(debug):
     try:
-        fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json()
+        os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
-        smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
+        os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
+        fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
+        smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
         if smtpport is None:
             smtphostport = smtphost
         else:
             smtphostport = "%s/%d" % (smtphost, smtpport)
+        os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
+        lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
+        if lmtpport is None:
+            lmtphostport = lmtphost
+        else:
+            lmtphostport = "%s/%d" % (lmtphost, lmtpport)
         for fetch in fetches:
             fetchmailrc = ""
             options = "options antispam 501, 504, 550, 553, 554"
             options += " ssl" if fetch["tls"] else ""
             options += " keep" if fetch["keep"] else " fetchall"
+            folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"')
             fetchmailrc += RC_LINE.format(
                 user_email=escape_rc_string(fetch["user_email"]),
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=smtphostport,
+                smtphost=smtphostport if fetch['scan'] else lmtphostport,
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
-                options=options
+                options=options,
+                folders=folders,
+                lmtp='' if fetch['scan'] else 'lmtp',
             )
             if debug:
                 print(fetchmailrc)
@@ -86,15 +103,29 @@ def run(debug):
                         user_info in error_message):
                     print(error_message)
             finally:
-                requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]),
+                requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']),
-                    json=error_message.split("\n")[0]
+                    json=error_message.split('\n')[0]
                 )
     except Exception:
         traceback.print_exc()
 
 
 if __name__ == "__main__":
+    id_fetchmail = getpwnam('fetchmail')
+    Path('/data/fetchids').touch()
+    os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+    os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+    os.chmod("/data/fetchids", 0o700)
+    os.setgid(id_fetchmail.pw_gid)
+    os.setuid(id_fetchmail.pw_uid)
     while True:
-        time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60)))
+        delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
+        print("Sleeping for {} seconds".format(delay))
+        time.sleep(delay)
+
+        if not os.environ.get("FETCHMAIL_ENABLED", 'True') in ('True', 'true'):
+            print("Fetchmail disabled, skipping...")
+            continue
+
         run(os.environ.get("DEBUG", None) == "True")
         sys.stdout.flush()

setup/Dockerfile

@@ -1,24 +1,21 @@
-ARG DISTRO=alpine:3.14.5
+# syntax=docker/dockerfile-upstream:1.4.3
-FROM $DISTRO
+
-ARG VERSION
+# setup image
-ENV TZ Etc/UTC
+FROM base
+
+ARG VERSION=local
 LABEL version=$VERSION
 
-
-RUN mkdir -p /app
-WORKDIR /app
-
-COPY requirements.txt requirements.txt
-RUN apk add --no-cache curl python3 py3-pip \
-    && pip3 install -r requirements.txt
-
-COPY server.py ./server.py
-COPY main.py ./main.py
 COPY flavors /data/flavors
 COPY templates /data/templates
 COPY static ./static
+COPY server.py ./server.py
+COPY main.py ./main.py
+
+RUN echo $VERSION >> /version
 
 EXPOSE 80/tcp
+HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost/
+USER mailu
 
 CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app
-RUN echo $VERSION >> /version

setup/flavors/compose/docker-compose.yml

@@ -157,8 +157,11 @@ services:
     env_file: {{ env }}
     volumes:
       - "{{ root }}/data/fetchmail:/data"
-    {% if resolver_enabled %}
     depends_on:
+      - admin
+      - smtp
+      - imap
+    {% if resolver_enabled %}
       - resolver
     dns:
       - {{ dns }}

setup/flavors/compose/mailu.env

@@ -79,6 +79,9 @@ RELAYNETS=
 # Will relay all outgoing mails if configured
 RELAYHOST={{ relayhost }}
 
+# Show fetchmail functionality in admin interface
+FETCHMAIL_ENABLED={{ fetchmail_enabled or 'False' }}
+
 # Fetchmail delay
 FETCHMAIL_DELAY={{ fetchmail_delay or '600' }}
 

setup/requirements.txt

@@ -1,12 +0,0 @@
-Flask==1.0.2
-Flask-Bootstrap==3.3.7.1
-gunicorn==19.9.0
-redis==3.2.1
-Jinja2==3.0.3
-MarkupSafe==2.1.0
-Werkzeug==2.0.3
-click==8.0.3
-dominate==2.6.0
-itsdangerous==2.0.1
-redis==3.2.1
-visitor==0.1.3

tests/build.hcl

@@ -106,6 +106,9 @@ target "docs" {
 target "setup" {
   inherits = ["defaults"]
   context = "setup/"
+  contexts = {
+    base = "target:base"
+  }
   tags = tag("setup")
 }
 

tests/compose/filters/02_email_antispam.sh

@@ -0,0 +1,7 @@
+# GTUBE should be blocked, see https://rspamd.com/doc/gtube_patterns.html
+python3 tests/email_test.py "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X"
+if [ $? -eq 25 ]; then
+	exit 0
+else
+	exit 1
+fi

towncrier/newsfragments/1231.bugfix

@@ -0,0 +1 @@
+Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231)

towncrier/newsfragments/2127.feature

@@ -0,0 +1 @@
+Add FETCHMAIL_ENABLED to toggle the fetchmail functionality in the admin interface

towncrier/newsfragments/2246.bugfix

@@ -0,0 +1 @@
+Fetchmail: Missing support for '*_ADDRESS' env vars

towncrier/newsfragments/2525.feature

@@ -0,0 +1 @@
+Switch to GrapheneOS's hardened_malloc

towncrier/newsfragments/2539.misc

@@ -0,0 +1 @@
+Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. Please ensure that your folder overrides/rspamd is owned by 1000:1000

towncrier/newsfragments/711.feature

@@ -0,0 +1 @@
+Allow other folders to be synced by fetchmail

webmails/Dockerfile

@@ -52,7 +52,7 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/
 
 # snappymail
 
-ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.0/snappymail-2.21.0.tar.gz
+ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.3/snappymail-2.21.3.tar.gz
 
 RUN set -euxo pipefail \
   ; mkdir  /var/www/snappymail \
@@ -82,7 +82,7 @@ RUN set -euxo pipefail \
 # common
 COPY start.py /
 COPY php.ini /defaults/
-COPY php-webmail.conf /etc/php81/php-fpm.d/php-webmail.conf
+COPY php-webmail.conf /etc/php81/php-fpm.d/
 COPY nginx-webmail.conf /conf/
 COPY snuffleupagus.rules /etc/snuffleupagus.rules.tpl