diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 3b5dafc3..4d10e4ca 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -126,7 +126,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -182,7 +182,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -244,7 +244,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -307,7 +307,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -370,7 +370,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Install python packages @@ -416,7 +416,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Push image to Docker @@ -461,7 +461,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Push image to Docker diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile index 613aa6c0..33b9bdb9 100644 --- a/core/admin/assets/Dockerfile +++ b/core/admin/assets/Dockerfile @@ -5,7 +5,6 @@ FROM node:16-alpine3.16 WORKDIR /work COPY package.json ./ -COPY webpack.config.js ./ RUN set -euxo pipefail \ ; npm config set update-notifier false \ @@ -17,6 +16,7 @@ RUN set -euxo pipefail \ done COPY assets/ ./assets/ +COPY webpack.config.js ./ RUN set -euxo pipefail \ ; node_modules/.bin/webpack-cli --color diff --git a/core/admin/assets/assets/app.js b/core/admin/assets/assets/app.js index ad2b543a..661e0242 100644 --- a/core/admin/assets/assets/app.js +++ b/core/admin/assets/assets/app.js @@ -1,8 +1,3 @@ -require('./app.css'); - -import logo from './mailu.png'; -import modules from "./*.json"; - // Inspired from https://github.com/mehdibo/hibp-js/blob/master/hibp.js function sha1(string) { var buffer = new TextEncoder("utf-8").encode(string); diff --git a/core/admin/assets/assets/vendor.js b/core/admin/assets/assets/vendor.js index 906448cf..819240ad 100644 --- a/core/admin/assets/assets/vendor.js +++ b/core/admin/assets/assets/vendor.js @@ -1,5 +1,5 @@ // AdminLTE -import 'admin-lte/plugins/jquery/jquery.min.js'; +window.$ = window.jQuery = require('admin-lte/plugins/jquery/jquery.min.js'); import 'admin-lte/plugins/bootstrap/js/bootstrap.bundle.min.js'; import 'admin-lte/build/scss/adminlte.scss'; import 'admin-lte/build/js/AdminLTE.js'; @@ -18,7 +18,7 @@ import 'admin-lte/plugins/datatables/jquery.dataTables.min.js'; import 'admin-lte/plugins/datatables-bs4/js/dataTables.bootstrap4.min.js'; import 'admin-lte/plugins/datatables-responsive/js/dataTables.responsive.min.js'; import 'admin-lte/plugins/datatables-responsive/js/responsive.bootstrap4.min.js'; +import modules from "./*.json"; // clipboard.js -import 'clipboard/dist/clipboard.min.js'; - +window.ClipboardJS = require('clipboard/dist/clipboard.min.js'); diff --git a/core/admin/assets/webpack.config.js b/core/admin/assets/webpack.config.js index 25c966c1..8c3d6c27 100644 --- a/core/admin/assets/webpack.config.js +++ b/core/admin/assets/webpack.config.js @@ -9,7 +9,7 @@ module.exports = { mode: 'production', entry: { app: { - import: './assets/app.js', + import: ['./assets/app.css', './assets/mailu.png', './assets/app.js'], dependOn: 'vendor', }, vendor: './assets/vendor.js', diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index b941e95c..d282bfc2 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -13,17 +13,19 @@ DEFAULT_CONFIG = { 'RATELIMIT_STORAGE_URL': '', 'DEBUG': False, 'DEBUG_PROFILER': False, + 'DEBUG_TB_INTERCEPT_REDIRECTS': False, 'DEBUG_ASSETS': '', 'DOMAIN_REGISTRATION': False, 'TEMPLATES_AUTO_RELOAD': True, 'MEMORY_SESSIONS': False, + 'FETCHMAIL_ENABLED': False, # Database settings 'DB_FLAVOR': None, 'DB_USER': 'mailu', 'DB_PW': None, 'DB_HOST': 'database', 'DB_NAME': 'mailu', - 'SQLITE_DATABASE_FILE':'data/main.db', + 'SQLITE_DATABASE_FILE': 'data/main.db', 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', 'SQLALCHEMY_TRACK_MODIFICATIONS': False, # Statistics management @@ -60,7 +62,7 @@ DEFAULT_CONFIG = { # Web settings 'SITENAME': 'Mailu', 'WEBSITE': 'https://mailu.io', - 'ADMIN' : 'none', + 'ADMIN': 'none', 'WEB_ADMIN': '/admin', 'WEB_WEBMAIL': '/webmail', 'WEBMAIL': 'none', @@ -73,7 +75,7 @@ DEFAULT_CONFIG = { 'SESSION_KEY_BITS': 128, 'SESSION_TIMEOUT': 3600, 'PERMANENT_SESSION_LIFETIME': 30*24*3600, - 'SESSION_COOKIE_SECURE': True, + 'SESSION_COOKIE_SECURE': None, 'CREDENTIAL_ROUNDS': 12, 'TLS_PERMISSIVE': True, 'TZ': 'Etc/UTC', @@ -156,6 +158,8 @@ class ConfigManager: self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3' self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True + if self.config['SESSION_COOKIE_SECURE'] is None: + self.config['SESSION_COOKIE_SECURE'] = self.config['TLS_FLAVOR'] != 'notls' self.config['SESSION_PERMANENT'] = True self.config['SESSION_TIMEOUT'] = int(self.config['SESSION_TIMEOUT']) self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME']) diff --git a/core/admin/mailu/internal/views/fetch.py b/core/admin/mailu/internal/views/fetch.py index 1945b9c7..e813c33b 100644 --- a/core/admin/mailu/internal/views/fetch.py +++ b/core/admin/mailu/internal/views/fetch.py @@ -12,10 +12,12 @@ def fetch_list(): "id": fetch.id, "tls": fetch.tls, "keep": fetch.keep, + "scan": fetch.scan, "user_email": fetch.user_email, "protocol": fetch.protocol, "host": fetch.host, "port": fetch.port, + "folders": fetch.folders, "username": fetch.username, "password": fetch.password } for fetch in models.Fetch.query.all() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 48ce8b33..4b048c45 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -771,6 +771,8 @@ class Fetch(Base): username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) keep = db.Column(db.Boolean, nullable=False, default=False) + scan = db.Column(db.Boolean, nullable=False, default=False) + folders = db.Column(CommaSeparatedList, nullable=True, default=list) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) diff --git a/core/admin/mailu/sso/views/languages.py b/core/admin/mailu/sso/views/languages.py index ff65af45..19764519 100644 --- a/core/admin/mailu/sso/views/languages.py +++ b/core/admin/mailu/sso/views/languages.py @@ -1,7 +1,7 @@ from mailu.sso import sso import flask -@sso.route('/language/', methods=['POST']) +@sso.route('/language/', methods=['GET','POST']) def set_language(language=None): if language: flask.session['language'] = language diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index beb44092..ec19bb0b 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -41,6 +41,16 @@ class MultipleEmailAddressesVerify(object): if not pattern.match(field.data.replace(" ", "")): raise validators.ValidationError(self.message) +class MultipleFoldersVerify(object): + """ Ensure that we have CSV formated data """ + def __init__(self,message=_('Invalid list of folders.')): + self.message = message + + def __call__(self, form, field): + pattern = re.compile(r'^\w+(\s*,\s*\w+)*$') + if not pattern.match(field.data.replace(" ", "")): + raise validators.ValidationError(self.message) + class ConfirmationForm(flask_wtf.FlaskForm): submit = fields.SubmitField(_('Confirm')) @@ -164,11 +174,13 @@ class FetchForm(flask_wtf.FlaskForm): ('imap', 'IMAP'), ('pop3', 'POP3') ]) host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()]) - port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)]) - tls = fields.BooleanField(_('Enable TLS')) + port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993) + tls = fields.BooleanField(_('Enable TLS'), default=True) username = fields.StringField(_('Username'), [validators.DataRequired()]) password = fields.PasswordField(_('Password')) keep = fields.BooleanField(_('Keep emails on the server')) + scan = fields.BooleanField(_('Rescan emails locally')) + folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk') submit = fields.SubmitField(_('Submit')) diff --git a/core/admin/mailu/ui/templates/fetch/create.html b/core/admin/mailu/ui/templates/fetch/create.html index 00698329..69584d15 100644 --- a/core/admin/mailu/ui/templates/fetch/create.html +++ b/core/admin/mailu/ui/templates/fetch/create.html @@ -24,6 +24,8 @@ {%- call macros.card(title="Settings") %} {{ macros.form_field(form.keep) }} + {{ macros.form_field(form.scan) }} + {{ macros.form_field(form.folders) }} {%- endcall %} {{ macros.form_field(form.submit) }} diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html index 7a527ce8..74d3a02f 100644 --- a/core/admin/mailu/ui/templates/fetch/list.html +++ b/core/admin/mailu/ui/templates/fetch/list.html @@ -20,6 +20,8 @@ {% trans %}Endpoint{% endtrans %} {% trans %}Username{% endtrans %} {% trans %}Keep emails{% endtrans %} + {% trans %}Rescan emails{% endtrans %} + {% trans %}Folders{% endtrans %} {% trans %}Last check{% endtrans %} {% trans %}Status{% endtrans %} {% trans %}Created{% endtrans %} @@ -36,6 +38,8 @@ {{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }} {{ fetch.username }} {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} + {% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} + {{ fetch.folders | join(',') }} {{ fetch.last_check | format_datetime or '-' }} {{ fetch.error or '-' }} {{ fetch.created_at | format_date }} diff --git a/core/admin/mailu/ui/templates/sidebar.html b/core/admin/mailu/ui/templates/sidebar.html index a57a2abe..54448c8b 100644 --- a/core/admin/mailu/ui/templates/sidebar.html +++ b/core/admin/mailu/ui/templates/sidebar.html @@ -31,12 +31,14 @@

{% trans %}Auto-reply{% endtrans %}

+ {%- if config["FETCHMAIL_ENABLED"] %}

  • {% trans %}Fetched accounts{% endtrans %}
    • + {%- endif %}
  • diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index ec208af1..3c4d629d 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -1,5 +1,6 @@ -from mailu import models +from mailu import models, utils from mailu.ui import ui, forms, access +from flask import current_app as app import flask import flask_login @@ -10,6 +11,8 @@ import wtforms @ui.route('/fetch/list/', methods=['GET']) @access.owner(models.User, 'user_email') def fetch_list(user_email): + if not app.config['FETCHMAIL_ENABLED']: + flask.abort(404) user_email = user_email or flask_login.current_user.email user = models.User.query.get(user_email) or flask.abort(404) return flask.render_template('fetch/list.html', user=user) @@ -19,13 +22,18 @@ def fetch_list(user_email): @ui.route('/fetch/create/', methods=['GET', 'POST']) @access.owner(models.User, 'user_email') def fetch_create(user_email): + if not app.config['FETCHMAIL_ENABLED']: + flask.abort(404) user_email = user_email or flask_login.current_user.email user = models.User.query.get(user_email) or flask.abort(404) form = forms.FetchForm() form.password.validators = [wtforms.validators.DataRequired()] + utils.formatCSVField(form.folders) if form.validate_on_submit(): fetch = models.Fetch(user=user) form.populate_obj(fetch) + if form.folders.data: + fetch.folders = form.folders.data.replace(' ','').split(',') models.db.session.add(fetch) models.db.session.commit() flask.flash('Fetch configuration created') @@ -37,12 +45,17 @@ def fetch_create(user_email): @ui.route('/fetch/edit/', methods=['GET', 'POST']) @access.owner(models.Fetch, 'fetch_id') def fetch_edit(fetch_id): + if not app.config['FETCHMAIL_ENABLED']: + flask.abort(404) fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) form = forms.FetchForm(obj=fetch) + utils.formatCSVField(form.folders) if form.validate_on_submit(): if not form.password.data: form.password.data = fetch.password form.populate_obj(fetch) + if form.folders.data: + fetch.folders = form.folders.data.replace(' ','').split(',') models.db.session.commit() flask.flash('Fetch configuration updated') return flask.redirect( @@ -55,6 +68,8 @@ def fetch_edit(fetch_id): @access.confirmation_required("delete a fetched account") @access.owner(models.Fetch, 'fetch_id') def fetch_delete(fetch_id): + if not app.config['FETCHMAIL_ENABLED']: + flask.abort(404) fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) user = fetch.user models.db.session.delete(fetch) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index 85a5c2db..c7d252a9 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -64,10 +64,11 @@ def user_edit(user_email): form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=max_quota_bytes)] if form.validate_on_submit(): - if msg := utils.isBadOrPwned(form): - flask.flash(msg, "error") - return flask.render_template('user/edit.html', form=form, user=user, - domain=user.domain, max_quota_bytes=max_quota_bytes) + if form.pw.data: + if msg := utils.isBadOrPwned(form): + flask.flash(msg, "error") + return flask.render_template('user/edit.html', form=form, user=user, + domain=user.domain, max_quota_bytes=max_quota_bytes) form.populate_obj(user) if form.pw.data: user.set_password(form.pw.data) @@ -99,11 +100,7 @@ def user_settings(user_email): user_email_or_current = user_email or flask_login.current_user.email user = models.User.query.get(user_email_or_current) or flask.abort(404) form = forms.UserSettingsForm(obj=user) - if isinstance(form.forward_destination.data,str): - data = form.forward_destination.data.replace(" ","").split(",") - else: - data = form.forward_destination.data - form.forward_destination.data = ", ".join(data) + utils.formatCSVField(form.forward_destination) if form.validate_on_submit(): form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",") form.populate_obj(user) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index f160fe3f..b432192d 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -518,3 +518,10 @@ def isBadOrPwned(form): if breaches > 0: return f"This password appears in {breaches} data breaches! It is not unique; please change it." return None + +def formatCSVField(field): + if isinstance(field.data,str): + data = field.data.replace(" ","").split(",") + else: + data = field.data + field.data = ", ".join(data) diff --git a/core/admin/migrations/versions/f4f0f89e0047_.py b/core/admin/migrations/versions/f4f0f89e0047_.py new file mode 100644 index 00000000..8d20063c --- /dev/null +++ b/core/admin/migrations/versions/f4f0f89e0047_.py @@ -0,0 +1,25 @@ +""" Add fetch.scan and fetch.folders + +Revision ID: f4f0f89e0047 +Revises: 8f9ea78776f4 +Create Date: 2022-11-13 16:29:01.246509 + +""" + +# revision identifiers, used by Alembic. +revision = 'f4f0f89e0047' +down_revision = '8f9ea78776f4' + +from alembic import op +import sqlalchemy as sa +import mailu + +def upgrade(): + with op.batch_alter_table('fetch') as batch: + batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false())) + batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True)) + +def downgrade(): + with op.batch_alter_table('fetch') as batch: + batch.drop_column('fetch', 'folders') + batch.drop_column('fetch', 'scan') diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 4ab76e74..cf05fba3 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -48,7 +48,7 @@ sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Docker # assets cp "${assets}/package.json" . -cp -r "${assets}/assets/" . +cp -r "${assets}/assets" ./assets awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile @@ -65,7 +65,7 @@ RUN set -euxo pipefail \ ; ln -s /app/start.py / ENV \ - FLASK_ENV="development" \ + FLASK_DEBUG="true" \ MEMORY_SESSIONS="true" \ RATELIMIT_STORAGE_URL="memory://" \ SESSION_COOKIE_SECURE="false" \ @@ -73,7 +73,7 @@ ENV \ DEBUG="true" \ DEBUG_PROFILER="${DEV_PROFILER}" \ DEBUG_ASSETS="/app/static" \ - DEBUG_TB_ENABLED="true" \ + DEBUG_TB_INTERCEPT_REDIRECTS=False \ \ IMAP_ADDRESS="127.0.0.1" \ POP3_ADDRESS="127.0.0.1" \ @@ -82,7 +82,7 @@ ENV \ REDIS_ADDRESS="127.0.0.1" \ WEBMAIL_ADDRESS="127.0.0.1" -CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null && flask run --host=0.0.0.0 --port=8080"] +CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"] EOF # build diff --git a/core/admin/start.py b/core/admin/start.py index 3cb5c422..e2163398 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -2,8 +2,15 @@ import os import logging as log +from pwd import getpwnam import sys +os.system("chown mailu:mailu -R /dkim") +os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu") +mailu_id = getpwnam('mailu') +os.setgid(mailu_id.pw_gid) +os.setuid(mailu_id.pw_uid) + log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO")) os.system("flask mailu advertise") diff --git a/core/base/Dockerfile b/core/base/Dockerfile index ca62e773..20d17f30 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile-upstream:1.4.3 # base system image (intermediate) -ARG DISTRO=alpine:3.16.2 +ARG DISTRO=alpine:3.16.3 FROM $DISTRO as system ENV TZ=Etc/UTC LANG=C.UTF-8 @@ -12,7 +12,16 @@ ARG MAILU_GID=1000 RUN set -euxo pipefail \ ; addgroup -Sg ${MAILU_GID} mailu \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ - ; apk add --no-cache bash ca-certificates curl python3 tzdata + ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \ + ; machine="$(uname -m)" \ + ; ! [[ "${machine}" == x86_64 ]] \ + || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc + +ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so +ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" +ENV CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions" +ENV CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2" +ENV LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" WORKDIR /app @@ -78,6 +87,7 @@ FROM system COPY --from=build /app/venv/ /app/venv/ COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/ +RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn ENV VIRTUAL_ENV=/app/venv ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" diff --git a/core/none/Dockerfile b/core/none/Dockerfile index f06cc31c..d605cb07 100644 --- a/core/none/Dockerfile +++ b/core/none/Dockerfile @@ -10,5 +10,5 @@ RUN echo $VERSION >/version HEALTHCHECK CMD true -USER app +USER mailu CMD ["/bin/bash", "-c", "sleep infinity"] diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 537d996d..37de1df9 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -33,4 +33,7 @@ while True: log.warning("Admin is not up just yet, retrying in 1 second") # Run rspamd -os.execv("/usr/sbin/rspamd", ["rspamd", "-i", "-f"]) +os.system("mkdir -m 755 -p /run/rspamd") +os.system("chown rspamd:rspamd /run/rspamd") +os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown rspamd:rspamd") +os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"]) diff --git a/docs/configuration.rst b/docs/configuration.rst index d411d2c7..1a40bf65 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -104,6 +104,9 @@ support or e.g. mismatching TLS versions to deliver emails to Mailu. .. _fetchmail: +When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is enabled in the admin interface. +The container itself still needs to be deployed manually. ``FETCHMAIL_ENABLED`` defaults to ``True``. + The ``FETCHMAIL_DELAY`` is a delay (in seconds) for the fetchmail service to go and fetch new email if available. Do not use too short delays if you do not want to be blacklisted by external services, but not too long delays if you @@ -287,6 +290,10 @@ The admin service stores configurations in a database. - ``DB_USER``: the database user for mailu admin service. (when not ``sqlite``) - ``DB_NAME``: the database name for mailu admin service. (when not ``sqlite``) +Alternatively, if you need more control, you can use a `DB URL`_ : do not set any of the ``DB_`` settings and set ``SQLALCHEMY_DATABASE_URI`` instead. + +.. _`DB URL`: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls + The roundcube service stores configurations in a database. - ``ROUNDCUBE_DB_FLAVOR``: the database type for roundcube service. (``sqlite``, ``postgresql``, ``mysql``) diff --git a/docs/webadministration.rst b/docs/webadministration.rst index e17d12f0..fde4a271 100644 --- a/docs/webadministration.rst +++ b/docs/webadministration.rst @@ -157,7 +157,11 @@ You can add a fetched account by clicking on the `Add an account` button on the * Keep emails on the server. When ticked, retains the email message in the email account after retrieving it. -Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after 10 minutes. +* Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...). + +* Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled. + +Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``. Authentication tokens diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 32751ed7..62bd7124 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -2,11 +2,14 @@ import time import os +from pathlib import Path +from pwd import getpwnam import tempfile import shlex import subprocess import re import requests +from socrate import system import sys import traceback @@ -14,6 +17,7 @@ import traceback FETCHMAIL = """ fetchmail -N \ --idfile /data/fetchids --uidl \ + --pidfile /dev/shm/fetchmail.pid \ --sslcertck --sslcertpath /etc/ssl/certs \ -f {} """ @@ -24,7 +28,9 @@ poll "{host}" proto {protocol} port {port} user "{username}" password "{password}" is "{user_email}" smtphost "{smtphost}" + {folders} {options} + {lmtp} """ @@ -48,26 +54,37 @@ def fetchmail(fetchmailrc): def run(debug): try: - fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json() - smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None) + os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp") + os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") + fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json() + smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None) if smtpport is None: smtphostport = smtphost else: smtphostport = "%s/%d" % (smtphost, smtpport) + os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525") + lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None) + if lmtpport is None: + lmtphostport = lmtphost + else: + lmtphostport = "%s/%d" % (lmtphost, lmtpport) for fetch in fetches: fetchmailrc = "" options = "options antispam 501, 504, 550, 553, 554" options += " ssl" if fetch["tls"] else "" options += " keep" if fetch["keep"] else " fetchall" + folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"') fetchmailrc += RC_LINE.format( user_email=escape_rc_string(fetch["user_email"]), protocol=fetch["protocol"], host=escape_rc_string(fetch["host"]), port=fetch["port"], - smtphost=smtphostport, + smtphost=smtphostport if fetch['scan'] else lmtphostport, username=escape_rc_string(fetch["username"]), password=escape_rc_string(fetch["password"]), - options=options + options=options, + folders=folders, + lmtp='' if fetch['scan'] else 'lmtp', ) if debug: print(fetchmailrc) @@ -86,15 +103,29 @@ def run(debug): user_info in error_message): print(error_message) finally: - requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]), - json=error_message.split("\n")[0] + requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']), + json=error_message.split('\n')[0] ) except Exception: traceback.print_exc() if __name__ == "__main__": + id_fetchmail = getpwnam('fetchmail') + Path('/data/fetchids').touch() + os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid) + os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid) + os.chmod("/data/fetchids", 0o700) + os.setgid(id_fetchmail.pw_gid) + os.setuid(id_fetchmail.pw_uid) while True: - time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60))) + delay = int(os.environ.get("FETCHMAIL_DELAY", 60)) + print("Sleeping for {} seconds".format(delay)) + time.sleep(delay) + + if not os.environ.get("FETCHMAIL_ENABLED", 'True') in ('True', 'true'): + print("Fetchmail disabled, skipping...") + continue + run(os.environ.get("DEBUG", None) == "True") sys.stdout.flush() diff --git a/setup/Dockerfile b/setup/Dockerfile index 85e5f55b..a410871d 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -1,24 +1,21 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION -ENV TZ Etc/UTC +# syntax=docker/dockerfile-upstream:1.4.3 + +# setup image +FROM base + +ARG VERSION=local LABEL version=$VERSION - -RUN mkdir -p /app -WORKDIR /app - -COPY requirements.txt requirements.txt -RUN apk add --no-cache curl python3 py3-pip \ - && pip3 install -r requirements.txt - -COPY server.py ./server.py -COPY main.py ./main.py COPY flavors /data/flavors COPY templates /data/templates COPY static ./static +COPY server.py ./server.py +COPY main.py ./main.py + +RUN echo $VERSION >> /version EXPOSE 80/tcp +HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost/ +USER mailu CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app -RUN echo $VERSION >> /version diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index c7f37e01..b6c99ca5 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -157,8 +157,11 @@ services: env_file: {{ env }} volumes: - "{{ root }}/data/fetchmail:/data" - {% if resolver_enabled %} depends_on: + - admin + - smtp + - imap + {% if resolver_enabled %} - resolver dns: - {{ dns }} diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index df8c4a4d..cc99912e 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -79,6 +79,9 @@ RELAYNETS= # Will relay all outgoing mails if configured RELAYHOST={{ relayhost }} +# Show fetchmail functionality in admin interface +FETCHMAIL_ENABLED={{ fetchmail_enabled or 'False' }} + # Fetchmail delay FETCHMAIL_DELAY={{ fetchmail_delay or '600' }} diff --git a/setup/requirements.txt b/setup/requirements.txt deleted file mode 100644 index b6f9f713..00000000 --- a/setup/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -Flask==1.0.2 -Flask-Bootstrap==3.3.7.1 -gunicorn==19.9.0 -redis==3.2.1 -Jinja2==3.0.3 -MarkupSafe==2.1.0 -Werkzeug==2.0.3 -click==8.0.3 -dominate==2.6.0 -itsdangerous==2.0.1 -redis==3.2.1 -visitor==0.1.3 diff --git a/tests/build.hcl b/tests/build.hcl index fd546ae4..d657cbb7 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -106,6 +106,9 @@ target "docs" { target "setup" { inherits = ["defaults"] context = "setup/" + contexts = { + base = "target:base" + } tags = tag("setup") } diff --git a/tests/compose/filters/02_email_antispam.sh b/tests/compose/filters/02_email_antispam.sh new file mode 100755 index 00000000..ac2653a3 --- /dev/null +++ b/tests/compose/filters/02_email_antispam.sh @@ -0,0 +1,7 @@ +# GTUBE should be blocked, see https://rspamd.com/doc/gtube_patterns.html +python3 tests/email_test.py "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" +if [ $? -eq 25 ]; then + exit 0 +else + exit 1 +fi diff --git a/towncrier/newsfragments/1231.bugfix b/towncrier/newsfragments/1231.bugfix new file mode 100644 index 00000000..333ae35f --- /dev/null +++ b/towncrier/newsfragments/1231.bugfix @@ -0,0 +1 @@ +Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231) diff --git a/towncrier/newsfragments/2127.feature b/towncrier/newsfragments/2127.feature new file mode 100644 index 00000000..dd4951dd --- /dev/null +++ b/towncrier/newsfragments/2127.feature @@ -0,0 +1 @@ +Add FETCHMAIL_ENABLED to toggle the fetchmail functionality in the admin interface \ No newline at end of file diff --git a/towncrier/newsfragments/2246.bugfix b/towncrier/newsfragments/2246.bugfix new file mode 100644 index 00000000..92e90ac6 --- /dev/null +++ b/towncrier/newsfragments/2246.bugfix @@ -0,0 +1 @@ +Fetchmail: Missing support for '*_ADDRESS' env vars diff --git a/towncrier/newsfragments/2525.feature b/towncrier/newsfragments/2525.feature new file mode 100644 index 00000000..634733c7 --- /dev/null +++ b/towncrier/newsfragments/2525.feature @@ -0,0 +1 @@ +Switch to GrapheneOS's hardened_malloc diff --git a/towncrier/newsfragments/2539.misc b/towncrier/newsfragments/2539.misc new file mode 100644 index 00000000..10e3954e --- /dev/null +++ b/towncrier/newsfragments/2539.misc @@ -0,0 +1 @@ +Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. Please ensure that your folder overrides/rspamd is owned by 1000:1000 diff --git a/towncrier/newsfragments/711.feature b/towncrier/newsfragments/711.feature new file mode 100644 index 00000000..aa605aa2 --- /dev/null +++ b/towncrier/newsfragments/711.feature @@ -0,0 +1 @@ +Allow other folders to be synced by fetchmail diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 0ec12384..2ae8a8f0 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -52,7 +52,7 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/ # snappymail -ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.0/snappymail-2.21.0.tar.gz +ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.3/snappymail-2.21.3.tar.gz RUN set -euxo pipefail \ ; mkdir /var/www/snappymail \ @@ -82,7 +82,7 @@ RUN set -euxo pipefail \ # common COPY start.py / COPY php.ini /defaults/ -COPY php-webmail.conf /etc/php81/php-fpm.d/php-webmail.conf +COPY php-webmail.conf /etc/php81/php-fpm.d/ COPY nginx-webmail.conf /conf/ COPY snuffleupagus.rules /etc/snuffleupagus.rules.tpl