diff --git a/core/admin/mailu/internal/views/fetch.py b/core/admin/mailu/internal/views/fetch.py index 1945b9c7..e813c33b 100644 --- a/core/admin/mailu/internal/views/fetch.py +++ b/core/admin/mailu/internal/views/fetch.py @@ -12,10 +12,12 @@ def fetch_list(): "id": fetch.id, "tls": fetch.tls, "keep": fetch.keep, + "scan": fetch.scan, "user_email": fetch.user_email, "protocol": fetch.protocol, "host": fetch.host, "port": fetch.port, + "folders": fetch.folders, "username": fetch.username, "password": fetch.password } for fetch in models.Fetch.query.all() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 48ce8b33..4b048c45 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -771,6 +771,8 @@ class Fetch(Base): username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) keep = db.Column(db.Boolean, nullable=False, default=False) + scan = db.Column(db.Boolean, nullable=False, default=False) + folders = db.Column(CommaSeparatedList, nullable=True, default=list) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index beb44092..fa81adc3 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -41,6 +41,15 @@ class MultipleEmailAddressesVerify(object): if not pattern.match(field.data.replace(" ", "")): raise validators.ValidationError(self.message) +class MultipleFoldersVerify(object): + def __init__(self,message=_('Invalid list of folders.')): + self.message = message + + def __call__(self, form, field): + pattern = re.compile(r'^\w+(\s*,\s*\w+)*$') + if not pattern.match(field.data.replace(" ", "")): + raise validators.ValidationError(self.message) + class ConfirmationForm(flask_wtf.FlaskForm): submit = fields.SubmitField(_('Confirm')) @@ -164,11 +173,13 @@ class FetchForm(flask_wtf.FlaskForm): ('imap', 'IMAP'), ('pop3', 'POP3') ]) host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()]) - port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)]) - tls = fields.BooleanField(_('Enable TLS')) + port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993) + tls = fields.BooleanField(_('Enable TLS'), default=True) username = fields.StringField(_('Username'), [validators.DataRequired()]) password = fields.PasswordField(_('Password')) keep = fields.BooleanField(_('Keep emails on the server')) + scan = fields.BooleanField(_('Rescan emails locally')) + folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk') submit = fields.SubmitField(_('Submit')) diff --git a/core/admin/mailu/ui/templates/fetch/create.html b/core/admin/mailu/ui/templates/fetch/create.html index 00698329..69584d15 100644 --- a/core/admin/mailu/ui/templates/fetch/create.html +++ b/core/admin/mailu/ui/templates/fetch/create.html @@ -24,6 +24,8 @@ {%- call macros.card(title="Settings") %} {{ macros.form_field(form.keep) }} + {{ macros.form_field(form.scan) }} + {{ macros.form_field(form.folders) }} {%- endcall %} {{ macros.form_field(form.submit) }} diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html index 7a527ce8..e502d96a 100644 --- a/core/admin/mailu/ui/templates/fetch/list.html +++ b/core/admin/mailu/ui/templates/fetch/list.html @@ -20,6 +20,8 @@ {% trans %}Endpoint{% endtrans %} {% trans %}Username{% endtrans %} {% trans %}Keep emails{% endtrans %} + {% trans %}Rescan emails{% endtrans %} + {% trans %}Folders{% endtrans %} {% trans %}Last check{% endtrans %} {% trans %}Status{% endtrans %} {% trans %}Created{% endtrans %} @@ -36,6 +38,8 @@ {{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }} {{ fetch.username }} {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} + {% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} + {{ fetch.folders.data | join(',') }} {{ fetch.last_check | format_datetime or '-' }} {{ fetch.error or '-' }} {{ fetch.created_at | format_date }} diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index ca837a8e..3c4d629d 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -1,4 +1,4 @@ -from mailu import models +from mailu import models, utils from mailu.ui import ui, forms, access from flask import current_app as app @@ -28,9 +28,12 @@ def fetch_create(user_email): user = models.User.query.get(user_email) or flask.abort(404) form = forms.FetchForm() form.password.validators = [wtforms.validators.DataRequired()] + utils.formatCSVField(form.folders) if form.validate_on_submit(): fetch = models.Fetch(user=user) form.populate_obj(fetch) + if form.folders.data: + fetch.folders = form.folders.data.replace(' ','').split(',') models.db.session.add(fetch) models.db.session.commit() flask.flash('Fetch configuration created') @@ -46,10 +49,13 @@ def fetch_edit(fetch_id): flask.abort(404) fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) form = forms.FetchForm(obj=fetch) + utils.formatCSVField(form.folders) if form.validate_on_submit(): if not form.password.data: form.password.data = fetch.password form.populate_obj(fetch) + if form.folders.data: + fetch.folders = form.folders.data.replace(' ','').split(',') models.db.session.commit() flask.flash('Fetch configuration updated') return flask.redirect( diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index b1b42c17..c7d252a9 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -100,11 +100,7 @@ def user_settings(user_email): user_email_or_current = user_email or flask_login.current_user.email user = models.User.query.get(user_email_or_current) or flask.abort(404) form = forms.UserSettingsForm(obj=user) - if isinstance(form.forward_destination.data,str): - data = form.forward_destination.data.replace(" ","").split(",") - else: - data = form.forward_destination.data - form.forward_destination.data = ", ".join(data) + utils.formatCSVField(form.forward_destination) if form.validate_on_submit(): form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",") form.populate_obj(user) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index f160fe3f..b432192d 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -518,3 +518,10 @@ def isBadOrPwned(form): if breaches > 0: return f"This password appears in {breaches} data breaches! It is not unique; please change it." return None + +def formatCSVField(field): + if isinstance(field.data,str): + data = field.data.replace(" ","").split(",") + else: + data = field.data + field.data = ", ".join(data) diff --git a/core/admin/migrations/versions/f4f0f89e0047_.py b/core/admin/migrations/versions/f4f0f89e0047_.py new file mode 100644 index 00000000..8d20063c --- /dev/null +++ b/core/admin/migrations/versions/f4f0f89e0047_.py @@ -0,0 +1,25 @@ +""" Add fetch.scan and fetch.folders + +Revision ID: f4f0f89e0047 +Revises: 8f9ea78776f4 +Create Date: 2022-11-13 16:29:01.246509 + +""" + +# revision identifiers, used by Alembic. +revision = 'f4f0f89e0047' +down_revision = '8f9ea78776f4' + +from alembic import op +import sqlalchemy as sa +import mailu + +def upgrade(): + with op.batch_alter_table('fetch') as batch: + batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false())) + batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True)) + +def downgrade(): + with op.batch_alter_table('fetch') as batch: + batch.drop_column('fetch', 'folders') + batch.drop_column('fetch', 'scan') diff --git a/docs/webadministration.rst b/docs/webadministration.rst index e17d12f0..fde4a271 100644 --- a/docs/webadministration.rst +++ b/docs/webadministration.rst @@ -157,7 +157,11 @@ You can add a fetched account by clicking on the `Add an account` button on the * Keep emails on the server. When ticked, retains the email message in the email account after retrieving it. -Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after 10 minutes. +* Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...). + +* Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled. + +Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``. Authentication tokens diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 3a82a124..62bd7124 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -2,11 +2,14 @@ import time import os +from pathlib import Path +from pwd import getpwnam import tempfile import shlex import subprocess import re import requests +from socrate import system import sys import traceback @@ -14,6 +17,7 @@ import traceback FETCHMAIL = """ fetchmail -N \ --idfile /data/fetchids --uidl \ + --pidfile /dev/shm/fetchmail.pid \ --sslcertck --sslcertpath /etc/ssl/certs \ -f {} """ @@ -24,7 +28,9 @@ poll "{host}" proto {protocol} port {port} user "{username}" password "{password}" is "{user_email}" smtphost "{smtphost}" + {folders} {options} + {lmtp} """ @@ -48,26 +54,37 @@ def fetchmail(fetchmailrc): def run(debug): try: - fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json() - smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None) + os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp") + os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") + fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json() + smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None) if smtpport is None: smtphostport = smtphost else: smtphostport = "%s/%d" % (smtphost, smtpport) + os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525") + lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None) + if lmtpport is None: + lmtphostport = lmtphost + else: + lmtphostport = "%s/%d" % (lmtphost, lmtpport) for fetch in fetches: fetchmailrc = "" options = "options antispam 501, 504, 550, 553, 554" options += " ssl" if fetch["tls"] else "" options += " keep" if fetch["keep"] else " fetchall" + folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"') fetchmailrc += RC_LINE.format( user_email=escape_rc_string(fetch["user_email"]), protocol=fetch["protocol"], host=escape_rc_string(fetch["host"]), port=fetch["port"], - smtphost=smtphostport, + smtphost=smtphostport if fetch['scan'] else lmtphostport, username=escape_rc_string(fetch["username"]), password=escape_rc_string(fetch["password"]), - options=options + options=options, + folders=folders, + lmtp='' if fetch['scan'] else 'lmtp', ) if debug: print(fetchmailrc) @@ -86,14 +103,21 @@ def run(debug): user_info in error_message): print(error_message) finally: - requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]), - json=error_message.split("\n")[0] + requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']), + json=error_message.split('\n')[0] ) except Exception: traceback.print_exc() if __name__ == "__main__": + id_fetchmail = getpwnam('fetchmail') + Path('/data/fetchids').touch() + os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid) + os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid) + os.chmod("/data/fetchids", 0o700) + os.setgid(id_fetchmail.pw_gid) + os.setuid(id_fetchmail.pw_uid) while True: delay = int(os.environ.get("FETCHMAIL_DELAY", 60)) print("Sleeping for {} seconds".format(delay)) diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 6dac166b..11596729 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -157,8 +157,11 @@ services: env_file: {{ env }} volumes: - "{{ root }}/data/fetchmail:/data" - {% if resolver_enabled %} depends_on: + - admin + - smtp + - imap + {% if resolver_enabled %} - resolver dns: - {{ dns }} diff --git a/towncrier/newsfragments/1231.bugfix b/towncrier/newsfragments/1231.bugfix new file mode 100644 index 00000000..333ae35f --- /dev/null +++ b/towncrier/newsfragments/1231.bugfix @@ -0,0 +1 @@ +Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231) diff --git a/towncrier/newsfragments/2246.bugfix b/towncrier/newsfragments/2246.bugfix new file mode 100644 index 00000000..92e90ac6 --- /dev/null +++ b/towncrier/newsfragments/2246.bugfix @@ -0,0 +1 @@ +Fetchmail: Missing support for '*_ADDRESS' env vars diff --git a/towncrier/newsfragments/711.feature b/towncrier/newsfragments/711.feature new file mode 100644 index 00000000..aa605aa2 --- /dev/null +++ b/towncrier/newsfragments/711.feature @@ -0,0 +1 @@ +Allow other folders to be synced by fetchmail