diff --git a/core/admin/mailu/internal/views/fetch.py b/core/admin/mailu/internal/views/fetch.py
index 1945b9c7..e813c33b 100644
--- a/core/admin/mailu/internal/views/fetch.py
+++ b/core/admin/mailu/internal/views/fetch.py
@@ -12,10 +12,12 @@ def fetch_list():
"id": fetch.id,
"tls": fetch.tls,
"keep": fetch.keep,
+ "scan": fetch.scan,
"user_email": fetch.user_email,
"protocol": fetch.protocol,
"host": fetch.host,
"port": fetch.port,
+ "folders": fetch.folders,
"username": fetch.username,
"password": fetch.password
} for fetch in models.Fetch.query.all()
diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py
index 48ce8b33..4b048c45 100644
--- a/core/admin/mailu/models.py
+++ b/core/admin/mailu/models.py
@@ -771,6 +771,8 @@ class Fetch(Base):
username = db.Column(db.String(255), nullable=False)
password = db.Column(db.String(255), nullable=False)
keep = db.Column(db.Boolean, nullable=False, default=False)
+ scan = db.Column(db.Boolean, nullable=False, default=False)
+ folders = db.Column(CommaSeparatedList, nullable=True, default=list)
last_check = db.Column(db.DateTime, nullable=True)
error = db.Column(db.String(1023), nullable=True)
diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py
index beb44092..fa81adc3 100644
--- a/core/admin/mailu/ui/forms.py
+++ b/core/admin/mailu/ui/forms.py
@@ -41,6 +41,15 @@ class MultipleEmailAddressesVerify(object):
if not pattern.match(field.data.replace(" ", "")):
raise validators.ValidationError(self.message)
+class MultipleFoldersVerify(object):
+ def __init__(self,message=_('Invalid list of folders.')):
+ self.message = message
+
+ def __call__(self, form, field):
+ pattern = re.compile(r'^\w+(\s*,\s*\w+)*$')
+ if not pattern.match(field.data.replace(" ", "")):
+ raise validators.ValidationError(self.message)
+
class ConfirmationForm(flask_wtf.FlaskForm):
submit = fields.SubmitField(_('Confirm'))
@@ -164,11 +173,13 @@ class FetchForm(flask_wtf.FlaskForm):
('imap', 'IMAP'), ('pop3', 'POP3')
])
host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()])
- port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)])
- tls = fields.BooleanField(_('Enable TLS'))
+ port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993)
+ tls = fields.BooleanField(_('Enable TLS'), default=True)
username = fields.StringField(_('Username'), [validators.DataRequired()])
password = fields.PasswordField(_('Password'))
keep = fields.BooleanField(_('Keep emails on the server'))
+ scan = fields.BooleanField(_('Rescan emails locally'))
+ folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk')
submit = fields.SubmitField(_('Submit'))
diff --git a/core/admin/mailu/ui/templates/fetch/create.html b/core/admin/mailu/ui/templates/fetch/create.html
index 00698329..69584d15 100644
--- a/core/admin/mailu/ui/templates/fetch/create.html
+++ b/core/admin/mailu/ui/templates/fetch/create.html
@@ -24,6 +24,8 @@
{%- call macros.card(title="Settings") %}
{{ macros.form_field(form.keep) }}
+ {{ macros.form_field(form.scan) }}
+ {{ macros.form_field(form.folders) }}
{%- endcall %}
{{ macros.form_field(form.submit) }}
diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html
index 7a527ce8..e502d96a 100644
--- a/core/admin/mailu/ui/templates/fetch/list.html
+++ b/core/admin/mailu/ui/templates/fetch/list.html
@@ -20,6 +20,8 @@
{% trans %}Endpoint{% endtrans %} |
{% trans %}Username{% endtrans %} |
{% trans %}Keep emails{% endtrans %} |
+ {% trans %}Rescan emails{% endtrans %} |
+ {% trans %}Folders{% endtrans %} |
{% trans %}Last check{% endtrans %} |
{% trans %}Status{% endtrans %} |
{% trans %}Created{% endtrans %} |
@@ -36,6 +38,8 @@
{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }} |
{{ fetch.username }} |
{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} |
+ {% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} |
+ {{ fetch.folders.data | join(',') }} |
{{ fetch.last_check | format_datetime or '-' }} |
{{ fetch.error or '-' }} |
{{ fetch.created_at | format_date }} |
diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py
index ca837a8e..3c4d629d 100644
--- a/core/admin/mailu/ui/views/fetches.py
+++ b/core/admin/mailu/ui/views/fetches.py
@@ -1,4 +1,4 @@
-from mailu import models
+from mailu import models, utils
from mailu.ui import ui, forms, access
from flask import current_app as app
@@ -28,9 +28,12 @@ def fetch_create(user_email):
user = models.User.query.get(user_email) or flask.abort(404)
form = forms.FetchForm()
form.password.validators = [wtforms.validators.DataRequired()]
+ utils.formatCSVField(form.folders)
if form.validate_on_submit():
fetch = models.Fetch(user=user)
form.populate_obj(fetch)
+ if form.folders.data:
+ fetch.folders = form.folders.data.replace(' ','').split(',')
models.db.session.add(fetch)
models.db.session.commit()
flask.flash('Fetch configuration created')
@@ -46,10 +49,13 @@ def fetch_edit(fetch_id):
flask.abort(404)
fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
form = forms.FetchForm(obj=fetch)
+ utils.formatCSVField(form.folders)
if form.validate_on_submit():
if not form.password.data:
form.password.data = fetch.password
form.populate_obj(fetch)
+ if form.folders.data:
+ fetch.folders = form.folders.data.replace(' ','').split(',')
models.db.session.commit()
flask.flash('Fetch configuration updated')
return flask.redirect(
diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py
index b1b42c17..c7d252a9 100644
--- a/core/admin/mailu/ui/views/users.py
+++ b/core/admin/mailu/ui/views/users.py
@@ -100,11 +100,7 @@ def user_settings(user_email):
user_email_or_current = user_email or flask_login.current_user.email
user = models.User.query.get(user_email_or_current) or flask.abort(404)
form = forms.UserSettingsForm(obj=user)
- if isinstance(form.forward_destination.data,str):
- data = form.forward_destination.data.replace(" ","").split(",")
- else:
- data = form.forward_destination.data
- form.forward_destination.data = ", ".join(data)
+ utils.formatCSVField(form.forward_destination)
if form.validate_on_submit():
form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",")
form.populate_obj(user)
diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py
index f160fe3f..b432192d 100644
--- a/core/admin/mailu/utils.py
+++ b/core/admin/mailu/utils.py
@@ -518,3 +518,10 @@ def isBadOrPwned(form):
if breaches > 0:
return f"This password appears in {breaches} data breaches! It is not unique; please change it."
return None
+
+def formatCSVField(field):
+ if isinstance(field.data,str):
+ data = field.data.replace(" ","").split(",")
+ else:
+ data = field.data
+ field.data = ", ".join(data)
diff --git a/core/admin/migrations/versions/f4f0f89e0047_.py b/core/admin/migrations/versions/f4f0f89e0047_.py
new file mode 100644
index 00000000..8d20063c
--- /dev/null
+++ b/core/admin/migrations/versions/f4f0f89e0047_.py
@@ -0,0 +1,25 @@
+""" Add fetch.scan and fetch.folders
+
+Revision ID: f4f0f89e0047
+Revises: 8f9ea78776f4
+Create Date: 2022-11-13 16:29:01.246509
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f4f0f89e0047'
+down_revision = '8f9ea78776f4'
+
+from alembic import op
+import sqlalchemy as sa
+import mailu
+
+def upgrade():
+ with op.batch_alter_table('fetch') as batch:
+ batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
+ batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True))
+
+def downgrade():
+ with op.batch_alter_table('fetch') as batch:
+ batch.drop_column('fetch', 'folders')
+ batch.drop_column('fetch', 'scan')
diff --git a/docs/webadministration.rst b/docs/webadministration.rst
index e17d12f0..fde4a271 100644
--- a/docs/webadministration.rst
+++ b/docs/webadministration.rst
@@ -157,7 +157,11 @@ You can add a fetched account by clicking on the `Add an account` button on the
* Keep emails on the server. When ticked, retains the email message in the email account after retrieving it.
-Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after 10 minutes.
+* Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...).
+
+* Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled.
+
+Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``.
Authentication tokens
diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py
index 3a82a124..62bd7124 100755
--- a/optional/fetchmail/fetchmail.py
+++ b/optional/fetchmail/fetchmail.py
@@ -2,11 +2,14 @@
import time
import os
+from pathlib import Path
+from pwd import getpwnam
import tempfile
import shlex
import subprocess
import re
import requests
+from socrate import system
import sys
import traceback
@@ -14,6 +17,7 @@ import traceback
FETCHMAIL = """
fetchmail -N \
--idfile /data/fetchids --uidl \
+ --pidfile /dev/shm/fetchmail.pid \
--sslcertck --sslcertpath /etc/ssl/certs \
-f {}
"""
@@ -24,7 +28,9 @@ poll "{host}" proto {protocol} port {port}
user "{username}" password "{password}"
is "{user_email}"
smtphost "{smtphost}"
+ {folders}
{options}
+ {lmtp}
"""
@@ -48,26 +54,37 @@ def fetchmail(fetchmailrc):
def run(debug):
try:
- fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json()
- smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
+ os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
+ os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
+ fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
+ smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
if smtpport is None:
smtphostport = smtphost
else:
smtphostport = "%s/%d" % (smtphost, smtpport)
+ os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
+ lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
+ if lmtpport is None:
+ lmtphostport = lmtphost
+ else:
+ lmtphostport = "%s/%d" % (lmtphost, lmtpport)
for fetch in fetches:
fetchmailrc = ""
options = "options antispam 501, 504, 550, 553, 554"
options += " ssl" if fetch["tls"] else ""
options += " keep" if fetch["keep"] else " fetchall"
+ folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"')
fetchmailrc += RC_LINE.format(
user_email=escape_rc_string(fetch["user_email"]),
protocol=fetch["protocol"],
host=escape_rc_string(fetch["host"]),
port=fetch["port"],
- smtphost=smtphostport,
+ smtphost=smtphostport if fetch['scan'] else lmtphostport,
username=escape_rc_string(fetch["username"]),
password=escape_rc_string(fetch["password"]),
- options=options
+ options=options,
+ folders=folders,
+ lmtp='' if fetch['scan'] else 'lmtp',
)
if debug:
print(fetchmailrc)
@@ -86,14 +103,21 @@ def run(debug):
user_info in error_message):
print(error_message)
finally:
- requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]),
- json=error_message.split("\n")[0]
+ requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']),
+ json=error_message.split('\n')[0]
)
except Exception:
traceback.print_exc()
if __name__ == "__main__":
+ id_fetchmail = getpwnam('fetchmail')
+ Path('/data/fetchids').touch()
+ os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+ os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+ os.chmod("/data/fetchids", 0o700)
+ os.setgid(id_fetchmail.pw_gid)
+ os.setuid(id_fetchmail.pw_uid)
while True:
delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
print("Sleeping for {} seconds".format(delay))
diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml
index 6dac166b..11596729 100644
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@@ -157,8 +157,11 @@ services:
env_file: {{ env }}
volumes:
- "{{ root }}/data/fetchmail:/data"
- {% if resolver_enabled %}
depends_on:
+ - admin
+ - smtp
+ - imap
+ {% if resolver_enabled %}
- resolver
dns:
- {{ dns }}
diff --git a/towncrier/newsfragments/1231.bugfix b/towncrier/newsfragments/1231.bugfix
new file mode 100644
index 00000000..333ae35f
--- /dev/null
+++ b/towncrier/newsfragments/1231.bugfix
@@ -0,0 +1 @@
+Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231)
diff --git a/towncrier/newsfragments/2246.bugfix b/towncrier/newsfragments/2246.bugfix
new file mode 100644
index 00000000..92e90ac6
--- /dev/null
+++ b/towncrier/newsfragments/2246.bugfix
@@ -0,0 +1 @@
+Fetchmail: Missing support for '*_ADDRESS' env vars
diff --git a/towncrier/newsfragments/711.feature b/towncrier/newsfragments/711.feature
new file mode 100644
index 00000000..aa605aa2
--- /dev/null
+++ b/towncrier/newsfragments/711.feature
@@ -0,0 +1 @@
+Allow other folders to be synced by fetchmail