2529: Improve fetchmail r=mergify[bot] a=nextgens

## What type of PR?

enhancement

## What does this PR do?

Improve fetchmail:
- allow delivery via LMTP (faster, bypassing the filters)
- allow several folders to be retrieved
- run fetchmail as non-root
- tweak the compose file to ensure we have all the dependencies

### Related issue(s)
- closes #1231 
- closes #2246 
- closes #711

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [ ] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.

Co-authored-by: Florent Daigniere <nextgens@freenetproject.org>
Co-authored-by: Florent Daigniere <nextgens@users.noreply.github.com>
main
bors[bot] 2 years ago committed by GitHub
commit 553b02fb3d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -12,10 +12,12 @@ def fetch_list():
"id": fetch.id,
"tls": fetch.tls,
"keep": fetch.keep,
"scan": fetch.scan,
"user_email": fetch.user_email,
"protocol": fetch.protocol,
"host": fetch.host,
"port": fetch.port,
"folders": fetch.folders,
"username": fetch.username,
"password": fetch.password
} for fetch in models.Fetch.query.all()

@ -771,6 +771,8 @@ class Fetch(Base):
username = db.Column(db.String(255), nullable=False)
password = db.Column(db.String(255), nullable=False)
keep = db.Column(db.Boolean, nullable=False, default=False)
scan = db.Column(db.Boolean, nullable=False, default=False)
folders = db.Column(CommaSeparatedList, nullable=True, default=list)
last_check = db.Column(db.DateTime, nullable=True)
error = db.Column(db.String(1023), nullable=True)

@ -41,6 +41,15 @@ class MultipleEmailAddressesVerify(object):
if not pattern.match(field.data.replace(" ", "")):
raise validators.ValidationError(self.message)
class MultipleFoldersVerify(object):
def __init__(self,message=_('Invalid list of folders.')):
self.message = message
def __call__(self, form, field):
pattern = re.compile(r'^\w+(\s*,\s*\w+)*$')
if not pattern.match(field.data.replace(" ", "")):
raise validators.ValidationError(self.message)
class ConfirmationForm(flask_wtf.FlaskForm):
submit = fields.SubmitField(_('Confirm'))
@ -164,11 +173,13 @@ class FetchForm(flask_wtf.FlaskForm):
('imap', 'IMAP'), ('pop3', 'POP3')
])
host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()])
port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)])
tls = fields.BooleanField(_('Enable TLS'))
port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993)
tls = fields.BooleanField(_('Enable TLS'), default=True)
username = fields.StringField(_('Username'), [validators.DataRequired()])
password = fields.PasswordField(_('Password'))
keep = fields.BooleanField(_('Keep emails on the server'))
scan = fields.BooleanField(_('Rescan emails locally'))
folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk')
submit = fields.SubmitField(_('Submit'))

@ -24,6 +24,8 @@
{%- call macros.card(title="Settings") %}
{{ macros.form_field(form.keep) }}
{{ macros.form_field(form.scan) }}
{{ macros.form_field(form.folders) }}
{%- endcall %}
{{ macros.form_field(form.submit) }}

@ -20,6 +20,8 @@
<th>{% trans %}Endpoint{% endtrans %}</th>
<th>{% trans %}Username{% endtrans %}</th>
<th>{% trans %}Keep emails{% endtrans %}</th>
<th>{% trans %}Rescan emails{% endtrans %}</th>
<th>{% trans %}Folders{% endtrans %}</th>
<th>{% trans %}Last check{% endtrans %}</th>
<th>{% trans %}Status{% endtrans %}</th>
<th>{% trans %}Created{% endtrans %}</th>
@ -36,6 +38,8 @@
<td>{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }}</td>
<td>{{ fetch.username }}</td>
<td data-sort="{{ fetch.keep }}">{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
<td data-sort="{{ fetch.scan }}">{% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
<td>{{ fetch.folders.data | join(',') }}</td>
<td>{{ fetch.last_check | format_datetime or '-' }}</td>
<td>{{ fetch.error or '-' }}</td>
<td data-sort="{{ fetch.created_at or '0000-00-00' }}">{{ fetch.created_at | format_date }}</td>

@ -1,4 +1,4 @@
from mailu import models
from mailu import models, utils
from mailu.ui import ui, forms, access
from flask import current_app as app
@ -28,9 +28,12 @@ def fetch_create(user_email):
user = models.User.query.get(user_email) or flask.abort(404)
form = forms.FetchForm()
form.password.validators = [wtforms.validators.DataRequired()]
utils.formatCSVField(form.folders)
if form.validate_on_submit():
fetch = models.Fetch(user=user)
form.populate_obj(fetch)
if form.folders.data:
fetch.folders = form.folders.data.replace(' ','').split(',')
models.db.session.add(fetch)
models.db.session.commit()
flask.flash('Fetch configuration created')
@ -46,10 +49,13 @@ def fetch_edit(fetch_id):
flask.abort(404)
fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
form = forms.FetchForm(obj=fetch)
utils.formatCSVField(form.folders)
if form.validate_on_submit():
if not form.password.data:
form.password.data = fetch.password
form.populate_obj(fetch)
if form.folders.data:
fetch.folders = form.folders.data.replace(' ','').split(',')
models.db.session.commit()
flask.flash('Fetch configuration updated')
return flask.redirect(

@ -100,11 +100,7 @@ def user_settings(user_email):
user_email_or_current = user_email or flask_login.current_user.email
user = models.User.query.get(user_email_or_current) or flask.abort(404)
form = forms.UserSettingsForm(obj=user)
if isinstance(form.forward_destination.data,str):
data = form.forward_destination.data.replace(" ","").split(",")
else:
data = form.forward_destination.data
form.forward_destination.data = ", ".join(data)
utils.formatCSVField(form.forward_destination)
if form.validate_on_submit():
form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",")
form.populate_obj(user)

@ -518,3 +518,10 @@ def isBadOrPwned(form):
if breaches > 0:
return f"This password appears in {breaches} data breaches! It is not unique; please change it."
return None
def formatCSVField(field):
if isinstance(field.data,str):
data = field.data.replace(" ","").split(",")
else:
data = field.data
field.data = ", ".join(data)

@ -0,0 +1,25 @@
""" Add fetch.scan and fetch.folders
Revision ID: f4f0f89e0047
Revises: 8f9ea78776f4
Create Date: 2022-11-13 16:29:01.246509
"""
# revision identifiers, used by Alembic.
revision = 'f4f0f89e0047'
down_revision = '8f9ea78776f4'
from alembic import op
import sqlalchemy as sa
import mailu
def upgrade():
with op.batch_alter_table('fetch') as batch:
batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True))
def downgrade():
with op.batch_alter_table('fetch') as batch:
batch.drop_column('fetch', 'folders')
batch.drop_column('fetch', 'scan')

@ -157,7 +157,11 @@ You can add a fetched account by clicking on the `Add an account` button on the
* Keep emails on the server. When ticked, retains the email message in the email account after retrieving it.
Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after 10 minutes.
* Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...).
* Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled.
Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``.
Authentication tokens

@ -2,11 +2,14 @@
import time
import os
from pathlib import Path
from pwd import getpwnam
import tempfile
import shlex
import subprocess
import re
import requests
from socrate import system
import sys
import traceback
@ -14,6 +17,7 @@ import traceback
FETCHMAIL = """
fetchmail -N \
--idfile /data/fetchids --uidl \
--pidfile /dev/shm/fetchmail.pid \
--sslcertck --sslcertpath /etc/ssl/certs \
-f {}
"""
@ -24,7 +28,9 @@ poll "{host}" proto {protocol} port {port}
user "{username}" password "{password}"
is "{user_email}"
smtphost "{smtphost}"
{folders}
{options}
{lmtp}
"""
@ -48,26 +54,37 @@ def fetchmail(fetchmailrc):
def run(debug):
try:
fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json()
smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
if smtpport is None:
smtphostport = smtphost
else:
smtphostport = "%s/%d" % (smtphost, smtpport)
os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
if lmtpport is None:
lmtphostport = lmtphost
else:
lmtphostport = "%s/%d" % (lmtphost, lmtpport)
for fetch in fetches:
fetchmailrc = ""
options = "options antispam 501, 504, 550, 553, 554"
options += " ssl" if fetch["tls"] else ""
options += " keep" if fetch["keep"] else " fetchall"
folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"')
fetchmailrc += RC_LINE.format(
user_email=escape_rc_string(fetch["user_email"]),
protocol=fetch["protocol"],
host=escape_rc_string(fetch["host"]),
port=fetch["port"],
smtphost=smtphostport,
smtphost=smtphostport if fetch['scan'] else lmtphostport,
username=escape_rc_string(fetch["username"]),
password=escape_rc_string(fetch["password"]),
options=options
options=options,
folders=folders,
lmtp='' if fetch['scan'] else 'lmtp',
)
if debug:
print(fetchmailrc)
@ -86,14 +103,21 @@ def run(debug):
user_info in error_message):
print(error_message)
finally:
requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]),
json=error_message.split("\n")[0]
requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']),
json=error_message.split('\n')[0]
)
except Exception:
traceback.print_exc()
if __name__ == "__main__":
id_fetchmail = getpwnam('fetchmail')
Path('/data/fetchids').touch()
os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
os.chmod("/data/fetchids", 0o700)
os.setgid(id_fetchmail.pw_gid)
os.setuid(id_fetchmail.pw_uid)
while True:
delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
print("Sleeping for {} seconds".format(delay))

@ -157,8 +157,11 @@ services:
env_file: {{ env }}
volumes:
- "{{ root }}/data/fetchmail:/data"
{% if resolver_enabled %}
depends_on:
- admin
- smtp
- imap
{% if resolver_enabled %}
- resolver
dns:
- {{ dns }}

@ -0,0 +1 @@
Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231)

@ -0,0 +1 @@
Fetchmail: Missing support for '*_ADDRESS' env vars

@ -0,0 +1 @@
Allow other folders to be synced by fetchmail
Loading…
Cancel
Save