Improve fetchmail

core/admin/mailu/internal/views/fetch.py

@@ -12,10 +12,12 @@ def fetch_list():
             "id": fetch.id,
             "tls": fetch.tls,
             "keep": fetch.keep,
+            "scan": fetch.scan,
             "user_email": fetch.user_email,
             "protocol": fetch.protocol,
             "host": fetch.host,
             "port": fetch.port,
+            "folders": fetch.folders,
             "username": fetch.username,
             "password": fetch.password
         } for fetch in models.Fetch.query.all()

core/admin/mailu/models.py

@@ -771,6 +771,8 @@ class Fetch(Base):
     username = db.Column(db.String(255), nullable=False)
     password = db.Column(db.String(255), nullable=False)
     keep = db.Column(db.Boolean, nullable=False, default=False)
+    scan = db.Column(db.Boolean, nullable=False, default=False)
+    folders = db.Column(CommaSeparatedList, nullable=True, default=list)
     last_check = db.Column(db.DateTime, nullable=True)
     error = db.Column(db.String(1023), nullable=True)
 

core/admin/mailu/ui/forms.py

@@ -41,6 +41,15 @@ class MultipleEmailAddressesVerify(object):
         if not pattern.match(field.data.replace(" ", "")):
             raise validators.ValidationError(self.message)
 
+class MultipleFoldersVerify(object):
+    def __init__(self,message=_('Invalid list of folders.')):
+        self.message = message
+
+    def __call__(self, form, field):
+        pattern = re.compile(r'^\w+(,\w+)*$')
+        if not pattern.match(field.data.replace(" ", "")):
+            raise validators.ValidationError(self.message)
+
 class ConfirmationForm(flask_wtf.FlaskForm):
     submit = fields.SubmitField(_('Confirm'))
 
@@ -164,11 +173,13 @@ class FetchForm(flask_wtf.FlaskForm):
         ('imap', 'IMAP'), ('pop3', 'POP3')
     ])
     host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()])
-    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)])
+    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993)
-    tls = fields.BooleanField(_('Enable TLS'))
+    tls = fields.BooleanField(_('Enable TLS'), default=True)
     username = fields.StringField(_('Username'), [validators.DataRequired()])
     password = fields.PasswordField(_('Password'))
     keep = fields.BooleanField(_('Keep emails on the server'))
+    scan = fields.BooleanField(_('Rescan emails locally'))
+    folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk')
     submit = fields.SubmitField(_('Submit'))
 
 

core/admin/mailu/ui/templates/fetch/create.html

@@ -24,6 +24,8 @@
 
   {%- call macros.card(title="Settings") %}
   {{ macros.form_field(form.keep) }}
+  {{ macros.form_field(form.scan) }}
+  {{ macros.form_field(form.folders) }}
   {%- endcall %}
 
   {{ macros.form_field(form.submit) }}

core/admin/mailu/ui/templates/fetch/list.html

@@ -20,6 +20,8 @@
     <th>{% trans %}Endpoint{% endtrans %}</th>
     <th>{% trans %}Username{% endtrans %}</th>
     <th>{% trans %}Keep emails{% endtrans %}</th>
+    <th>{% trans %}Rescan emails{% endtrans %}</th>
+    <th>{% trans %}Folders{% endtrans %}</th>
     <th>{% trans %}Last check{% endtrans %}</th>
     <th>{% trans %}Status{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
@@ -36,6 +38,8 @@
     <td>{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }}</td>
     <td>{{ fetch.username }}</td>
     <td data-sort="{{ fetch.keep }}">{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td data-sort="{{ fetch.scan }}">{% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td>{% for folder in fetch.folders %}{{ folder }},{% endfor %}</td>
     <td>{{ fetch.last_check | format_datetime or '-' }}</td>
     <td>{{ fetch.error or '-' }}</td>
     <td data-sort="{{ fetch.created_at or '0000-00-00' }}">{{ fetch.created_at | format_date }}</td>

core/admin/mailu/ui/views/fetches.py

@@ -26,6 +26,8 @@ def fetch_create(user_email):
     if form.validate_on_submit():
         fetch = models.Fetch(user=user)
         form.populate_obj(fetch)
+        if form.folders.data:
+            fetch.folders = form.folders.data.replace(' ','').split(',')
         models.db.session.add(fetch)
         models.db.session.commit()
         flask.flash('Fetch configuration created')
@@ -43,6 +45,8 @@ def fetch_edit(fetch_id):
         if not form.password.data:
             form.password.data = fetch.password
         form.populate_obj(fetch)
+        if form.folders.data:
+            fetch.folders = form.folders.data.replace(' ','').split(',')
         models.db.session.commit()
         flask.flash('Fetch configuration updated')
         return flask.redirect(

core/admin/migrations/versions/f4f0f89e0047_.py

@@ -0,0 +1,25 @@
+"""empty message
+
+Revision ID: f4f0f89e0047
+Revises: 8f9ea78776f4
+Create Date: 2022-11-13 16:29:01.246509
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f4f0f89e0047'
+down_revision = '8f9ea78776f4'
+
+from alembic import op
+import sqlalchemy as sa
+import mailu
+
+def upgrade():
+    with op.batch_alter_table('fetch') as batch:
+        batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
+        batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True))
+
+def downgrade():
+    with op.batch_alter_table('fetch') as batch:
+        batch.drop_column('fetch', 'folders')
+        batch.drop_column('fetch', 'scan')

optional/fetchmail/fetchmail.py

@@ -2,11 +2,14 @@
 
 import time
 import os
+from pathlib import Path
+from pwd import getpwnam
 import tempfile
 import shlex
 import subprocess
 import re
 import requests
+from socrate import system
 import sys
 import traceback
 
@@ -14,6 +17,7 @@ import traceback
 FETCHMAIL = """
 fetchmail -N \
     --idfile /data/fetchids --uidl \
+    --pidfile /dev/shm/fetchmail.pid \
     --sslcertck --sslcertpath /etc/ssl/certs \
     -f {}
 """
@@ -24,7 +28,9 @@ poll "{host}" proto {protocol}  port {port}
     user "{username}" password "{password}"
     is "{user_email}"
     smtphost "{smtphost}"
+    {folders}
     {options}
+    {lmtp}
 """
 
 
@@ -48,26 +54,37 @@ def fetchmail(fetchmailrc):
 
 def run(debug):
     try:
-        fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json()
+        os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
-        smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
+        os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
+        fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
+        smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
         if smtpport is None:
             smtphostport = smtphost
         else:
             smtphostport = "%s/%d" % (smtphost, smtpport)
+        os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
+        lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
+        if lmtpport is None:
+            lmtphostport = lmtphost
+        else:
+            lmtphostport = "%s/%d" % (lmtphost, lmtpport)
         for fetch in fetches:
             fetchmailrc = ""
             options = "options antispam 501, 504, 550, 553, 554"
             options += " ssl" if fetch["tls"] else ""
             options += " keep" if fetch["keep"] else " fetchall"
+            folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"')
             fetchmailrc += RC_LINE.format(
                 user_email=escape_rc_string(fetch["user_email"]),
                 protocol=fetch["protocol"],
                 host=escape_rc_string(fetch["host"]),
                 port=fetch["port"],
-                smtphost=smtphostport,
+                smtphost=smtphostport if fetch['scan'] else lmtphostport,
                 username=escape_rc_string(fetch["username"]),
                 password=escape_rc_string(fetch["password"]),
-                options=options
+                options=options,
+                folders=folders,
+                lmtp='' if fetch['scan'] else 'lmtp',
             )
             if debug:
                 print(fetchmailrc)
@@ -86,7 +103,7 @@ def run(debug):
                         user_info in error_message):
                     print(error_message)
             finally:
-                requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]),
+                requests.post("http://" + os.environ["ADMIN_ADDRESS"] + "/internal/fetch/{}".format(fetch["id"]),
                     json=error_message.split("\n")[0]
                 )
     except Exception:
@@ -94,6 +111,13 @@ def run(debug):
 
 
 if __name__ == "__main__":
+    id_fetchmail = getpwnam('fetchmail')
+    Path('/data/fetchids').touch()
+    os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+    os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
+    os.chmod("/data/fetchids", 0o700)
+    os.setgid(id_fetchmail.pw_gid)
+    os.setuid(id_fetchmail.pw_uid)
     while True:
         time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60)))
         run(os.environ.get("DEBUG", None) == "True")

setup/flavors/compose/docker-compose.yml

@@ -157,8 +157,11 @@ services:
     env_file: {{ env }}
     volumes:
       - "{{ root }}/data/fetchmail:/data"
-    {% if resolver_enabled %}
     depends_on:
+      - admin
+      - smtp
+      - imap
+    {% if resolver_enabled %}
       - resolver
     dns:
       - {{ dns }}

towncrier/newsfragments/1231.bugfix

@@ -0,0 +1 @@
+Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231)

towncrier/newsfragments/2246.bugfix

@@ -0,0 +1 @@
+Fetchmail: Missing support for '*_ADDRESS' env vars

towncrier/newsfragments/711.feature

@@ -0,0 +1 @@
+Allow other folders to be synced by fetchmail