Merge branch 'master' of https://github.com/Mailu/Mailu into webmail-hardening

main
Florent Daigniere 2 years ago
commit 9e61a33cb2

@ -126,7 +126,7 @@ jobs:
password: ${{ secrets.Docker_Password }} password: ${{ secrets.Docker_Password }}
- name: Helper to convert docker org to lowercase - name: Helper to convert docker org to lowercase
id: string id: string
uses: ASzc/change-string-case-action@v2 uses: ASzc/change-string-case-action@v5
with: with:
string: ${{ github.repository_owner }} string: ${{ github.repository_owner }}
- name: Build all docker images - name: Build all docker images
@ -182,7 +182,7 @@ jobs:
password: ${{ secrets.Docker_Password }} password: ${{ secrets.Docker_Password }}
- name: Helper to convert docker org to lowercase - name: Helper to convert docker org to lowercase
id: string id: string
uses: ASzc/change-string-case-action@v2 uses: ASzc/change-string-case-action@v5
with: with:
string: ${{ github.repository_owner }} string: ${{ github.repository_owner }}
- name: Build all docker images - name: Build all docker images
@ -244,7 +244,7 @@ jobs:
password: ${{ secrets.Docker_Password }} password: ${{ secrets.Docker_Password }}
- name: Helper to convert docker org to lowercase - name: Helper to convert docker org to lowercase
id: string id: string
uses: ASzc/change-string-case-action@v2 uses: ASzc/change-string-case-action@v5
with: with:
string: ${{ github.repository_owner }} string: ${{ github.repository_owner }}
- name: Build all docker images - name: Build all docker images
@ -307,7 +307,7 @@ jobs:
password: ${{ secrets.Docker_Password }} password: ${{ secrets.Docker_Password }}
- name: Helper to convert docker org to lowercase - name: Helper to convert docker org to lowercase
id: string id: string
uses: ASzc/change-string-case-action@v2 uses: ASzc/change-string-case-action@v5
with: with:
string: ${{ github.repository_owner }} string: ${{ github.repository_owner }}
- name: Build all docker images - name: Build all docker images
@ -370,7 +370,7 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }} password: ${{ secrets.GITHUB_TOKEN }}
- name: Helper to convert docker org to lowercase - name: Helper to convert docker org to lowercase
id: string id: string
uses: ASzc/change-string-case-action@v2 uses: ASzc/change-string-case-action@v5
with: with:
string: ${{ github.repository_owner }} string: ${{ github.repository_owner }}
- name: Install python packages - name: Install python packages
@ -416,7 +416,7 @@ jobs:
password: ${{ secrets.Docker_Password }} password: ${{ secrets.Docker_Password }}
- name: Helper to convert docker org to lowercase - name: Helper to convert docker org to lowercase
id: string id: string
uses: ASzc/change-string-case-action@v2 uses: ASzc/change-string-case-action@v5
with: with:
string: ${{ github.repository_owner }} string: ${{ github.repository_owner }}
- name: Push image to Docker - name: Push image to Docker
@ -461,7 +461,7 @@ jobs:
password: ${{ secrets.Docker_Password }} password: ${{ secrets.Docker_Password }}
- name: Helper to convert docker org to lowercase - name: Helper to convert docker org to lowercase
id: string id: string
uses: ASzc/change-string-case-action@v2 uses: ASzc/change-string-case-action@v5
with: with:
string: ${{ github.repository_owner }} string: ${{ github.repository_owner }}
- name: Push image to Docker - name: Push image to Docker

@ -5,7 +5,6 @@ FROM node:16-alpine3.16
WORKDIR /work WORKDIR /work
COPY package.json ./ COPY package.json ./
COPY webpack.config.js ./
RUN set -euxo pipefail \ RUN set -euxo pipefail \
; npm config set update-notifier false \ ; npm config set update-notifier false \
@ -17,6 +16,7 @@ RUN set -euxo pipefail \
done done
COPY assets/ ./assets/ COPY assets/ ./assets/
COPY webpack.config.js ./
RUN set -euxo pipefail \ RUN set -euxo pipefail \
; node_modules/.bin/webpack-cli --color ; node_modules/.bin/webpack-cli --color

@ -1,8 +1,3 @@
require('./app.css');
import logo from './mailu.png';
import modules from "./*.json";
// Inspired from https://github.com/mehdibo/hibp-js/blob/master/hibp.js // Inspired from https://github.com/mehdibo/hibp-js/blob/master/hibp.js
function sha1(string) { function sha1(string) {
var buffer = new TextEncoder("utf-8").encode(string); var buffer = new TextEncoder("utf-8").encode(string);

@ -1,5 +1,5 @@
// AdminLTE // AdminLTE
import 'admin-lte/plugins/jquery/jquery.min.js'; window.$ = window.jQuery = require('admin-lte/plugins/jquery/jquery.min.js');
import 'admin-lte/plugins/bootstrap/js/bootstrap.bundle.min.js'; import 'admin-lte/plugins/bootstrap/js/bootstrap.bundle.min.js';
import 'admin-lte/build/scss/adminlte.scss'; import 'admin-lte/build/scss/adminlte.scss';
import 'admin-lte/build/js/AdminLTE.js'; import 'admin-lte/build/js/AdminLTE.js';
@ -18,7 +18,7 @@ import 'admin-lte/plugins/datatables/jquery.dataTables.min.js';
import 'admin-lte/plugins/datatables-bs4/js/dataTables.bootstrap4.min.js'; import 'admin-lte/plugins/datatables-bs4/js/dataTables.bootstrap4.min.js';
import 'admin-lte/plugins/datatables-responsive/js/dataTables.responsive.min.js'; import 'admin-lte/plugins/datatables-responsive/js/dataTables.responsive.min.js';
import 'admin-lte/plugins/datatables-responsive/js/responsive.bootstrap4.min.js'; import 'admin-lte/plugins/datatables-responsive/js/responsive.bootstrap4.min.js';
import modules from "./*.json";
// clipboard.js // clipboard.js
import 'clipboard/dist/clipboard.min.js'; window.ClipboardJS = require('clipboard/dist/clipboard.min.js');

@ -9,7 +9,7 @@ module.exports = {
mode: 'production', mode: 'production',
entry: { entry: {
app: { app: {
import: './assets/app.js', import: ['./assets/app.css', './assets/mailu.png', './assets/app.js'],
dependOn: 'vendor', dependOn: 'vendor',
}, },
vendor: './assets/vendor.js', vendor: './assets/vendor.js',

@ -13,17 +13,19 @@ DEFAULT_CONFIG = {
'RATELIMIT_STORAGE_URL': '', 'RATELIMIT_STORAGE_URL': '',
'DEBUG': False, 'DEBUG': False,
'DEBUG_PROFILER': False, 'DEBUG_PROFILER': False,
'DEBUG_TB_INTERCEPT_REDIRECTS': False,
'DEBUG_ASSETS': '', 'DEBUG_ASSETS': '',
'DOMAIN_REGISTRATION': False, 'DOMAIN_REGISTRATION': False,
'TEMPLATES_AUTO_RELOAD': True, 'TEMPLATES_AUTO_RELOAD': True,
'MEMORY_SESSIONS': False, 'MEMORY_SESSIONS': False,
'FETCHMAIL_ENABLED': False,
# Database settings # Database settings
'DB_FLAVOR': None, 'DB_FLAVOR': None,
'DB_USER': 'mailu', 'DB_USER': 'mailu',
'DB_PW': None, 'DB_PW': None,
'DB_HOST': 'database', 'DB_HOST': 'database',
'DB_NAME': 'mailu', 'DB_NAME': 'mailu',
'SQLITE_DATABASE_FILE':'data/main.db', 'SQLITE_DATABASE_FILE': 'data/main.db',
'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db',
'SQLALCHEMY_TRACK_MODIFICATIONS': False, 'SQLALCHEMY_TRACK_MODIFICATIONS': False,
# Statistics management # Statistics management
@ -60,7 +62,7 @@ DEFAULT_CONFIG = {
# Web settings # Web settings
'SITENAME': 'Mailu', 'SITENAME': 'Mailu',
'WEBSITE': 'https://mailu.io', 'WEBSITE': 'https://mailu.io',
'ADMIN' : 'none', 'ADMIN': 'none',
'WEB_ADMIN': '/admin', 'WEB_ADMIN': '/admin',
'WEB_WEBMAIL': '/webmail', 'WEB_WEBMAIL': '/webmail',
'WEBMAIL': 'none', 'WEBMAIL': 'none',
@ -73,7 +75,7 @@ DEFAULT_CONFIG = {
'SESSION_KEY_BITS': 128, 'SESSION_KEY_BITS': 128,
'SESSION_TIMEOUT': 3600, 'SESSION_TIMEOUT': 3600,
'PERMANENT_SESSION_LIFETIME': 30*24*3600, 'PERMANENT_SESSION_LIFETIME': 30*24*3600,
'SESSION_COOKIE_SECURE': True, 'SESSION_COOKIE_SECURE': None,
'CREDENTIAL_ROUNDS': 12, 'CREDENTIAL_ROUNDS': 12,
'TLS_PERMISSIVE': True, 'TLS_PERMISSIVE': True,
'TZ': 'Etc/UTC', 'TZ': 'Etc/UTC',
@ -156,6 +158,8 @@ class ConfigManager:
self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3' self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3'
self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_SAMESITE'] = 'Strict'
self.config['SESSION_COOKIE_HTTPONLY'] = True self.config['SESSION_COOKIE_HTTPONLY'] = True
if self.config['SESSION_COOKIE_SECURE'] is None:
self.config['SESSION_COOKIE_SECURE'] = self.config['TLS_FLAVOR'] != 'notls'
self.config['SESSION_PERMANENT'] = True self.config['SESSION_PERMANENT'] = True
self.config['SESSION_TIMEOUT'] = int(self.config['SESSION_TIMEOUT']) self.config['SESSION_TIMEOUT'] = int(self.config['SESSION_TIMEOUT'])
self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME']) self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME'])

@ -12,10 +12,12 @@ def fetch_list():
"id": fetch.id, "id": fetch.id,
"tls": fetch.tls, "tls": fetch.tls,
"keep": fetch.keep, "keep": fetch.keep,
"scan": fetch.scan,
"user_email": fetch.user_email, "user_email": fetch.user_email,
"protocol": fetch.protocol, "protocol": fetch.protocol,
"host": fetch.host, "host": fetch.host,
"port": fetch.port, "port": fetch.port,
"folders": fetch.folders,
"username": fetch.username, "username": fetch.username,
"password": fetch.password "password": fetch.password
} for fetch in models.Fetch.query.all() } for fetch in models.Fetch.query.all()

@ -771,6 +771,8 @@ class Fetch(Base):
username = db.Column(db.String(255), nullable=False) username = db.Column(db.String(255), nullable=False)
password = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False)
keep = db.Column(db.Boolean, nullable=False, default=False) keep = db.Column(db.Boolean, nullable=False, default=False)
scan = db.Column(db.Boolean, nullable=False, default=False)
folders = db.Column(CommaSeparatedList, nullable=True, default=list)
last_check = db.Column(db.DateTime, nullable=True) last_check = db.Column(db.DateTime, nullable=True)
error = db.Column(db.String(1023), nullable=True) error = db.Column(db.String(1023), nullable=True)

@ -1,7 +1,7 @@
from mailu.sso import sso from mailu.sso import sso
import flask import flask
@sso.route('/language/<language>', methods=['POST']) @sso.route('/language/<language>', methods=['GET','POST'])
def set_language(language=None): def set_language(language=None):
if language: if language:
flask.session['language'] = language flask.session['language'] = language

@ -41,6 +41,16 @@ class MultipleEmailAddressesVerify(object):
if not pattern.match(field.data.replace(" ", "")): if not pattern.match(field.data.replace(" ", "")):
raise validators.ValidationError(self.message) raise validators.ValidationError(self.message)
class MultipleFoldersVerify(object):
""" Ensure that we have CSV formated data """
def __init__(self,message=_('Invalid list of folders.')):
self.message = message
def __call__(self, form, field):
pattern = re.compile(r'^\w+(\s*,\s*\w+)*$')
if not pattern.match(field.data.replace(" ", "")):
raise validators.ValidationError(self.message)
class ConfirmationForm(flask_wtf.FlaskForm): class ConfirmationForm(flask_wtf.FlaskForm):
submit = fields.SubmitField(_('Confirm')) submit = fields.SubmitField(_('Confirm'))
@ -164,11 +174,13 @@ class FetchForm(flask_wtf.FlaskForm):
('imap', 'IMAP'), ('pop3', 'POP3') ('imap', 'IMAP'), ('pop3', 'POP3')
]) ])
host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()]) host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()])
port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)]) port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993)
tls = fields.BooleanField(_('Enable TLS')) tls = fields.BooleanField(_('Enable TLS'), default=True)
username = fields.StringField(_('Username'), [validators.DataRequired()]) username = fields.StringField(_('Username'), [validators.DataRequired()])
password = fields.PasswordField(_('Password')) password = fields.PasswordField(_('Password'))
keep = fields.BooleanField(_('Keep emails on the server')) keep = fields.BooleanField(_('Keep emails on the server'))
scan = fields.BooleanField(_('Rescan emails locally'))
folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk')
submit = fields.SubmitField(_('Submit')) submit = fields.SubmitField(_('Submit'))

@ -24,6 +24,8 @@
{%- call macros.card(title="Settings") %} {%- call macros.card(title="Settings") %}
{{ macros.form_field(form.keep) }} {{ macros.form_field(form.keep) }}
{{ macros.form_field(form.scan) }}
{{ macros.form_field(form.folders) }}
{%- endcall %} {%- endcall %}
{{ macros.form_field(form.submit) }} {{ macros.form_field(form.submit) }}

@ -20,6 +20,8 @@
<th>{% trans %}Endpoint{% endtrans %}</th> <th>{% trans %}Endpoint{% endtrans %}</th>
<th>{% trans %}Username{% endtrans %}</th> <th>{% trans %}Username{% endtrans %}</th>
<th>{% trans %}Keep emails{% endtrans %}</th> <th>{% trans %}Keep emails{% endtrans %}</th>
<th>{% trans %}Rescan emails{% endtrans %}</th>
<th>{% trans %}Folders{% endtrans %}</th>
<th>{% trans %}Last check{% endtrans %}</th> <th>{% trans %}Last check{% endtrans %}</th>
<th>{% trans %}Status{% endtrans %}</th> <th>{% trans %}Status{% endtrans %}</th>
<th>{% trans %}Created{% endtrans %}</th> <th>{% trans %}Created{% endtrans %}</th>
@ -36,6 +38,8 @@
<td>{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }}</td> <td>{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }}</td>
<td>{{ fetch.username }}</td> <td>{{ fetch.username }}</td>
<td data-sort="{{ fetch.keep }}">{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td> <td data-sort="{{ fetch.keep }}">{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
<td data-sort="{{ fetch.scan }}">{% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
<td>{{ fetch.folders | join(',') }}</td>
<td>{{ fetch.last_check | format_datetime or '-' }}</td> <td>{{ fetch.last_check | format_datetime or '-' }}</td>
<td>{{ fetch.error or '-' }}</td> <td>{{ fetch.error or '-' }}</td>
<td data-sort="{{ fetch.created_at or '0000-00-00' }}">{{ fetch.created_at | format_date }}</td> <td data-sort="{{ fetch.created_at or '0000-00-00' }}">{{ fetch.created_at | format_date }}</td>

@ -31,12 +31,14 @@
<p>{% trans %}Auto-reply{% endtrans %}</p> <p>{% trans %}Auto-reply{% endtrans %}</p>
</a> </a>
</li> </li>
{%- if config["FETCHMAIL_ENABLED"] %}
<li class="nav-item" role="none"> <li class="nav-item" role="none">
<a href="{{ url_for('.fetch_list') }}" class="nav-link" role="menuitem"> <a href="{{ url_for('.fetch_list') }}" class="nav-link" role="menuitem">
<i class="nav-icon fas fa-download"></i> <i class="nav-icon fas fa-download"></i>
<p>{% trans %}Fetched accounts{% endtrans %}</p> <p>{% trans %}Fetched accounts{% endtrans %}</p>
</a> </a>
</li> </li>
{%- endif %}
<li class="nav-item" role="none"> <li class="nav-item" role="none">
<a href="{{ url_for('.token_list') }}" class="nav-link" role="menuitem"> <a href="{{ url_for('.token_list') }}" class="nav-link" role="menuitem">
<i class="nav-icon fas fa-ticket-alt"></i> <i class="nav-icon fas fa-ticket-alt"></i>

@ -1,5 +1,6 @@
from mailu import models from mailu import models, utils
from mailu.ui import ui, forms, access from mailu.ui import ui, forms, access
from flask import current_app as app
import flask import flask
import flask_login import flask_login
@ -10,6 +11,8 @@ import wtforms
@ui.route('/fetch/list/<path:user_email>', methods=['GET']) @ui.route('/fetch/list/<path:user_email>', methods=['GET'])
@access.owner(models.User, 'user_email') @access.owner(models.User, 'user_email')
def fetch_list(user_email): def fetch_list(user_email):
if not app.config['FETCHMAIL_ENABLED']:
flask.abort(404)
user_email = user_email or flask_login.current_user.email user_email = user_email or flask_login.current_user.email
user = models.User.query.get(user_email) or flask.abort(404) user = models.User.query.get(user_email) or flask.abort(404)
return flask.render_template('fetch/list.html', user=user) return flask.render_template('fetch/list.html', user=user)
@ -19,13 +22,18 @@ def fetch_list(user_email):
@ui.route('/fetch/create/<path:user_email>', methods=['GET', 'POST']) @ui.route('/fetch/create/<path:user_email>', methods=['GET', 'POST'])
@access.owner(models.User, 'user_email') @access.owner(models.User, 'user_email')
def fetch_create(user_email): def fetch_create(user_email):
if not app.config['FETCHMAIL_ENABLED']:
flask.abort(404)
user_email = user_email or flask_login.current_user.email user_email = user_email or flask_login.current_user.email
user = models.User.query.get(user_email) or flask.abort(404) user = models.User.query.get(user_email) or flask.abort(404)
form = forms.FetchForm() form = forms.FetchForm()
form.password.validators = [wtforms.validators.DataRequired()] form.password.validators = [wtforms.validators.DataRequired()]
utils.formatCSVField(form.folders)
if form.validate_on_submit(): if form.validate_on_submit():
fetch = models.Fetch(user=user) fetch = models.Fetch(user=user)
form.populate_obj(fetch) form.populate_obj(fetch)
if form.folders.data:
fetch.folders = form.folders.data.replace(' ','').split(',')
models.db.session.add(fetch) models.db.session.add(fetch)
models.db.session.commit() models.db.session.commit()
flask.flash('Fetch configuration created') flask.flash('Fetch configuration created')
@ -37,12 +45,17 @@ def fetch_create(user_email):
@ui.route('/fetch/edit/<fetch_id>', methods=['GET', 'POST']) @ui.route('/fetch/edit/<fetch_id>', methods=['GET', 'POST'])
@access.owner(models.Fetch, 'fetch_id') @access.owner(models.Fetch, 'fetch_id')
def fetch_edit(fetch_id): def fetch_edit(fetch_id):
if not app.config['FETCHMAIL_ENABLED']:
flask.abort(404)
fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
form = forms.FetchForm(obj=fetch) form = forms.FetchForm(obj=fetch)
utils.formatCSVField(form.folders)
if form.validate_on_submit(): if form.validate_on_submit():
if not form.password.data: if not form.password.data:
form.password.data = fetch.password form.password.data = fetch.password
form.populate_obj(fetch) form.populate_obj(fetch)
if form.folders.data:
fetch.folders = form.folders.data.replace(' ','').split(',')
models.db.session.commit() models.db.session.commit()
flask.flash('Fetch configuration updated') flask.flash('Fetch configuration updated')
return flask.redirect( return flask.redirect(
@ -55,6 +68,8 @@ def fetch_edit(fetch_id):
@access.confirmation_required("delete a fetched account") @access.confirmation_required("delete a fetched account")
@access.owner(models.Fetch, 'fetch_id') @access.owner(models.Fetch, 'fetch_id')
def fetch_delete(fetch_id): def fetch_delete(fetch_id):
if not app.config['FETCHMAIL_ENABLED']:
flask.abort(404)
fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
user = fetch.user user = fetch.user
models.db.session.delete(fetch) models.db.session.delete(fetch)

@ -64,6 +64,7 @@ def user_edit(user_email):
form.quota_bytes.validators = [ form.quota_bytes.validators = [
wtforms.validators.NumberRange(max=max_quota_bytes)] wtforms.validators.NumberRange(max=max_quota_bytes)]
if form.validate_on_submit(): if form.validate_on_submit():
if form.pw.data:
if msg := utils.isBadOrPwned(form): if msg := utils.isBadOrPwned(form):
flask.flash(msg, "error") flask.flash(msg, "error")
return flask.render_template('user/edit.html', form=form, user=user, return flask.render_template('user/edit.html', form=form, user=user,
@ -99,11 +100,7 @@ def user_settings(user_email):
user_email_or_current = user_email or flask_login.current_user.email user_email_or_current = user_email or flask_login.current_user.email
user = models.User.query.get(user_email_or_current) or flask.abort(404) user = models.User.query.get(user_email_or_current) or flask.abort(404)
form = forms.UserSettingsForm(obj=user) form = forms.UserSettingsForm(obj=user)
if isinstance(form.forward_destination.data,str): utils.formatCSVField(form.forward_destination)
data = form.forward_destination.data.replace(" ","").split(",")
else:
data = form.forward_destination.data
form.forward_destination.data = ", ".join(data)
if form.validate_on_submit(): if form.validate_on_submit():
form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",") form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",")
form.populate_obj(user) form.populate_obj(user)

@ -518,3 +518,10 @@ def isBadOrPwned(form):
if breaches > 0: if breaches > 0:
return f"This password appears in {breaches} data breaches! It is not unique; please change it." return f"This password appears in {breaches} data breaches! It is not unique; please change it."
return None return None
def formatCSVField(field):
if isinstance(field.data,str):
data = field.data.replace(" ","").split(",")
else:
data = field.data
field.data = ", ".join(data)

@ -0,0 +1,25 @@
""" Add fetch.scan and fetch.folders
Revision ID: f4f0f89e0047
Revises: 8f9ea78776f4
Create Date: 2022-11-13 16:29:01.246509
"""
# revision identifiers, used by Alembic.
revision = 'f4f0f89e0047'
down_revision = '8f9ea78776f4'
from alembic import op
import sqlalchemy as sa
import mailu
def upgrade():
with op.batch_alter_table('fetch') as batch:
batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True))
def downgrade():
with op.batch_alter_table('fetch') as batch:
batch.drop_column('fetch', 'folders')
batch.drop_column('fetch', 'scan')

@ -48,7 +48,7 @@ sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Docker
# assets # assets
cp "${assets}/package.json" . cp "${assets}/package.json" .
cp -r "${assets}/assets/" . cp -r "${assets}/assets" ./assets
awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js
sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile
@ -65,7 +65,7 @@ RUN set -euxo pipefail \
; ln -s /app/start.py / ; ln -s /app/start.py /
ENV \ ENV \
FLASK_ENV="development" \ FLASK_DEBUG="true" \
MEMORY_SESSIONS="true" \ MEMORY_SESSIONS="true" \
RATELIMIT_STORAGE_URL="memory://" \ RATELIMIT_STORAGE_URL="memory://" \
SESSION_COOKIE_SECURE="false" \ SESSION_COOKIE_SECURE="false" \
@ -73,7 +73,7 @@ ENV \
DEBUG="true" \ DEBUG="true" \
DEBUG_PROFILER="${DEV_PROFILER}" \ DEBUG_PROFILER="${DEV_PROFILER}" \
DEBUG_ASSETS="/app/static" \ DEBUG_ASSETS="/app/static" \
DEBUG_TB_ENABLED="true" \ DEBUG_TB_INTERCEPT_REDIRECTS=False \
\ \
IMAP_ADDRESS="127.0.0.1" \ IMAP_ADDRESS="127.0.0.1" \
POP3_ADDRESS="127.0.0.1" \ POP3_ADDRESS="127.0.0.1" \
@ -82,7 +82,7 @@ ENV \
REDIS_ADDRESS="127.0.0.1" \ REDIS_ADDRESS="127.0.0.1" \
WEBMAIL_ADDRESS="127.0.0.1" WEBMAIL_ADDRESS="127.0.0.1"
CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null && flask run --host=0.0.0.0 --port=8080"] CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"]
EOF EOF
# build # build

@ -2,8 +2,15 @@
import os import os
import logging as log import logging as log
from pwd import getpwnam
import sys import sys
os.system("chown mailu:mailu -R /dkim")
os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
mailu_id = getpwnam('mailu')
os.setgid(mailu_id.pw_gid)
os.setuid(mailu_id.pw_uid)
log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO")) log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
os.system("flask mailu advertise") os.system("flask mailu advertise")

@ -1,7 +1,7 @@
# syntax=docker/dockerfile-upstream:1.4.3 # syntax=docker/dockerfile-upstream:1.4.3
# base system image (intermediate) # base system image (intermediate)
ARG DISTRO=alpine:3.16.2 ARG DISTRO=alpine:3.16.3
FROM $DISTRO as system FROM $DISTRO as system
ENV TZ=Etc/UTC LANG=C.UTF-8 ENV TZ=Etc/UTC LANG=C.UTF-8
@ -12,7 +12,16 @@ ARG MAILU_GID=1000
RUN set -euxo pipefail \ RUN set -euxo pipefail \
; addgroup -Sg ${MAILU_GID} mailu \ ; addgroup -Sg ${MAILU_GID} mailu \
; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \
; apk add --no-cache bash ca-certificates curl python3 tzdata ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \
; machine="$(uname -m)" \
; ! [[ "${machine}" == x86_64 ]] \
|| apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc
ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so
ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
ENV CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
ENV CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
ENV LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now"
WORKDIR /app WORKDIR /app
@ -78,6 +87,7 @@ FROM system
COPY --from=build /app/venv/ /app/venv/ COPY --from=build /app/venv/ /app/venv/
COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/ COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/
RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn
ENV VIRTUAL_ENV=/app/venv ENV VIRTUAL_ENV=/app/venv
ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

@ -10,5 +10,5 @@ RUN echo $VERSION >/version
HEALTHCHECK CMD true HEALTHCHECK CMD true
USER app USER mailu
CMD ["/bin/bash", "-c", "sleep infinity"] CMD ["/bin/bash", "-c", "sleep infinity"]

@ -33,4 +33,7 @@ while True:
log.warning("Admin is not up just yet, retrying in 1 second") log.warning("Admin is not up just yet, retrying in 1 second")
# Run rspamd # Run rspamd
os.execv("/usr/sbin/rspamd", ["rspamd", "-i", "-f"]) os.system("mkdir -m 755 -p /run/rspamd")
os.system("chown rspamd:rspamd /run/rspamd")
os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown rspamd:rspamd")
os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"])

@ -104,6 +104,9 @@ support or e.g. mismatching TLS versions to deliver emails to Mailu.
.. _fetchmail: .. _fetchmail:
When ``FETCHMAIL_ENABLED`` is set to ``True``, the fetchmail functionality is enabled in the admin interface.
The container itself still needs to be deployed manually. ``FETCHMAIL_ENABLED`` defaults to ``True``.
The ``FETCHMAIL_DELAY`` is a delay (in seconds) for the fetchmail service to The ``FETCHMAIL_DELAY`` is a delay (in seconds) for the fetchmail service to
go and fetch new email if available. Do not use too short delays if you do not go and fetch new email if available. Do not use too short delays if you do not
want to be blacklisted by external services, but not too long delays if you want to be blacklisted by external services, but not too long delays if you
@ -287,6 +290,10 @@ The admin service stores configurations in a database.
- ``DB_USER``: the database user for mailu admin service. (when not ``sqlite``) - ``DB_USER``: the database user for mailu admin service. (when not ``sqlite``)
- ``DB_NAME``: the database name for mailu admin service. (when not ``sqlite``) - ``DB_NAME``: the database name for mailu admin service. (when not ``sqlite``)
Alternatively, if you need more control, you can use a `DB URL`_ : do not set any of the ``DB_`` settings and set ``SQLALCHEMY_DATABASE_URI`` instead.
.. _`DB URL`: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls
The roundcube service stores configurations in a database. The roundcube service stores configurations in a database.
- ``ROUNDCUBE_DB_FLAVOR``: the database type for roundcube service. (``sqlite``, ``postgresql``, ``mysql``) - ``ROUNDCUBE_DB_FLAVOR``: the database type for roundcube service. (``sqlite``, ``postgresql``, ``mysql``)

@ -157,7 +157,11 @@ You can add a fetched account by clicking on the `Add an account` button on the
* Keep emails on the server. When ticked, retains the email message in the email account after retrieving it. * Keep emails on the server. When ticked, retains the email message in the email account after retrieving it.
Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after 10 minutes. * Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...).
* Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled.
Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``.
Authentication tokens Authentication tokens

@ -2,11 +2,14 @@
import time import time
import os import os
from pathlib import Path
from pwd import getpwnam
import tempfile import tempfile
import shlex import shlex
import subprocess import subprocess
import re import re
import requests import requests
from socrate import system
import sys import sys
import traceback import traceback
@ -14,6 +17,7 @@ import traceback
FETCHMAIL = """ FETCHMAIL = """
fetchmail -N \ fetchmail -N \
--idfile /data/fetchids --uidl \ --idfile /data/fetchids --uidl \
--pidfile /dev/shm/fetchmail.pid \
--sslcertck --sslcertpath /etc/ssl/certs \ --sslcertck --sslcertpath /etc/ssl/certs \
-f {} -f {}
""" """
@ -24,7 +28,9 @@ poll "{host}" proto {protocol} port {port}
user "{username}" password "{password}" user "{username}" password "{password}"
is "{user_email}" is "{user_email}"
smtphost "{smtphost}" smtphost "{smtphost}"
{folders}
{options} {options}
{lmtp}
""" """
@ -48,26 +54,37 @@ def fetchmail(fetchmailrc):
def run(debug): def run(debug):
try: try:
fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json() os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp")
smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None) os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin")
fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json()
smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None)
if smtpport is None: if smtpport is None:
smtphostport = smtphost smtphostport = smtphost
else: else:
smtphostport = "%s/%d" % (smtphost, smtpport) smtphostport = "%s/%d" % (smtphost, smtpport)
os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525")
lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None)
if lmtpport is None:
lmtphostport = lmtphost
else:
lmtphostport = "%s/%d" % (lmtphost, lmtpport)
for fetch in fetches: for fetch in fetches:
fetchmailrc = "" fetchmailrc = ""
options = "options antispam 501, 504, 550, 553, 554" options = "options antispam 501, 504, 550, 553, 554"
options += " ssl" if fetch["tls"] else "" options += " ssl" if fetch["tls"] else ""
options += " keep" if fetch["keep"] else " fetchall" options += " keep" if fetch["keep"] else " fetchall"
folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"')
fetchmailrc += RC_LINE.format( fetchmailrc += RC_LINE.format(
user_email=escape_rc_string(fetch["user_email"]), user_email=escape_rc_string(fetch["user_email"]),
protocol=fetch["protocol"], protocol=fetch["protocol"],
host=escape_rc_string(fetch["host"]), host=escape_rc_string(fetch["host"]),
port=fetch["port"], port=fetch["port"],
smtphost=smtphostport, smtphost=smtphostport if fetch['scan'] else lmtphostport,
username=escape_rc_string(fetch["username"]), username=escape_rc_string(fetch["username"]),
password=escape_rc_string(fetch["password"]), password=escape_rc_string(fetch["password"]),
options=options options=options,
folders=folders,
lmtp='' if fetch['scan'] else 'lmtp',
) )
if debug: if debug:
print(fetchmailrc) print(fetchmailrc)
@ -86,15 +103,29 @@ def run(debug):
user_info in error_message): user_info in error_message):
print(error_message) print(error_message)
finally: finally:
requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]), requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']),
json=error_message.split("\n")[0] json=error_message.split('\n')[0]
) )
except Exception: except Exception:
traceback.print_exc() traceback.print_exc()
if __name__ == "__main__": if __name__ == "__main__":
id_fetchmail = getpwnam('fetchmail')
Path('/data/fetchids').touch()
os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid)
os.chmod("/data/fetchids", 0o700)
os.setgid(id_fetchmail.pw_gid)
os.setuid(id_fetchmail.pw_uid)
while True: while True:
time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60))) delay = int(os.environ.get("FETCHMAIL_DELAY", 60))
print("Sleeping for {} seconds".format(delay))
time.sleep(delay)
if not os.environ.get("FETCHMAIL_ENABLED", 'True') in ('True', 'true'):
print("Fetchmail disabled, skipping...")
continue
run(os.environ.get("DEBUG", None) == "True") run(os.environ.get("DEBUG", None) == "True")
sys.stdout.flush() sys.stdout.flush()

@ -1,24 +1,21 @@
ARG DISTRO=alpine:3.14.5 # syntax=docker/dockerfile-upstream:1.4.3
FROM $DISTRO
ARG VERSION # setup image
ENV TZ Etc/UTC FROM base
ARG VERSION=local
LABEL version=$VERSION LABEL version=$VERSION
RUN mkdir -p /app
WORKDIR /app
COPY requirements.txt requirements.txt
RUN apk add --no-cache curl python3 py3-pip \
&& pip3 install -r requirements.txt
COPY server.py ./server.py
COPY main.py ./main.py
COPY flavors /data/flavors COPY flavors /data/flavors
COPY templates /data/templates COPY templates /data/templates
COPY static ./static COPY static ./static
COPY server.py ./server.py
COPY main.py ./main.py
RUN echo $VERSION >> /version
EXPOSE 80/tcp EXPOSE 80/tcp
HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost/
USER mailu
CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app
RUN echo $VERSION >> /version

@ -157,8 +157,11 @@ services:
env_file: {{ env }} env_file: {{ env }}
volumes: volumes:
- "{{ root }}/data/fetchmail:/data" - "{{ root }}/data/fetchmail:/data"
{% if resolver_enabled %}
depends_on: depends_on:
- admin
- smtp
- imap
{% if resolver_enabled %}
- resolver - resolver
dns: dns:
- {{ dns }} - {{ dns }}

@ -79,6 +79,9 @@ RELAYNETS=
# Will relay all outgoing mails if configured # Will relay all outgoing mails if configured
RELAYHOST={{ relayhost }} RELAYHOST={{ relayhost }}
# Show fetchmail functionality in admin interface
FETCHMAIL_ENABLED={{ fetchmail_enabled or 'False' }}
# Fetchmail delay # Fetchmail delay
FETCHMAIL_DELAY={{ fetchmail_delay or '600' }} FETCHMAIL_DELAY={{ fetchmail_delay or '600' }}

@ -1,12 +0,0 @@
Flask==1.0.2
Flask-Bootstrap==3.3.7.1
gunicorn==19.9.0
redis==3.2.1
Jinja2==3.0.3
MarkupSafe==2.1.0
Werkzeug==2.0.3
click==8.0.3
dominate==2.6.0
itsdangerous==2.0.1
redis==3.2.1
visitor==0.1.3

@ -106,6 +106,9 @@ target "docs" {
target "setup" { target "setup" {
inherits = ["defaults"] inherits = ["defaults"]
context = "setup/" context = "setup/"
contexts = {
base = "target:base"
}
tags = tag("setup") tags = tag("setup")
} }

@ -0,0 +1,7 @@
# GTUBE should be blocked, see https://rspamd.com/doc/gtube_patterns.html
python3 tests/email_test.py "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X"
if [ $? -eq 25 ]; then
exit 0
else
exit 1
fi

@ -0,0 +1 @@
Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231)

@ -0,0 +1 @@
Add FETCHMAIL_ENABLED to toggle the fetchmail functionality in the admin interface

@ -0,0 +1 @@
Fetchmail: Missing support for '*_ADDRESS' env vars

@ -0,0 +1 @@
Switch to GrapheneOS's hardened_malloc

@ -0,0 +1 @@
Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. Please ensure that your folder overrides/rspamd is owned by 1000:1000

@ -0,0 +1 @@
Allow other folders to be synced by fetchmail

@ -52,7 +52,7 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/
# snappymail # snappymail
ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.0/snappymail-2.21.0.tar.gz ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.3/snappymail-2.21.3.tar.gz
RUN set -euxo pipefail \ RUN set -euxo pipefail \
; mkdir /var/www/snappymail \ ; mkdir /var/www/snappymail \
@ -82,7 +82,7 @@ RUN set -euxo pipefail \
# common # common
COPY start.py / COPY start.py /
COPY php.ini /defaults/ COPY php.ini /defaults/
COPY php-webmail.conf /etc/php81/php-fpm.d/php-webmail.conf COPY php-webmail.conf /etc/php81/php-fpm.d/
COPY nginx-webmail.conf /conf/ COPY nginx-webmail.conf /conf/
COPY snuffleupagus.rules /etc/snuffleupagus.rules.tpl COPY snuffleupagus.rules /etc/snuffleupagus.rules.tpl

Loading…
Cancel
Save