Merge branch 'master' into feat-reply-startdate

master
kaiyou 6 years ago
commit c6846fd8db

@ -0,0 +1,10 @@
rules:
default: null
branches:
master:
protection:
required_status_checks:
contexts:
- continuous-integration/travis-ci
required_pull_request_reviews:
required_approving_review_count: 2

@ -8,4 +8,15 @@ env:
- VERSION=$TRAVIS_BRANCH
script:
- docker-compose -f tests/build.yml -p Mailu build
# Default to mailu for DOCKER_ORG
- if [ -z "$DOCKER_ORG" ]; then export DOCKER_ORG="mailu"; fi
- docker-compose -f tests/build.yml build
- tests/compose/test-script.sh
deploy:
provider: script
script: bash tests/deploy.sh
on:
all_branches: true
condition: -n $DOCKER_UN

@ -17,5 +17,6 @@ COPY start.sh /start.sh
RUN pybabel compile -d mailu/translations
EXPOSE 80/tcp
VOLUME ["/data"]
CMD ["/start.sh"]

@ -11,7 +11,6 @@ import os
import docker
import socket
import uuid
import redis
from werkzeug.contrib import fixers
@ -89,9 +88,6 @@ manager.add_command('db', flask_migrate.MigrateCommand)
babel = flask_babel.Babel(app)
translations = list(map(str, babel.list_translations()))
# Quota manager
quota = redis.Redis.from_url(app.config.get("QUOTA_STORAGE_URL"))
@babel.localeselector
def get_locale():
return flask.request.accept_languages.best_match(translations)

@ -6,7 +6,8 @@ import socket
import flask
internal = flask.Blueprint('internal', __name__)
internal = flask.Blueprint('internal', __name__, template_folder='templates')
@internal.app_errorhandler(RateLimitExceeded)
def rate_limit_handler(e):
@ -17,6 +18,7 @@ def rate_limit_handler(e):
response.headers['Auth-Wait'] = '3'
return response
@limiter.request_filter
def whitelist_webmail():
try:
@ -26,4 +28,4 @@ def whitelist_webmail():
return False
from mailu.internal import views
from mailu.internal.views import *

@ -8,8 +8,6 @@ require "regex";
require "relational";
require "date";
require "comparator-i;ascii-numeric";
require "vnd.dovecot.extdata";
require "vnd.dovecot.execute";
require "spamtestplus";
require "editheader";
require "index";
@ -20,22 +18,20 @@ if header :index 2 :matches "Received" "from * by * for <*>; *"
addheader "Delivered-To" "<${3}>";
}
if allof (string :is "${extdata.spam_enabled}" "1",
spamtest :percent :value "gt" :comparator "i;ascii-numeric" "${extdata.spam_threshold}")
{% if user.spam_enabled %}
if spamtest :percent :value "gt" :comparator "i;ascii-numeric" "{{ user.spam_threshold }}"
{
setflag "\\seen";
fileinto :create "Junk";
stop;
}
{% endif %}
if exists "X-Virus" {
discard;
stop;
}
if allof (string :is "${extdata.reply_enabled}" "1",
currentdate :value "ge" "date" "${extdata.reply_startdate}",
currentdate :value "le" "date" "${extdata.reply_enddate}")
{
vacation :days 1 :subject "${extdata.reply_subject}" "${extdata.reply_body}";
}
{% if user.reply_active and %}
vacation :days 1 :subject "{{ user.reply_subject }}" "{{ user.reply_body }}";
{% endif %}

@ -0,0 +1,3 @@
__all__ = [
'auth', 'postfix', 'dovecot', 'fetch'
]

@ -4,7 +4,6 @@ from mailu.internal import internal, nginx
import flask
import flask_login
import base64
import urllib
@internal.route("/auth/email")

@ -0,0 +1,40 @@
from mailu import db, models
from mailu.internal import internal
import flask
@internal.route("/dovecot/passdb/<user_email>")
def dovecot_passdb_dict(user_email):
user = models.User.query.get(user_email) or flask.abort(404)
return flask.jsonify({
"password": user.password,
})
@internal.route("/dovecot/userdb/<user_email>")
def dovecot_userdb_dict(user_email):
user = models.User.query.get(user_email) or flask.abort(404)
return flask.jsonify({
"quota_rule": "*:bytes={}".format(user.quota_bytes)
})
@internal.route("/dovecot/quota/<ns>/<user_email>", methods=["POST"])
def dovecot_quota(ns, user_email):
user = models.User.query.get(user_email) or flask.abort(404)
if ns == "storage":
user.quota_bytes_used = flask.request.get_json()
db.session.commit()
return flask.jsonify(None)
@internal.route("/dovecot/sieve/name/<script>/<user_email>")
def dovecot_sieve_name(script, user_email):
return flask.jsonify(script)
@internal.route("/dovecot/sieve/data/default/<user_email>")
def dovecot_sieve_data(user_email):
user = models.User.query.get(user_email) or flask.abort(404)
return flask.jsonify(flask.render_template("default.sieve", user=user))

@ -0,0 +1,32 @@
from mailu import db, models
from mailu.internal import internal
import flask
import datetime
@internal.route("/fetch")
def fetch_list():
return flask.jsonify([
{
"id": fetch.id,
"tls": fetch.tls,
"keep": fetch.keep,
"user_email": fetch.user_email,
"protocol": fetch.protocol,
"host": fetch.host,
"port": fetch.port,
"username": fetch.username,
"password": fetch.password
} for fetch in models.Fetch.query.all()
])
@internal.route("/fetch/<fetch_id>", methods=["POST"])
def fetch_done(fetch_id):
fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
fetch.last_check = datetime.datetime.now()
fetch.error_message = str(flask.request.get_json())
db.session.add(fetch)
db.session.commit()
return ""

@ -0,0 +1,54 @@
from mailu import db, models
from mailu.internal import internal
import flask
@internal.route("/postfix/domain/<domain_name>")
def postfix_mailbox_domain(domain_name):
domain = models.Domain.query.get(domain_name) or flask.abort(404)
return flask.jsonify(domain.name)
@internal.route("/postfix/mailbox/<email>")
def postfix_mailbox_map(email):
user = models.User.query.get(email) or flask.abort(404)
return flask.jsonify(user.email)
@internal.route("/postfix/alias/<alias>")
def postfix_alias_map(alias):
localpart, domain = alias.split('@', 1) if '@' in alias else (None, alias)
alternative = models.Alternative.query.get(domain)
if alternative:
domain = alternative.domain_name
email = '{}@{}'.format(localpart, domain)
if localpart is None:
return flask.jsonify(domain)
else:
alias_obj = models.Alias.resolve(localpart, domain)
if alias_obj:
return flask.jsonify(",".join(alias_obj.destination))
user_obj = models.User.query.get(email)
if user_obj:
return flask.jsonify(user_obj.destination)
return flask.abort(404)
@internal.route("/postfix/transport/<email>")
def postfix_transport(email):
localpart, domain = email.split('@', 1) if '@' in email else (None, email)
relay = models.Relay.query.get(domain) or flask.abort(404)
return flask.jsonify("smtp:[{}]".format(relay.smtp))
@internal.route("/postfix/sender/<sender>")
def postfix_sender(sender):
""" Simply reject any sender that pretends to be from a local domain
"""
localpart, domain_name = sender.split('@', 1) if '@' in sender else (None, sender)
domain = models.Domain.query.get(domain_name)
alternative = models.Alternative.query.get(domain_name)
if domain or alternative:
return flask.jsonify("REJECT")
return flask.abort(404)

@ -1,11 +1,11 @@
from mailu import app, db, dkim, login_manager, quota
from mailu import app, db, dkim, login_manager
from sqlalchemy.ext import declarative
from passlib import context, hash
from datetime import datetime, date
from email.mime import text
import sqlalchemy
import re
import time
import os
@ -235,6 +235,7 @@ class User(Base, Email):
backref=db.backref('users', cascade='all, delete-orphan'))
password = db.Column(db.String(255), nullable=False)
quota_bytes = db.Column(db.Integer(), nullable=False, default=10**9)
quota_bytes_used = db.Column(db.Integer(), nullable=False, default=0)
global_admin = db.Column(db.Boolean(), nullable=False, default=False)
enabled = db.Column(db.Boolean(), nullable=False, default=True)
@ -268,8 +269,23 @@ class User(Base, Email):
return self.email
@property
def quota_bytes_used(self):
return quota.get(self.email + "/quota/storage") or 0
def destination(self):
if self.forward_enabled:
result = self.self.forward_destination
if self.forward_keep:
result += ',' + self.email
return result
else:
return self.email
@property
def reply_active(self):
now = datetime.datetime.now()
return (
self.reply_enabled and
self.reply_startdate < now and
self.reply_enddate > now
)
scheme_dict = {'SHA512-CRYPT': "sha512_crypt",
'SHA256-CRYPT': "sha256_crypt",
@ -331,6 +347,22 @@ class Alias(Base, Email):
wildcard = db.Column(db.Boolean(), nullable=False, default=False)
destination = db.Column(CommaSeparatedList, nullable=False, default=[])
@classmethod
def resolve(cls, localpart, domain_name):
return cls.query.filter(
sqlalchemy.and_(cls.domain_name == domain_name,
sqlalchemy.or_(
sqlalchemy.and_(
cls.wildcard == False,
cls.localpart == localpart
), sqlalchemy.and_(
cls.wildcard == True,
sqlalchemy.bindparam("l", localpart).like(cls.localpart)
)
)
)
).first()
class Token(Base):
""" A token is an application password for a given user.

@ -0,0 +1,28 @@
""" Add a column for used quota
Revision ID: 25fd6c7bcb4a
Revises: 049fed905da7
Create Date: 2018-07-25 21:56:09.729153
"""
# revision identifiers, used by Alembic.
revision = '25fd6c7bcb4a'
down_revision = '049fed905da7'
from alembic import op
import sqlalchemy as sa
from alembic import op
import sqlalchemy as sa
def upgrade():
with op.batch_alter_table('user') as batch:
batch.add_column(sa.Column('quota_bytes_used', sa.Integer(), nullable=False, server_default='0'))
def downgrade():
with op.batch_alter_table('user') as batch:
batch.drop_column('user', 'quota_bytes_used')

@ -2,4 +2,4 @@
python manage.py advertise
python manage.py db upgrade
gunicorn -w 4 -b 0.0.0.0:80 -b [::]:80 --access-logfile - --error-logfile - --preload mailu:app
gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload mailu:app

@ -1,14 +1,15 @@
FROM alpine:3.7
FROM alpine:3.8
RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories \
&& apk add --no-cache \
dovecot dovecot-sqlite dovecot-pigeonhole-plugin dovecot-pigeonhole-plugin-extdata \
dovecot-fts-lucene rspamd-client@testing python py-jinja2
RUN apk add --no-cache \
dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \
python3 py3-pip \
&& pip3 install --upgrade pip \
&& pip3 install jinja2 podop tenacity
COPY conf /conf
COPY sieve /var/lib/dovecot
COPY start.py /start.py
EXPOSE 110/tcp 143/tcp 993/tcp 4190/tcp 2525/tcp
VOLUME ["/data", "/mail"]
CMD /start.py

@ -0,0 +1,5 @@
uri = proxy:/tmp/podop.socket:auth
iterate_disable = yes
default_pass_scheme = plain
password_key = passdb/%u
user_key = userdb/%u

@ -1,18 +0,0 @@
driver = sqlite
connect = /data/main.db
# Return the user hashed password
password_query = \
SELECT NULL as password, 'Y' as nopassword, '{% if POD_ADDRESS_RANGE %}{{ POD_ADDRESS_RANGE }}{% else %}{{ FRONT_ADDRESS }}{% if WEBMAIL_ADDRESS %},{{ WEBMAIL_ADDRESS }}{% endif %}{% endif %}' as allow_nets \
FROM user \
WHERE user.email = '%u'
# Mostly get the user quota
user_query = \
SELECT '*:bytes=' || user.quota_bytes AS quota_rule \
FROM user \
WHERE user.email = '%u'
# For using doveadm -A:
iterate_query = \
SELECT user.email AS user FROM user

@ -7,17 +7,6 @@ postmaster_address = {{ POSTMASTER }}@{{ DOMAIN }}
hostname = {{ HOSTNAMES.split(",")[0] }}
submission_host = {{ FRONT_ADDRESS }}
service dict {
unix_listener dict {
group = mail
mode = 0660
}
}
dict {
sieve = sqlite:/etc/dovecot/pigeonhole-sieve.dict
}
###############
# Full-text search
###############
@ -50,28 +39,18 @@ mail_plugins = $mail_plugins quota quota_clone zlib
namespace inbox {
inbox = yes
mailbox Trash {
{% for mailbox in ("Trash", "Drafts", "Sent", "Junk") %}
mailbox {{ mailbox }} {
auto = subscribe
special_use = \Trash
}
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Sent {
auto = subscribe
special_use = \Sent
}
mailbox Junk {
auto = subscribe
special_use = \Junk
special_use = \{{ mailbox }}
}
{% endfor %}
}
plugin {
quota = count:User quota
quota_vsizes = yes
quota_clone_dict = redis:host={{ REDIS_ADDRESS }}:port=6379:db=1
quota_clone_dict = proxy:/tmp/podop.socket:quota
{% if COMPRESSION in [ 'gz', 'bz2' ] %}
zlib_save = {{ COMPRESSION }}
@ -87,16 +66,15 @@ plugin {
###############
auth_mechanisms = plain login
disable_plaintext_auth = no
ssl_protocols = !SSLv3
passdb {
driver = sql
args = /etc/dovecot/dovecot-sql.conf.ext
driver = dict
args = /etc/dovecot/auth.conf
}
userdb {
driver = sql
args = /etc/dovecot/dovecot-sql.conf.ext
driver = dict
args = /etc/dovecot/auth.conf
}
service auth {
@ -117,7 +95,6 @@ service auth-worker {
###############
# IMAP & POP
###############
protocol imap {
mail_plugins = $mail_plugins imap_quota imap_sieve
}
@ -135,7 +112,6 @@ service imap-login {
###############
# Delivery
###############
protocol lmtp {
mail_plugins = $mail_plugins sieve
recipient_delimiter = {{ RECIPIENT_DELIMITER }}
@ -147,11 +123,9 @@ service lmtp {
}
}
###############
# Filtering
###############
service managesieve-login {
inet_listener sieve {
port = 4190
@ -162,16 +136,13 @@ service managesieve {
}
plugin {
sieve = file:~/sieve;active=~/.dovecot.sieve
sieve_plugins = sieve_extdata sieve_imapsieve sieve_extprograms
sieve_global_extensions = +vnd.dovecot.extdata +spamtest +spamtestplus +vnd.dovecot.execute +editheader
sieve_before = /var/lib/dovecot/before.sieve
sieve_default = /var/lib/dovecot/default.sieve
sieve_after = /var/lib/dovecot/after.sieve
sieve_extdata_dict_uri = proxy::sieve
sieve = dict:proxy:/tmp/podop.socket:sieve
sieve_plugins = sieve_imapsieve sieve_extprograms
sieve_extensions = +spamtest +spamtestplus +editheader
sieve_global_extensions = +vnd.dovecot.execute
# Sieve execute
sieve_execute_bin_dir = /var/lib/dovecot/bin
sieve_execute_bin_dir = /conf/bin
# Send vacation replies even for aliases
# See the Pigeonhole documentation about warnings: http://wiki2.dovecot.org/Pigeonhole/Sieve/Extensions/Vacation
@ -190,11 +161,11 @@ plugin {
# Learn from spam
imapsieve_mailbox1_name = Junk
imapsieve_mailbox1_causes = COPY
imapsieve_mailbox1_before = file:/var/lib/dovecot/report-spam.sieve
imapsieve_mailbox1_before = file:/conf/report-spam.sieve
imapsieve_mailbox2_name = *
imapsieve_mailbox2_from = Junk
imapsieve_mailbox2_causes = COPY
imapsieve_mailbox2_before = file:/var/lib/dovecot/report-ham.sieve
imapsieve_mailbox2_before = file:/conf/report-ham.sieve
}
###############

@ -1,51 +0,0 @@
connect = /data/main.db
map {
pattern = priv/spam_enabled
table = user
username_field = email
value_field = spam_enabled
}
map {
pattern = priv/spam_threshold
table = user
username_field = email
value_field = spam_threshold
}
map {
pattern = priv/reply_enabled
table = user
username_field = email
value_field = reply_enabled
}
map {
pattern = priv/reply_subject
table = user
username_field = email
value_field = reply_subject
}
map {
pattern = priv/reply_body
table = user
username_field = email
value_field = reply_body
}
map {
pattern = priv/reply_enddate
table = user
username_field = email
value_field = reply_enddate
}
map {
pattern = priv/reply_startdate
table = user
username_field = email
value_field = reply_startdate
}

@ -1,21 +1,40 @@
#!/usr/bin/python
#!/usr/bin/python3
import jinja2
import os
import socket
import glob
import multiprocessing
import tenacity
from tenacity import retry
from podop import run_server
def start_podop():
os.setuid(8)
run_server(3 if "DEBUG" in os.environ else 0, "dovecot", "/tmp/podop.socket", [
("quota", "url", "http://admin/internal/dovecot/§"),
("auth", "url", "http://admin/internal/dovecot/§"),
("sieve", "url", "http://admin/internal/dovecot/§"),
])
convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
# Actual startup script
@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
def resolve():
os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front"))
os.environ["REDIS_ADDRESS"] = socket.gethostbyname(os.environ.get("REDIS_ADDRESS", "redis"))
if os.environ["WEBMAIL"] != "none":
os.environ["WEBMAIL_ADDRESS"] = socket.gethostbyname(os.environ.get("WEBMAIL_ADDRESS", "webmail"))
for dovecot_file in glob.glob("/conf/*"):
# Actual startup script
resolve()
for dovecot_file in glob.glob("/conf/*.conf"):
convert(dovecot_file, os.path.join("/etc/dovecot", os.path.basename(dovecot_file)))
# Run postfix
# Run Podop, then postfix
multiprocessing.Process(target=start_podop).start()
os.system("chown -R mail:mail /mail /var/lib/dovecot")
os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

@ -1,10 +1,14 @@
FROM alpine:3.7
FROM alpine:3.8
RUN apk add --no-cache nginx nginx-mod-mail python py-jinja2 certbot openssl
RUN apk add --no-cache certbot nginx nginx-mod-mail openssl \
python py-jinja2 py-requests-toolbelt py-pip \
&& pip install --upgrade pip \
&& pip install idna
COPY conf /conf
COPY *.py /
EXPOSE 80/tcp 443/tcp 110/tcp 143/tcp 465/tcp 587/tcp 993/tcp 995/tcp 25/tcp 10025/tcp 10143/tcp
VOLUME ["/certs"]
CMD /start.py

@ -1,5 +1,5 @@
# This is an idle image to dynamically replace any component if disabled.
FROM alpine
FROM alpine:3.8
CMD sleep 1000000d

@ -1,10 +1,14 @@
FROM alpine:3.7
FROM alpine:3.8
RUN apk add --no-cache postfix postfix-sqlite postfix-pcre rsyslog python py-jinja2
RUN apk add --no-cache postfix postfix-pcre rsyslog \
python3 py3-pip \
&& pip3 install --upgrade pip \
&& pip3 install jinja2 podop tenacity
COPY conf /conf
COPY start.py /start.py
EXPOSE 25/tcp 10025/tcp
VOLUME ["/data"]
CMD /start.py

@ -2,6 +2,8 @@
# General
###############
debug_peer_list = 0.0.0.0/0
# Main domain and hostname
mydomain = {{ DOMAIN }}
myhostname = {{ HOSTNAMES.split(",")[0] }}
@ -19,8 +21,8 @@ mynetworks = 127.0.0.1/32 [::1]/128 {{ RELAYNETS }}
# Empty alias list to override the configuration variable and disable NIS
alias_maps =
# SQLite configuration
sql = sqlite:${config_directory}/
# Podop configuration
podop = socketmap:unix:/tmp/podop.socket:
# Only accept virtual emails
mydestination =
@ -56,13 +58,14 @@ smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
# The alias map actually returns both aliases and local mailboxes, which is
# required for reject_unlisted_sender to work properly
virtual_alias_maps = ${sql}sqlite-virtual_alias_maps.cf
virtual_mailbox_domains = ${sql}sqlite-virtual_mailbox_domains.cf
virtual_mailbox_maps = $virtual_alias_maps
virtual_alias_domains =
virtual_alias_maps = ${podop}alias
virtual_mailbox_domains = ${podop}domain
virtual_mailbox_maps = ${podop}mailbox
# Mails are transported if required, then forwarded to Dovecot for delivery
relay_domains = ${sql}sqlite-transport.cf
transport_maps = ${sql}sqlite-transport.cf
relay_domains = ${podop}transport
transport_maps = ${podop}transport
virtual_transport = lmtp:inet:{{ HOST_LMTP }}
# In order to prevent Postfix from running DNS query, enforce the use of the
@ -82,15 +85,20 @@ smtpd_sender_login_maps = $virtual_alias_maps
# Restrictions for incoming SMTP, other restrictions are applied in master.cf
smtpd_helo_required = yes
smtpd_recipient_restrictions =
smtpd_client_restrictions =
permit_mynetworks,
check_sender_access ${sql}sqlite-reject-spoofed.cf,
check_sender_access ${podop}sender,
reject_non_fqdn_sender,
reject_unknown_sender_domain,
reject_unknown_recipient_domain,
reject_unverified_recipient,
permit
smtpd_relay_restrictions =
permit_mynetworks,
permit_sasl_authenticated,
reject_unauth_destination
unverified_recipient_reject_reason = Address lookup failure
###############

@ -7,7 +7,8 @@ smtp inet n - n - - smtpd
# Internal SMTP service
10025 inet n - n - - smtpd
-o smtpd_sasl_auth_enable=yes
-o smtpd_recipient_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
-o smtpd_client_restrictions=reject_unlisted_sender,reject_authenticated_sender_login_mismatch,permit
-o smtpd_reject_unlisted_recipient={% if REJECT_UNLISTED_RECIPIENT %}{{ REJECT_UNLISTED_RECIPIENT }}{% else %}no{% endif %}
-o cleanup_service_name=outclean
outclean unix n - n - 0 cleanup
-o header_checks=pcre:/etc/postfix/outclean_header_filter.cf

@ -1,5 +0,0 @@
dbpath = /data/main.db
query =
SELECT 'REJECT' FROM domain WHERE name='%s'
UNION
SELECT 'REJECT' FROM alternative WHERE name='%s'

@ -1,3 +0,0 @@
dbpath = /data/main.db
query =
SELECT 'smtp:['||smtp||']' FROM relay WHERE name='%s'

@ -1,23 +0,0 @@
dbpath = /data/main.db
query =
SELECT destination
FROM
(SELECT destination, email, wildcard, localpart, localpart||'@'||alternative.name AS alt_email FROM alias LEFT JOIN alternative ON alias.domain_name = alternative.domain_name
UNION
SELECT (CASE WHEN forward_enabled=1 THEN (CASE WHEN forward_keep=1 THEN email||',' ELSE '' END)||forward_destination ELSE email END) AS destination, email, 0 as wildcard, localpart, localpart||'@'||alternative.name as alt_email FROM user LEFT JOIN alternative ON user.domain_name = alternative.domain_name
UNION
SELECT '@'||domain_name as destination, '@'||name as email, 0 as wildcard, '' as localpart, NULL AS alt_email FROM alternative)
WHERE
(
wildcard = 0
AND
(email = '%s' OR alt_email = '%s')
) OR (
wildcard = 1
AND
'%s' LIKE email
)
ORDER BY
wildcard ASC,
length(localpart) DESC
LIMIT 1

@ -1,5 +0,0 @@
dbpath = /data/main.db
query =
SELECT name FROM domain WHERE name='%s'
UNION
SELECT name FROM alternative WHERE name='%s'

@ -1,15 +1,35 @@
#!/usr/bin/python
#!/usr/bin/python3
import jinja2
import os
import socket
import glob
import shutil
import tenacity
import multiprocessing
from tenacity import retry
from podop import run_server
def start_podop():
os.setuid(100)
run_server(3 if "DEBUG" in os.environ else 0, "postfix", "/tmp/podop.socket", [
("transport", "url", "http://admin/internal/postfix/transport/§"),
("alias", "url", "http://admin/internal/postfix/alias/§"),
("domain", "url", "http://admin/internal/postfix/domain/§"),
("mailbox", "url", "http://admin/internal/postfix/mailbox/§"),
("sender", "url", "http://admin/internal/postfix/sender/§")
])
convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
# Actual startup script
@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
def resolve():
os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front"))
# Actual startup script
resolve()
os.environ["HOST_ANTISPAM"] = os.environ.get("HOST_ANTISPAM", "antispam:11332")
os.environ["HOST_LMTP"] = os.environ.get("HOST_LMTP", "imap:2525")
@ -32,7 +52,8 @@ for map_file in glob.glob("/overrides/*.map"):
convert("/conf/rsyslog.conf", "/etc/rsyslog.conf")
# Run postfix
# Run Podop and Postfix
multiprocessing.Process(target=start_podop).start()
if os.path.exists("/var/run/rsyslogd.pid"):
os.remove("/var/run/rsyslogd.pid")
os.system("/usr/lib/postfix/post-install meta_directory=/etc/postfix create-missing")

@ -0,0 +1,14 @@
FROM python:3-alpine
COPY requirements.txt /requirements.txt
RUN pip install -r /requirements.txt \
&& apk add --no-cache nginx \
&& mkdir /run/nginx
COPY ./nginx.conf /etc/nginx/conf.d/default.conf
COPY . /docs
RUN sphinx-build /docs /build
CMD nginx -g "daemon off;"

@ -132,3 +132,6 @@ REAL_IP_HEADER=
# IPs for nginx set_real_ip_from (CIDR list separated by commas)
REAL_IP_FROM=
# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no)
REJECT_UNLISTED_RECIPIENT=

@ -39,7 +39,6 @@ services:
restart: always
env_file: .env
volumes:
- "$ROOT/data:/data"
- "$ROOT/mail:/mail"
- "$ROOT/overrides:/overrides"
depends_on:
@ -50,7 +49,6 @@ services:
restart: always
env_file: .env
volumes:
- "$ROOT/data:/data"
- "$ROOT/overrides:/overrides"
depends_on:
- front
@ -104,5 +102,3 @@ services:
image: mailu/fetchmail:$VERSION
restart: always
env_file: .env
volumes:
- "$ROOT/data:/data"

@ -7,7 +7,7 @@ templates_path = ['_templates']
source_suffix = '.rst'
master_doc = 'index'
project = 'Mailu'
copyright = '2017, Mailu authors'
copyright = '2018, Mailu authors'
author = 'Mailu authors'
version = release = 'latest'
language = None
@ -23,7 +23,7 @@ htmlhelp_basename = 'Mailudoc'
# to template names.
html_sidebars = {
'**': [
'relations.html', # needs 'show_related': True theme option to display
'relations.html',
'searchbox.html',
]
}
@ -36,24 +36,3 @@ html_context = {
'github_version': 'master',
'conf_py_path': '/docs/'
}
# Upload function when the script is called directly
if __name__ == "__main__":
import os, sys, paramiko
build_dir, hostname, username, password, dest_dir = sys.argv[1:]
transport = paramiko.Transport((hostname, 22))
transport.connect(username=username, password=password)
sftp = paramiko.SFTPClient.from_transport(transport)
os.chdir(build_dir)
for dirpath, dirnames, filenames in os.walk("."):
remote_path = os.path.join(dest_dir, dirpath)
try:
sftp.mkdir(remote_path)
except:
pass
for filename in filenames:
sftp.put(
os.path.join(dirpath, filename),
os.path.join(remote_path, filename)
)

@ -89,3 +89,20 @@ Any change to the files will automatically restart the Web server and reload the
When using the development environment, a debugging toolbar is displayed on the right side
of the screen, that you can open to access query details, internal variables, etc.
Documentation
-------------
Documentation is maintained in the ``docs`` directory and are maintained as `reStructuredText`_ files. It is possible to run a local documentation server for reviewing purposes, using Docker:
.. code-block:: bash
cd <Mailu repo>
docker build -t docs docs
docker run -p 127.0.0.1:8080:80 docs
You can now read the local documentation by navigating to http://localhost:8080.
.. note:: After modifying the documentation, the image needs to be rebuild and the container restarted for the changes to become visible.
.. _`reStructuredText`: http://docutils.sourceforge.net/rst.html

@ -0,0 +1,5 @@
server {
listen 80;
listen [::]:80;
root /build;
}

@ -2,5 +2,3 @@ recommonmark
Sphinx
sphinx-autobuild
sphinx-rtd-theme
sphinxcontrib-versioning
paramiko

@ -6,6 +6,7 @@ One of Mailu use cases is as part of a larger services platform, where maybe oth
In such a configuration, one would usually run a frontend reverse proxy to serve all Web contents based on criteria like the requested hostname (virtual hosts) and/or the requested path. Mailu Web frontend is disabled in the default setup for security reasons, it is however expected that most users will enable it at some point. Also, due to Docker Compose configuration structure, it is impossible for us to make disabling the Web frontend completely available through a configuration variable. This guide was written to help users setup such an architecture.
There are basically three options, from the most to the least recommended one:
- have Mailu Web frontend listen locally and use your own Web frontend on top of it
- override Mailu Web frontend configuration
- disable Mailu Web frontend completely and use your own

@ -1,4 +1,4 @@
FROM alpine:edge
FROM alpine:3.8
RUN apk add --no-cache clamav rsyslog wget clamav-libunrar
@ -6,5 +6,6 @@ COPY conf /etc/clamav
COPY start.sh /start.sh
EXPOSE 3310/tcp
VOLUME ["/data"]
CMD ["/start.sh"]

@ -6,5 +6,6 @@ RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/re
COPY radicale.conf /radicale.conf
EXPOSE 5232/tcp
VOLUME ["/data"]
CMD radicale -f -S -C /radicale.conf

@ -1,7 +1,9 @@
FROM python:alpine
FROM python:3-alpine
RUN apk add --no-cache fetchmail ca-certificates
RUN apk add --no-cache fetchmail ca-certificates \
&& pip install requests
COPY fetchmail.py /fetchmail.py
USER fetchmail
CMD ["/fetchmail.py"]

@ -1,12 +1,12 @@
#!/usr/bin/env python
import sqlite3
import time
import os
import tempfile
import shlex
import subprocess
import re
import requests
FETCHMAIL = """
@ -15,6 +15,7 @@ fetchmail -N \
-f {}
"""
RC_LINE = """
poll "{host}" proto {protocol} port {port}
user "{username}" password "{password}"
@ -24,10 +25,12 @@ poll "{host}" proto {protocol} port {port}
sslproto 'AUTO'
"""
def extract_host_port(host_and_port, default_port):
host, _, port = re.match('^(.*)(:([0-9]*))?$', host_and_port).groups()
return host, int(port) if port else default_port
def escape_rc_string(arg):
return arg.replace("\\", "\\\\").replace('"', '\\"')
@ -41,30 +44,26 @@ def fetchmail(fetchmailrc):
return output
def run(connection, cursor, debug):
cursor.execute("""
SELECT user_email, protocol, host, port, tls, username, password, keep
FROM fetch
""")
def run(debug):
fetches = requests.get("http://admin/internal/fetch").json()
smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None)
if smtpport is None:
smtphostport = smtphost
else:
smtphostport = "%s/%d" % (smtphost, smtpport)
for line in cursor.fetchall():
for fetch in fetches:
fetchmailrc = ""
user_email, protocol, host, port, tls, username, password, keep = line
options = "options antispam 501, 504, 550, 553, 554"
options += " ssl" if tls else ""
options += " keep" if keep else " fetchall"
options += " ssl" if fetch["tls"] else ""
options += " keep" if fetch["keep"] else " fetchall"
fetchmailrc += RC_LINE.format(
user_email=escape_rc_string(user_email),
protocol=protocol,
host=escape_rc_string(host),
port=port,
user_email=escape_rc_string(fetch["user_email"]),
protocol=fetch["protocol"],
host=escape_rc_string(fetch["host"]),
port=fetch["port"],
smtphost=smtphostport,
username=escape_rc_string(username),
password=escape_rc_string(password),
username=escape_rc_string(fetch["username"]),
password=escape_rc_string(fetch["password"]),
options=options
)
if debug:
@ -77,26 +76,20 @@ def run(connection, cursor, debug):
# No mail is not an error
if not error_message.startswith("fetchmail: No mail"):
print(error_message)
user_info = "for %s at %s" % (user_email, host)
user_info = "for %s at %s" % (fetch["user_email"], fetch["host"])
# Number of messages seen is not a error as well
if ("messages" in error_message and
"(seen " in error_message and
user_info in error_message):
print(error_message)
finally:
cursor.execute("""
UPDATE fetch SET error=?, last_check=datetime('now')
WHERE user_email=?
""", (error_message.split("\n")[0], user_email))
connection.commit()
requests.post("http://admin/internal/fetch/{}".format(fetch["id"]),
json=error_message.split("\n")[0]
)
if __name__ == "__main__":
debug = os.environ.get("DEBUG", None) == "True"
db_path = os.environ.get("DB_PATH", "/data/main.db")
connection = sqlite3.connect(db_path)
while True:
cursor = connection.cursor()
run(connection, cursor, debug)
cursor.close()
time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60)))
run(os.environ.get("DEBUG", None) == "True")

@ -1,6 +1,8 @@
FROM alpine:edge
FROM alpine:3.8
RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates
RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \
&& pip install --upgrade pip \
&& pip install tenacity
RUN mkdir /run/rspamd
@ -12,4 +14,6 @@ RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
EXPOSE 11332/tcp 11334/tcp
VOLUME ["/var/lib/rspamd"]
CMD /start.py

@ -0,0 +1,4 @@
try_fallback = true;
path = "/dkim/$domain.$selector.key";
selector = "dkim"
use_esld = false;

@ -4,11 +4,17 @@ import jinja2
import os
import socket
import glob
import tenacity
from tenacity import retry
convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ))
# Actual startup script
@retry(stop=tenacity.stop_after_attempt(100), wait=tenacity.wait_random(min=2, max=5))
def resolve():
os.environ["FRONT_ADDRESS"] = socket.gethostbyname(os.environ.get("FRONT_ADDRESS", "front"))
# Actual startup script
resolve()
if "HOST_REDIS" not in os.environ: os.environ["HOST_REDIS"] = "redis"
for rspamd_file in glob.glob("/conf/*"):

@ -3,45 +3,54 @@ version: '3'
services:
front:
image: mailu/nginx:$VERSION
image: $DOCKER_ORG/nginx:$VERSION
build: ../core/nginx
imap:
image: mailu/dovecot:$VERSION
image: $DOCKER_ORG/dovecot:$VERSION
build: ../core/dovecot
smtp:
image: mailu/postfix:$VERSION
image: $DOCKER_ORG/postfix:$VERSION
build: ../core/postfix
antispam:
image: mailu/rspamd:$VERSION
image: $DOCKER_ORG/rspamd:$VERSION
build: ../services/rspamd
antivirus:
image: mailu/clamav:$VERSION
image: $DOCKER_ORG/clamav:$VERSION
build: ../optional/clamav
webdav:
image: mailu/radicale:$VERSION
image: $DOCKER_ORG/radicale:$VERSION
build: ../optional/radicale
admin:
image: mailu/admin:$VERSION
image: $DOCKER_ORG/admin:$VERSION
build: ../core/admin
roundcube:
image: mailu/roundcube:$VERSION
image: $DOCKER_ORG/roundcube:$VERSION
build: ../webmails/roundcube
rainloop:
image: mailu/rainloop:$VERSION
image: $DOCKER_ORG/rainloop:$VERSION
build: ../webmails/rainloop
fetchmail:
image: mailu/fetchmail:$VERSION
image: $DOCKER_ORG/fetchmail:$VERSION
build: ../services/fetchmail
none:
image: mailu/none:$VERSION
image: $DOCKER_ORG/none:$VERSION
build: ../core/none
docs:
image: $DOCKER_ORG/docs:$VERSION
build: ../docs
setup:
image: $DOCKER_ORG/setup:$VERSION
build: ../setup

@ -0,0 +1,134 @@
# Mailu main configuration file
#
# Most configuration variables can be modified through the Web interface,
# these few settings must however be configured before starting the mail
# server and require a restart upon change.
###################################
# Common configuration variables
###################################
# Set this to the path where Mailu data and configuration is stored
ROOT=/mailu
# Mailu version to run (1.0, 1.1, etc. or master)
#VERSION=master
# Set to a randomly generated 16 bytes string
SECRET_KEY=ChangeMeChangeMe
# Address where listening ports should bind
BIND_ADDRESS4=127.0.0.1
#BIND_ADDRESS6=::1
# Main mail domain
DOMAIN=mailu.io
# Hostnames for this server, separated with comas
HOSTNAMES=mail.mailu.io,alternative.mailu.io,yetanother.mailu.io
# Postmaster local part (will append the main mail domain)
POSTMASTER=admin
# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt)
TLS_FLAVOR=cert
# Authentication rate limit (per source IP address)
AUTH_RATELIMIT=10/minute;1000/hour
# Opt-out of statistics, replace with "True" to opt out
DISABLE_STATISTICS=False
###################################
# Optional features
###################################
# Expose the admin interface (value: true, false)
ADMIN=false
# Choose which webmail to run if any (values: roundcube, rainloop, none)
WEBMAIL=none
# Dav server implementation (value: radicale, none)
WEBDAV=none
# Antivirus solution (value: clamav, none)
ANTIVIRUS=none
###################################
# Mail settings
###################################
# Message size limit in bytes
# Default: accept messages up to 50MB
MESSAGE_SIZE_LIMIT=50000000
# Networks granted relay permissions, make sure that you include your Docker
# internal network (default to 172.17.0.0/16)
RELAYNETS=172.16.0.0/12
# Will relay all outgoing mails if configured
RELAYHOST=
# Fetchmail delay
FETCHMAIL_DELAY=600
# Recipient delimiter, character used to delimiter localpart from custom address part
# e.g. localpart+custom@domain;tld
RECIPIENT_DELIMITER=+
# DMARC rua and ruf email
DMARC_RUA=admin
DMARC_RUF=admin
# Welcome email, enable and set a topic and body if you wish to send welcome
# emails to all users.
WELCOME=false
WELCOME_SUBJECT=Welcome to your new email account
WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly!
# Maildir Compression
# choose compression-method, default: none (value: bz2, gz)
COMPRESSION=
# change compression-level, default: 6 (value: 1-9)
COMPRESSION_LEVEL=
###################################
# Web settings
###################################
# Path to the admin interface if enabled
WEB_ADMIN=/admin
# Path to the webmail if enabled
WEB_WEBMAIL=/webmail
# Website name
SITENAME=Mailu
# Linked Website URL
WEBSITE=https://mailu.io
# Registration reCaptcha settings (warning, this has some privacy impact)
# RECAPTCHA_PUBLIC_KEY=
# RECAPTCHA_PRIVATE_KEY=
# Domain registration, uncomment to enable
# DOMAIN_REGISTRATION=true
###################################
# Advanced settings
###################################
# Docker-compose project name, this will prepended to containers names.
#COMPOSE_PROJECT_NAME=mailu
# Default password scheme used for newly created accounts and changed passwords
# (value: SHA512-CRYPT, SHA256-CRYPT, MD5-CRYPT, CRYPT)
PASSWORD_SCHEME=SHA512-CRYPT
# Header to take the real ip from
REAL_IP_HEADER=
# IPs for nginx set_real_ip_from (CIDR list separated by commas)
REAL_IP_FROM=

@ -0,0 +1,99 @@
version: '2'
services:
front:
image: $DOCKER_ORG/nginx:$VERSION
restart: 'no'
env_file: $PWD/.env
ports:
- "$BIND_ADDRESS4:80:80"
- "$BIND_ADDRESS4:443:443"
- "$BIND_ADDRESS4:110:110"
- "$BIND_ADDRESS4:143:143"
- "$BIND_ADDRESS4:993:993"
- "$BIND_ADDRESS4:995:995"
- "$BIND_ADDRESS4:25:25"
- "$BIND_ADDRESS4:465:465"
- "$BIND_ADDRESS4:587:587"
volumes:
- "$ROOT/certs:/certs"
redis:
image: redis:alpine
restart: 'no'
volumes:
- "$ROOT/redis:/data"
imap:
image: $DOCKER_ORG/dovecot:$VERSION
restart: 'no'
env_file: $PWD/.env
volumes:
- "$ROOT/data:/data"
- "$ROOT/mail:/mail"
- "$ROOT/overrides:/overrides"
depends_on:
- front
smtp:
image: $DOCKER_ORG/postfix:$VERSION
restart: 'no'
env_file: $PWD/.env
volumes:
- "$ROOT/data:/data"
- "$ROOT/overrides:/overrides"
depends_on:
- front
antispam:
image: $DOCKER_ORG/rspamd:$VERSION
restart: 'no'
env_file: $PWD/.env
volumes:
- "$ROOT/filter:/var/lib/rspamd"
- "$ROOT/dkim:/dkim"
- "$ROOT/overrides/rspamd:/etc/rspamd/override.d"
depends_on:
- front
antivirus:
image: $DOCKER_ORG/$ANTIVIRUS:$VERSION
restart: 'no'
env_file: $PWD/.env
volumes:
- "$ROOT/filter:/data"
webdav:
image: $DOCKER_ORG/$WEBDAV:$VERSION
restart: 'no'
env_file: $PWD/.env
volumes:
- "$ROOT/dav:/data"
admin:
image: $DOCKER_ORG/admin:$VERSION
restart: 'no'
env_file: $PWD/.env
volumes:
- "$ROOT/data:/data"
- "$ROOT/dkim:/dkim"
- /var/run/docker.sock:/var/run/docker.sock:ro
depends_on:
- redis
webmail:
image: "$DOCKER_ORG/$WEBMAIL:$VERSION"
restart: 'no'
env_file: $PWD/.env
volumes:
- "$ROOT/webmail:/data"
depends_on:
- imap
fetchmail:
image: $DOCKER_ORG/fetchmail:$VERSION
restart: 'no'
env_file: $PWD/.env
volumes:
- "$ROOT/data:/data"

@ -0,0 +1,57 @@
#!/bin/bash
containers=(
webmail
imap
smtp
antispam
admin
redis
antivirus
webdav
# fetchmail
front
)
# Time to sleep in minutes after starting the containers
WAIT=1
containers_check() {
status=0
for container in "${containers[@]}"; do
name="${DOCKER_ORG}_${container}_1"
echo "Checking $name"
docker inspect "$name" | grep '"Status": "running"' || status=1
done
docker ps -a
return $status
}
container_logs() {
for container in "${containers[@]}"; do
name="${DOCKER_ORG}_${container}_1"
echo "Showing logs for $name"
docker container logs "$name"
done
}
clean() {
docker-compose -f tests/compose/run.yml -p $DOCKER_ORG down || exit 1
rm -fv .env
}
# Cleanup before callig exit
die() {
clean
exit $1
}
for file in tests/compose/*.env ; do
cp $file .env
docker-compose -f tests/compose/run.yml -p $DOCKER_ORG up -d
echo -e "\nSleeping for ${WAIT} minutes" # Clean terminal distortion from docker-compose in travis
travis_wait sleep ${WAIT}m || sleep ${WAIT}m #Fallback sleep for local run
container_logs
containers_check || die 1
clean
done

@ -0,0 +1,4 @@
#!/bin/bash
docker login -u $DOCKER_UN -p $DOCKER_PW
docker-compose -f tests/build.yml push

@ -0,0 +1,17 @@
import smtplib
import sys
from email import mime
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
msg = mime.multipart.MIMEMultipart()
msg['Subject'] = 'Test email'
msg['From'] = sys.argv[1]
msg['To'] = sys.argv[2]
msg.preamble = 'Test email'
s = smtplib.SMTP('localhost')
s.set_debuglevel(1)
s.send_message(msg)
s.quit()

@ -1,4 +1,4 @@
FROM php:5-apache
FROM php:7.2-apache
RUN apt-get update && apt-get install -y \
unzip python3 python3-jinja2
@ -24,4 +24,7 @@ COPY default.ini /default.ini
COPY start.py /start.py
EXPOSE 80/tcp
VOLUME ["/data"]
CMD /start.py

@ -18,4 +18,7 @@ os.makedirs(base + "configs", exist_ok=True)
convert("/default.ini", "/data/_data_/_default_/domains/default.ini")
convert("/config.ini", "/data/_data_/_default_/configs/config.ini")
os.system("chown -R www-data:www-data /data")
os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"])

@ -1,11 +1,8 @@
FROM php:7.0-apache
FROM php:7.2-apache
RUN apt-get update && apt-get install -y \
libfreetype6-dev \
libjpeg62-turbo-dev \
libmcrypt-dev \
libpng-dev \
&& docker-php-ext-install pdo_mysql mcrypt zip
zlib1g-dev \
&& docker-php-ext-install zip
ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.3.7/roundcubemail-1.3.7-complete.tar.gz
@ -28,4 +25,7 @@ COPY config.inc.php /var/www/html/config/
COPY start.sh /start.sh
EXPOSE 80/tcp
VOLUME ["/data"]
CMD ["/start.sh"]

Loading…
Cancel
Save