From 5e32447f07a2148cc1800a49ce7c9c7a19049d31 Mon Sep 17 00:00:00 2001 From: Jon Wilson Date: Mon, 21 Sep 2020 15:06:43 +0100 Subject: [PATCH 01/65] Change unbound logfile to the empty string This is defined to send log messages to stderr, which is what we want - fixes #1536 ("Could not open logfile /dev/stdout: Permission denied") --- optional/unbound/unbound.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optional/unbound/unbound.conf b/optional/unbound/unbound.conf index 8abd4325..6c8fc64d 100644 --- a/optional/unbound/unbound.conf +++ b/optional/unbound/unbound.conf @@ -2,7 +2,7 @@ server: verbosity: 1 interface: 0.0.0.0 interface: ::0 - logfile: /dev/stdout + logfile: "" do-ip4: yes do-ip6: yes do-udp: yes From ef71bc04cb27f6ed0779584afcc8690fbf20048a Mon Sep 17 00:00:00 2001 From: Patryk Tech Date: Thu, 1 Oct 2020 13:51:06 +0300 Subject: [PATCH 02/65] Update docs/reverse.rst with Traefik v2+ info --- docs/reverse.rst | 41 ++++++++++++++++++++++++++++++-- towncrier/newsfragments/1503.doc | 1 + 2 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/1503.doc diff --git a/docs/reverse.rst b/docs/reverse.rst index 9a7a7dc0..f3b3e7bd 100644 --- a/docs/reverse.rst +++ b/docs/reverse.rst @@ -154,7 +154,40 @@ Add the respective Traefik labels for your domain/configuration, like If your Traefik is configured to automatically request certificates from *letsencrypt*, then you’ll have a certificate for ``mail.your.doma.in`` now. However, ``mail.your.doma.in`` might only be the location where you want the Mailu web-interfaces to live — your mail should be sent/received from ``your.doma.in``, and this is the ``DOMAIN`` in your ``.env``? -To support that use-case, Traefik can request ``SANs`` for your domain. Lets add something like +To support that use-case, Traefik can request ``SANs`` for your domain. The configuration for this will depend on your Traefik version. + +---- + +Traefik 2.x using labels configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Add the appropriate labels for your domain(s) to the ``front`` container in ``docker-compose.yml``. + +.. code-block:: yaml + + services: + front: + labels: + # Enable TLS + - "traefik.http.routers.mailu-secure.tls" + # Your main domain + - "traefik.http.routers.mailu-secure.tls.domains[0].main=your.doma.in" + # Optional SANs for your main domain + - "traefik.http.routers.mailu-secure.tls.domains[0].sans=mail.your.doma.in,webmail.your.doma.in,smtp.your.doma.in" + # Optionally add other domains + - "traefik.http.routers.mailu-secure.tls.domains[1].main=mail.other.doma.in" + - "traefik.http.routers.mailu-secure.tls.domains[1].sans=mail2.other.doma.in,mail3.other.doma.in" + # Your ACME certificate resolver + - "traefik.http.routers.mailu-secure.tls.certResolver=foo" + +Of course, be sure to define the Certificate Resolver ``foo`` in the static configuration as well. + +Alternatively, you can define SANs in the Traefik static configuration using routers, or in the static configuration using entrypoints. Refer to the Traefik documentation for more details. + +Traefik 1.x with TOML configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Lets add something like .. code-block:: yaml @@ -163,7 +196,11 @@ To support that use-case, Traefik can request ``SANs`` for your domain. Lets add main = "your.doma.in" # this is the same as $TRAEFIK_DOMAIN! sans = ["mail.your.doma.in", "webmail.your.doma.in", "smtp.your.doma.in"] -to your ``traefik.toml``. You might need to clear your ``acme.json``, if a certificate for one of these domains already exists. +to your ``traefik.toml``. + +---- + +You might need to clear your ``acme.json``, if a certificate for one of these domains already exists. You will need some solution which dumps the certificates in ``acme.json``, so you can include them in the ``mailu/front`` container. One such example is ``mailu/traefik-certdumper``, which has been adapted for use in Mailu. You can add it to your ``docker-compose.yml`` like: diff --git a/towncrier/newsfragments/1503.doc b/towncrier/newsfragments/1503.doc new file mode 100644 index 00000000..9c59feb7 --- /dev/null +++ b/towncrier/newsfragments/1503.doc @@ -0,0 +1 @@ +Add documentation for Traefik 2 in Reverse Proxy \ No newline at end of file From 32f6a23a95de6d5ee2d64eecc452469967289fdf Mon Sep 17 00:00:00 2001 From: cbachert Date: Fri, 30 Oct 2020 17:12:34 +0000 Subject: [PATCH 03/65] Remove rspamd unused env var from start script Environment variable FRONT_ADDRESS is unused in rspamd FRONT_ADDRESS references were removed with commit 8172f3e in PR #727 --- core/rspamd/start.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index bde708f2..e2e72bcb 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -10,7 +10,6 @@ log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) # Actual startup script -os.environ["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") os.environ["REDIS_ADDRESS"] = system.get_host_address_from_environment("REDIS", "redis") if os.environ.get("ANTIVIRUS") == 'clamav': From 98a6ffb497e0e368808614219c6b099dcc828c0b Mon Sep 17 00:00:00 2001 From: lub Date: Thu, 17 Sep 2020 19:33:55 +0200 Subject: [PATCH 04/65] add compression via xz and lz4 --- core/dovecot/conf/dovecot.conf | 2 +- docs/compose/.env | 2 +- setup/flavors/compose/mailu.env | 2 +- tests/compose/core/mailu.env | 2 +- tests/compose/fetchmail/mailu.env | 2 +- tests/compose/filters/mailu.env | 2 +- tests/compose/rainloop/mailu.env | 2 +- tests/compose/roundcube/mailu.env | 2 +- tests/compose/webdav/mailu.env | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 81811cdb..ab5cb43a 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -50,7 +50,7 @@ plugin { fts_autoindex_exclude = \Trash {% endif %} - {% if COMPRESSION in [ 'gz', 'bz2' ] %} + {% if COMPRESSION in [ 'gz', 'bz2', 'xz', 'lz4' ] %} zlib_save = {{ COMPRESSION }} {% endif %} diff --git a/docs/compose/.env b/docs/compose/.env index 7f91c270..b4a8b218 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -97,7 +97,7 @@ WELCOME_SUBJECT=Welcome to your new email account WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly! # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 44452e36..04148b40 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -86,7 +86,7 @@ WELCOME_SUBJECT={{ welcome_subject or 'Welcome to your new email account' }} WELCOME_BODY={{ welcome_body or 'Welcome to your new email account, if you can read this, then it is configured properly!' }} # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION={{ compression }} # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL={{ compression_level }} diff --git a/tests/compose/core/mailu.env b/tests/compose/core/mailu.env index b13e57c5..edea6a5c 100644 --- a/tests/compose/core/mailu.env +++ b/tests/compose/core/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/fetchmail/mailu.env b/tests/compose/fetchmail/mailu.env index 636a09a9..4a53ec46 100644 --- a/tests/compose/fetchmail/mailu.env +++ b/tests/compose/fetchmail/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/filters/mailu.env b/tests/compose/filters/mailu.env index b6d5ca8f..1106deb0 100644 --- a/tests/compose/filters/mailu.env +++ b/tests/compose/filters/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/rainloop/mailu.env b/tests/compose/rainloop/mailu.env index 9c31c8bb..d02b98f2 100644 --- a/tests/compose/rainloop/mailu.env +++ b/tests/compose/rainloop/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/roundcube/mailu.env index dc503268..e1005487 100644 --- a/tests/compose/roundcube/mailu.env +++ b/tests/compose/roundcube/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/webdav/mailu.env b/tests/compose/webdav/mailu.env index 90fb25b1..58b9810a 100644 --- a/tests/compose/webdav/mailu.env +++ b/tests/compose/webdav/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: bz2, gz) +# choose compression-method, default: none (value: gz, bz2, xz, lz4) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= From 2606ace1dfddcc32c00b640a95300b9a0c820635 Mon Sep 17 00:00:00 2001 From: lub Date: Sat, 21 Nov 2020 12:39:42 +0100 Subject: [PATCH 05/65] add changelog for #1694 --- towncrier/newsfragments/1694.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/1694.feature diff --git a/towncrier/newsfragments/1694.feature b/towncrier/newsfragments/1694.feature new file mode 100644 index 00000000..41548707 --- /dev/null +++ b/towncrier/newsfragments/1694.feature @@ -0,0 +1 @@ +Support configuring xz and lz4 compression for dovecot. From a1a527f201c25ab553ea3d41f37c70a390abe340 Mon Sep 17 00:00:00 2001 From: Grace <30454698+c4lliope@users.noreply.github.com> Date: Mon, 4 Jan 2021 02:37:48 -0500 Subject: [PATCH 06/65] Warn people off of the documentation's K8s recipe. Based on a discussion on Matrix (2020.12.19), the helm charts are the way to go. --- docs/kubernetes/mailu/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/kubernetes/mailu/index.rst b/docs/kubernetes/mailu/index.rst index 5d3502a7..0af3942e 100644 --- a/docs/kubernetes/mailu/index.rst +++ b/docs/kubernetes/mailu/index.rst @@ -3,6 +3,10 @@ Kubernetes setup ================ +> Hold up! +> These instructions are not recommended for setting up Mailu in a production Kubernetes environment. +> Please see [the Helm Chart documentation](https://github.com/Mailu/helm-charts/blob/master/mailu/README.md). + Prequisites ----------- From 906a051925766058feec60c0b768e274ccd7c862 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 6 Feb 2021 17:23:05 +0100 Subject: [PATCH 07/65] Make rainloop use internal auth --- core/admin/mailu/internal/nginx.py | 19 ++++++++++----- core/admin/mailu/internal/views/auth.py | 12 ++++++++++ core/admin/mailu/models.py | 10 ++++++++ core/admin/mailu/ui/views/base.py | 3 +++ core/nginx/conf/nginx.conf | 20 ++++++++++++++++ webmails/rainloop/Dockerfile | 1 + webmails/rainloop/application.ini | 2 ++ webmails/rainloop/sso.php | 31 +++++++++++++++++++++++++ webmails/rainloop/start.py | 1 + 9 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 webmails/rainloop/sso.php diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 1e0b16c2..a41543d3 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -7,7 +7,6 @@ import ipaddress import socket import tenacity - SUPPORTED_AUTH_METHODS = ["none", "plain"] @@ -49,11 +48,19 @@ def handle_authentication(headers): user = models.User.query.get(user_email) status = False if user: - for token in user.tokens: - if (token.check_password(password) and - (not token.ip or token.ip == ip)): - status = True - if user.check_password(password): + # webmails + if len(password) == 64 and ip == app.config['WEBMAIL_ADDRESS']: + if user.verify_temp_token(password): + status = True + + # All tokens are 32 characters hex lowercase + if len(password) == 32: + for token in user.tokens: + if (token.check_password(password) and + (not token.ip or token.ip == ip)): + status = True + break + if not status and user.check_password(password): status = True if status: if protocol == "imap" and not user.enable_imap: diff --git a/core/admin/mailu/internal/views/auth.py b/core/admin/mailu/internal/views/auth.py index 825dba56..338141d8 100644 --- a/core/admin/mailu/internal/views/auth.py +++ b/core/admin/mailu/internal/views/auth.py @@ -43,6 +43,18 @@ def admin_authentication(): return "" return flask.abort(403) +@internal.route("/auth/user") +def user_authentication(): + """ Fails if the user is not authenticated. + """ + if (not flask_login.current_user.is_anonymous + and flask_login.current_user.enabled): + response = flask.Response() + response.headers["X-User"] = flask_login.current_user.get_id() + response.headers["X-User-Token"] = models.User.get_temp_token(flask_login.current_user.get_id()) + return response + return flask.abort(403) + @internal.route("/auth/basic") def basic_authentication(): diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index bbc00f2d..6230e252 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -12,6 +12,7 @@ import re import time import os import glob +import hmac import smtplib import idna import dns @@ -425,6 +426,15 @@ class User(Base, Email): user = cls.query.get(email) return user if (user and user.enabled and user.check_password(password)) else None + @classmethod + def get_temp_token(cls, email): + user = cls.query.get(email) + return hmac.new(bytearray(app.secret_key,'utf-8'), bytearray("{}|{}".format(datetime.utcnow().strftime("%Y%m%d"), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None + + def verify_temp_token(self, token): + return hmac.compare_digest(b''.fromhex(self.get_temp_token(self.email)), b''.fromhex(token)) + + class Alias(Base, Email): """ An alias is an email address that redirects to some destination. diff --git a/core/admin/mailu/ui/views/base.py b/core/admin/mailu/ui/views/base.py index 7501a883..1aff7db1 100644 --- a/core/admin/mailu/ui/views/base.py +++ b/core/admin/mailu/ui/views/base.py @@ -47,6 +47,9 @@ def announcement(): flask.flash('Your announcement was sent', 'success') return flask.render_template('announcement.html', form=form) +@ui.route('/webmail', methods=['GET']) +def webmail(): + return flask.redirect(app.config['WEB_WEBMAIL']) @ui.route('/client', methods=['GET']) def client(): diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index df598c94..a7a9e134 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -133,7 +133,27 @@ http { {% endif %} include /etc/nginx/proxy.conf; client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; + auth_request /internal/auth/user; proxy_pass http://$webmail; + error_page 403 @webmail_login; + } + location {{ WEB_WEBMAIL }}/sso.php { + {% if WEB_WEBMAIL != '/' %} + rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent; + rewrite ^{{ WEB_WEBMAIL }}/(.*) /$1 break; + {% endif %} + include /etc/nginx/proxy.conf; + client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; + auth_request /internal/auth/user; + auth_request_set $user $upstream_http_x_user; + auth_request_set $token $upstream_http_x_user_token; + proxy_set_header X-Remote-User $user; + proxy_set_header X-Remote-User-Token $token; + proxy_pass http://$webmail; + error_page 403 @webmail_login; + } + location @webmail_login { + return 302 {{ WEB_ADMIN }}/ui/login?next=ui.webmail; } {% endif %} diff --git a/webmails/rainloop/Dockerfile b/webmails/rainloop/Dockerfile index c67c7496..640d7176 100644 --- a/webmails/rainloop/Dockerfile +++ b/webmails/rainloop/Dockerfile @@ -35,6 +35,7 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists COPY include.php /var/www/html/include.php +COPY sso.php /var/www/html/sso.php COPY php.ini /php.ini COPY application.ini /application.ini diff --git a/webmails/rainloop/application.ini b/webmails/rainloop/application.ini index 5bd9943d..bc953af1 100644 --- a/webmails/rainloop/application.ini +++ b/webmails/rainloop/application.ini @@ -7,6 +7,8 @@ attachment_size_limit = {{ MAX_FILESIZE }} allow_admin_panel = Off [labs] +custom_login_link='sso.php' +custom_logout_link='{{ WEB_ADMIN }}/ui/logout' allow_gravatar = Off [contacts] diff --git a/webmails/rainloop/sso.php b/webmails/rainloop/sso.php new file mode 100644 index 00000000..2415f45c --- /dev/null +++ b/webmails/rainloop/sso.php @@ -0,0 +1,31 @@ + Date: Sat, 6 Feb 2021 18:14:58 +0100 Subject: [PATCH 08/65] Make roundcube use internal auth --- webmails/roundcube/Dockerfile | 1 + webmails/roundcube/config.inc.php | 4 ++- webmails/roundcube/mailu.php | 59 +++++++++++++++++++++++++++++++ webmails/roundcube/start.py | 2 ++ 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 webmails/roundcube/mailu.php diff --git a/webmails/roundcube/Dockerfile b/webmails/roundcube/Dockerfile index 79b911b0..da355cc3 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/roundcube/Dockerfile @@ -46,6 +46,7 @@ RUN apt-get update && apt-get install -y \ COPY php.ini /php.ini COPY config.inc.php /var/www/html/config/ +COPY mailu.php /var/www/html/plugins/mailu/mailu.php COPY start.py /start.py EXPOSE 80/tcp diff --git a/webmails/roundcube/config.inc.php b/webmails/roundcube/config.inc.php index eb40047a..bb1a5e84 100644 --- a/webmails/roundcube/config.inc.php +++ b/webmails/roundcube/config.inc.php @@ -17,7 +17,8 @@ $config['plugins'] = array( 'markasjunk', 'managesieve', 'enigma', - 'carddav' + 'carddav', + 'mailu' ); $front = getenv('FRONT_ADDRESS') ? getenv('FRONT_ADDRESS') : 'front'; @@ -37,6 +38,7 @@ $config['managesieve_usetls'] = false; // Customization settings $config['support_url'] = getenv('WEB_ADMIN') ? '../..' . getenv('WEB_ADMIN') : ''; +$config['sso_logout_url'] = getenv('WEB_ADMIN').'/ui/logout'; $config['product_name'] = 'Mailu Webmail'; // We access the IMAP and SMTP servers locally with internal names, SSL diff --git a/webmails/roundcube/mailu.php b/webmails/roundcube/mailu.php new file mode 100644 index 00000000..bb4d65e9 --- /dev/null +++ b/webmails/roundcube/mailu.php @@ -0,0 +1,59 @@ +add_hook('startup', array($this, 'startup')); + $this->add_hook('authenticate', array($this, 'authenticate')); + $this->add_hook('login_after', array($this, 'login')); + $this->add_hook('login_failed', array($this, 'login_failed')); + $this->add_hook('logout_after', array($this, 'logout')); + } + + function startup($args) + { + if (empty($_SESSION['user_id'])) { + $args['action'] = 'login'; + } + + return $args; + } + + function authenticate($args) + { + if (!in_array('HTTP_X_REMOTE_USER', $_SERVER) || !in_array('HTTP_X_REMOTE_USER_TOKEN', $_SERVER)) { + header('HTTP/1.0 403 Forbidden'); + die(); + } + $args['user'] = $_SERVER['HTTP_X_REMOTE_USER']; + $args['pass'] = $_SERVER['HTTP_X_REMOTE_USER_TOKEN']; + + $args['cookiecheck'] = false; + $args['valid'] = true; + + return $args; + } + + function logout($args) { + // Redirect to global SSO logout path. + $this->load_config(); + + $sso_logout_url = rcmail::get_instance()->config->get('sso_logout_url'); + header("Location: " . $sso_logout_url, true); + exit; + } + + function login($args) + { + header('Location: index.php'); + exit(); + } + function login_failed($args) + { + header('Location: sso.php'); + exit(); + } + +} diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 649f3324..9ce383c8 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -39,6 +39,8 @@ conf.jinja("/php.ini", os.environ, "/usr/local/etc/php/conf.d/roundcube.ini") os.system("mkdir -p /data/gpg /var/www/html/logs") os.system("touch /var/www/html/logs/errors.log") os.system("chown -R www-data:www-data /var/www/html/logs") +os.system("chmod -R a+rX /var/www/html/") +os.system("ln -s /var/www/html/index.php /var/www/html/sso.php") try: print("Initializing database") From 99c7420f923a8b2c44860d25c196fcf2c00280e4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 7 Feb 2021 17:51:19 +0100 Subject: [PATCH 09/65] towncrier --- towncrier/newsfragments/783.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/783.feature diff --git a/towncrier/newsfragments/783.feature b/towncrier/newsfragments/783.feature new file mode 100644 index 00000000..fcafceef --- /dev/null +++ b/towncrier/newsfragments/783.feature @@ -0,0 +1 @@ +Centralize the authentication of webmails behind the admin interface From ef637f51b7f620c88dff403cf1fdb4a74ccbe663 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 7 Feb 2021 17:58:19 +0100 Subject: [PATCH 10/65] derive the SSO keys from a KDF --- core/admin/mailu/__init__.py | 3 +++ core/admin/mailu/models.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 4de3e580..b37b4493 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -3,6 +3,7 @@ import flask_bootstrap from mailu import utils, debug, models, manage, configuration +import hmac def create_app_from_config(config): """ Create a new application based on the given configuration @@ -24,6 +25,8 @@ def create_app_from_config(config): utils.proxy.init_app(app) utils.migrate.init_app(app, models.db) + app.temp_token_key = hmac.new(bytearray(app.secret_key, 'utf-8'), bytearray('WEBMAIL_TEMP_TOKEN_KEY', 'utf-8'), 'sha256').digest() + # Initialize debugging tools if app.config.get("DEBUG"): debug.toolbar.init_app(app) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6230e252..c3f79b92 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -429,10 +429,10 @@ class User(Base, Email): @classmethod def get_temp_token(cls, email): user = cls.query.get(email) - return hmac.new(bytearray(app.secret_key,'utf-8'), bytearray("{}|{}".format(datetime.utcnow().strftime("%Y%m%d"), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None + return hmac.new(app.temp_token_key, bytearray("{}|{}".format(datetime.utcnow().strftime("%Y%m%d"), email), 'utf-8'), 'sha256').hexdigest() if (user and user.enabled) else None def verify_temp_token(self, token): - return hmac.compare_digest(b''.fromhex(self.get_temp_token(self.email)), b''.fromhex(token)) + return hmac.compare_digest(self.get_temp_token(self.email), token) From b49554bec1156c69c5f24c5ca62cbb9a818ac8a7 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 7 Feb 2021 18:08:58 +0100 Subject: [PATCH 11/65] merge artifact --- core/admin/mailu/internal/nginx.py | 2 +- core/admin/mailu/ui/views/base.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index a41543d3..b2ee3fba 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -54,7 +54,7 @@ def handle_authentication(headers): status = True # All tokens are 32 characters hex lowercase - if len(password) == 32: + if not status and len(password) == 32: for token in user.tokens: if (token.check_password(password) and (not token.ip or token.ip == ip)): diff --git a/core/admin/mailu/ui/views/base.py b/core/admin/mailu/ui/views/base.py index 1aff7db1..457fd7fb 100644 --- a/core/admin/mailu/ui/views/base.py +++ b/core/admin/mailu/ui/views/base.py @@ -1,6 +1,7 @@ from mailu import models from mailu.ui import ui, forms, access +from flask import current_app as app import flask import flask_login From 2e749abe6118f9a2c414ac239efff099dc3fdb1a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 25 Jan 2021 13:19:21 +0100 Subject: [PATCH 12/65] DNS records for client autoconfiguration (RFC6186) --- core/admin/mailu/ui/templates/domain/details.html | 12 ++++++++++++ towncrier/newsfragments/224.feature | 1 + 2 files changed, 13 insertions(+) create mode 100644 towncrier/newsfragments/224.feature diff --git a/core/admin/mailu/ui/templates/domain/details.html b/core/admin/mailu/ui/templates/domain/details.html index 65c6ec1a..5822ffbc 100644 --- a/core/admin/mailu/ui/templates/domain/details.html +++ b/core/admin/mailu/ui/templates/domain/details.html @@ -50,5 +50,17 @@ 
_dmarc.{{ domain.name }}. 600 IN TXT "v=DMARC1; p=reject;{% if config["DMARC_RUA"] %} rua=mailto:{{ config["DMARC_RUA"] }}@{{ config["DOMAIN"] }};{% endif %}{% if config["DMARC_RUF"] %} ruf=mailto:{{ config["DMARC_RUF"] }}@{{ config["DOMAIN"] }};{% endif %} adkim=s; aspf=s"
{% endif %} + + {% trans %}DNS client auto-configuration (RFC6186) entries{% endtrans %} + + 
_submission._tcp.{{ domain.name }}. 600 IN SRV 1 1 587 {{ config["HOSTNAMES"].split(',')[0] }}.
+ 
_imap._tcp.{{ domain.name }}. 600 IN SRV 100 1 143 {{ config["HOSTNAMES"].split(',')[0] }}.
+ 
_pop3._tcp.{{ domain.name }}. 600 IN SRV 100 1 110 {{ config["HOSTNAMES"].split(',')[0] }}.
+{% if config["TLS_FLAVOR"] != "notls" %} + 
_submissions._tcp.{{ domain.name }}. 600 IN SRV 10 1 465 {{ config["HOSTNAMES"].split(',')[0] }}.
+ 
_imaps._tcp.{{ domain.name }}. 600 IN SRV 10 1 993 {{ config["HOSTNAMES"].split(',')[0] }}.
+ 
_pop3s._tcp.{{ domain.name }}. 600 IN SRV 10 1 995 {{ config["HOSTNAMES"].split(',')[0] }}.
+{% endif %} + {% endcall %} {% endblock %} diff --git a/towncrier/newsfragments/224.feature b/towncrier/newsfragments/224.feature new file mode 100644 index 00000000..9a2f479b --- /dev/null +++ b/towncrier/newsfragments/224.feature @@ -0,0 +1 @@ +Add instructions on how to create DNS records for email client auto-configuration (RFC6186 style) From 80f939cf1ac3fe9863825de1360ee7c5d67926f6 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 8 Feb 2021 10:16:03 +0100 Subject: [PATCH 13/65] Revert to the old behaviour when ADMIN=false --- core/nginx/conf/nginx.conf | 10 +++++++--- webmails/rainloop/application.ini | 4 +++- webmails/roundcube/config.inc.php | 10 ++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/core/nginx/conf/nginx.conf b/core/nginx/conf/nginx.conf index a7a9e134..81f1ac0d 100644 --- a/core/nginx/conf/nginx.conf +++ b/core/nginx/conf/nginx.conf @@ -133,10 +133,12 @@ http { {% endif %} include /etc/nginx/proxy.conf; client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; - auth_request /internal/auth/user; proxy_pass http://$webmail; + {% if ADMIN == 'true' %} + auth_request /internal/auth/user; error_page 403 @webmail_login; } + location {{ WEB_WEBMAIL }}/sso.php { {% if WEB_WEBMAIL != '/' %} rewrite ^({{ WEB_WEBMAIL }})$ $1/ permanent; @@ -152,11 +154,13 @@ http { proxy_pass http://$webmail; error_page 403 @webmail_login; } + location @webmail_login { return 302 {{ WEB_ADMIN }}/ui/login?next=ui.webmail; } - {% endif %} - + {% else %} + } + {% endif %}{% endif %} {% if ADMIN == 'true' %} location {{ WEB_ADMIN }} { return 301 {{ WEB_ADMIN }}/ui; diff --git a/webmails/rainloop/application.ini b/webmails/rainloop/application.ini index bc953af1..0504f174 100644 --- a/webmails/rainloop/application.ini +++ b/webmails/rainloop/application.ini @@ -7,9 +7,11 @@ attachment_size_limit = {{ MAX_FILESIZE }} allow_admin_panel = Off [labs] +allow_gravatar = Off +{% if ADMIN == "true" %} custom_login_link='sso.php' custom_logout_link='{{ WEB_ADMIN }}/ui/logout' -allow_gravatar = Off +{% endif %} [contacts] enable = On diff --git a/webmails/roundcube/config.inc.php b/webmails/roundcube/config.inc.php index bb1a5e84..d8028db3 100644 --- a/webmails/roundcube/config.inc.php +++ b/webmails/roundcube/config.inc.php @@ -17,8 +17,7 @@ $config['plugins'] = array( 'markasjunk', 'managesieve', 'enigma', - 'carddav', - 'mailu' + 'carddav' ); $front = getenv('FRONT_ADDRESS') ? getenv('FRONT_ADDRESS') : 'front'; @@ -37,8 +36,11 @@ $config['managesieve_host'] = $imap; $config['managesieve_usetls'] = false; // Customization settings -$config['support_url'] = getenv('WEB_ADMIN') ? '../..' . getenv('WEB_ADMIN') : ''; -$config['sso_logout_url'] = getenv('WEB_ADMIN').'/ui/logout'; +if (filter_var(getenv('ADMIN'), FILTER_VALIDATE_BOOLEAN, FILTER_NULL_ON_FAILURE)) { + array_push($config['plugins'], 'mailu'); + $config['support_url'] = getenv('WEB_ADMIN') ? '../..' . getenv('WEB_ADMIN') : ''; + $config['sso_logout_url'] = getenv('WEB_ADMIN').'/ui/logout'; +} $config['product_name'] = 'Mailu Webmail'; // We access the IMAP and SMTP servers locally with internal names, SSL From 0917a6817f845c01a2247d74f111abc2ef54e51a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 8 Feb 2021 10:17:43 +0100 Subject: [PATCH 14/65] Set ADMIN=false to ensure that the tests pass --- tests/compose/rainloop/mailu.env | 2 +- tests/compose/roundcube/mailu.env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compose/rainloop/mailu.env b/tests/compose/rainloop/mailu.env index 9c31c8bb..47b0b934 100644 --- a/tests/compose/rainloop/mailu.env +++ b/tests/compose/rainloop/mailu.env @@ -51,7 +51,7 @@ DISABLE_STATISTICS=False ################################### # Expose the admin interface (value: true, false) -ADMIN=true +ADMIN=false # Choose which webmail to run if any (values: roundcube, rainloop, none) WEBMAIL=rainloop diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/roundcube/mailu.env index dc503268..887cebbd 100644 --- a/tests/compose/roundcube/mailu.env +++ b/tests/compose/roundcube/mailu.env @@ -51,7 +51,7 @@ DISABLE_STATISTICS=False ################################### # Expose the admin interface (value: true, false) -ADMIN=true +ADMIN=false # Choose which webmail to run if any (values: roundcube, rainloop, none) WEBMAIL=roundcube From e8f70c12dcdce1064ea063cc5ddcae28a5221613 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 8 Feb 2021 10:22:25 +0100 Subject: [PATCH 15/65] avoid a warning --- webmails/roundcube/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 9ce383c8..45b2aa76 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -40,7 +40,7 @@ os.system("mkdir -p /data/gpg /var/www/html/logs") os.system("touch /var/www/html/logs/errors.log") os.system("chown -R www-data:www-data /var/www/html/logs") os.system("chmod -R a+rX /var/www/html/") -os.system("ln -s /var/www/html/index.php /var/www/html/sso.php") +os.system("ln -sf /var/www/html/index.php /var/www/html/sso.php") try: print("Initializing database") From 6c4fa5432f3f040cc16fca870f0382939ae2817b Mon Sep 17 00:00:00 2001 From: Nils Vogels Date: Thu, 11 Feb 2021 12:03:07 +0100 Subject: [PATCH 16/65] Provide fix in postgresql container for CVE-2021-23240, CVE-2021-3156, CVE-2021-23239 --- optional/postgresql/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/optional/postgresql/Dockerfile b/optional/postgresql/Dockerfile index 95048147..ff25a66f 100644 --- a/optional/postgresql/Dockerfile +++ b/optional/postgresql/Dockerfile @@ -3,6 +3,7 @@ FROM $DISTRO # python3 shared with most images RUN apk add --no-cache \ python3 py3-pip bash py3-multidict \ + && apk add --upgrade sudo \ && pip3 install --upgrade pip # Shared layer between nginx, dovecot, postfix, postgresql, rspamd, unbound, rainloop, roundcube From 3b7ecb3a8b4c466ad2f37276107bb49efe978758 Mon Sep 17 00:00:00 2001 From: Nils Vogels Date: Thu, 11 Feb 2021 12:12:06 +0100 Subject: [PATCH 17/65] Add changelog --- towncrier/newsfragments/1760.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 towncrier/newsfragments/1760.bugfix diff --git a/towncrier/newsfragments/1760.bugfix b/towncrier/newsfragments/1760.bugfix new file mode 100644 index 00000000..9d6f38af --- /dev/null +++ b/towncrier/newsfragments/1760.bugfix @@ -0,0 +1,2 @@ +Fix CVE-2021-23240, CVE-2021-3156 and CVE-2021-23239 for postgresql +by force-upgrading sudo. From 1a365f469c85adfe125f04a4b29d6cdabe436709 Mon Sep 17 00:00:00 2001 From: lub Date: Fri, 12 Feb 2021 12:18:22 +0100 Subject: [PATCH 18/65] check for `ipv6_enabled` in the compose template Checking only `ipv6` isn't sufficient, because it has a default value. --- setup/flavors/compose/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 3fbfb862..155e1180 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -26,7 +26,7 @@ services: {% if bind4 %} - "{{ bind4 }}:{{ port }}:{{ port }}" {% endif %} - {% if bind6 %} + {% if ipv6_enabled and bind6 %} - "{{ bind6 }}:{{ port }}:{{ port }}" {% endif %} {% endfor %} From b55b53b781350addad83fe93ac68c5c6c7188847 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 26 Feb 2021 20:51:58 +0100 Subject: [PATCH 19/65] optimize generation of transport nexthop --- core/admin/mailu/internal/views/postfix.py | 48 +++++++++++++++++++--- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index a5507830..7f8418cf 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -2,6 +2,7 @@ from mailu import models from mailu.internal import internal import flask +import idna import re import srslib @@ -37,11 +38,48 @@ def postfix_transport(email): return flask.abort(404) localpart, domain_name = models.Email.resolve_domain(email) relay = models.Relay.query.get(domain_name) or flask.abort(404) - ret = "smtp:[{0}]".format(relay.smtp) - if ":" in relay.smtp: - split = relay.smtp.split(':') - ret = "smtp:[{0}]:{1}".format(split[0], split[1]) - return flask.jsonify(ret) + target = relay.smtp.lower() + port = None + if use_mx := target.startswith('mx:'): + target = target[3:] + if target.startswith('['): + if use_mx or ']' not in target: + # invalid target (mx: and []) + flask.abort(400) + host, rest = target[1:].split(']', 1) + if rest.startswith(':'): + port = rest[1:] + elif rest: + # invalid target (rest should be :port) + flask.abort(400) + else: + if ':' in target: + host, port = target.rsplit(':', 1) + else: + host = target + if not host: + host = relay.name.lower() + use_mx = True + if ':' in host: + host = f'ipv6:{host}' + else: + try: + host = idna.encode(host).decode('ascii') + except idna.IDNAError: + # invalid target (fqdn not encodable) + flask.abort(400) + if port is not None: + try: + port = int(port, 10) + if port == 25: + port = None + except ValueError: + # invalid target (port should be numeric) + flask.abort(400) + if not use_mx: + host = f'[{host}]' + port = '' if port is None else f':{port}' + return flask.jsonify(f'smtp:{host}{port}') @internal.route("/postfix/recipient/map/") From 75baa1da993363f1fe6bef10758956502369befd Mon Sep 17 00:00:00 2001 From: ronivay Date: Thu, 18 Mar 2021 09:46:27 +0200 Subject: [PATCH 20/65] Update fail2ban documentation --- docs/faq.rst | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 59a841dc..14dcc4ed 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -528,25 +528,42 @@ The above will block flagged IPs for a week, you can of course change it to you actionstart = iptables -N f2b-bad-auth iptables -A f2b-bad-auth -j RETURN - iptables -I FORWARD -p tcp -m multiport --dports 1:1024 -j f2b-bad-auth + iptables -I DOCKER-USER -p tcp -m multiport --dports 1:1024 -j f2b-bad-auth - actionstop = iptables -D FORWARD -p tcp -m multiport --dports 1:1024 -j f2b-bad-auth + actionstop = iptables -D DOCKER-USER -p tcp -m multiport --dports 1:1024 -j f2b-bad-auth iptables -F f2b-bad-auth iptables -X f2b-bad-auth - actioncheck = iptables -n -L FORWARD | grep -q 'f2b-bad-auth[ \t]' + actioncheck = iptables -n -L DOCKER-USER | grep -q 'f2b-bad-auth[ \t]' actionban = iptables -I f2b-bad-auth 1 -s -j DROP actionunban = iptables -D f2b-bad-auth -s -j DROP -5. Restart Fail2Ban +Using DOCKER-USER chain ensures that blocked IPs are processed in correct order with Docker. See more in: https://docs.docker.com/network/iptables/ + +5. Configure and restart Fail2Ban service + +Make sure Fail2Ban is started after Docker service by adding partial override which appends this to existing configuration.. + +.. code-block:: bash + + sudo systemctl edit fail2ban + +Add override and save file. + +.. code-block:: bash + + [Unit] + After=docker.service + +Restart service. .. code-block:: bash sudo systemctl restart fail2ban -*Issue reference:* `85`_, `116`_, `171`_, `584`_, `592`_. +*Issue reference:* `85`_, `116`_, `171`_, `584`_, `592`_, `1727`_. Users can't change their password from webmail `````````````````````````````````````````````` @@ -670,7 +687,7 @@ iptables -t nat -A POSTROUTING -o eth0 -p tcp --dport 25 -j SNAT --to Date: Fri, 19 Mar 2021 09:59:16 +0100 Subject: [PATCH 21/65] Update faq.rst Some spelling improvements. --- docs/faq.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 14dcc4ed..9c4f1d75 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -540,24 +540,24 @@ The above will block flagged IPs for a week, you can of course change it to you actionunban = iptables -D f2b-bad-auth -s -j DROP -Using DOCKER-USER chain ensures that blocked IPs are processed in correct order with Docker. See more in: https://docs.docker.com/network/iptables/ +Using DOCKER-USER chain ensures that the blocked IPs are processed in the correct order with Docker. See more in: https://docs.docker.com/network/iptables/ -5. Configure and restart Fail2Ban service +5. Configure and restart the Fail2Ban service -Make sure Fail2Ban is started after Docker service by adding partial override which appends this to existing configuration.. +Make sure Fail2Ban is started after the Docker service by adding a partial override which appends this to the existing configuration. .. code-block:: bash sudo systemctl edit fail2ban -Add override and save file. +Add the override and save the file. .. code-block:: bash [Unit] After=docker.service -Restart service. +Restart the Fail2Ban service. .. code-block:: bash From c6d0ef229fe2d06fee304b704e1c2bb1857f8067 Mon Sep 17 00:00:00 2001 From: Vincent Kling Date: Fri, 19 Mar 2021 10:46:42 +0100 Subject: [PATCH 22/65] Update messages.po --- core/admin/mailu/translations/nl/LC_MESSAGES/messages.po | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/translations/nl/LC_MESSAGES/messages.po b/core/admin/mailu/translations/nl/LC_MESSAGES/messages.po index 8d7b5054..c7f6db5e 100644 --- a/core/admin/mailu/translations/nl/LC_MESSAGES/messages.po +++ b/core/admin/mailu/translations/nl/LC_MESSAGES/messages.po @@ -528,7 +528,7 @@ msgstr "Alternatieve naam" #: mailu/ui/forms.py:70 msgid "Relayed domain name" -msgstr "Relayed domainnaam" +msgstr "Relayed domeinnaam" #: mailu/ui/forms.py:71 mailu/ui/templates/relay/list.html:18 msgid "Remote host" @@ -536,7 +536,7 @@ msgstr "Externe host" #: mailu/ui/templates/sidebar.html:54 msgid "Relayed domains" -msgstr "Relayed domainen" +msgstr "Relayed domeinen" #: mailu/ui/templates/alternative/create.html:4 msgid "Create alternative domain" From 1f2aee278c1a2dc1ab0d232433f448264a29ba9f Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Thu, 13 May 2021 18:59:18 +0200 Subject: [PATCH 23/65] Reflect override settings for postfix Also added a stumbling stone when changing postfix.cf --- docs/faq.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 9c4f1d75..5b13f191 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -258,9 +258,11 @@ Postfix, Dovecot, Nginx and Rspamd support overriding configuration files. Overr correct syntax. The following file names will be taken as override configuration: - `Postfix`_ : - - ``main.cf`` as ``$ROOT/overrides/postfix/postfix.cf`` - - ``master.cf`` as ``$ROOT/overrides/postfix/postfix.master`` + - ``main.cf`` as ``$ROOT/overrides/postfix.cf`` + - ``master.cf`` as ``$ROOT/overrides/postfix.master`` - All ``$ROOT/overrides/postfix/*.map`` files + - For both ``postfix.cf`` and ``postfix.master``, you need to put one configuration per line, as they are fed line-by-line + to postfix. - `Dovecot`_ - ``dovecot.conf`` in dovecot sub-directory; - `Nginx`_ - All ``*.conf`` files in the ``nginx`` sub-directory; - `Rspamd`_ - All files in the ``rspamd`` sub-directory. From ae9206e968b10de6c456a76f914f8d86dff02f6f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 10 Feb 2021 13:51:07 +0100 Subject: [PATCH 24/65] Implement a simple credential cache --- core/admin/mailu/models.py | 23 +++++++++++++++++++++++ towncrier/newsfragments/1194.feature | 1 + 2 files changed, 24 insertions(+) create mode 100644 towncrier/newsfragments/1194.feature diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a63c33a5..c7787e74 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -305,6 +305,7 @@ class User(Base, Email): """ __tablename__ = "user" _ctx = None + _credential_cache = {} domain = db.relationship(Domain, backref=db.backref('users', cascade='all, delete-orphan')) @@ -382,6 +383,17 @@ class User(Base, Email): return User._ctx def check_password(self, password): + cache_result = self._credential_cache.get(self.get_id()) + current_salt = self.password.split('$')[3] if len(self.password.split('$')) == 5 else None + if cache_result and current_salt: + cache_salt, cache_hash = cache_result + if cache_salt == current_salt: + return hash.pbkdf2_sha256.verify(password, cache_hash) + else: + # the cache is local per gunicorn; the password has changed + # so the local cache can be invalidated + del self._credential_cache[self.get_id()] + reference = self.password # strip {scheme} if that's something mailu has added # passlib will identify *crypt based hashes just fine @@ -396,6 +408,17 @@ class User(Base, Email): self.password = new_hash db.session.add(self) db.session.commit() + + if result: + """The credential cache uses a low number of rounds to be fast. +While it's not meant to be persisted to cold-storage, no additional measures +are taken to ensure it isn't (mlock(), encrypted swap, ...) on the basis that +we have little control over GC and string interning anyways. + + An attacker that can dump the process' memory is likely to find credentials +in clear-text regardless of the presence of the cache. + """ + self._credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) return result def set_password(self, password, hash_scheme=None, raw=False): diff --git a/towncrier/newsfragments/1194.feature b/towncrier/newsfragments/1194.feature new file mode 100644 index 00000000..0cd2a9e9 --- /dev/null +++ b/towncrier/newsfragments/1194.feature @@ -0,0 +1 @@ +Add a credential cache to speedup authentication requests. From f52984e4c337e5a101aea82a0654fd731ab94164 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 10 Feb 2021 16:10:10 +0100 Subject: [PATCH 25/65] In fact it could be global --- core/admin/mailu/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index c7787e74..a5ee1f57 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -17,7 +17,7 @@ import dns db = flask_sqlalchemy.SQLAlchemy() - +_credential_cache = {} class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) @@ -383,7 +383,7 @@ class User(Base, Email): return User._ctx def check_password(self, password): - cache_result = self._credential_cache.get(self.get_id()) + cache_result = _credential_cache.get(self.get_id()) current_salt = self.password.split('$')[3] if len(self.password.split('$')) == 5 else None if cache_result and current_salt: cache_salt, cache_hash = cache_result @@ -392,7 +392,7 @@ class User(Base, Email): else: # the cache is local per gunicorn; the password has changed # so the local cache can be invalidated - del self._credential_cache[self.get_id()] + del _credential_cache[self.get_id()] reference = self.password # strip {scheme} if that's something mailu has added @@ -418,7 +418,7 @@ we have little control over GC and string interning anyways. An attacker that can dump the process' memory is likely to find credentials in clear-text regardless of the presence of the cache. """ - self._credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) + _credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) return result def set_password(self, password, hash_scheme=None, raw=False): From 875308d40518dde4000a4f09286059e07f36de71 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 4 Jun 2021 09:51:58 +0200 Subject: [PATCH 26/65] Revert "In fact it could be global" This reverts commit f52984e4c337e5a101aea82a0654fd731ab94164. --- core/admin/mailu/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index a5ee1f57..c7787e74 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -17,7 +17,7 @@ import dns db = flask_sqlalchemy.SQLAlchemy() -_credential_cache = {} + class IdnaDomain(db.TypeDecorator): """ Stores a Unicode string in it's IDNA representation (ASCII only) @@ -383,7 +383,7 @@ class User(Base, Email): return User._ctx def check_password(self, password): - cache_result = _credential_cache.get(self.get_id()) + cache_result = self._credential_cache.get(self.get_id()) current_salt = self.password.split('$')[3] if len(self.password.split('$')) == 5 else None if cache_result and current_salt: cache_salt, cache_hash = cache_result @@ -392,7 +392,7 @@ class User(Base, Email): else: # the cache is local per gunicorn; the password has changed # so the local cache can be invalidated - del _credential_cache[self.get_id()] + del self._credential_cache[self.get_id()] reference = self.password # strip {scheme} if that's something mailu has added @@ -418,7 +418,7 @@ we have little control over GC and string interning anyways. An attacker that can dump the process' memory is likely to find credentials in clear-text regardless of the presence of the cache. """ - _credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) + self._credential_cache[self.get_id()] = (self.password.split('$')[3], hash.pbkdf2_sha256.using(rounds=1).hash(password)) return result def set_password(self, password, hash_scheme=None, raw=False): From ffa75620799669f7fe23aeaa75e4b90d55c1fe95 Mon Sep 17 00:00:00 2001 From: Linus Gasser Date: Mon, 7 Jun 2021 07:57:30 +0200 Subject: [PATCH 27/65] configurations changed place in 1.8 --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 5b13f191..e42bf309 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -258,8 +258,8 @@ Postfix, Dovecot, Nginx and Rspamd support overriding configuration files. Overr correct syntax. The following file names will be taken as override configuration: - `Postfix`_ : - - ``main.cf`` as ``$ROOT/overrides/postfix.cf`` - - ``master.cf`` as ``$ROOT/overrides/postfix.master`` + - ``main.cf`` as ``$ROOT/overrides/postfix/postfix.cf`` + - ``master.cf`` as ``$ROOT/overrides/postfix/postfix.master`` - All ``$ROOT/overrides/postfix/*.map`` files - For both ``postfix.cf`` and ``postfix.master``, you need to put one configuration per line, as they are fed line-by-line to postfix. From 2316ef1162d400fc486ad72df62fbd1fbd46afce Mon Sep 17 00:00:00 2001 From: lub Date: Wed, 16 Jun 2021 14:21:55 +0200 Subject: [PATCH 28/65] update compression algorithms for dovecot 3.3.14 xz is deprecated; lz4 and zstd were not present in our configs before --- core/dovecot/conf/dovecot.conf | 2 +- setup/flavors/compose/mailu.env | 2 +- tests/compose/core/mailu.env | 2 +- tests/compose/fetchmail/mailu.env | 2 +- tests/compose/filters/mailu.env | 2 +- tests/compose/rainloop/mailu.env | 2 +- tests/compose/roundcube/mailu.env | 2 +- tests/compose/webdav/mailu.env | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index ab5cb43a..50657088 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -50,7 +50,7 @@ plugin { fts_autoindex_exclude = \Trash {% endif %} - {% if COMPRESSION in [ 'gz', 'bz2', 'xz', 'lz4' ] %} + {% if COMPRESSION in [ 'gz', 'bz2', 'lz4', 'zstd' ] %} zlib_save = {{ COMPRESSION }} {% endif %} diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 04148b40..0aabf478 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -86,7 +86,7 @@ WELCOME_SUBJECT={{ welcome_subject or 'Welcome to your new email account' }} WELCOME_BODY={{ welcome_body or 'Welcome to your new email account, if you can read this, then it is configured properly!' }} # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION={{ compression }} # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL={{ compression_level }} diff --git a/tests/compose/core/mailu.env b/tests/compose/core/mailu.env index edea6a5c..a78515b8 100644 --- a/tests/compose/core/mailu.env +++ b/tests/compose/core/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/fetchmail/mailu.env b/tests/compose/fetchmail/mailu.env index 4a53ec46..afb57751 100644 --- a/tests/compose/fetchmail/mailu.env +++ b/tests/compose/fetchmail/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/filters/mailu.env b/tests/compose/filters/mailu.env index 1106deb0..4c8c219d 100644 --- a/tests/compose/filters/mailu.env +++ b/tests/compose/filters/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/rainloop/mailu.env b/tests/compose/rainloop/mailu.env index d02b98f2..08b0f8a4 100644 --- a/tests/compose/rainloop/mailu.env +++ b/tests/compose/rainloop/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/roundcube/mailu.env index e1005487..faf1198f 100644 --- a/tests/compose/roundcube/mailu.env +++ b/tests/compose/roundcube/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= diff --git a/tests/compose/webdav/mailu.env b/tests/compose/webdav/mailu.env index 58b9810a..939f453b 100644 --- a/tests/compose/webdav/mailu.env +++ b/tests/compose/webdav/mailu.env @@ -92,7 +92,7 @@ DMARC_RUF=admin # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= From 40ad3ca0325b6920128624665bf0164670f6e31d Mon Sep 17 00:00:00 2001 From: lub Date: Wed, 16 Jun 2021 14:56:53 +0200 Subject: [PATCH 29/65] only load zlib when compression is used --- core/dovecot/conf/dovecot.conf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/dovecot/conf/dovecot.conf b/core/dovecot/conf/dovecot.conf index 50657088..6b97a086 100644 --- a/core/dovecot/conf/dovecot.conf +++ b/core/dovecot/conf/dovecot.conf @@ -21,7 +21,10 @@ mail_access_groups = mail maildir_stat_dirs = yes mailbox_list_index = yes mail_vsize_bg_after_count = 100 -mail_plugins = $mail_plugins quota quota_clone zlib{{ ' ' }} +mail_plugins = $mail_plugins quota quota_clone{{ ' ' }} + {%- if COMPRESSION -%} + zlib{{ ' ' }} + {%- endif %} {%- if (FULL_TEXT_SEARCH or '').lower() not in ['off', 'false', '0'] -%} fts fts_xapian {%- endif %} From 18f5a2fc11191168c436e862f8f060ba4aeed9c0 Mon Sep 17 00:00:00 2001 From: lub Date: Wed, 16 Jun 2021 15:01:55 +0200 Subject: [PATCH 30/65] update newsfragment #1694 --- towncrier/newsfragments/1694.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/towncrier/newsfragments/1694.feature b/towncrier/newsfragments/1694.feature index 41548707..f7e2013e 100644 --- a/towncrier/newsfragments/1694.feature +++ b/towncrier/newsfragments/1694.feature @@ -1 +1 @@ -Support configuring xz and lz4 compression for dovecot. +Support configuring lz4 and zstd compression for dovecot. From 587901ca51462bf32a4d6df314667a0081a36378 Mon Sep 17 00:00:00 2001 From: lub Date: Wed, 16 Jun 2021 15:03:09 +0200 Subject: [PATCH 31/65] fix comment in compose .env --- docs/compose/.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/compose/.env b/docs/compose/.env index b4a8b218..27822c37 100644 --- a/docs/compose/.env +++ b/docs/compose/.env @@ -97,7 +97,7 @@ WELCOME_SUBJECT=Welcome to your new email account WELCOME_BODY=Welcome to your new email account, if you can read this, then it is configured properly! # Maildir Compression -# choose compression-method, default: none (value: gz, bz2, xz, lz4) +# choose compression-method, default: none (value: gz, bz2, lz4, zstd) COMPRESSION= # change compression-level, default: 6 (value: 1-9) COMPRESSION_LEVEL= From a1fd44fced2c65a1c379b445fb65f2a5a24efd24 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 16 Jun 2021 16:19:31 +0200 Subject: [PATCH 32/65] added lmtp: prefix and documentation --- core/admin/mailu/internal/views/postfix.py | 39 ++++-- docs/webadministration.rst | 139 +++++++++++---------- 2 files changed, 101 insertions(+), 77 deletions(-) diff --git a/core/admin/mailu/internal/views/postfix.py b/core/admin/mailu/internal/views/postfix.py index 7f8418cf..c358c37f 100644 --- a/core/admin/mailu/internal/views/postfix.py +++ b/core/admin/mailu/internal/views/postfix.py @@ -36,15 +36,23 @@ def postfix_alias_map(alias): def postfix_transport(email): if email == '*' or re.match("(^|.*@)\[.*\]$", email): return flask.abort(404) - localpart, domain_name = models.Email.resolve_domain(email) + _, domain_name = models.Email.resolve_domain(email) relay = models.Relay.query.get(domain_name) or flask.abort(404) target = relay.smtp.lower() port = None - if use_mx := target.startswith('mx:'): + use_lmtp = False + use_mx = False + # strip prefixes mx: and lmtp: + if target.startswith('mx:'): target = target[3:] + use_mx = True + elif target.startswith('lmtp:'): + target = target[5:] + use_lmtp = True + # split host:port or [host]:port if target.startswith('['): if use_mx or ']' not in target: - # invalid target (mx: and []) + # invalid target (mx: and [] or missing ]) flask.abort(400) host, rest = target[1:].split(']', 1) if rest.startswith(':'): @@ -57,29 +65,38 @@ def postfix_transport(email): host, port = target.rsplit(':', 1) else: host = target + # default for empty host part is mx:domain if not host: - host = relay.name.lower() - use_mx = True + if not use_lmtp: + host = relay.name.lower() + use_mx = True + else: + # lmtp: needs a host part + flask.abort(400) + # detect ipv6 address or encode host if ':' in host: host = f'ipv6:{host}' else: try: host = idna.encode(host).decode('ascii') except idna.IDNAError: - # invalid target (fqdn not encodable) + # invalid host (fqdn not encodable) flask.abort(400) + # validate port if port is not None: try: port = int(port, 10) - if port == 25: - port = None except ValueError: - # invalid target (port should be numeric) + # invalid port (should be numeric) flask.abort(400) - if not use_mx: + # create transport + transport = 'lmtp' if use_lmtp else 'smtp' + # use [] when not using MX lookups or host is an ipv6 address + if host.startswith('ipv6:') or (not use_lmtp and not use_mx): host = f'[{host}]' + # create port suffix port = '' if port is None else f':{port}' - return flask.jsonify(f'smtp:{host}{port}') + return flask.jsonify(f'{transport}:{host}{port}') @internal.route("/postfix/recipient/map/") diff --git a/docs/webadministration.rst b/docs/webadministration.rst index 070eb473..86ce41c0 100644 --- a/docs/webadministration.rst +++ b/docs/webadministration.rst @@ -1,7 +1,7 @@ Web administration interface ============================ -The web administration interface is the main website for maintaining your Mailu installation. +The web administration interface is the main website for maintaining your Mailu installation. For brevity the web administration interface will now be mentioned as admin gui. It offers the following configuration options: @@ -30,13 +30,13 @@ It offers the following configuration options: * Configure all email domains served by Mailu, including: * generating dkim and dmarc keys for a domain. - + * view email domain information on how to configure your SPF, DMARC, DKIM and MX dns records for an email domain. - + * Add new email domains. - + * For existing domains, configure users, quotas, aliases, administrators and alternative domain names. - + * access the webmail site. * lookup settings for configuring your email client. @@ -49,7 +49,7 @@ The admin GUI is by default accessed via the URL `https:///admin`, wh To login the admin GUI enter the email address and password of an user. Only global administrator users have access to all configuration settings and the Rspamd webgui. Other users will be presented with settings for only their account, and domains they are managers of. -To create a user who is a global administrator for a new installation, the Mailu.env file can be adapted. +To create a user who is a global administrator for a new installation, the Mailu.env file can be adapted. For more information see the section 'Admin account - automatic creation' in :ref:`the configuration reference `. The following sections are only accessible for global administrators: @@ -69,7 +69,7 @@ The following sections are only accessible for global administrators: Settings -------- -After logging in the web administration interface, the settings page is loaded. +After logging in the web administration interface, the settings page is loaded. On the settings page the settings of the currently logged in user can be changed. Changes are saved and effective immediately after clicking the Save Settings button at the bottom of the page. @@ -77,27 +77,27 @@ Changes are saved and effective immediately after clicking the Save Settings but Display name ```````````` -On the settings page the displayed name can be changed of the logged in user. +On the settings page the displayed name can be changed of the logged in user. This display name is only used within the web administration interface. Antispam ```````` -Under the section `Antispam` the spam filter can be enabled or disabled for the logged in user. By default the spam filter is enabled. +Under the section `Antispam` the spam filter can be enabled or disabled for the logged in user. By default the spam filter is enabled. When the spam filter is disabled, all received email messages will go to the inbox folder of the logged in user. The exception to this rule, are email messages with an extremely high spam score. These email messages are always rejected by Rspamd. When the spam filter is enabled, received email messages will be moved to the logged in user's inbox folder or junk folder depending on the user defined spam filter tolerance. -The user defined spam filter tolerance determines when an email is classified as ham (moved to the inbox folder) or spam (moved to the junk folder). -The default value is 80%. The lower the spam filter tolerance, the more false positives (ham classified as spam). The higher the spam filter tolerance, the more false negatives (spam classified as ham). +The user defined spam filter tolerance determines when an email is classified as ham (moved to the inbox folder) or spam (moved to the junk folder). +The default value is 80%. The lower the spam filter tolerance, the more false positives (ham classified as spam). The higher the spam filter tolerance, the more false negatives (spam classified as ham). For more information see the :ref:`antispam documentation `. Auto-forward ````````````` -Under the section `Auto-forward`, the automatic forwarding of received email messages can be enabled. When enabled, all received email messages are forwarded to the specified email address. +Under the section `Auto-forward`, the automatic forwarding of received email messages can be enabled. When enabled, all received email messages are forwarded to the specified email address. The option "Keep a copy of the emails" can be ticked, to keep a copy of the received email message in the inbox folder. @@ -107,7 +107,7 @@ In the destination textbox, the email addresses can be entered for automatic for Update password --------------- -On the `update password` page, the password of the logged in user can be changed. Changes are effective immediately. +On the `update password` page, the password of the logged in user can be changed. Changes are effective immediately. .. _webadministration_auto-reply: @@ -117,7 +117,7 @@ Auto-reply On the `auto-reply` page, automatic replies can be configured. This is also known as out of office (ooo) or out of facility (oof) replies. -To enable automatic replies tick the checkbox 'Enable automatic reply'. +To enable automatic replies tick the checkbox 'Enable automatic reply'. Under Reply subject the email subject for automatic replies can be configured. When a reply subject is entered, this subject will be used for the automatic reply. @@ -130,12 +130,12 @@ E.g. if the email subject of the received email message is "how are you?", then Fetched accounts ---------------- -This page is only available when the Fetchmail container is part of your Mailu deployment. +This page is only available when the Fetchmail container is part of your Mailu deployment. Fetchmail can be enabled when creating the docker-compose.yml file with the setup utility (https://setup.mailu.io). On the `fetched accounts` page you can configure email accounts from which email messages will be retrieved. -Only unread email messages are retrieved from the specified email account. -By default Fetchmail will retrieve email messages every 10 minutes. This can be changed in the Mailu.env file. +Only unread email messages are retrieved from the specified email account. +By default Fetchmail will retrieve email messages every 10 minutes. This can be changed in the Mailu.env file. For more information on changing the polling interval see :ref:`the configuration reference `. @@ -149,7 +149,7 @@ You can add a fetched account by clicking on the `Add an account` button on the * Enable TLS. Tick this setting if the email server requires TLS/SSL instead of STARTTLS. -* Username. The user name for logging in to the email server. Normally this is the email address or the email address' local-part (the part before @). +* Username. The user name for logging in to the email server. Normally this is the email address or the email address' local-part (the part before @). * Password. The password for logging in to the email server. @@ -166,8 +166,8 @@ The purpose of an authentication token is to create a unique and strong password The application will use this authentication token instead of the logged in user's password for sending/receiving email. This allows safe access to the logged in user's email account. At any moment, the authentication token can be deleted so that the application has no access to the logged in user's email account anymore. -By clicking on the New token button on the top right of the page, a new authentication token can be created. On this page the generated authentication token will only be displayed once. -After saving the application token it is not possible anymore to view the unique password. +By clicking on the New token button on the top right of the page, a new authentication token can be created. On this page the generated authentication token will only be displayed once. +After saving the application token it is not possible anymore to view the unique password. The comment field can be used to enter a description for the authentication token. For example the name of the application the application token is created for. @@ -198,9 +198,9 @@ A global administrator can change `any setting` in the admin GUI. Be careful tha Relayed domains --------------- -On the `relayed domains list` page, destination domains can be added that Mailu will relay email messages for without authentication. -This means that for these destination domains, other email clients or email servers can send email via Mailu unauthenticated via port 25 to this destination domain. -For example if the destination domain example.com is added. Any emails to example.com (john@example.com) will be relayed to example.com. +On the `relayed domains list` page, destination domains can be added that Mailu will relay email messages for without authentication. +This means that for these destination domains, other email clients or email servers can send email via Mailu unauthenticated via port 25 to this destination domain. +For example if the destination domain example.com is added. Any emails to example.com (john@example.com) will be relayed to example.com. Example scenario's are: * relay domain from a backup server. @@ -212,30 +212,37 @@ Example scenario's are: On the new relayed domain page the following options can be entered for a new relayed domain: -* Relayed domain name. The domain name that is relayed. Email messages addressed to this domain (To: John@example.com), will be forwarded to this domain. - No authentication is required. +* Relayed domain name. The domain name that is relayed. Email messages addressed to this domain (To: John@example.com), will be forwarded to this domain. + No authentication is required. -* Remote host (optional). The SMPT server that will be used for relaying the email message. - When this field is blank, the Mailu server will directly send the email message to the relayed domain. - As value can be entered either a hostname or IP address of the SMPT server. - By default port 25 is used. To use a different port append ":port number" to the Remote Host. For example: - 123.45.67.90:2525. +* Remote host (optional). The host that will be used for relaying the email message. + When this field is blank, the Mailu server will directly send the email message to the mail server of the relayed domain. + When a remote host is specified it can be prefixed by ``mx:`` or ``lmtp:`` and followed by a port number: ``:port``). + + ================ ===================================== ========================= + Remote host Description postfix transport:nexthop + ================ ===================================== ========================= + empty use MX of relay domain smtp:domain + :port use MX of relay domain and use port smtp:domain:port + target resolve A/AAAA of target smtp:[target] + target:port resolve A/AAAA of target and use port smtp:[target]:port + mx:target resolve MX of target smtp:target + mx:target:port resolve MX of target and use port smtp:target:port + lmtp:target resolve A/AAAA of target lmtp:target + lmtp:target:port resolve A/AAAA of target and use port lmtp:target:port + ================ ===================================== ========================= + + `target` can also be an IPv4 or IPv6 address (an IPv6 address must be enclosed in []: ``[2001:DB8::]``). * Comment. A text field where a comment can be entered to describe the entry. Changes are effective immediately after clicking the Save button. -NOTE: Due to bug `1588`_ email messages fail to be relayed if no Remote Host is configured. -As a workaround the HOSTNAME or IP Address of the SMPT server of the relayed domain can be entered as Remote Host. -Please note that no MX lookup is performed when entering a hostname as Remote Host. You can use the MX lookup on mxtoolbox.com to find the hostname and IP Address of the SMTP server. - -.. _`1588`: https://github.com/Mailu/Mailu/issues/1588 - Antispam -------- The menu item Antispam opens the Rspamd webgui. For more information how spam filtering works in Mailu see the :ref:`Spam filtering page `. -The spam filtering page also contains a section that describes how to create a local blacklist for blocking email messages from specific domains. +The spam filtering page also contains a section that describes how to create a local blacklist for blocking email messages from specific domains. The Rspamd webgui offers basic functions for setting metric actions, scores, viewing statistics and learning. The following settings are not persisent and are *lost* when the Antispam container is recreated or restarted: @@ -266,31 +273,31 @@ On the `Mail domains` page all the domains served by Mailu are configured. Via t Details ``````` -This page is also accessible for domain managers. On the details page all DNS settings are displayed for configuring your DNS server. It contains information on what to configure as MX record and SPF record. On this page it is also possible to (re-)generate the keys for DKIM and DMARC. The option for generating keys for DKIM and DMARC is only available for global administrators. After generating the keys for DKIM and DMARC, this page will also show the DNS records for configuring the DKIM/DMARC records on the DNS server. +This page is also accessible for domain managers. On the details page all DNS settings are displayed for configuring your DNS server. It contains information on what to configure as MX record and SPF record. On this page it is also possible to (re-)generate the keys for DKIM and DMARC. The option for generating keys for DKIM and DMARC is only available for global administrators. After generating the keys for DKIM and DMARC, this page will also show the DNS records for configuring the DKIM/DMARC records on the DNS server. Edit -```` +```` -This page is only accessible for global administrators. On the edit page, the global settings for the domain can be changed. +This page is only accessible for global administrators. On the edit page, the global settings for the domain can be changed. * Maximum user count. The maximum amount of users that can be created under this domain. Once this limit is reached it is not possible anymore to add users to the domain; and it is also not possible for users to self-register. - + * Maximum alias count. The maximum amount of aliases that can be created for an email account. - + * Maximum user quota. The maximum amount of quota that can be assigned to a user. When creating or editing a user, this sets the limit on the maximum amount of quota that can be assigned to the user. - -* Enable sign-up. When this option is ticked, self-registration is enabled. When the Admin GUI is accessed, in the menu list the option Signup becomes available. - Obviously this menu item is only visible when signed out. On the Signup page a user can create an email account. - If your Admin GUI is available to the public internet, this means your Mailu installation basically becomes a free email provider. + +* Enable sign-up. When this option is ticked, self-registration is enabled. When the Admin GUI is accessed, in the menu list the option Signup becomes available. + Obviously this menu item is only visible when signed out. On the Signup page a user can create an email account. + If your Admin GUI is available to the public internet, this means your Mailu installation basically becomes a free email provider. Use this option with care! - + * Comment. Description for the domain. This description is visible on the parent domains list page. Delete `````` -This page is only accessible for global administrators. This page allows you to delete the domain. The Admin GUI will ask for confirmation if the domain must be really deleted. +This page is only accessible for global administrators. This page allows you to delete the domain. The Admin GUI will ask for confirmation if the domain must be really deleted. Users @@ -326,7 +333,7 @@ For adding a new user the following options can be configured. * Enabled. Tick this checkbox to enable the user account. When an user is disabled, the user is unable to login to the Admin GUI or webmail or access his email via IMAP/POP3 or send mail. The email inbox of the user is still retained. This option can be used to temporarily suspend an user account. - + * Quota. The maximum quota for the user's email box. * Allow IMAP access. When ticked, allows email retrieval via the IMAP protocol. @@ -337,7 +344,7 @@ For adding a new user the following options can be configured. Aliases ``````` -This page is also accessible for domain managers. On the aliases page, aliases can be added for email addresses. An alias is a way to disguise another email address. +This page is also accessible for domain managers. On the aliases page, aliases can be added for email addresses. An alias is a way to disguise another email address. Everything sent to an alias email address is actually received in the primary email account's inbox of the destination email address. Aliases can diversify a single email account without having to create multiple email addresses (users). It is also possible to add multiple email addresses to the destination field. All incoming mails will be sent to each users inbox in this case. @@ -348,11 +355,11 @@ The following options are available when adding an alias: * Use SQL LIKE Syntax (e.g. for catch-all aliases). When this option is ticked, you can use SQL LIKE syntax as alias. The SQL LIKE syntax is used to match text values against a pattern using wildcards. There are two wildcards that can be used with SQL LIKE syntax: - + * % - The percent sign represents zero, one, or multiple characters * _ - The underscore represents a single character - - Examples are: + + Examples are: * a% - Finds any values that start with "a" * %a - Finds any values that end with "a" * %or% - Finds any values that have "or" in any position @@ -369,7 +376,7 @@ The following options are available when adding an alias: Managers ```````` -This page is also accessible for domain managers. On the `managers list` page, managers can be added for the domain and can be deleted. +This page is also accessible for domain managers. On the `managers list` page, managers can be added for the domain and can be deleted. Managers have access to configuration settings of the domain. On the `add manager` page you can click on the manager email text box to access a drop down list of users that can be made a manager of the domain. @@ -377,11 +384,11 @@ On the `add manager` page you can click on the manager email text box to access Alternatives ```````````` -This page is only accessible for global administrators. On the alternatives page, alternative domains can be added for the domain. +This page is only accessible for global administrators. On the alternatives page, alternative domains can be added for the domain. An alternative domain acts as a copy of a given domain. -Everything sent to an alternative domain, is actually received in the domain the alternative is created for. -This allows you to receive emails for multiple domains while using a single domain. -For example if the main domain has the email address user@example.com, and the alternative domain is mymail.com, +Everything sent to an alternative domain, is actually received in the domain the alternative is created for. +This allows you to receive emails for multiple domains while using a single domain. +For example if the main domain has the email address user@example.com, and the alternative domain is mymail.com, then email send to user@mymail.com will end up in the email box of user@example.com. New domain @@ -392,16 +399,16 @@ This page is only accessible for global administrators. Via this page a new doma * domain name. The name of the domain. * Maximum user count. The maximum amount of users that can be created under this domain. Once this limit is reached it is not possible anymore to add users to the domain; and it is also not possible for users to self-register. - + * Maximum alias count. The maximum amount of aliases that can be made for an email account. - + * Maximum user quota. The maximum amount of quota that can be assigned to a user. When creating or editing a user, this sets the limit on the maximum amount of quota that can be assigned to the user. - -* Enable sign-up. When this option is ticked, self-registration is enabled. When the Admin GUI is accessed, in the menu list the option Signup becomes available. - Obviously this menu item is only visible when signed out. On the Signup page a user can create an email account. - If your Admin GUI is available to the public internet, this means your Mailu installation basically becomes a free email provider. + +* Enable sign-up. When this option is ticked, self-registration is enabled. When the Admin GUI is accessed, in the menu list the option Signup becomes available. + Obviously this menu item is only visible when signed out. On the Signup page a user can create an email account. + If your Admin GUI is available to the public internet, this means your Mailu installation basically becomes a free email provider. Use this option with care! - + * Comment. Description for the domain. This description is visible on the parent domains list page. @@ -414,7 +421,7 @@ The menu item `Webmail` opens the webmail page. This option is only available if Client setup ------------ -The menu item `Client setup` shows all settings for configuring your email client for connecting to Mailu. +The menu item `Client setup` shows all settings for configuring your email client for connecting to Mailu. Website From 49c5c0eba691791c1cc7e4e1b0b4746b38f57074 Mon Sep 17 00:00:00 2001 From: parisni Date: Fri, 18 Jun 2021 23:17:35 +0200 Subject: [PATCH 33/65] Split mailu / roundcube db config There is no reason to share the flavor since at least the dbname shall be different. --- webmails/roundcube/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 649f3324..36502eb6 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -10,7 +10,7 @@ log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576)) -db_flavor=os.environ.get("ROUNDCUBE_DB_FLAVOR",os.environ.get("DB_FLAVOR","sqlite")) +db_flavor=os.environ.get("ROUNDCUBE_DB_FLAVOR","sqlite") if db_flavor=="sqlite": os.environ["DB_DSNW"]="sqlite:////data/roundcube.db" elif db_flavor=="mysql": From 5386e33af303af9e46f18fd7e0c6379f37bd3858 Mon Sep 17 00:00:00 2001 From: parisni Date: Fri, 18 Jun 2021 23:21:24 +0200 Subject: [PATCH 34/65] Reformat python --- webmails/roundcube/start.py | 43 ++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 36502eb6..3e47ce69 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -8,31 +8,29 @@ import subprocess log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) -os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576)) +os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT")) * 0.66 / 1048576)) -db_flavor=os.environ.get("ROUNDCUBE_DB_FLAVOR","sqlite") -if db_flavor=="sqlite": - os.environ["DB_DSNW"]="sqlite:////data/roundcube.db" -elif db_flavor=="mysql": - os.environ["DB_DSNW"]="mysql://%s:%s@%s/%s" % ( - os.environ.get("ROUNDCUBE_DB_USER","roundcube"), +db_flavor = os.environ.get("ROUNDCUBE_DB_FLAVOR", "sqlite") +if db_flavor == "sqlite": + os.environ["DB_DSNW"] = "sqlite:////data/roundcube.db" +elif db_flavor == "mysql": + os.environ["DB_DSNW"] = "mysql://%s:%s@%s/%s" % ( + os.environ.get("ROUNDCUBE_DB_USER", "roundcube"), os.environ.get("ROUNDCUBE_DB_PW"), - os.environ.get("ROUNDCUBE_DB_HOST",os.environ.get("DB_HOST","database")), - os.environ.get("ROUNDCUBE_DB_NAME","roundcube") - ) -elif db_flavor=="postgresql": - os.environ["DB_DSNW"]="pgsql://%s:%s@%s/%s" % ( - os.environ.get("ROUNDCUBE_DB_USER","roundcube"), + os.environ.get("ROUNDCUBE_DB_HOST", os.environ.get("DB_HOST", "database")), + os.environ.get("ROUNDCUBE_DB_NAME", "roundcube") + ) +elif db_flavor == "postgresql": + os.environ["DB_DSNW"] = "pgsql://%s:%s@%s/%s" % ( + os.environ.get("ROUNDCUBE_DB_USER", "roundcube"), os.environ.get("ROUNDCUBE_DB_PW"), - os.environ.get("ROUNDCUBE_DB_HOST",os.environ.get("DB_HOST","database")), - os.environ.get("ROUNDCUBE_DB_NAME","roundcube") - ) + os.environ.get("ROUNDCUBE_DB_HOST", os.environ.get("DB_HOST", "database")), + os.environ.get("ROUNDCUBE_DB_NAME", "roundcube") + ) else: - print("Unknown ROUNDCUBE_DB_FLAVOR: %s",db_flavor) + print("Unknown ROUNDCUBE_DB_FLAVOR: %s", db_flavor) exit(1) - - conf.jinja("/php.ini", os.environ, "/usr/local/etc/php/conf.d/roundcube.ini") # Create dirs, setup permissions @@ -42,7 +40,8 @@ os.system("chown -R www-data:www-data /var/www/html/logs") try: print("Initializing database") - result=subprocess.check_output(["/var/www/html/bin/initdb.sh","--dir","/var/www/html/SQL"],stderr=subprocess.STDOUT) + result = subprocess.check_output(["/var/www/html/bin/initdb.sh", "--dir", "/var/www/html/SQL"], + stderr=subprocess.STDOUT) print(result.decode()) except subprocess.CalledProcessError as e: if "already exists" in e.stdout.decode(): @@ -53,7 +52,7 @@ except subprocess.CalledProcessError as e: try: print("Upgrading database") - subprocess.check_call(["/var/www/html/bin/update.sh","--version=?","-y"],stderr=subprocess.STDOUT) + subprocess.check_call(["/var/www/html/bin/update.sh", "--version=?", "-y"], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: quit(1) @@ -61,7 +60,7 @@ except subprocess.CalledProcessError as e: os.system("chown -R www-data:www-data /data") # Tail roundcube logs -subprocess.Popen(["tail","-f","-n","0","/var/www/html/logs/errors.log"]) +subprocess.Popen(["tail", "-f", "-n", "0", "/var/www/html/logs/errors.log"]) # Run apache os.execv("/usr/local/bin/apache2-foreground", ["apache2-foreground"]) From 278878d48db01ee581014dd8cc0d102f43fb12c4 Mon Sep 17 00:00:00 2001 From: parisni Date: Fri, 18 Jun 2021 23:36:14 +0200 Subject: [PATCH 35/65] Remove unused deps --- optional/postgresql/start.py | 1 - 1 file changed, 1 deletion(-) diff --git a/optional/postgresql/start.py b/optional/postgresql/start.py index 1f2f2a2b..d318d4d9 100755 --- a/optional/postgresql/start.py +++ b/optional/postgresql/start.py @@ -2,7 +2,6 @@ import anosql import psycopg2 -import jinja2 import glob import os import subprocess From d2803f6f4613df2948c19931793eede3a9c72362 Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 00:38:53 +0200 Subject: [PATCH 36/65] Update setup website --- optional/postgresql/start.py | 1 - setup/flavors/compose/mailu.env | 6 ++++++ setup/static/render.js | 21 +++++++++++++++++++++ setup/templates/steps/database.html | 14 +++++++++++++- 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/optional/postgresql/start.py b/optional/postgresql/start.py index d318d4d9..e34e157e 100755 --- a/optional/postgresql/start.py +++ b/optional/postgresql/start.py @@ -37,7 +37,6 @@ if not os.listdir("/data"): rec.write("restore_command = 'gunzip < /backup/wal_archive/%f > %p'\n") rec.write("standby_mode = off\n") os.system("chown postgres:postgres /data/recovery.conf") - #os.system("sudo -u postgres pg_ctl start -D /data -o '-h \"''\" '") else: # Bootstrap the database os.system("sudo -u postgres initdb -D /data") diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 44452e36..150c70a3 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -175,3 +175,9 @@ DB_HOST={{ db_url }} DB_NAME={{ db_name }} {% endif %} +{% if (postgresql == 'external' or db_flavor == 'mysql') and webmail_type == 'roundcube' %} +ROUNDCUBE_DB_USER={{ roundcube_db_user }} +ROUNDCUBE_DB_PW={{ roundcube_db_pw }} +ROUNDCUBE_DB_HOST={{ roundcube_db_url }} +ROUNDCUBE_DB_NAME={{ roundcube_db_name }} +{% endif %} diff --git a/setup/static/render.js b/setup/static/render.js index a817c4f0..0a0a6675 100644 --- a/setup/static/render.js +++ b/setup/static/render.js @@ -57,6 +57,13 @@ $(document).ready(function() { $("#db_pw").prop('required',true); $("#db_url").prop('required',true); $("#db_name").prop('required',true); + if ($("#webmail").val() == 'roundcube') { + $("#roundcube_external_db").show(); + $("#roundcube_db_user").prop('required',true); + $("#roundcube_db_pw").prop('required',true); + $("#roundcube_db_url").prop('required',true); + $("#roundcube_db_name").prop('required',true); + } } else if (this.value == 'mysql') { $("#postgres_db").hide(); $("#external_db").show(); @@ -64,6 +71,13 @@ $(document).ready(function() { $("#db_pw").prop('required',true); $("#db_url").prop('required',true); $("#db_name").prop('required',true); + if ($("#webmail").val() == 'roundcube') { + $("#roundcube_external_db").show(); + $("#roundcube_db_user").prop('required',true); + $("#roundcube_db_pw").prop('required',true); + $("#roundcube_db_url").prop('required',true); + $("#roundcube_db_name").prop('required',true); + } } }); $("#external_psql").change(function() { @@ -73,6 +87,13 @@ $(document).ready(function() { $("#db_pw").prop('required',true); $("#db_url").prop('required',true); $("#db_name").prop('required',true); + if ($("#webmail").val() == 'roundcube') { + $("#roundcube_external_db").show(); + $("#roundcube_db_user").prop('required',true); + $("#roundcube_db_pw").prop('required',true); + $("#roundcube_db_url").prop('required',true); + $("#roundcube_db_name").prop('required',true); + } } else { $("#external_db").hide(); } diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html index ad5411ab..d7184110 100644 --- a/setup/templates/steps/database.html +++ b/setup/templates/steps/database.html @@ -28,7 +28,7 @@
From 84e59c0a6e4d84b1a91c8e84293a7abd12259f6d Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 01:22:23 +0200 Subject: [PATCH 37/65] Add missing roundcube_db_flavor --- setup/flavors/compose/mailu.env | 1 + 1 file changed, 1 insertion(+) diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index 150c70a3..4dce34f1 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -176,6 +176,7 @@ DB_NAME={{ db_name }} {% endif %} {% if (postgresql == 'external' or db_flavor == 'mysql') and webmail_type == 'roundcube' %} +ROUNDCUBE_DB_FLAVOR={{ db_flavor }} ROUNDCUBE_DB_USER={{ roundcube_db_user }} ROUNDCUBE_DB_PW={{ roundcube_db_pw }} ROUNDCUBE_DB_HOST={{ roundcube_db_url }} From 14307c83c13877234d3daec11999f26d5bb0efff Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 09:12:46 +0200 Subject: [PATCH 38/65] Document databases variable and deprecation --- docs/configuration.rst | 20 ++++++++++++++++++++ docs/database.rst | 3 ++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index e08675a8..16ea23c3 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -195,4 +195,24 @@ resolved. This can be used to rely on DNS based service discovery with changing When using ``*_ADDRESS``, the hostnames must be full-qualified hostnames. Otherwise nginx will not be able to resolve the hostnames. +Database settings +----------------- + +The admin service stores configurations in a database. + +- ``DB_FLAVOR``: the database type for mailu admin service. (``sqlite``, ``postgresql``, ``mysql``) +- ``DB_HOST``: the database host for mailu admin service. (when not ``sqlite``) +- ``DB_PORT``: the database port for mailu admin service. (when not ``sqlite``) +- ``DB_PW``: the database password for mailu admin service. (when not ``sqlite``) +- ``DB_USER``: the database user for mailu admin service. (when not ``sqlite``) +- ``DB_NAME``: the database name for mailu admin service. (when not ``sqlite``) + +The roundcube service stores configurations in a database. + +- ``ROUNDCUBE_DB_FLAVOR``: the database type for roundcube service. (``sqlite``, ``postgresql``, ``mysql``) +- ``ROUNDCUBE_DB_HOST``: the database host for roundcube service. (when not ``sqlite``) +- ``ROUNDCUBE_DB_PORT``: the database port for roundcube service. (when not ``sqlite``) +- ``ROUNDCUBE_DB_PW``: the database password for roundcube service. (when not ``sqlite``) +- ``ROUNDCUBE_DB_USER``: the database user for roundcube service. (when not ``sqlite``) +- ``ROUNDCUBE_DB_NAME``: the database name for roundcube service. (when not ``sqlite``) diff --git a/docs/database.rst b/docs/database.rst index b2526d6f..c13ca0bf 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -8,7 +8,8 @@ This functionality should still be considered experimental! Mailu Postgresql ---------------- -Mailu optionally comes with a pre-configured Postgresql image. +Mailu optionally comes with a pre-configured Postgresql image, wich as of 1.8 is deprecated +will be removed in 1.9 This images has the following features: - Automatic creation of users, db, extensions and password; From a9548e4cbd31a9cd1b04414fc3ad4ca668f05bf6 Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 09:20:23 +0200 Subject: [PATCH 39/65] Remove mailu/roundcube shared host --- webmails/roundcube/start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webmails/roundcube/start.py b/webmails/roundcube/start.py index 3e47ce69..f87e460f 100755 --- a/webmails/roundcube/start.py +++ b/webmails/roundcube/start.py @@ -17,14 +17,14 @@ elif db_flavor == "mysql": os.environ["DB_DSNW"] = "mysql://%s:%s@%s/%s" % ( os.environ.get("ROUNDCUBE_DB_USER", "roundcube"), os.environ.get("ROUNDCUBE_DB_PW"), - os.environ.get("ROUNDCUBE_DB_HOST", os.environ.get("DB_HOST", "database")), + os.environ.get("ROUNDCUBE_DB_HOST", "database"), os.environ.get("ROUNDCUBE_DB_NAME", "roundcube") ) elif db_flavor == "postgresql": os.environ["DB_DSNW"] = "pgsql://%s:%s@%s/%s" % ( os.environ.get("ROUNDCUBE_DB_USER", "roundcube"), os.environ.get("ROUNDCUBE_DB_PW"), - os.environ.get("ROUNDCUBE_DB_HOST", os.environ.get("DB_HOST", "database")), + os.environ.get("ROUNDCUBE_DB_HOST", "database"), os.environ.get("ROUNDCUBE_DB_NAME", "roundcube") ) else: From f4c76d49c1862894eba341eca318629a2137f274 Mon Sep 17 00:00:00 2001 From: parisni Date: Sat, 19 Jun 2021 09:30:32 +0200 Subject: [PATCH 40/65] Add changelog entry --- towncrier/newsfragments/1831.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/1831.bugfix diff --git a/towncrier/newsfragments/1831.bugfix b/towncrier/newsfragments/1831.bugfix new file mode 100644 index 00000000..1094be34 --- /dev/null +++ b/towncrier/newsfragments/1831.bugfix @@ -0,0 +1 @@ +Fix roundcube environment configuration for databases \ No newline at end of file From 58235bcc44e9a7449229ea0d851e35846a5ce176 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 08:25:15 +0000 Subject: [PATCH 41/65] Switch to github actions for CI/CD --- .github/worfklows/CI.yml | 276 +++++++++++++++++++++++ .travis.yml | 56 ----- bors.toml | 3 +- tests/compose/filters/00_create_users.sh | 5 + 4 files changed, 283 insertions(+), 57 deletions(-) create mode 100644 .github/worfklows/CI.yml delete mode 100644 .travis.yml create mode 100755 tests/compose/filters/00_create_users.sh diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml new file mode 100644 index 00000000..3dcaf096 --- /dev/null +++ b/.github/worfklows/CI.yml @@ -0,0 +1,276 @@ + + +name: CI +on: + push: + branches: + - staging + - testing + - '1.5' + - '1.6' + - '1.7' + - '1.8' + - master + # version tags, e.g. 1.7.1 + - '[1-9].[0-9].[0-9]' + # pre-releases, e.g. 1.8-pre1 + - 1.8-pre[0-9] + # test branches, e.g. test-debian + - test-* + +############################################### +# REQUIRED secrets +# DOCKER_UN: ${{ secrets.Docker_Login }} +# Username of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_PW: ${{ secrets.Docker_Password }} +# Password of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_ORG: ${{ secrets.DOCKER_ORG }} +# The docker repository where the images are pushed to. +# +# Add the above secrets to your github repo to determine where the images will be pushed. +################################################ + +jobs: + build: + name: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - name: check docker-compose version + run: docker-compose -v + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + # In this step, this action saves a list of existing images, + # the cache is created without them in the post run. + # It also restores the cache if it exists. + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: build all docker images + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: docker-compose -f tests/build.yml build + +#NOTE: It appears the filter test depends on the core test. The filter test requires an email user +#that is created by the core test. + core-test: + name: core test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test core suite + run: python tests/compose/test.py core 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + filter-test: + name: filter test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: 'test clamvav' + run: python tests/compose/test.py filters 2 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + fetch-test: + name: fetch test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test fetch + run: python tests/compose/test.py fetchmail 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + rainloop-test: + name: rainloop test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test rainloop + run: python tests/compose/test.py rainloop 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + roundcube-test: + name: roundcube test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test roundcube + run: python tests/compose/test.py roundcube 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + webdav-test: + name: webdav test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test webdav + run: python tests/compose/test.py webdav 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + deploy: + name: deploy step + runs-on: ubuntu-latest + needs: + - build + - core-test + - filter-test + - fetch-test + - rainloop-test + - roundcube-test + - webdav-test + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + - name: build all docker images + run: docker-compose -f tests/build.yml build + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + - name: deploy built docker images + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: bash tests/deploy.sh + + # This job is watched by bors. It only complets if building,testing and deploy worked. + ci-success: + name: CI-Done + #Returns true when none of the **previous** steps have failed or been canceled. + if: ${{ success() }} + needs: + - deploy + runs-on: ubuntu-latest + steps: + - name: CI/CD succeeded. + run: exit 0 + + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f2a85630..00000000 --- a/.travis.yml +++ /dev/null @@ -1,56 +0,0 @@ -branches: - only: - - staging - - testing - - '1.5' - - '1.6' - - '1.7' - - '1.8' - - master - # version tags, e.g. 1.7.1 - - /^1\.[5678]\.\d+$/ - # pre-releases, e.g. 1.8-pre1 - - /^1\.8-pre\d+$/ - # test branches, e.g. test-debian - - /^test-[\w\-\.]+$/ - -sudo: required -services: docker -addons: - apt: - packages: - - docker-ce - -env: - - MAILU_VERSION=${TRAVIS_BRANCH////-} - -language: python -python: - - "3.6" -install: - - pip install -r tests/requirements.txt - - sudo curl -L https://github.com/docker/compose/releases/download/1.23.0-rc3/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose - - sudo chmod +x /usr/local/bin/docker-compose - -before_script: - - docker-compose -v - - echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - - docker-compose -f tests/build.yml build - - sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - -script: -# test.py, test name and timeout between start and tests. - - python tests/compose/test.py core 1 - - python tests/compose/test.py fetchmail 1 - - travis_wait python tests/compose/test.py filters 10 - - python tests/compose/test.py rainloop 1 - - python tests/compose/test.py roundcube 1 - - python tests/compose/test.py webdav 1 - -deploy: - provider: script - script: bash tests/deploy.sh - on: - all_branches: true - condition: -n $DOCKER_UN diff --git a/bors.toml b/bors.toml index 5279fe72..272a6047 100644 --- a/bors.toml +++ b/bors.toml @@ -1,3 +1,4 @@ status = [ - "continuous-integration/travis-ci/push" + "CI-Done" ] + diff --git a/tests/compose/filters/00_create_users.sh b/tests/compose/filters/00_create_users.sh new file mode 100755 index 00000000..3c581685 --- /dev/null +++ b/tests/compose/filters/00_create_users.sh @@ -0,0 +1,5 @@ +echo "Creating user required for next test ..." +# Should not fail and update the password; update mode +docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu admin admin mailu.io 'password' --mode=update || exit 1 +docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu user user mailu.io 'password' || exit 1 +echo "User created successfully" From 54dd4cf224b510f657f0b07696cebf9e8d89d7c2 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 19:16:56 +0000 Subject: [PATCH 42/65] Added new docker repo for test image. Adapted deploy script to use env var for test repo name. Modified travis references to github actions references in docs. Added changelog entry. --- .github/worfklows/CI.yml | 4 +++- docs/contributors/environment.rst | 14 +++++++------- docs/contributors/workflow.rst | 2 +- docs/faq.rst | 2 +- tests/deploy.sh | 2 +- towncrier/newsfragments/1828.misc | 1 + 6 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 towncrier/newsfragments/1828.misc diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml index 3dcaf096..421fec3c 100644 --- a/.github/worfklows/CI.yml +++ b/.github/worfklows/CI.yml @@ -26,7 +26,8 @@ on: # Password of docker login for pushing the images to repo $DOCKER_ORG # DOCKER_ORG: ${{ secrets.DOCKER_ORG }} # The docker repository where the images are pushed to. -# +# DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} +# The docker repository for test images. Only used for the branch TESTING (BORS try). # Add the above secrets to your github repo to determine where the images will be pushed. ################################################ @@ -256,6 +257,7 @@ jobs: DOCKER_UN: ${{ secrets.Docker_Login }} DOCKER_PW: ${{ secrets.Docker_Password }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index 26c04d0b..cef71c6c 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -178,9 +178,9 @@ In the case of a PR from a fellow team member, a single review is enough to initiate merging. In all other cases, two approving reviews are required. There is also a possibility to set the ``review/need2`` to require a second review. -After Travis successfully tests the PR and the required amount of reviews are acquired, +After the Github Action workflow successfully tests the PR and the required amount of reviews are acquired, Mergify will trigger with a ``bors r+`` command. Bors will batch any approved PR's, -merges them with master in a staging branch where Travis builds and tests the result. +merges them with master in a staging branch where the Github Action workflow builds and tests the result. After a successful test, the actual master gets fast-forwarded to that point. System requirements @@ -201,16 +201,16 @@ us on `Matrix`_. Test images ``````````` -All PR's automatically get build by Travis, controlled by `bors-ng`_. +All PR's automatically get build by a Github Action workflow, controlled by `bors-ng`_. Some primitive auto testing is done. The resulting images get uploaded to Docker hub, under the -tag name ``mailutest/:pr-``. +tag name ``mailuci/:pr-``. For example, to test PR #500 against master, reviewers can use: .. code-block:: bash - export DOCKER_ORG="mailutest" + export DOCKER_ORG="mailuci" export MAILU_VERSION="pr-500" docker-compose pull docker-compose up -d @@ -232,8 +232,8 @@ after Bors confirms a successful build. When bors try fails ``````````````````` -Sometimes Travis fails when another PR triggers a ``bors try`` command, -before Travis cloned the git repository. +Sometimes the Github Action workflow fails when another PR triggers a ``bors try`` command, +before the Github Action workflow cloned the git repository. Inspect the build log in the link provided by *bors-ng* to find out the cause. If you see something like the following error on top of the logs, feel free to write a comment with ``bors retry``. diff --git a/docs/contributors/workflow.rst b/docs/contributors/workflow.rst index 16dcef52..31ffd793 100644 --- a/docs/contributors/workflow.rst +++ b/docs/contributors/workflow.rst @@ -41,7 +41,7 @@ PR Workflow ----------- All pull requests have to be against the main ``master`` branch. -The PR gets build by Travis and some primitive auto-testing is done. +The PR gets build by a Github Action workflow and some primitive auto-testing is done. Test images get uploaded to a separate section in Docker hub. Reviewers will check the PR and test the resulting images. See the :ref:`testing` section for more info. diff --git a/docs/faq.rst b/docs/faq.rst index 9c4f1d75..c4cea444 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -61,7 +61,7 @@ have to prevent pushing out something quickly. We currently maintain a strict work flow: #. Someone writes a solution and sends a pull request; -#. We use Travis-CI for some very basic building and testing; +#. We use Github actions for some very basic building and testing; #. The pull request needs to be code-reviewed and tested by at least two members from the contributors team. diff --git a/tests/deploy.sh b/tests/deploy.sh index 21aec444..a836417b 100755 --- a/tests/deploy.sh +++ b/tests/deploy.sh @@ -5,7 +5,7 @@ # Retag in case of `bors try` if [ "$TRAVIS_BRANCH" = "testing" ]; then - export DOCKER_ORG="mailutest" + export DOCKER_ORG=$DOCKER_ORG_TESTS # Commit message is like "Try #99". # This sets the version tag to "pr-99" export MAILU_VERSION="pr-${TRAVIS_COMMIT_MESSAGE//[!0-9]/}" diff --git a/towncrier/newsfragments/1828.misc b/towncrier/newsfragments/1828.misc new file mode 100644 index 00000000..09da59ad --- /dev/null +++ b/towncrier/newsfragments/1828.misc @@ -0,0 +1 @@ +Switched from Travis to Github actions for CI/CD. Improved CI workflow to perform all tests in parallel. From c6a38bbbccafd8e2623645da6c5759c9a23b4382 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Sat, 26 Jun 2021 21:50:55 +0200 Subject: [PATCH 43/65] Update CI.yml --- .github/worfklows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml index 421fec3c..e0462e1e 100644 --- a/.github/worfklows/CI.yml +++ b/.github/worfklows/CI.yml @@ -1,5 +1,3 @@ - - name: CI on: push: From fb30a62629b6372450b34b6c9a7238d40013ef58 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Sat, 26 Jun 2021 21:52:09 +0200 Subject: [PATCH 44/65] Create CI.yml --- .github/workflows/CI.yml | 276 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 .github/workflows/CI.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 00000000..e0462e1e --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,276 @@ +name: CI +on: + push: + branches: + - staging + - testing + - '1.5' + - '1.6' + - '1.7' + - '1.8' + - master + # version tags, e.g. 1.7.1 + - '[1-9].[0-9].[0-9]' + # pre-releases, e.g. 1.8-pre1 + - 1.8-pre[0-9] + # test branches, e.g. test-debian + - test-* + +############################################### +# REQUIRED secrets +# DOCKER_UN: ${{ secrets.Docker_Login }} +# Username of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_PW: ${{ secrets.Docker_Password }} +# Password of docker login for pushing the images to repo $DOCKER_ORG +# DOCKER_ORG: ${{ secrets.DOCKER_ORG }} +# The docker repository where the images are pushed to. +# DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} +# The docker repository for test images. Only used for the branch TESTING (BORS try). +# Add the above secrets to your github repo to determine where the images will be pushed. +################################################ + +jobs: + build: + name: build + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - name: check docker-compose version + run: docker-compose -v + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + # In this step, this action saves a list of existing images, + # the cache is created without them in the post run. + # It also restores the cache if it exists. + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: build all docker images + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + run: docker-compose -f tests/build.yml build + +#NOTE: It appears the filter test depends on the core test. The filter test requires an email user +#that is created by the core test. + core-test: + name: core test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test core suite + run: python tests/compose/test.py core 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + filter-test: + name: filter test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: 'test clamvav' + run: python tests/compose/test.py filters 2 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + fetch-test: + name: fetch test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test fetch + run: python tests/compose/test.py fetchmail 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + rainloop-test: + name: rainloop test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test rainloop + run: python tests/compose/test.py rainloop 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + roundcube-test: + name: roundcube test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test roundcube + run: python tests/compose/test.py roundcube 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + webdav-test: + name: webdav test + runs-on: ubuntu-latest + needs: + - build + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test webdav + run: python tests/compose/test.py webdav 1 + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + + deploy: + name: deploy step + runs-on: ubuntu-latest + needs: + - build + - core-test + - filter-test + - fetch-test + - rainloop-test + - roundcube-test + - webdav-test + steps: + - uses: actions/checkout@v2 + - name: Extract branch name + shell: bash + run: | + echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + - name: install python packages + run: python3 -m pip install -r tests/requirements.txt + - uses: satackey/action-docker-layer-caching@v0.0.11 + # Ignore the failure of a step and avoid terminating the job. + continue-on-error: true + - name: copy all certs + run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: login docker + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin + - name: build all docker images + run: docker-compose -f tests/build.yml build + env: + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + - name: deploy built docker images + env: + DOCKER_UN: ${{ secrets.Docker_Login }} + DOCKER_PW: ${{ secrets.Docker_Password }} + DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} + MAILU_VERSION: ${{ env.BRANCH }} + TRAVIS_BRANCH: ${{ env.BRANCH }} + TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + run: bash tests/deploy.sh + + # This job is watched by bors. It only complets if building,testing and deploy worked. + ci-success: + name: CI-Done + #Returns true when none of the **previous** steps have failed or been canceled. + if: ${{ success() }} + needs: + - deploy + runs-on: ubuntu-latest + steps: + - name: CI/CD succeeded. + run: exit 0 + + From 006da4c5e409526ccf5fb3e62ac512d3d5e79593 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman <52963853+Diman0@users.noreply.github.com> Date: Sat, 26 Jun 2021 21:52:51 +0200 Subject: [PATCH 45/65] My mistake. A typo --- .github/worfklows/CI.yml | 276 --------------------------------------- 1 file changed, 276 deletions(-) delete mode 100644 .github/worfklows/CI.yml diff --git a/.github/worfklows/CI.yml b/.github/worfklows/CI.yml deleted file mode 100644 index e0462e1e..00000000 --- a/.github/worfklows/CI.yml +++ /dev/null @@ -1,276 +0,0 @@ -name: CI -on: - push: - branches: - - staging - - testing - - '1.5' - - '1.6' - - '1.7' - - '1.8' - - master - # version tags, e.g. 1.7.1 - - '[1-9].[0-9].[0-9]' - # pre-releases, e.g. 1.8-pre1 - - 1.8-pre[0-9] - # test branches, e.g. test-debian - - test-* - -############################################### -# REQUIRED secrets -# DOCKER_UN: ${{ secrets.Docker_Login }} -# Username of docker login for pushing the images to repo $DOCKER_ORG -# DOCKER_PW: ${{ secrets.Docker_Password }} -# Password of docker login for pushing the images to repo $DOCKER_ORG -# DOCKER_ORG: ${{ secrets.DOCKER_ORG }} -# The docker repository where the images are pushed to. -# DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} -# The docker repository for test images. Only used for the branch TESTING (BORS try). -# Add the above secrets to your github repo to determine where the images will be pushed. -################################################ - -jobs: - build: - name: build - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - name: check docker-compose version - run: docker-compose -v - - name: login docker - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - # In this step, this action saves a list of existing images, - # the cache is created without them in the post run. - # It also restores the cache if it exists. - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: build all docker images - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - run: docker-compose -f tests/build.yml build - -#NOTE: It appears the filter test depends on the core test. The filter test requires an email user -#that is created by the core test. - core-test: - name: core test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test core suite - run: python tests/compose/test.py core 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - filter-test: - name: filter test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: 'test clamvav' - run: python tests/compose/test.py filters 2 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - fetch-test: - name: fetch test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test fetch - run: python tests/compose/test.py fetchmail 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - rainloop-test: - name: rainloop test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test rainloop - run: python tests/compose/test.py rainloop 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - roundcube-test: - name: roundcube test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test roundcube - run: python tests/compose/test.py roundcube 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - webdav-test: - name: webdav test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: test webdav - run: python tests/compose/test.py webdav 1 - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - deploy: - name: deploy step - runs-on: ubuntu-latest - needs: - - build - - core-test - - filter-test - - fetch-test - - rainloop-test - - roundcube-test - - webdav-test - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: login docker - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - - name: build all docker images - run: docker-compose -f tests/build.yml build - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - name: deploy built docker images - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - DOCKER_ORG_TESTS: ${{ secrets.DOCKER_ORG_TESTS }} - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - TRAVIS_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - run: bash tests/deploy.sh - - # This job is watched by bors. It only complets if building,testing and deploy worked. - ci-success: - name: CI-Done - #Returns true when none of the **previous** steps have failed or been canceled. - if: ${{ success() }} - needs: - - deploy - runs-on: ubuntu-latest - steps: - - name: CI/CD succeeded. - run: exit 0 - - From 606c039a6ffbcb63bb28bb1b422d0e622ebc0088 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 21:00:51 +0000 Subject: [PATCH 46/65] Switch back to sequential workflow --- .github/workflows/CI.yml | 126 ++--------------------- tests/compose/filters/00_create_users.sh | 5 - 2 files changed, 6 insertions(+), 125 deletions(-) delete mode 100755 tests/compose/filters/00_create_users.sh diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e0462e1e..8cd9a8d3 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -30,8 +30,8 @@ on: ################################################ jobs: - build: - name: build + build-test: + name: build and test runs-on: ubuntu-latest steps: @@ -63,24 +63,6 @@ jobs: DOCKER_ORG: ${{ secrets.DOCKER_ORG }} run: docker-compose -f tests/build.yml build -#NOTE: It appears the filter test depends on the core test. The filter test requires an email user -#that is created by the core test. - core-test: - name: core test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - name: copy all certs run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: test core suite @@ -90,74 +72,20 @@ jobs: TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - filter-test: - name: filter test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: 'test clamvav' run: python tests/compose/test.py filters 2 env: MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - fetch-test: - name: fetch test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test fetch run: python tests/compose/test.py fetchmail 1 env: MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - rainloop-test: - name: rainloop test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test rainloop run: python tests/compose/test.py rainloop 1 env: @@ -165,49 +93,13 @@ jobs: TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - roundcube-test: - name: roundcube test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - name: test roundcube run: python tests/compose/test.py roundcube 1 env: MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - webdav-test: - name: webdav test - runs-on: ubuntu-latest - needs: - - build - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' + - name: test webdav run: python tests/compose/test.py webdav 1 env: @@ -219,13 +111,7 @@ jobs: name: deploy step runs-on: ubuntu-latest needs: - - build - - core-test - - filter-test - - fetch-test - - rainloop-test - - roundcube-test - - webdav-test + - build-test steps: - uses: actions/checkout@v2 - name: Extract branch name diff --git a/tests/compose/filters/00_create_users.sh b/tests/compose/filters/00_create_users.sh deleted file mode 100755 index 3c581685..00000000 --- a/tests/compose/filters/00_create_users.sh +++ /dev/null @@ -1,5 +0,0 @@ -echo "Creating user required for next test ..." -# Should not fail and update the password; update mode -docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu admin admin mailu.io 'password' --mode=update || exit 1 -docker-compose -f tests/compose/filters/docker-compose.yml exec -T admin flask mailu user user mailu.io 'password' || exit 1 -echo "User created successfully" From 24200ddb670cf500d83695241bb158a4324e94b7 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 26 Jun 2021 21:49:37 +0000 Subject: [PATCH 47/65] Forgot to remove duplicate steps when switching back to sequential workflow --- .github/workflows/CI.yml | 35 +++-------------------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8cd9a8d3..71897c52 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -30,7 +30,7 @@ on: ################################################ jobs: - build-test: + build-test-deploy: name: build and test runs-on: ubuntu-latest @@ -106,36 +106,7 @@ jobs: MAILU_VERSION: ${{ env.BRANCH }} TRAVIS_BRANCH: ${{ env.BRANCH }} DOCKER_ORG: ${{ secrets.DOCKER_ORG }} - - deploy: - name: deploy step - runs-on: ubuntu-latest - needs: - - build-test - steps: - - uses: actions/checkout@v2 - - name: Extract branch name - shell: bash - run: | - echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV - - name: install python packages - run: python3 -m pip install -r tests/requirements.txt - - uses: satackey/action-docker-layer-caching@v0.0.11 - # Ignore the failure of a step and avoid terminating the job. - continue-on-error: true - - name: copy all certs - run: sudo -- sh -c 'mkdir -p /mailu && cp -r tests/certs /mailu && chmod 600 /mailu/certs/*' - - name: login docker - env: - DOCKER_UN: ${{ secrets.Docker_Login }} - DOCKER_PW: ${{ secrets.Docker_Password }} - run: echo "$DOCKER_PW" | docker login --username $DOCKER_UN --password-stdin - - name: build all docker images - run: docker-compose -f tests/build.yml build - env: - MAILU_VERSION: ${{ env.BRANCH }} - TRAVIS_BRANCH: ${{ env.BRANCH }} - DOCKER_ORG: ${{ secrets.DOCKER_ORG }} + - name: deploy built docker images env: DOCKER_UN: ${{ secrets.Docker_Login }} @@ -153,7 +124,7 @@ jobs: #Returns true when none of the **previous** steps have failed or been canceled. if: ${{ success() }} needs: - - deploy + - build-test-deploy runs-on: ubuntu-latest steps: - name: CI/CD succeeded. From b560d1f36998c4c2f65875cbb88f8fbc319ffbe2 Mon Sep 17 00:00:00 2001 From: Nicolas Paris Date: Sun, 27 Jun 2021 10:38:32 +0200 Subject: [PATCH 48/65] Improve english Co-authored-by: decentral1se <1991377+decentral1se@users.noreply.github.com> --- docs/database.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/database.rst b/docs/database.rst index c13ca0bf..0f8318d5 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -8,8 +8,8 @@ This functionality should still be considered experimental! Mailu Postgresql ---------------- -Mailu optionally comes with a pre-configured Postgresql image, wich as of 1.8 is deprecated -will be removed in 1.9 +Mailu optionally comes with a pre-configured Postgresql image, which as of 1.8, is deprecated +and will be removed in 1.9. This images has the following features: - Automatic creation of users, db, extensions and password; From c0c8c4a55113237274ca4b4e32c9c46e5297998f Mon Sep 17 00:00:00 2001 From: Nicolas Paris Date: Sun, 27 Jun 2021 10:46:28 +0200 Subject: [PATCH 49/65] Fix typo Co-authored-by: decentral1se <1991377+decentral1se@users.noreply.github.com> --- setup/templates/steps/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html index d7184110..2727687f 100644 --- a/setup/templates/steps/database.html +++ b/setup/templates/steps/database.html @@ -46,7 +46,7 @@ - + From ab7264df0c9a115a7e9cf63cd4ff55e380c3e7f7 Mon Sep 17 00:00:00 2001 From: Nicolas Paris Date: Sun, 27 Jun 2021 10:46:41 +0200 Subject: [PATCH 50/65] Fix typo Co-authored-by: decentral1se <1991377+decentral1se@users.noreply.github.com> --- setup/templates/steps/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html index 2727687f..be32fe0b 100644 --- a/setup/templates/steps/database.html +++ b/setup/templates/steps/database.html @@ -44,7 +44,7 @@ - + From 7386257dedda678419e495c28e382251f7212bfa Mon Sep 17 00:00:00 2001 From: Nicolas Paris Date: Sun, 27 Jun 2021 10:47:28 +0200 Subject: [PATCH 51/65] Fix typo Co-authored-by: decentral1se <1991377+decentral1se@users.noreply.github.com> --- setup/templates/steps/database.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html index be32fe0b..5e942061 100644 --- a/setup/templates/steps/database.html +++ b/setup/templates/steps/database.html @@ -28,7 +28,7 @@