From dc9e2a3e70954d460963e25b43889d0e55508024 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 11:34:58 +0100 Subject: [PATCH 01/68] Upgrade Snappymail to 2.21 and merge the webmail containers --- .github/workflows/build_test_deploy.yml | 6 +- setup/flavors/compose/docker-compose.yml | 2 +- setup/flavors/stack/docker-compose.yml | 2 +- tests/build.hcl | 20 +-- towncrier/newsfragments/2526.misc | 1 + webmails/{roundcube => }/Dockerfile | 48 ++++--- ...ginx-roundcube.conf => nginx-webmail.conf} | 2 +- .../php-roundcube.conf => php-webmail.conf} | 6 +- webmails/{roundcube/config => }/php.ini | 2 +- webmails/snappymail/Dockerfile | 54 -------- webmails/snappymail/config.py | 16 --- .../snappymail/config/php-snappymail.conf | 118 ------------------ webmails/snappymail/defaults/default.ini | 15 --- webmails/snappymail/defaults/default.json | 50 ++++++++ webmails/snappymail/defaults/php.ini | 5 - webmails/snappymail/start.py | 34 ----- webmails/{roundcube => }/start.py | 31 +++-- 17 files changed, 120 insertions(+), 292 deletions(-) create mode 100644 towncrier/newsfragments/2526.misc rename webmails/{roundcube => }/Dockerfile (55%) rename webmails/{roundcube/config/nginx-roundcube.conf => nginx-webmail.conf} (97%) rename webmails/{roundcube/config/php-roundcube.conf => php-webmail.conf} (98%) rename webmails/{roundcube/config => }/php.ini (77%) delete mode 100644 webmails/snappymail/Dockerfile delete mode 100755 webmails/snappymail/config.py delete mode 100644 webmails/snappymail/config/php-snappymail.conf delete mode 100644 webmails/snappymail/defaults/default.ini create mode 100644 webmails/snappymail/defaults/default.json delete mode 100644 webmails/snappymail/defaults/php.ini delete mode 100755 webmails/snappymail/start.py rename webmails/{roundcube => }/start.py (71%) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index d1395bec..3b5dafc3 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -340,7 +340,7 @@ jobs: strategy: fail-fast: false matrix: - target: ["core", "fetchmail", "filters", "snappymail", "roundcube", "webdav"] + target: ["core", "fetchmail", "filters", "webmail", "webdav"] time: ["2"] include: - target: "filters" @@ -394,7 +394,7 @@ jobs: strategy: fail-fast: false matrix: - target: ["setup", "docs", "fetchmail", "roundcube", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx", "snappymail"] + target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx"] steps: - uses: actions/checkout@v3 - name: Retrieve global variables @@ -439,7 +439,7 @@ jobs: strategy: fail-fast: false matrix: - target: ["setup", "docs", "fetchmail", "roundcube", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx", "snappymail"] + target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx"] steps: - uses: actions/checkout@v3 - name: Retrieve global variables diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 6dac166b..c7f37e01 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -168,7 +168,7 @@ services: # Webmail {% if webmail_type != 'none' %} webmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}{{ webmail_type }}:${MAILU_VERSION:-{{ version }}} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}webmail:${MAILU_VERSION:-{{ version }}} restart: always env_file: {{ env }} volumes: diff --git a/setup/flavors/stack/docker-compose.yml b/setup/flavors/stack/docker-compose.yml index 89da923c..809362df 100644 --- a/setup/flavors/stack/docker-compose.yml +++ b/setup/flavors/stack/docker-compose.yml @@ -119,7 +119,7 @@ services: {% if webmail_type != 'none' %} webmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}{{ webmail_type }}:${MAILU_VERSION:-{{ version }}} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}webmail:${MAILU_VERSION:-{{ version }}} env_file: {{ env }} volumes: - "{{ root }}/webmail:/data" diff --git a/tests/build.hcl b/tests/build.hcl index 34955270..fd546ae4 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -36,8 +36,7 @@ group "default" { "imap", "smtp", - "snappymail", - "roundcube", + "webmail", "antivirus", "fetchmail", @@ -169,24 +168,15 @@ target "smtp" { } # ----------------------------------------------------------------------------------------- -# Webmail images +# Webmail image # ----------------------------------------------------------------------------------------- -target "snappymail" { +target "webmail" { inherits = ["defaults"] - context = "webmails/snappymail/" + context = "webmails/" contexts = { base = "target:base" } - tags = tag("snappymail") -} - -target "roundcube" { - inherits = ["defaults"] - context = "webmails/roundcube/" - contexts = { - base = "target:base" - } - tags = tag("roundcube") + tags = tag("webmail") } # ----------------------------------------------------------------------------------------- diff --git a/towncrier/newsfragments/2526.misc b/towncrier/newsfragments/2526.misc new file mode 100644 index 00000000..9425e88a --- /dev/null +++ b/towncrier/newsfragments/2526.misc @@ -0,0 +1 @@ +Upgrade Snappymail to 2.21 and merge the webmail containers diff --git a/webmails/roundcube/Dockerfile b/webmails/Dockerfile similarity index 55% rename from webmails/roundcube/Dockerfile rename to webmails/Dockerfile index 8db6f984..ccc2da0c 100644 --- a/webmails/roundcube/Dockerfile +++ b/webmails/Dockerfile @@ -1,6 +1,5 @@ # syntax=docker/dockerfile-upstream:1.4.3 -#roundcube image FROM base ARG VERSION @@ -19,6 +18,7 @@ RUN set -euxo pipefail \ ; mkdir -p /run/nginx \ ; mkdir -p /conf +# roundcube ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.5.3/roundcubemail-1.5.3-complete.tar.gz ENV CARDDAV_URL https://github.com/mstilkerich/rcmcarddav/releases/download/v4.4.3/carddav-v4.4.3.tar.gz @@ -26,26 +26,44 @@ RUN set -euxo pipefail \ ; cd /var/www \ ; curl -sL ${ROUNDCUBE_URL} | tar xz \ ; curl -sL ${CARDDAV_URL} | tar xz \ - ; mv roundcubemail-* webmail \ - ; mkdir -p /var/www/webmail/config \ - ; mv carddav webmail/plugins/ \ - ; cd webmail \ + ; mv roundcubemail-* roundcube \ + ; mkdir -p /var/www/roundcube/config \ + ; mv carddav roundcube/plugins/ \ + ; cd roundcube \ ; rm -rf CHANGELOG.md SECURITY.md INSTALL LICENSE README.md UPGRADING composer.json-dist installer composer.* \ - ; ln -sf index.php /var/www/webmail/sso.php \ - ; chmod -R u+w,a+rX /var/www/webmail \ - ; chown -R nginx:nginx /var/www/webmail \ + ; ln -sf index.php /var/www/roundcube/sso.php \ + ; chmod -R u+w,a+rX /var/www/roundcube \ + ; chown -R nginx:nginx /var/www/roundcube \ ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query} +COPY roundcube/config/config.inc.php /conf/ +COPY roundcube/login/mailu.php /var/www/roundcube/plugins/mailu/ +COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/config.inc.php -# nginx / PHP config files -COPY config/nginx-roundcube.conf /conf/ -COPY config/php-roundcube.conf /etc/php81/php-fpm.d/roundcube.conf -COPY config/php.ini /conf/ -COPY config/config.inc.php /conf/ -COPY login/mailu.php /var/www/webmail/plugins/mailu/ -COPY config/config.inc.carddav.php /var/www/webmail/plugins/carddav/config.inc.php +# snappymail +ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.0/snappymail-2.21.0.tar.gz + +RUN set -euxo pipefail \ + ; mkdir /var/www/snappymail \ + ; cd /var/www/snappymail \ + ; curl -sL ${SNAPPYMAIL_URL} | tar xz \ + ; chmod -R u+w,a+rX /var/www/snappymail \ + ; chown -R nginx:nginx /var/www/snappymail + +# SnappyMail login +COPY snappymail/login/include.php /var/www/snappymail/ +COPY snappymail/login/sso.php /var/www/snappymail/ + +# Parsed and moved at startup +COPY snappymail/defaults/application.ini /defaults/ +COPY snappymail/defaults/default.json /defaults/ + +# common COPY start.py / +COPY php.ini /defaults/ +COPY php-webmail.conf /etc/php81/php-fpm.d/php-webmail.conf +COPY nginx-webmail.conf /conf/ EXPOSE 80/tcp VOLUME /data diff --git a/webmails/roundcube/config/nginx-roundcube.conf b/webmails/nginx-webmail.conf similarity index 97% rename from webmails/roundcube/config/nginx-roundcube.conf rename to webmails/nginx-webmail.conf index 80268340..5e5f8ec3 100644 --- a/webmails/roundcube/config/nginx-roundcube.conf +++ b/webmails/nginx-webmail.conf @@ -2,7 +2,7 @@ server { listen 80 default_server; listen [::]:80 default_server; - root /var/www/webmail; + root /var/www/{{ WEBMAIL }}; include /etc/nginx/mime.types; diff --git a/webmails/roundcube/config/php-roundcube.conf b/webmails/php-webmail.conf similarity index 98% rename from webmails/roundcube/config/php-roundcube.conf rename to webmails/php-webmail.conf index ac0c3375..47c1f6dd 100644 --- a/webmails/roundcube/config/php-roundcube.conf +++ b/webmails/php-webmail.conf @@ -1,7 +1,7 @@ -; Start a new pool named 'roundcube'. +; Start a new pool named 'php'. ; the variable $pool can be used in any directive and will be replaced by the -; pool name ('roundcube' here) -[roundcube] +; pool name ('php' here) +[php] ; Redirect worker stdout and stderr into main error log. If not set, stdout and ; stderr will be redirected to /dev/null according to FastCGI specs. diff --git a/webmails/roundcube/config/php.ini b/webmails/php.ini similarity index 77% rename from webmails/roundcube/config/php.ini rename to webmails/php.ini index 9f45dc80..af9ce8c5 100644 --- a/webmails/roundcube/config/php.ini +++ b/webmails/php.ini @@ -2,7 +2,7 @@ expose_php=Off date.timezone={{ TZ }} upload_max_filesize = {{ MAX_FILESIZE }}M post_max_size = {{ MAX_FILESIZE }}M -suhosin.session.encrypt=Off session.auto_start=Off mbstring.func_overload=Off file_uploads=On +error_reporting = E_ALL & ~E_DEPRECATED & ~E_STRICT diff --git a/webmails/snappymail/Dockerfile b/webmails/snappymail/Dockerfile deleted file mode 100644 index 3bc4ef53..00000000 --- a/webmails/snappymail/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -# syntax=docker/dockerfile-upstream:1.4.3 - -#snappymail image -FROM base - -ARG VERSION -LABEL version=$VERSION - -RUN set -euxo pipefail \ - ; apk add --no-cache \ - nginx curl \ - php81 php81-fpm php81-mbstring php81-zip php81-xml php81-simplexml \ - php81-dom php81-curl php81-exif gd php81-gd php81-iconv php81-intl php81-openssl \ - php81-pdo_sqlite php81-pdo php81-sodium libsodium php81-tidy php81-pecl-uuid \ - ; ln -s /usr/bin/php81 /usr/bin/php \ - ; rm /etc/nginx/http.d/default.conf \ - ; rm /etc/php81/php-fpm.d/www.conf \ - ; mkdir -p /run/nginx \ - ; mkdir -p /var/www/webmail \ - ; mkdir -p /config - -# nginx / PHP config files -COPY config/nginx-snappymail.conf /config/ -COPY config/php-snappymail.conf /etc/php81/php-fpm.d/snappymail.conf - -# Parsed and moved at startup -COPY defaults/php.ini /defaults/ -COPY defaults/application.ini /defaults/ -COPY defaults/default.ini /defaults/ - -# Install Snappymail from source -ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.19.4/snappymail-2.19.4.tar.gz -# Note. This is the last working snappymail version. 2.19.6 up to 2.20.6 do not work. - -RUN set -euxo pipefail \ - ; cd /var/www/webmail \ - ; curl -sL ${SNAPPYMAIL_URL} | tar xz \ - ; chmod -R u+w,a+rX /var/www/webmail \ - ; chown -R nginx:nginx /var/www/webmail - -# SnappyMail login -COPY login/include.php /var/www/webmail/ -COPY login/sso.php /var/www/webmail/ - -COPY start.py / -COPY config.py / - -EXPOSE 80/tcp -VOLUME ["/data"] - -CMD /start.py - -HEALTHCHECK CMD curl -f -L http://localhost/ping || exit 1 -RUN echo $VERSION >> /version diff --git a/webmails/snappymail/config.py b/webmails/snappymail/config.py deleted file mode 100755 index f9fa363c..00000000 --- a/webmails/snappymail/config.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 - -import os -import logging as log -import sys - -from socrate import system, conf - -args = os.environ.copy() - -log.basicConfig(stream=sys.stderr, level=args.get("LOG_LEVEL", "WARNING")) - -# Build final configuration paths -conf.jinja("/config/nginx-snappymail.conf", args, "/etc/nginx/http.d/snappymail.conf") -if os.path.exists("/var/run/nginx.pid"): - os.system("nginx -s reload") diff --git a/webmails/snappymail/config/php-snappymail.conf b/webmails/snappymail/config/php-snappymail.conf deleted file mode 100644 index 74b1889f..00000000 --- a/webmails/snappymail/config/php-snappymail.conf +++ /dev/null @@ -1,118 +0,0 @@ -; Start a new pool named 'snappymail'. -; the variable $pool can be used in any directive and will be replaced by the -; pool name ('snappymail' here) -[snappymail] - -; Redirect worker stdout and stderr into main error log. If not set, stdout and -; stderr will be redirected to /dev/null according to FastCGI specs. -; Default value: no. -catch_workers_output = 1 - -; Unix user/group of processes -; Note: The user is mandatory. If the group is not set, the default user's group -; will be used. -user = nginx -group = nginx - -; The address on which to accept FastCGI requests. -; Valid syntaxes are: -; 'ip.add.re.ss:port' - to listen on a TCP socket to a specific IPv4 address on -; a specific port; -; '[ip:6:addr:ess]:port' - to listen on a TCP socket to a specific IPv6 address on -; a specific port; -; 'port' - to listen on a TCP socket to all addresses -; (IPv6 and IPv4-mapped) on a specific port; -; '/path/to/unix/socket' - to listen on a unix socket. -; Note: This value is mandatory. -listen = /var/run/php8-fpm.sock - -; Set permissions for unix socket, if one is used. In Linux, read/write -; permissions must be set in order to allow connections from a web server. Many -; BSD-derived systems allow connections regardless of permissions. -; Default Values: user and group are set as the running user -; mode is set to 0660 -listen.owner = nginx -listen.group = nginx -listen.mode = 0660 - -; Choose how the process manager will control the number of child processes. -; Possible Values: -; static - a fixed number (pm.max_children) of child processes; -; dynamic - the number of child processes are set dynamically based on the -; following directives. With this process management, there will be -; always at least 1 children. -; pm.max_children - the maximum number of children that can -; be alive at the same time. -; pm.start_servers - the number of children created on startup. -; pm.min_spare_servers - the minimum number of children in 'idle' -; state (waiting to process). If the number -; of 'idle' processes is less than this -; number then some children will be created. -; pm.max_spare_servers - the maximum number of children in 'idle' -; state (waiting to process). If the number -; of 'idle' processes is greater than this -; number then some children will be killed. -; ondemand - no children are created at startup. Children will be forked when -; new requests will connect. The following parameter are used: -; pm.max_children - the maximum number of children that -; can be alive at the same time. -; pm.process_idle_timeout - The number of seconds after which -; an idle process will be killed. -; Note: This value is mandatory. -pm = ondemand - -; The number of child processes to be created when pm is set to 'static' and the -; maximum number of child processes when pm is set to 'dynamic' or 'ondemand'. -; This value sets the limit on the number of simultaneous requests that will be -; served. Equivalent to the ApacheMaxClients directive with mpm_prefork. -; Equivalent to the PHP_FCGI_CHILDREN environment variable in the original PHP -; CGI. The below defaults are based on a server without much resources. Don't -; forget to tweak pm.* to fit your needs. -; Note: Used when pm is set to 'static', 'dynamic' or 'ondemand' -; Note: This value is mandatory. -pm.max_children = 5 - -; The number of child processes created on startup. -; Note: Used only when pm is set to 'dynamic' -; Default Value: min_spare_servers + (max_spare_servers - min_spare_servers) / 2 -; pm.start_servers = 2 - -; The desired minimum number of idle server processes. -; Note: Used only when pm is set to 'dynamic' -; Note: Mandatory when pm is set to 'dynamic' -; pm.min_spare_servers = 1 - -; The desired maximum number of idle server processes. -; Note: Used only when pm is set to 'dynamic' -; Note: Mandatory when pm is set to 'dynamic' -; pm.max_spare_servers = 3 - -; This sets the maximum time in seconds a script is allowed to run before it is -; terminated by the parser. This helps prevent poorly written scripts from tying up -; the server. The default setting is 30s. -; Note: Used only when pm is set to 'ondemand' -pm.process_idle_timeout = 10s - -; The number of requests each child process should execute before respawning. -; This can be useful to work around memory leaks in 3rd party libraries. For endless -; request processing specify '0'. -; Equivalent to PHP_FCGI_MAX_REQUESTS. Default value: 0. -; Noted: Used only when pm is set to 'ondemand' -pm.max_requests = 200 - -; The ping URI to call the monitoring page of FPM. If this value is not set, no -; URI will be recognized as a ping page. This could be used to test from outside -; that FPM is alive and responding, or to -; - create a graph of FPM availability (rrd or such); -; - remove a server from a group if it is not responding (load balancing); -; - trigger alerts for the operating team (24/7). -; Note: The value must start with a leading slash (/). The value can be -; anything, but it may not be a good idea to use the .php extension or it -; may conflict with a real PHP file. -; Default Value: not set -ping.path = /ping - -; This directive may be used to customize the response of a ping request. The -; response is formatted as text/plain with a 200 response code. -; Default Value: pong -;ping.response = pong diff --git a/webmails/snappymail/defaults/default.ini b/webmails/snappymail/defaults/default.ini deleted file mode 100644 index be9a0969..00000000 --- a/webmails/snappymail/defaults/default.ini +++ /dev/null @@ -1,15 +0,0 @@ -imap_host = "{{ FRONT_ADDRESS }}" -imap_port = 10143 -imap_secure = "None" -imap_short_login = Off -sieve_use = On -sieve_allow_raw = Off -sieve_host = "{{ IMAP_ADDRESS }}" -sieve_port = 4190 -sieve_secure = "None" -smtp_host = "{{ FRONT_ADDRESS }}" -smtp_port = 10025 -smtp_secure = "None" -smtp_short_login = Off -smtp_auth = On -smtp_php_mail = Off diff --git a/webmails/snappymail/defaults/default.json b/webmails/snappymail/defaults/default.json new file mode 100644 index 00000000..ecbf116c --- /dev/null +++ b/webmails/snappymail/defaults/default.json @@ -0,0 +1,50 @@ +{ + "name": "*", + "IMAP": { + "host": "{{ FRONT_ADDRESS }}", + "port": 10143, + "secure": 0, + "shortLogin": false, + "ssl": { + "verify_peer": false, + "verify_peer_name": false, + "allow_self_signed": false, + "SNI_enabled": true, + "disable_compression": true, + "security_level": 1 + } + }, + "SMTP": { + "host": "{{ FRONT_ADDRESS }}", + "port": 10025, + "secure": 0, + "shortLogin": false, + "ssl": { + "verify_peer": false, + "verify_peer_name": false, + "allow_self_signed": false, + "SNI_enabled": true, + "disable_compression": true, + "security_level": 1 + }, + "useAuth": true, + "setSender": false, + "usePhpMail": false + }, + "Sieve": { + "host": "{{ IMAP_ADDRESS }}", + "port": 4190, + "secure": 0, + "shortLogin": false, + "ssl": { + "verify_peer": false, + "verify_peer_name": false, + "allow_self_signed": false, + "SNI_enabled": true, + "disable_compression": true, + "security_level": 1 + }, + "enabled": true + }, + "whiteList": "" +} diff --git a/webmails/snappymail/defaults/php.ini b/webmails/snappymail/defaults/php.ini deleted file mode 100644 index d3d4d9f1..00000000 --- a/webmails/snappymail/defaults/php.ini +++ /dev/null @@ -1,5 +0,0 @@ -expose_php=Off -date.timezone={{ TZ }} -upload_max_filesize = {{ MAX_FILESIZE }}M -post_max_size = {{ MAX_FILESIZE }}M - diff --git a/webmails/snappymail/start.py b/webmails/snappymail/start.py deleted file mode 100755 index 5307f23b..00000000 --- a/webmails/snappymail/start.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -import os -import shutil -import logging as log -import sys -import subprocess - -from socrate import system, conf - -log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "WARNING")) - -# Actual startup script -os.environ["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front")) -os.environ["IMAP_ADDRESS"] = system.resolve_address(os.environ.get("HOST_IMAP", "imap")) - -os.environ["MAX_FILESIZE"] = str(int(int(os.environ.get("MESSAGE_SIZE_LIMIT"))*0.66/1048576)) - -base = "/data/_data_/_default_/" -shutil.rmtree(base + "domains/", ignore_errors=True) -os.makedirs(base + "domains", exist_ok=True) -os.makedirs(base + "configs", exist_ok=True) - -conf.jinja("/defaults/default.ini", os.environ, "/data/_data_/_default_/domains/default.ini") -conf.jinja("/defaults/application.ini", os.environ, "/data/_data_/_default_/configs/application.ini") -conf.jinja("/defaults/php.ini", os.environ, "/etc/php81/php.ini") -# Start the fastcgi process manager now that config files have been adjusted -os.system("php-fpm81") - -os.system("chown -R nginx:nginx /data") -os.system("chmod -R a+rX /var/www/webmail/") - -subprocess.call(["/config.py"]) -os.execv("/usr/sbin/nginx", ["nginx", "-g", "daemon off;"]) diff --git a/webmails/roundcube/start.py b/webmails/start.py similarity index 71% rename from webmails/roundcube/start.py rename to webmails/start.py index b5a4dca5..464d50e8 100755 --- a/webmails/roundcube/start.py +++ b/webmails/start.py @@ -4,9 +4,10 @@ import os import logging import sys import subprocess +import shutil import hmac -from socrate import conf +from socrate import conf, system env = os.environ @@ -17,6 +18,8 @@ context = {} context.update(env) context["MAX_FILESIZE"] = str(int(int(env.get("MESSAGE_SIZE_LIMIT", "50000000")) * 0.66 / 1048576)) +context["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front")) +context["IMAP_ADDRESS"] = system.resolve_address(os.environ.get("HOST_IMAP", "imap")) db_flavor = env.get("ROUNDCUBE_DB_FLAVOR", "sqlite") if db_flavor == "sqlite": @@ -52,7 +55,7 @@ context['SECRET_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUN # roundcube plugins # (using "dict" because it is ordered and "set" is not) -plugins = dict((p, None) for p in env.get("ROUNDCUBE_PLUGINS", "").replace(" ", "").split(",") if p and os.path.isdir(os.path.join("/var/www/webmail/plugins", p))) +plugins = dict((p, None) for p in env.get("ROUNDCUBE_PLUGINS", "").replace(" ", "").split(",") if p and os.path.isdir(os.path.join("/var/www/roundcube/plugins", p))) if plugins: plugins["mailu"] = None else: @@ -67,15 +70,14 @@ context["INCLUDES"] = sorted(inc for inc in os.listdir("/overrides") if inc.ends context["SESSION_TIMEOUT_MINUTES"] = max(int(env.get("SESSION_TIMEOUT", "3600")) // 60, 1) # create config files -conf.jinja("/conf/php.ini", context, "/etc/php81/php.ini") -conf.jinja("/conf/config.inc.php", context, "/var/www/webmail/config/config.inc.php") +conf.jinja("/conf/config.inc.php", context, "/var/www/roundcube/config/config.inc.php") # create dirs os.system("mkdir -p /data/gpg") print("Initializing database") try: - result = subprocess.check_output(["/var/www/webmail/bin/initdb.sh", "--dir", "/var/www/webmail/SQL"], + result = subprocess.check_output(["/var/www/roundcube/bin/initdb.sh", "--dir", "/var/www/roundcube/SQL"], stderr=subprocess.STDOUT) print(result.decode()) except subprocess.CalledProcessError as exc: @@ -88,22 +90,31 @@ except subprocess.CalledProcessError as exc: print("Upgrading database") try: - subprocess.check_call(["/var/www/webmail/bin/update.sh", "--version=?", "-y"], stderr=subprocess.STDOUT) + subprocess.check_call(["/var/www/roundcube/bin/update.sh", "--version=?", "-y"], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: exit(4) else: print("Cleaning database") try: - subprocess.check_call(["/var/www/webmail/bin/cleandb.sh"], stderr=subprocess.STDOUT) + subprocess.check_call(["/var/www/roundcube/bin/cleandb.sh"], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: exit(5) +base = "/data/_data_/_default_/" +shutil.rmtree(base + "domains/", ignore_errors=True) +os.makedirs(base + "domains", exist_ok=True) +os.makedirs(base + "configs", exist_ok=True) + +conf.jinja("/defaults/default.json", context, "/data/_data_/_default_/domains/default.json") +conf.jinja("/defaults/application.ini", context, "/data/_data_/_default_/configs/application.ini") +conf.jinja("/defaults/php.ini", context, "/etc/php81/php.ini") + # setup permissions -os.system("chown -R nginx:nginx /data") -os.system("chmod -R a+rX /var/www/webmail/") +os.system("chown -R nginx:nginx /data /var/www") +os.system("chmod -R a+rX /var/www/") # Configure nginx -conf.jinja("/conf/nginx-roundcube.conf", context, "/etc/nginx/http.d/roundcube.conf") +conf.jinja("/conf/nginx-webmail.conf", context, "/etc/nginx/http.d/webmail.conf") if os.path.exists("/var/run/nginx.pid"): os.system("nginx -s reload") From 1edef755f1a61312b2fee19fff0991e61f2798e3 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 11:40:23 +0100 Subject: [PATCH 02/68] Fix bug #2466 --- webmails/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index ccc2da0c..9a4f5c8e 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -12,6 +12,7 @@ RUN set -euxo pipefail \ php81-dom php81-curl php81-exif gd php81-gd php81-iconv php81-intl php81-openssl \ php81-pdo_sqlite php81-pdo_mysql php81-pdo_pgsql php81-pdo php81-sodium libsodium php81-tidy php81-pecl-uuid \ php81-pspell php81-pecl-imagick php81-opcache php81-session php81-sockets php81-fileinfo \ + aspell-uk aspell-ru aspell-fr aspell-de aspell-en \ ; rm /etc/nginx/http.d/default.conf \ ; rm /etc/php81/php-fpm.d/www.conf \ ; ln -s /usr/bin/php81 /usr/bin/php \ From 13adf4aeec639cc9f9f4010c7aff79c273480fc4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 11:46:59 +0100 Subject: [PATCH 03/68] Fix tests --- tests/compose/snappymail/docker-compose.yml | 106 -------------- tests/compose/snappymail/mailu.env | 138 ------------------ .../{roundcube => webmail}/docker-compose.yml | 0 .../compose/{roundcube => webmail}/mailu.env | 2 +- 4 files changed, 1 insertion(+), 245 deletions(-) delete mode 100644 tests/compose/snappymail/docker-compose.yml delete mode 100644 tests/compose/snappymail/mailu.env rename tests/compose/{roundcube => webmail}/docker-compose.yml (100%) rename tests/compose/{roundcube => webmail}/mailu.env (99%) diff --git a/tests/compose/snappymail/docker-compose.yml b/tests/compose/snappymail/docker-compose.yml deleted file mode 100644 index b9df7332..00000000 --- a/tests/compose/snappymail/docker-compose.yml +++ /dev/null @@ -1,106 +0,0 @@ -# This file is auto-generated by the Mailu configuration wizard. -# Please read the documentation before attempting any change. -# Generated for compose flavor - -version: '3.6' - -services: - - # External dependencies - redis: - image: redis:alpine - restart: always - volumes: - - "/mailu/redis:/data" - - # Core services - front: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}nginx:${MAILU_VERSION:-local} - restart: always - env_file: mailu.env - logging: - driver: json-file - ports: - - "127.0.0.1:80:80" - - "127.0.0.1:443:443" - - "127.0.0.1:25:25" - - "127.0.0.1:465:465" - - "127.0.0.1:587:587" - - "127.0.0.1:110:110" - - "127.0.0.1:995:995" - - "127.0.0.1:143:143" - - "127.0.0.1:993:993" - volumes: - - "/mailu/certs:/certs" - - admin: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}admin:${MAILU_VERSION:-local} - restart: always - env_file: mailu.env - volumes: - - "/mailu/data:/data" - - "/mailu/dkim:/dkim" - depends_on: - - redis - - resolver - dns: - - 192.168.203.254 - - imap: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}dovecot:${MAILU_VERSION:-local} - restart: always - env_file: mailu.env - volumes: - - "/mailu/mail:/mail" - - "/mailu/overrides:/overrides" - depends_on: - - front - - smtp: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}postfix:${MAILU_VERSION:-local} - restart: always - env_file: mailu.env - volumes: - - "/mailu/overrides:/overrides" - depends_on: - - front - - antispam: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}rspamd:${MAILU_VERSION:-local} - restart: always - env_file: mailu.env - volumes: - - "/mailu/filter:/var/lib/rspamd" - - "/mailu/dkim:/dkim" - - "/mailu/overrides/rspamd:/etc/rspamd/override.d" - depends_on: - - front - - # Optional services - - resolver: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}unbound:${MAILU_VERSION:-local} - env_file: mailu.env - restart: always - networks: - default: - ipv4_address: 192.168.203.254 - - # Webmail - webmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}snappymail:${MAILU_VERSION:-local} - restart: always - env_file: mailu.env - volumes: - - "/mailu/webmail:/data" - depends_on: - - imap - - -networks: - default: - driver: bridge - ipam: - driver: default - config: - - subnet: 192.168.203.0/24 diff --git a/tests/compose/snappymail/mailu.env b/tests/compose/snappymail/mailu.env deleted file mode 100644 index 50271fc7..00000000 --- a/tests/compose/snappymail/mailu.env +++ /dev/null @@ -1,138 +0,0 @@ -# Mailu main configuration file -# -# Generated for compose flavor -# -# This file is autogenerated by the configuration management wizard. -# For a detailed list of configuration variables, see the documentation at -# https://mailu.io - -################################### -# Common configuration variables -################################### - -# Set this to the path where Mailu data and configuration is stored -# This variable is now set directly in `docker-compose.yml by the setup utility -# ROOT=/mailu - -# Mailu version to run (1.0, 1.1, etc. or master) -#VERSION=master - -# Set to a randomly generated 16 bytes string -SECRET_KEY=V5J4SHRYVW9PZIQU - -# Address where listening ports should bind -# This variables are now set directly in `docker-compose.yml by the setup utility -# PUBLIC_IPV4= 127.0.0.1 (default: 127.0.0.1) -# PUBLIC_IPV6= (default: ::1) - -# Subnet of the docker network. This should not conflict with any networks to which your system is connected. (Internal and external!) -SUBNET=192.168.203.0/24 - -# Main mail domain -DOMAIN=mailu.io - -# Hostnames for this server, separated with comas -HOSTNAMES=localhost - -# Postmaster local part (will append the main mail domain) -POSTMASTER=admin - -# Choose how secure connections will behave (value: letsencrypt, cert, notls, mail, mail-letsencrypt) -TLS_FLAVOR=cert - -# Authentication rate limit (per source IP address) -AUTH_RATELIMIT=10/minute;1000/hour - -# Opt-out of statistics, replace with "True" to opt out -DISABLE_STATISTICS=False - -################################### -# Optional features -################################### - -# Expose the admin interface (value: true, false) -ADMIN=false - -# Choose which webmail to run if any (values: roundcube, snappymail, none) -WEBMAIL=snappymail - -# Dav server implementation (value: radicale, none) -WEBDAV=none - -# Antivirus solution (value: clamav, none) -#ANTIVIRUS=none - -#Antispam solution -ANTISPAM=none - -################################### -# Mail settings -################################### - -# Message size limit in bytes -# Default: accept messages up to 50MB -MESSAGE_SIZE_LIMIT=50000000 - -# Networks granted relay permissions -# Use this with care, all hosts in this networks will be able to send mail without authentication! -RELAYNETS= - -# Will relay all outgoing mails if configured -RELAYHOST= - -# Fetchmail delay -FETCHMAIL_DELAY=600 - -# Recipient delimiter, character used to delimiter localpart from custom address part -RECIPIENT_DELIMITER=+ - -# DMARC rua and ruf email -DMARC_RUA=admin -DMARC_RUF=admin - - -# Maildir Compression -# choose compression-method, default: none (value: gz, bz2, lz4, zstd) -COMPRESSION= -# change compression-level, default: 6 (value: 1-9) -COMPRESSION_LEVEL= - -################################### -# Web settings -################################### - -# Path to the admin interface if enabled -WEB_ADMIN=/admin - -# Path to the webmail if enabled -WEB_WEBMAIL=/webmail - -# Website name -SITENAME=Mailu - -# Linked Website URL -WEBSITE=https://mailu.io - - - -################################### -# Advanced settings -################################### - -# Log driver for front service. Possible values: -# json-file (default) -# journald (On systemd platforms, useful for Fail2Ban integration) -# syslog (Non systemd platforms, Fail2Ban integration. Disables `docker-compose log` for front!) -# LOG_DRIVER=json-file - -# Docker-compose project name, this will prepended to containers names. -COMPOSE_PROJECT_NAME=mailu - -# Header to take the real ip from -REAL_IP_HEADER= - -# IPs for nginx set_real_ip_from (CIDR list separated by commas) -REAL_IP_FROM= - -# choose wether mailu bounces (no) or rejects (yes) mail when recipient is unknown (value: yes, no) -REJECT_UNLISTED_RECIPIENT= diff --git a/tests/compose/roundcube/docker-compose.yml b/tests/compose/webmail/docker-compose.yml similarity index 100% rename from tests/compose/roundcube/docker-compose.yml rename to tests/compose/webmail/docker-compose.yml diff --git a/tests/compose/roundcube/mailu.env b/tests/compose/webmail/mailu.env similarity index 99% rename from tests/compose/roundcube/mailu.env rename to tests/compose/webmail/mailu.env index 7f000f2c..f87f3262 100644 --- a/tests/compose/roundcube/mailu.env +++ b/tests/compose/webmail/mailu.env @@ -54,7 +54,7 @@ DISABLE_STATISTICS=False ADMIN=false # Choose which webmail to run if any (values: roundcube, snappymail, none) -WEBMAIL=roundcube +WEBMAIL=snappymail # Dav server implementation (value: radicale, none) WEBDAV=none From ae64c6cc306bad0344d7c0af8082ecd8c0ef135d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 11:51:12 +0100 Subject: [PATCH 04/68] Doh --- tests/compose/webmail/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compose/webmail/docker-compose.yml b/tests/compose/webmail/docker-compose.yml index f2c43686..14d1dae9 100644 --- a/tests/compose/webmail/docker-compose.yml +++ b/tests/compose/webmail/docker-compose.yml @@ -88,7 +88,7 @@ services: # Webmail webmail: - image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}roundcube:${MAILU_VERSION:-local} + image: ${DOCKER_ORG:-mailu}/${DOCKER_PREFIX:-}webmail:${MAILU_VERSION:-local} restart: always env_file: mailu.env volumes: From a8d405cb487365ff50b1124eabed5e8308814ad3 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 12:25:03 +0100 Subject: [PATCH 05/68] Verify the gpg signature of webmails --- webmails/Dockerfile | 15 ++++- webmails/roundcube/pubkey.asc | 102 +++++++++++++++++++++++++++++++++ webmails/snappymail/pubkey.asc | 11 ++++ 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 webmails/roundcube/pubkey.asc create mode 100644 webmails/snappymail/pubkey.asc diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 9a4f5c8e..b967af5a 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -5,6 +5,9 @@ FROM base ARG VERSION LABEL version=$VERSION +COPY snappymail/pubkey.asc /tmp/snappymail.asc +COPY roundcube/pubkey.asc /tmp/roundcube.asc + RUN set -euxo pipefail \ ; apk add --no-cache \ nginx gpg gpg-agent \ @@ -16,6 +19,8 @@ RUN set -euxo pipefail \ ; rm /etc/nginx/http.d/default.conf \ ; rm /etc/php81/php-fpm.d/www.conf \ ; ln -s /usr/bin/php81 /usr/bin/php \ + ; gpg --import /tmp/snappymail.asc \ + ; gpg --import /tmp/roundcube.asc \ ; mkdir -p /run/nginx \ ; mkdir -p /conf @@ -25,7 +30,10 @@ ENV CARDDAV_URL https://github.com/mstilkerich/rcmcarddav/releases/download/v4.4 RUN set -euxo pipefail \ ; cd /var/www \ - ; curl -sL ${ROUNDCUBE_URL} | tar xz \ + ; curl -sLo /dev/shm/roundcube.tgz ${ROUNDCUBE_URL} \ + ; curl -sLo /dev/shm/roundcube.tgz.asc ${ROUNDCUBE_URL}.asc \ + ; gpg --status-fd 1 --verify /dev/shm/roundcube.tgz.asc \ + ; tar xzf /dev/shm/roundcube.tgz \ ; curl -sL ${CARDDAV_URL} | tar xz \ ; mv roundcubemail-* roundcube \ ; mkdir -p /var/www/roundcube/config \ @@ -48,7 +56,10 @@ ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2 RUN set -euxo pipefail \ ; mkdir /var/www/snappymail \ ; cd /var/www/snappymail \ - ; curl -sL ${SNAPPYMAIL_URL} | tar xz \ + ; curl -sLo /dev/shm/snappymail.tgz ${SNAPPYMAIL_URL} \ + ; curl -sLo /dev/shm/snappymail.tgz.asc ${SNAPPYMAIL_URL}.asc \ + ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc \ + ; tar xzf /dev/shm/snappymail.tgz \ ; chmod -R u+w,a+rX /var/www/snappymail \ ; chown -R nginx:nginx /var/www/snappymail diff --git a/webmails/roundcube/pubkey.asc b/webmails/roundcube/pubkey.asc new file mode 100644 index 00000000..3d4449c9 --- /dev/null +++ b/webmails/roundcube/pubkey.asc @@ -0,0 +1,102 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBFcNX2kBEACmCY1yOI8MUk0fHtMOqxzDwA/CH0yN2nQu/mNiwOzx9pCtpX2u +F//FAql2Ob8ZVpwichouC//y7+dpqhzF+1TQYKZP9wtR4f5Y5T4SEDMGS+mhsdvO +LBSSpbteLtwbWrWU7CGTx6ohGO15VYfLagVKUvKkslSXFgWAfH+VrD1x05AlNeio +rgbdHLZsh5+JhqiyOMg8lsLkUA5mwe75TLjMF7xS3BKqBlnE7grWUfBs3/5vhIiu +/vsmnLX98tbBk6ZY+FB0xuzqiA8rW1LCB0d8eIBHnU1Xi0n1ebEG2xqtxV2Kprvj +NZDIZfOrTRqoP0fe36PxWXGHoR7tntWyqXfC3ZWgw00S7wrp0f3YZAASVbj2863i +gMs06zSHhVKnKqo6r+eDRcie+CRvtRVlh3PKaluh1ea+ad8A3BK1F8MKEpm3zBAn +/RP+p0ZNa0K3IDkuacG/yJ8f+VAeJl5KYu6Uv3+jADbCUuZFbm8ZGDoT1qcxkATd +S35D26oe41STPRUMppb+aJFMbgFLQLE5lHPEROUG1I5trrV9cfi5zP4G1A9bc9Cj +B9m5kyz5tmST1WVYB2yFsngYCIRx2sbQwAY8z2JThTUUWL6KaJuwcFXInGQqjUU1 +GJHBGED0lduVnK3WgVKNLthABFMXJ34dzxPsiAJ68295OhUP9G4Qvo5DzQARAQAB +tClSb3VuZGN1YmUgRGV2ZWxvcGVycyA8ZGV2c0Byb3VuZGN1YmUubmV0PokCOQQT +AQgAIwUCVw1faQIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEFqyuqFB +xPfVN3IP/2ANH6mgd66Acz7AuUp9YhZ6A00VkrGfmdju9aA8LuEBdt2dUyUIvzzm +BqKbIfotbpn7lpJsDRV2L2alDUL0fvVcuH6vy1u/LrAOVXPuE0ACyRuwBIzmKV8g +iJYES5FOVVfjZh/k+rdWDj654ohOyQxPYiW/213/MNonbgodXk5H+jTMGxsVJHhi +VyRwiwzkFV9qozb+R/fCirCayHL6v0A0HWtAwXbHabZUoHXEY/XtQFnvEw1HR3u5 +1nIl17ClaKtoOeXh35ONXqu27Xzxw/skqOVUj3LNzZN7IhR4PzKaTCg4g6n1ngyU +VgrXIS6JLwLSyyurkdGCIKifW/5BqmikXdp6oJ6x3/nDzg7IzpEbipetiYsVVjZG +aZkuATC+Pj/kW/AmWYX9vxxEDnVEu6r71zMWIqiEzu+8JoO2IvvuU5tvbbMhRze7 +/tc/WxZSYOzaudb6Bi/4FX2x8l6FGiIP/xI6Gpyjd5HwRWYnUqv7pBqyzs0Z15vG +roYcayLaFAhLCxBnBhUVbwVoRif4h9ihPc6PndZp/nOIAOpNGVqZbXcoXjz+Ugvb +icGKul/q7t1vl+3cf0bBT8O918TvzVXJIixnW/f9rdPAGT0KtsE7B7UXxOkV3xpC +uh+kA0W8huJLaEWFZ5izBixkhzdLwITJD2VQ/TVuwHSI2A4kFnF5iQIiBBMBCAAM +BQJXDWCdBYMHhh+AAAoJED5UKNAmLFT4KOoQAJ7qQ25imKrnebNVQ7unSCDIcZ7n +wc7MGlOCmO0txGtDgaVZy2pvBd/zIliYtrGkbkDpMTTVds73/XofLJ+n41nNLPI7 +jDdVOnYpcu2bj74KUQRY+2WQ6riewsFUF52FtNOegsIj8JXmK58CPoW3M/uVZRdf +ISVAUHkQuP9YWJoeToB/RXqICCRX3DfUgFSbHaEVRqpln+mnljopNBrDMe9ZthC2 +6Py8HwhshtBiwcP9NlaGTeG+Ks2A7Ujt2BUgBWyN4ouf8ehmyjD5D9RCxjPh7lof +Ap8JhGpbd8Yu97Ax8bwZcHZ1ePx9NxcC+PFf6wK3jK464Vx7JTKk4gS3Ktk/+adA +b9dasn+/OOaWwzHkpBTUJP7gW1pv8xhA+Op2VqwRNqB2WfiqOHyydQSZKJVncdA6 +/p3p4ABluPtbe8L1SE0ZDEOGjXwTMxH3ssDLlQ4BlqlWzhudeNv9Tizd8tlgtBvg +VprEpWd++JovQs8MmEcoLaDS1DSglEsoRnrpCJ1vkacQZlN2wpv7PEEmH8SBaYU7 +xRZhRmc1arRFnelVo4OPzLTSMSFjZIdmMs8Lfzrw2fRGesrJGpb3DnVphwML1aXp +mSFHKuXDqDVMW+Ey437KadG/Bd92q4FEeyCjjoHYa2C86dZG1yMfuVVMfvVz0A+v +lSR6abLAK3f+VO1piQEcBBMBAgAGBQJXGG4NAAoJEL7mdKAZNZ3BLmkH/i03cRxM +WU9baZgpZ7IkIz77tJJdcW51dZKy04FhbFKH6Qlp6WcGHEPy6EZWRdktJlSXTc+T +/1lhlXeRPGesqvIAqnDfOayKf2rihBoAfPQCzxaJOAldt0KdDX6zGIYa4Xqappla +kPLHeCSKhGm8eYf7IQjiq3AoMRvtGDtv8ygrA7sN8vc7Ftr1fg3s8UaB8QULLRD4 +INRgxfuPG9St5V5zYV/3Xf/61uOlNfxxikx5PCHle4jKJGkP+smXON4l8+XPyhSG +US7aIGalr58acv0VZHFkTaCi+96s14df0XRENO5D4l5n18PiHQvh/th995ba96K/ +8jrcY7f8wjM0OYm5Ag0EVw1faQEQAPII9TY0LeEWP+4/FFQCBmgXR+aWjMK0O3fa +BuPzL/VVHQJ3i41PvvP+Osb7BYPFTxPWkvVF2J1bLZfH1wFq+hMfEOkGMGtBFOP2 +VxWEYxMondktMhKDHT5EppPwqsZYPqlNz6Sk/bW81IXKtSG/hvPyBDv1+GaHZlz+ +NJrKjVlBN+6U4noM2P9n/QPCd5VmkZMWzCfbtmGZKHspOJswMhcW28YvMmYTK+0b +ZcKCs2S2wgfM8d5EEeoYTXH6PqxfW3ezZXQ5ieM1sub59GnS+7gqxPEs+LyVQtxT +7dgCnZQ73tmQP3pG2Zx0pKQHK/hZk8R6aEaYtV1QlfUI1TMG1eH+xHXGSWFnCbiX +cGLltaLFBX11+qwF50FfYu8MRUM9rKW+ms2wBVmHuSGKgn0lglBGU2s/pPPw6Alu +GWa289vGdnztoQyY33L3u/la0wCBbM/8JxZYZdmTq1iL0oYuPbn3axfa6JCX9CwC +KQjOcJe8K+scRsSFI23M3ZySVgKpkOdhz9VfBZHTqMpbsTd8kNHBDu5J3C0v2NsV +gJsqI5c3cVtaGPL2NVdfjZ668aXs89JA0Sc9Q1ppiDQX2ArNbq0ZRG4pGfAP3zA9 +6RyfHTgM9PZ5M4BReeWJCYQb6UI8Uw/NlUYsMMMbi8yqhIkXCY0U7I0ZKtVUSHSR +W6gftdEhABEBAAGJAh8EGAEIAAkFAlcNX2kCGwwACgkQWrK6oUHE99XmpA/5AXxm +SfeyUcUUaMH+n1EJt7lH6u8Tg4WxoSpSoF/GrArEBfdDGmUog2kR8cgyTFKjtiuP +icCIapeezP2QMxWfm0TTITtFiHAUJZn0642SY4uXI/73Bwa0r5Vi1UevaFrRPkee +0Jt3Tg45nvkUNQBuRK81Wr2o+EuNiMgssd78MHiWjllVptFg0GnfE1VUeMeM8Rwa +QnVzVyYZbqe4jL20+QCba/zyrcQgcxZ/gtojADpPHojI2BQlsXnIhrSlXYXIDhmF +SCG4+RdUq+JVI8vjO42bHA51gGyvZR7Fh7tcdU++U6wbhF5gkzB3v+NjHxwmcI/t +pnrTP7nT1rZOUdyuKSJkcCUa3l8u+bqlxgQ3r+PJOXuW5Tn53HYkxdTSgzFwc9GS +SvyTZnz/JYE241Yf14Vjn8fZqPsN+uplc4b42G08gQi0Juni7W5dPo3Jl+7MgXJR +0vBtCEuZLJ49ZUpKwf0vS1aDDfMNA4ESs/TagIakUMGNH0tVsEm5YNMoNx9qZA3a +rJT+ZhpZNFBW94QU3hQ+hbtyR/0rO8BGlpA0XLhNoPUNhgWMobgWAIA9kEQilm1Y +tPDS5EHhsAiLi60/bIuti4T0nhxlgw+yfeb5kEnm5v5XYSj5w0XzfyGirfV80QP4 +7CE8GKy2q+e3xau15t/eVvMtYd2RDgykqIjvwtC5Ag0EVw1f/QEQAO2JeXBrzcBt +TeUcPA70W9quirv4wnXtUTwAGRXklK/OaKPruPTPJIQu6qdimJO+p6KbWP4mD8b9 +t7mWilDpJO3omZKqMqCRqd+TPp0rzvHde1QhwCNIByCIkrTjcsq2JuGTSEME09Aa +nOTE5/UeThTeXI+xvta63kpHgBolBunMUwPlde36KOUgWktr6NiCr3CQ1MtzDuBl +wEAi1/K8/mkIU5SXmmC7NOKQVsK/HCpuhkT0fZY4RGIHlauIiOs8vXvJ9kajkvF+ +HJcmsQ/8GuMELVKi/V9BnObCCL49EykK5s5VEF4guQ4r3ElbS/PXvE4OXL+0vmBR +YQFdVUdHNS36LErGzYIgghQIgDF1JS08EuoD86+fVHwwbupCp9SMQRWjrvWroipG +Sk6K3BJfM9deZhuMH2j2ab4OleHZdJH+4PLIa+NwXMhuvKPJPKXmP5c1Seu7AyON +hUQEU/lHEW03NvS4nh/ArM/za+dFplzSSaoUq8Qhr3AeyAVd+4PXgpbj7pIdfaBI +IADx/uFYLLcc/whD/2C2t37h3TIjR18IS05aiGHDJyZ9eV2K/wf8kZ7Xq4ix+6Or +Jt37g2/klHsvHo3kb+6XPpo263+pRj/bcA2vUA3c26cZ8nCsHu9K4aN4VN8DTTPS +YYT9940OfRh8CRCNlcVerfbjNAE3fgnbABEBAAGJBD4EGAEIAAkFAlcNX/0CGwIC +KQkQWrK6oUHE99XBXSAEGQEIAAYFAlcNX/0ACgkQwpRqlgnNVrRIXRAA48pg+pQG +aqghqsVPtRt4yZy3zc0RDr5vV3r00Tqutg7l1J/8gNm9NayyBX0BEY+bKvNPeNjl +gNkXCSH7eXX1mvUJuUUnbqJv+MT3roCcvLz6KLdQQdHarJSs4LmqF9/4NfHsSecg +jq3Y9fsG5sNf/a7BraIcdlOq92t0DlpAmAtm10ywUXJPc1uAxqd/2QyfuPQE/eoR +rmGnKR1W6FO1cAZYVWd3hyPAyr/EHHJonycpp8CKCe9CLu3iFXR8+GVq7ZiDVNk+ +MHMYg1Njfk3TY/UEUGXqFfTsD47S8fqEV/koWSSxTkSwPjwVP1z0yu9cV87ULeJN +LDdwyFvmTrQv71YkAD12CchRymqLxtItSF1QMiHBFXTICreYGk41pS89KNshgFpe +WfRq6WpPegUj1qdM/GJuBvSu7CTT2mpQQNk4maIIeUPcHRCA//H3WvXj3jMp3CFK +S82YYDkUW/XWkWIRmpALrX8gSYlthKFf24RZZFrAd7NfSq1Hy0RjAwtm0+LsRTtT +znzTUr2SocCEGqFjiczIJ/4zQ+25N2PPg1G5lCrIeE7VOifKD3jujMYiAEr6QUUm +Vldw7Rn0tmJIiq0bc3MbadUxrT0PJXxOlQpfV2ZjM76gMpvvSCe6o6mckDT4sT3G +4vfc02Pe4g4DYpVPlV/GE1T26NzK1Z3ONFzhLQ//abRaJKfy19+lNNJoGfGGLher +AdymumxmGZf74wS6xAlP+LwJldUA8iidSxM0gR6bmw8q2SO7dqziGreaPaFVmeUB +62rSXD0QSielIoRP1QZuD1ZO5tEZ2wxjcCnaBj2nG3bBj4RJ7FAD9CceSyPJFNYD +n6cvslV/MGzacMtTTIwdFJmHaoU86heADWkYIFm/jndYX6b/IdJDNOYDYA4m+5S8 +ANQ3uOuaBMDo4sOAUCeophdjZeyne2kIWR7kmWis5kFf/Criy6u+yPs+a7kt+PbI +2Uo1rmrNUiMiROkezbnZAEf/8wUi7KgRjZ6qfij/QM+0WMeUWu8NRqiS+KRLQIh7 +Y8f3u0ddlfGF7/UpAEXzv2KKpLO+SaUkvaatZucOD/hbDThqOVCtX7mQ03XTO9Pn +SHVSxBsJse4Jn/n6oCt6FT7wMbh3IuZTeU7kiT9VO8+M/ehUS0sIbwwsYrdAT2Od +/Txs7jWinvsuH/qsNFVDrxKKcFQi99m0Zm3IIo2DX5PUo9KvPO8xzZgFKQDOIKBw +1PNQr0xRqbI1dsFcaN2yqF4hrYYmn4bDJCOMHV3gxltFaLU/rj7atdIWGOPzw/1N +WQujs2OMoiJWTidcd/LTxbEvEDyS9vMiIXrAoadvRtBxmFqJfcmRhOrbKIcA4A65 +0dXJnhEe7eXkwBbfEzk= +=lBKd +-----END PGP PUBLIC KEY BLOCK----- diff --git a/webmails/snappymail/pubkey.asc b/webmails/snappymail/pubkey.asc new file mode 100644 index 00000000..9f295b79 --- /dev/null +++ b/webmails/snappymail/pubkey.asc @@ -0,0 +1,11 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: Hostname: +Version: Hockeypuck 2.1.0-184-g50f1108 + +xjMEYg0atBYJKwYBBAHaRw8BAQdA2S2tvGavChACjtBastsKRThD3rsBW1LUZLmN +Zbs4uaHNI1NuYXBweU1haWwgPHJlbGVhc2VzQHNuYXBweW1haWwuZXU+wpQEExYK +ADwWIQQQFuRweRRVQvi6EzVIIIuhMpDz6wUCYg0atAIbAwULCQgHAgMiAgEGFQoJ +CAsCBBYCAwECHgcCF4AACgkQSCCLoTKQ8+u9SAD/Q/IoAwjUkKDJBPq0RGwCFnl6 +FG/VHB97CvBSpGOxtIsBAMCwMhWlsaBHAEqbzxiN+cdlMYwV23+SWLUJ/XMFgukE +=vC/h +-----END PGP PUBLIC KEY BLOCK----- From 224f2f45081313fd9c162969e33652dc4fc679e3 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 14:01:01 +0100 Subject: [PATCH 06/68] This isn't used anymore The healthcheck is now done by fpm --- webmails/roundcube/login/mailu.php | 7 ------- 1 file changed, 7 deletions(-) diff --git a/webmails/roundcube/login/mailu.php b/webmails/roundcube/login/mailu.php index 0596ca9d..25d95a10 100644 --- a/webmails/roundcube/login/mailu.php +++ b/webmails/roundcube/login/mailu.php @@ -18,13 +18,6 @@ class mailu extends rcube_plugin $args['action'] = 'login'; } - $ua = $_SERVER['HTTP_USER_AGENT']; - $ra = $_SERVER['REMOTE_ADDR']; - if ($ua == 'health' and ($ra == '127.0.0.1' or $ra == '::1')) { - print('OK'); - exit(); - } - return $args; } From 7e722cd0c30325f603d228bba4903cedd4b46b1a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 14:10:50 +0100 Subject: [PATCH 07/68] fix #2250: ensure rainloop uses _ADDRESS --- webmails/start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webmails/start.py b/webmails/start.py index 464d50e8..c2bd87b7 100755 --- a/webmails/start.py +++ b/webmails/start.py @@ -18,8 +18,8 @@ context = {} context.update(env) context["MAX_FILESIZE"] = str(int(int(env.get("MESSAGE_SIZE_LIMIT", "50000000")) * 0.66 / 1048576)) -context["FRONT_ADDRESS"] = system.resolve_address(os.environ.get("HOST_FRONT", "front")) -context["IMAP_ADDRESS"] = system.resolve_address(os.environ.get("HOST_IMAP", "imap")) +context["FRONT_ADDRESS"] = system.get_host_address_from_environment("FRONT", "front") +context["IMAP_ADDRESS"] = system.get_host_address_from_environment("IMAP", "imap") db_flavor = env.get("ROUNDCUBE_DB_FLAVOR", "sqlite") if db_flavor == "sqlite": From 710dde1faf4846c5889fc77f4502c5d056b08678 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 14:27:32 +0100 Subject: [PATCH 08/68] Fix #948: ensure the admin panel is disabled --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/compose/webmail/01_ensure_admin_unreachable.sh diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh new file mode 100644 index 00000000..c4afc76a --- /dev/null +++ b/tests/compose/webmail/01_ensure_admin_unreachable.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +[[ `curl -I -so /dev/null -w "%{http_code}" http://localhost/` -ne 200 ]] && echo "The default page of rainloop hasn't returned 200!" >>/dev/stderr && exit 1 +[[ `curl -I -so /dev/null -w "%{http_code}" http://localhost/?admin` -ne 403 ]] && echo "The admin of rainloop is not disabled!" >>/dev/stderr && exit 1 From 50f94a282f391ec0034085749fd8b8fe289cfc42 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 14:35:17 +0100 Subject: [PATCH 09/68] doh --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/compose/webmail/01_ensure_admin_unreachable.sh diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh old mode 100644 new mode 100755 From 1379a583525351458d2675d3fc81840eec7f73fe Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 14:50:30 +0100 Subject: [PATCH 10/68] Basic hardening --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 6 +++--- webmails/nginx-webmail.conf | 11 ++++++++++- webmails/php.ini | 3 +++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh index c4afc76a..a3864b78 100755 --- a/tests/compose/webmail/01_ensure_admin_unreachable.sh +++ b/tests/compose/webmail/01_ensure_admin_unreachable.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash -[[ `curl -I -so /dev/null -w "%{http_code}" http://localhost/` -ne 200 ]] && echo "The default page of rainloop hasn't returned 200!" >>/dev/stderr && exit 1 -[[ `curl -I -so /dev/null -w "%{http_code}" http://localhost/?admin` -ne 403 ]] && echo "The admin of rainloop is not disabled!" >>/dev/stderr && exit 1 +[[ `curl -I -so /dev/null -w "%{http_code}" http://localhost/` -ne 200 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 +[[ `curl -I -so /dev/null -w "%{http_code}" http://localhost/?admin` -ne 403 ]] && echo "The admin of snappymail is not disabled!" >>/dev/stderr && exit 1 diff --git a/webmails/nginx-webmail.conf b/webmails/nginx-webmail.conf index 5e5f8ec3..b1149c49 100644 --- a/webmails/nginx-webmail.conf +++ b/webmails/nginx-webmail.conf @@ -16,6 +16,11 @@ server { # set maximum body size to configured limit client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; + fastcgi_hide_header X-Powered-By; + add_header X-Download-Options "noopen" always; + add_header X-Robots-Tag "none" always; + add_header X-Permitted-Cross-Domain-Policies "none" always; + add_header Referrer-Policy "no-referrer" always; location / { try_files $uri $uri/ /index.php$args; @@ -42,10 +47,14 @@ server { {% endif %} } - location ~ /\. { + location ~ (^|/)\. { deny all; } + location ~* ^/(config|temp|logs) { + deny all; + } + location ^~ /data { deny all; } diff --git a/webmails/php.ini b/webmails/php.ini index af9ce8c5..884dda72 100644 --- a/webmails/php.ini +++ b/webmails/php.ini @@ -6,3 +6,6 @@ session.auto_start=Off mbstring.func_overload=Off file_uploads=On error_reporting = E_ALL & ~E_DEPRECATED & ~E_STRICT +display_errors=Off +log_errors=On +zlib.output_compression=Off From 729838c8fe2a6ddf5515dac7c0cf7a8b35a14633 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 15:12:22 +0100 Subject: [PATCH 11/68] Grrr. --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh index a3864b78..3351b85b 100755 --- a/tests/compose/webmail/01_ensure_admin_unreachable.sh +++ b/tests/compose/webmail/01_ensure_admin_unreachable.sh @@ -1,4 +1,6 @@ #!/bin/bash -[[ `curl -I -so /dev/null -w "%{http_code}" http://localhost/` -ne 200 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 -[[ `curl -I -so /dev/null -w "%{http_code}" http://localhost/?admin` -ne 403 ]] && echo "The admin of snappymail is not disabled!" >>/dev/stderr && exit 1 +IP="$(docker inspect webmail_webmail_1|jq '.[0].NetworkSettings.Networks.webmail_default.IPAddress')" + +[[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/) -ne 200 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 +[[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/?admin) -ne 403 ]] && echo "The admin of snappymail is not disabled!" >>/dev/stderr && exit 1 From 6d8cc9083bbc44a84e8412c61cf539b6ce356f4e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 15:21:04 +0100 Subject: [PATCH 12/68] test --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh index 3351b85b..ff0e9f25 100755 --- a/tests/compose/webmail/01_ensure_admin_unreachable.sh +++ b/tests/compose/webmail/01_ensure_admin_unreachable.sh @@ -2,5 +2,8 @@ IP="$(docker inspect webmail_webmail_1|jq '.[0].NetworkSettings.Networks.webmail_default.IPAddress')" +echo "webmail is on $IP" +wget -S http://$IP/ &>/dev/stderr + [[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/) -ne 200 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 [[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/?admin) -ne 403 ]] && echo "The admin of snappymail is not disabled!" >>/dev/stderr && exit 1 From 4517ce23a6510e47b84ea3da33636fdd58ff9f81 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 15:28:01 +0100 Subject: [PATCH 13/68] Aliases be damned. --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh index ff0e9f25..76e45179 100755 --- a/tests/compose/webmail/01_ensure_admin_unreachable.sh +++ b/tests/compose/webmail/01_ensure_admin_unreachable.sh @@ -1,9 +1,6 @@ #!/bin/bash -IP="$(docker inspect webmail_webmail_1|jq '.[0].NetworkSettings.Networks.webmail_default.IPAddress')" - -echo "webmail is on $IP" -wget -S http://$IP/ &>/dev/stderr +IP="$(docker inspect webmail_webmail_1|jq -r '.[0].NetworkSettings.Networks.webmail_default.IPAddress')" [[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/) -ne 200 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 [[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/?admin) -ne 403 ]] && echo "The admin of snappymail is not disabled!" >>/dev/stderr && exit 1 From ad17b10c8e208338c2001b86d7604b8c8a883b89 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 15:31:47 +0100 Subject: [PATCH 14/68] redirects should be HTTP/302 --- webmails/roundcube/login/mailu.php | 8 ++++---- webmails/snappymail/login/sso.php | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/webmails/roundcube/login/mailu.php b/webmails/roundcube/login/mailu.php index 25d95a10..86de6562 100644 --- a/webmails/roundcube/login/mailu.php +++ b/webmails/roundcube/login/mailu.php @@ -28,7 +28,7 @@ class mailu extends rcube_plugin header('HTTP/1.0 403 Forbidden'); print('mailu sso failure'); } else { - header('Location: sso.php'); + header('Location: sso.php', 302); } exit(); } @@ -47,19 +47,19 @@ class mailu extends rcube_plugin { $this->load_config(); $sso_logout_url = rcmail::get_instance()->config->get('sso_logout_url'); - header('Location: ' . $sso_logout_url, true); + header('Location: ' . $sso_logout_url, true, 302); exit(); } function login($args) { - header('Location: index.php'); + header('Location: index.php', 302); exit(); } function login_failed($args) { - header('Location: sso.php'); + header('Location: sso.php', 302); exit(); } diff --git a/webmails/snappymail/login/sso.php b/webmails/snappymail/login/sso.php index e3d04824..254bb151 100644 --- a/webmails/snappymail/login/sso.php +++ b/webmails/snappymail/login/sso.php @@ -9,9 +9,9 @@ if (isset($_SERVER['HTTP_X_REMOTE_USER']) && isset($_SERVER['HTTP_X_REMOTE_USER_ $ssoHash = \RainLoop\Api::CreateUserSsoHash($email, $password); // redirect to webmail sso url - header('Location: index.php?sso&hash='.$ssoHash); + header('Location: index.php?sso&hash='.$ssoHash, 302); } else { - header('HTTP/1.0 403 Forbidden'); + header('HTTP/1.0 403 Forbidden', 403); } -?> \ No newline at end of file +?> From 225322fe88673d4aeb3baddb66c360772b937483 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 15:34:43 +0100 Subject: [PATCH 15/68] More hardening --- webmails/Dockerfile | 2 +- webmails/nginx-webmail.conf | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index b967af5a..8fd6f9f8 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -40,7 +40,7 @@ RUN set -euxo pipefail \ ; mv carddav roundcube/plugins/ \ ; cd roundcube \ ; rm -rf CHANGELOG.md SECURITY.md INSTALL LICENSE README.md UPGRADING composer.json-dist installer composer.* \ - ; ln -sf index.php /var/www/roundcube/sso.php \ + ; ln -sf index.php /var/www/roundcube/public_html/sso.php \ ; chmod -R u+w,a+rX /var/www/roundcube \ ; chown -R nginx:nginx /var/www/roundcube \ ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query} diff --git a/webmails/nginx-webmail.conf b/webmails/nginx-webmail.conf index b1149c49..37672ab1 100644 --- a/webmails/nginx-webmail.conf +++ b/webmails/nginx-webmail.conf @@ -2,7 +2,11 @@ server { listen 80 default_server; listen [::]:80 default_server; +{% if WEBMAIL == 'roundcube' %} + root /var/www/{{ WEBMAIL }}/public_html; +{% else %} root /var/www/{{ WEBMAIL }}; +{% endif %} include /etc/nginx/mime.types; From b488e576024eaf84f709a050e00081b1b8cda0c8 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 15:39:11 +0100 Subject: [PATCH 16/68] debug --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh index 76e45179..8f9667aa 100755 --- a/tests/compose/webmail/01_ensure_admin_unreachable.sh +++ b/tests/compose/webmail/01_ensure_admin_unreachable.sh @@ -2,5 +2,9 @@ IP="$(docker inspect webmail_webmail_1|jq -r '.[0].NetworkSettings.Networks.webmail_default.IPAddress')" +echo "webmail is on $IP" >/dev/stderr +wget -S -O - "http://$IP/" &>/dev/stderr +docker inspect webmail_webmail_1 &>/dev/stderr + [[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/) -ne 200 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 [[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/?admin) -ne 403 ]] && echo "The admin of snappymail is not disabled!" >>/dev/stderr && exit 1 From f3a91d1a180413e2e472bc0a1041ce86a5ad0b68 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 16:00:55 +0100 Subject: [PATCH 17/68] enable APCu --- webmails/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 8fd6f9f8..fed0f464 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -11,7 +11,7 @@ COPY roundcube/pubkey.asc /tmp/roundcube.asc RUN set -euxo pipefail \ ; apk add --no-cache \ nginx gpg gpg-agent \ - php81 php81-fpm php81-mbstring php81-zip php81-xml php81-simplexml \ + php81 php81-fpm php81-mbstring php81-zip php81-xml php81-simplexml php81-pecl-apc \ php81-dom php81-curl php81-exif gd php81-gd php81-iconv php81-intl php81-openssl \ php81-pdo_sqlite php81-pdo_mysql php81-pdo_pgsql php81-pdo php81-sodium libsodium php81-tidy php81-pecl-uuid \ php81-pspell php81-pecl-imagick php81-opcache php81-session php81-sockets php81-fileinfo \ From 7ebac75045e091a6bfac3cffefe16e6ce14f8965 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 16:07:08 +0100 Subject: [PATCH 18/68] fix tests --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 7 ++----- webmails/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh index 8f9667aa..7a3df370 100755 --- a/tests/compose/webmail/01_ensure_admin_unreachable.sh +++ b/tests/compose/webmail/01_ensure_admin_unreachable.sh @@ -2,9 +2,6 @@ IP="$(docker inspect webmail_webmail_1|jq -r '.[0].NetworkSettings.Networks.webmail_default.IPAddress')" -echo "webmail is on $IP" >/dev/stderr -wget -S -O - "http://$IP/" &>/dev/stderr -docker inspect webmail_webmail_1 &>/dev/stderr - -[[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/) -ne 200 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 +MAIN_RETURN_CODE=$(curl -I -so /dev/null -w "%{http_code}" http://$IP/) +[[ $MAIN_RETURN_CODE -ne 200 ]] && [[ $MAIN_RETURN_CODE -ne 302 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 [[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/?admin) -ne 403 ]] && echo "The admin of snappymail is not disabled!" >>/dev/stderr && exit 1 diff --git a/webmails/Dockerfile b/webmails/Dockerfile index fed0f464..90d56ad5 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -11,7 +11,7 @@ COPY roundcube/pubkey.asc /tmp/roundcube.asc RUN set -euxo pipefail \ ; apk add --no-cache \ nginx gpg gpg-agent \ - php81 php81-fpm php81-mbstring php81-zip php81-xml php81-simplexml php81-pecl-apc \ + php81 php81-fpm php81-mbstring php81-zip php81-xml php81-simplexml php81-pecl-apcu \ php81-dom php81-curl php81-exif gd php81-gd php81-iconv php81-intl php81-openssl \ php81-pdo_sqlite php81-pdo_mysql php81-pdo_pgsql php81-pdo php81-sodium libsodium php81-tidy php81-pecl-uuid \ php81-pspell php81-pecl-imagick php81-opcache php81-session php81-sockets php81-fileinfo \ From d7b80e94a422145a1ae3238d748694c6db2b29b4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 12 Nov 2022 16:21:28 +0100 Subject: [PATCH 19/68] try again. --- tests/compose/webmail/01_ensure_admin_unreachable.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/compose/webmail/01_ensure_admin_unreachable.sh b/tests/compose/webmail/01_ensure_admin_unreachable.sh index 7a3df370..4fd78a1b 100755 --- a/tests/compose/webmail/01_ensure_admin_unreachable.sh +++ b/tests/compose/webmail/01_ensure_admin_unreachable.sh @@ -3,5 +3,8 @@ IP="$(docker inspect webmail_webmail_1|jq -r '.[0].NetworkSettings.Networks.webmail_default.IPAddress')" MAIN_RETURN_CODE=$(curl -I -so /dev/null -w "%{http_code}" http://$IP/) -[[ $MAIN_RETURN_CODE -ne 200 ]] && [[ $MAIN_RETURN_CODE -ne 302 ]] && echo "The default page of snappymail hasn't returned 200!" >>/dev/stderr && exit 1 +[[ $MAIN_RETURN_CODE -ne 200 && $MAIN_RETURN_CODE -ne 302 ]] && echo "The default page of snappymail hasn't returned 200 but $MAIN_RETURN_CODE!" >>/dev/stderr && exit 1 [[ $(curl -I -so /dev/null -w "%{http_code}" http://$IP/?admin) -ne 403 ]] && echo "The admin of snappymail is not disabled!" >>/dev/stderr && exit 1 +echo "Everything OK" >/dev/stderr + +exit 0 From 06c0c78956efd18f4cb68724a12b5ae841afb84a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 13 Nov 2022 13:44:35 +0100 Subject: [PATCH 20/68] Hardening: run the http and php as different users --- webmails/Dockerfile | 9 +++++---- webmails/php-webmail.conf | 4 ++-- webmails/start.py | 3 +-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 90d56ad5..34085f4f 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -41,8 +41,9 @@ RUN set -euxo pipefail \ ; cd roundcube \ ; rm -rf CHANGELOG.md SECURITY.md INSTALL LICENSE README.md UPGRADING composer.json-dist installer composer.* \ ; ln -sf index.php /var/www/roundcube/public_html/sso.php \ - ; chmod -R u+w,a+rX /var/www/roundcube \ - ; chown -R nginx:nginx /var/www/roundcube \ + ; chown -R root:root /var/www/roundcube/ \ + ; chown -R mailu:mailu /var/www/roundcube/temp /var/www/roundcube/logs \ + ; chmod -R a+rX /var/www/roundcube \ ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query} COPY roundcube/config/config.inc.php /conf/ @@ -60,8 +61,8 @@ RUN set -euxo pipefail \ ; curl -sLo /dev/shm/snappymail.tgz.asc ${SNAPPYMAIL_URL}.asc \ ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc \ ; tar xzf /dev/shm/snappymail.tgz \ - ; chmod -R u+w,a+rX /var/www/snappymail \ - ; chown -R nginx:nginx /var/www/snappymail + ; chmod -R a+rX /var/www/snappymail \ + ; chown -R root:root /var/www/snappymail # SnappyMail login COPY snappymail/login/include.php /var/www/snappymail/ diff --git a/webmails/php-webmail.conf b/webmails/php-webmail.conf index 47c1f6dd..18a1f66e 100644 --- a/webmails/php-webmail.conf +++ b/webmails/php-webmail.conf @@ -11,8 +11,8 @@ catch_workers_output = 1 ; Unix user/group of processes ; Note: The user is mandatory. If the group is not set, the default user's group ; will be used. -user = nginx -group = nginx +user = mailu +group = mailu ; The address on which to accept FastCGI requests. ; Valid syntaxes are: diff --git a/webmails/start.py b/webmails/start.py index c2bd87b7..06b90351 100755 --- a/webmails/start.py +++ b/webmails/start.py @@ -110,8 +110,7 @@ conf.jinja("/defaults/application.ini", context, "/data/_data_/_default_/configs conf.jinja("/defaults/php.ini", context, "/etc/php81/php.ini") # setup permissions -os.system("chown -R nginx:nginx /data /var/www") -os.system("chmod -R a+rX /var/www/") +os.system("chown -R mailu:mailu /data") # Configure nginx conf.jinja("/conf/nginx-webmail.conf", context, "/etc/nginx/http.d/webmail.conf") From f2f430af5dbfbd22986226be3a0ba9bcf0cb7492 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 13 Nov 2022 14:07:40 +0100 Subject: [PATCH 21/68] Redirect the logs where they belong --- webmails/Dockerfile | 16 ++++++++++------ webmails/php.ini | 4 +++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 34085f4f..376399bf 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -41,9 +41,6 @@ RUN set -euxo pipefail \ ; cd roundcube \ ; rm -rf CHANGELOG.md SECURITY.md INSTALL LICENSE README.md UPGRADING composer.json-dist installer composer.* \ ; ln -sf index.php /var/www/roundcube/public_html/sso.php \ - ; chown -R root:root /var/www/roundcube/ \ - ; chown -R mailu:mailu /var/www/roundcube/temp /var/www/roundcube/logs \ - ; chmod -R a+rX /var/www/roundcube \ ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query} COPY roundcube/config/config.inc.php /conf/ @@ -60,9 +57,7 @@ RUN set -euxo pipefail \ ; curl -sLo /dev/shm/snappymail.tgz ${SNAPPYMAIL_URL} \ ; curl -sLo /dev/shm/snappymail.tgz.asc ${SNAPPYMAIL_URL}.asc \ ; gpg --status-fd 1 --verify /dev/shm/snappymail.tgz.asc \ - ; tar xzf /dev/shm/snappymail.tgz \ - ; chmod -R a+rX /var/www/snappymail \ - ; chown -R root:root /var/www/snappymail + ; tar xzf /dev/shm/snappymail.tgz # SnappyMail login COPY snappymail/login/include.php /var/www/snappymail/ @@ -72,6 +67,15 @@ COPY snappymail/login/sso.php /var/www/snappymail/ COPY snappymail/defaults/application.ini /defaults/ COPY snappymail/defaults/default.json /defaults/ +# set perms +RUN set -euxo pipefail \ + ; chmod -R a+rX /var/www/snappymail \ + ; chown -R root:root /var/www/snappymail \ + ; chown -R mailu:mailu /var/www/snappymail/data \ + ; chown -R root:root /var/www/roundcube/ \ + ; chown -R mailu:mailu /var/www/roundcube/temp /var/www/roundcube/logs \ + ; chmod -R a+rX /var/www/roundcube + # common COPY start.py / COPY php.ini /defaults/ diff --git a/webmails/php.ini b/webmails/php.ini index 884dda72..d9ba892c 100644 --- a/webmails/php.ini +++ b/webmails/php.ini @@ -5,7 +5,9 @@ post_max_size = {{ MAX_FILESIZE }}M session.auto_start=Off mbstring.func_overload=Off file_uploads=On -error_reporting = E_ALL & ~E_DEPRECATED & ~E_STRICT +error_reporting = E_ALL & ~E_DEPRECATED & ~E_STRICT & ~E_NOTICE display_errors=Off log_errors=On zlib.output_compression=Off +access.log = /dev/fd/2 +error_log = /dev/fd/2 From a508eeaafba574aa8b3b53c8c1b49557b11917fa Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 13 Nov 2022 14:16:44 +0100 Subject: [PATCH 22/68] Use /dev/shm for tmp --- webmails/roundcube/config/config.inc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/roundcube/config/config.inc.php b/webmails/roundcube/config/config.inc.php index d5213b32..6e5ea0bd 100644 --- a/webmails/roundcube/config/config.inc.php +++ b/webmails/roundcube/config/config.inc.php @@ -4,7 +4,7 @@ $config = array(); // Generals $config['db_dsnw'] = '{{ DB_DSNW }}'; -$config['temp_dir'] = '/tmp/'; +$config['temp_dir'] = '/dev/shm/'; $config['des_key'] = '{{ SECRET_KEY }}'; $config['cipher_method'] = 'AES-256-CBC'; $config['identities_level'] = 0; From 6b2cb95a7d75f905e282ae490bf41a5b350db885 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 13 Nov 2022 14:17:37 +0100 Subject: [PATCH 23/68] This is not required anymore --- .../snappymail/config/nginx-snappymail.conf | 63 ------------------- 1 file changed, 63 deletions(-) delete mode 100644 webmails/snappymail/config/nginx-snappymail.conf diff --git a/webmails/snappymail/config/nginx-snappymail.conf b/webmails/snappymail/config/nginx-snappymail.conf deleted file mode 100644 index 80268340..00000000 --- a/webmails/snappymail/config/nginx-snappymail.conf +++ /dev/null @@ -1,63 +0,0 @@ -server { - listen 80 default_server; - listen [::]:80 default_server; - - root /var/www/webmail; - - include /etc/nginx/mime.types; - - # /dev/stdout (Default), , off - access_log off; - - # /dev/stderr (Default), , debug, info, notice, warn, error, crit, alert, emerg - error_log /dev/stderr notice; - - index index.php; - - # set maximum body size to configured limit - client_max_body_size {{ MESSAGE_SIZE_LIMIT|int + 8388608 }}; - - location / { - try_files $uri $uri/ /index.php$args; - } - - location ~ \.php$ { - fastcgi_split_path_info ^(.+?\.php)(/.*)$; - if (!-f $document_root$fastcgi_script_name) { - return 404; - } - include /etc/nginx/fastcgi_params; - - fastcgi_intercept_errors on; - fastcgi_index index.php; - - fastcgi_keep_conn on; - - fastcgi_pass unix:/var/run/php8-fpm.sock; - fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; - {% if WEB_WEBMAIL == '/' %} - fastcgi_param SCRIPT_NAME $fastcgi_script_name; - {% else %} - fastcgi_param SCRIPT_NAME {{WEB_WEBMAIL}}/$fastcgi_script_name; - {% endif %} - } - - location ~ /\. { - deny all; - } - - location ^~ /data { - deny all; - } - - location = /ping { - allow 127.0.0.1; - allow ::1; - deny all; - - include /etc/nginx/fastcgi_params; - fastcgi_index index.php; - fastcgi_pass unix:/var/run/php8-fpm.sock; - fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; - } -} From 071ad15a97df5135b8edddec05d26a69ac6dc90b Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 13 Nov 2022 14:34:25 +0100 Subject: [PATCH 24/68] Better snappymail defaults --- webmails/nginx-webmail.conf | 6 +----- webmails/snappymail/defaults/application.ini | 11 +++++++++++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/webmails/nginx-webmail.conf b/webmails/nginx-webmail.conf index 37672ab1..1794a635 100644 --- a/webmails/nginx-webmail.conf +++ b/webmails/nginx-webmail.conf @@ -55,11 +55,7 @@ server { deny all; } - location ~* ^/(config|temp|logs) { - deny all; - } - - location ^~ /data { + location ~* /(config|temp|logs|data) { deny all; } diff --git a/webmails/snappymail/defaults/application.ini b/webmails/snappymail/defaults/application.ini index 71a19f35..85c8c7c6 100644 --- a/webmails/snappymail/defaults/application.ini +++ b/webmails/snappymail/defaults/application.ini @@ -5,6 +5,7 @@ attachment_size_limit = {{ MAX_FILESIZE }} [security] allow_admin_panel = Off +openpgp = On [labs] allow_gravatar = Off @@ -21,3 +22,13 @@ allow_sync = On [defaults] contacts_autosave = On + +[cache] +fast_cache_driver = "APCU" + +[imap] +use_move = On + +[labs] +image_exif_auto_rotate = On +try_to_detect_hidden_images = On From 56a106ad608948f3c4b0faee73c0f35845eeeda9 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 13 Nov 2022 15:25:46 +0100 Subject: [PATCH 25/68] Only one labs section in the conf file --- webmails/snappymail/defaults/application.ini | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/webmails/snappymail/defaults/application.ini b/webmails/snappymail/defaults/application.ini index 85c8c7c6..bcf544c5 100644 --- a/webmails/snappymail/defaults/application.ini +++ b/webmails/snappymail/defaults/application.ini @@ -9,12 +9,10 @@ openpgp = On [labs] allow_gravatar = Off -{% if WEB_WEBMAIL == '/' %} -custom_login_link='sso.php' -{% else %} -custom_login_link='{{ WEB_WEBMAIL }}/sso.php' -{% endif %} -custom_logout_link='/sso/logout' +image_exif_auto_rotate = On +try_to_detect_hidden_images = On +{% if WEB_WEBMAIL == '/' %}custom_login_link = "sso.php"{% else %}custom_login_link = "{{ WEB_WEBMAIL }}/sso.php"{% endif %} +custom_logout_link = "/sso/logout" [contacts] enable = On @@ -24,11 +22,8 @@ allow_sync = On contacts_autosave = On [cache] +enable = On fast_cache_driver = "APCU" [imap] use_move = On - -[labs] -image_exif_auto_rotate = On -try_to_detect_hidden_images = On From 08a9ab9a56c81072fd47680bb8b2e889443fe028 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 13 Nov 2022 17:15:50 +0100 Subject: [PATCH 26/68] Improve fetchmail --- core/admin/mailu/internal/views/fetch.py | 2 ++ core/admin/mailu/models.py | 2 ++ core/admin/mailu/ui/forms.py | 15 ++++++-- .../mailu/ui/templates/fetch/create.html | 2 ++ core/admin/mailu/ui/templates/fetch/list.html | 4 +++ core/admin/mailu/ui/views/fetches.py | 4 +++ .../migrations/versions/f4f0f89e0047_.py | 25 ++++++++++++++ optional/fetchmail/fetchmail.py | 34 ++++++++++++++++--- setup/flavors/compose/docker-compose.yml | 5 ++- towncrier/newsfragments/1231.bugfix | 1 + towncrier/newsfragments/2246.bugfix | 1 + towncrier/newsfragments/711.feature | 1 + 12 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 core/admin/migrations/versions/f4f0f89e0047_.py create mode 100644 towncrier/newsfragments/1231.bugfix create mode 100644 towncrier/newsfragments/2246.bugfix create mode 100644 towncrier/newsfragments/711.feature diff --git a/core/admin/mailu/internal/views/fetch.py b/core/admin/mailu/internal/views/fetch.py index 1945b9c7..e813c33b 100644 --- a/core/admin/mailu/internal/views/fetch.py +++ b/core/admin/mailu/internal/views/fetch.py @@ -12,10 +12,12 @@ def fetch_list(): "id": fetch.id, "tls": fetch.tls, "keep": fetch.keep, + "scan": fetch.scan, "user_email": fetch.user_email, "protocol": fetch.protocol, "host": fetch.host, "port": fetch.port, + "folders": fetch.folders, "username": fetch.username, "password": fetch.password } for fetch in models.Fetch.query.all() diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 48ce8b33..4b048c45 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -771,6 +771,8 @@ class Fetch(Base): username = db.Column(db.String(255), nullable=False) password = db.Column(db.String(255), nullable=False) keep = db.Column(db.Boolean, nullable=False, default=False) + scan = db.Column(db.Boolean, nullable=False, default=False) + folders = db.Column(CommaSeparatedList, nullable=True, default=list) last_check = db.Column(db.DateTime, nullable=True) error = db.Column(db.String(1023), nullable=True) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index beb44092..f6d03fc2 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -41,6 +41,15 @@ class MultipleEmailAddressesVerify(object): if not pattern.match(field.data.replace(" ", "")): raise validators.ValidationError(self.message) +class MultipleFoldersVerify(object): + def __init__(self,message=_('Invalid list of folders.')): + self.message = message + + def __call__(self, form, field): + pattern = re.compile(r'^\w+(,\w+)*$') + if not pattern.match(field.data.replace(" ", "")): + raise validators.ValidationError(self.message) + class ConfirmationForm(flask_wtf.FlaskForm): submit = fields.SubmitField(_('Confirm')) @@ -164,11 +173,13 @@ class FetchForm(flask_wtf.FlaskForm): ('imap', 'IMAP'), ('pop3', 'POP3') ]) host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()]) - port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)]) - tls = fields.BooleanField(_('Enable TLS')) + port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993) + tls = fields.BooleanField(_('Enable TLS'), default=True) username = fields.StringField(_('Username'), [validators.DataRequired()]) password = fields.PasswordField(_('Password')) keep = fields.BooleanField(_('Keep emails on the server')) + scan = fields.BooleanField(_('Rescan emails locally')) + folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk') submit = fields.SubmitField(_('Submit')) diff --git a/core/admin/mailu/ui/templates/fetch/create.html b/core/admin/mailu/ui/templates/fetch/create.html index 00698329..69584d15 100644 --- a/core/admin/mailu/ui/templates/fetch/create.html +++ b/core/admin/mailu/ui/templates/fetch/create.html @@ -24,6 +24,8 @@ {%- call macros.card(title="Settings") %} {{ macros.form_field(form.keep) }} + {{ macros.form_field(form.scan) }} + {{ macros.form_field(form.folders) }} {%- endcall %} {{ macros.form_field(form.submit) }} diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html index 7a527ce8..63968e96 100644 --- a/core/admin/mailu/ui/templates/fetch/list.html +++ b/core/admin/mailu/ui/templates/fetch/list.html @@ -20,6 +20,8 @@ {% trans %}Endpoint{% endtrans %} {% trans %}Username{% endtrans %} {% trans %}Keep emails{% endtrans %} + {% trans %}Rescan emails{% endtrans %} + {% trans %}Folders{% endtrans %} {% trans %}Last check{% endtrans %} {% trans %}Status{% endtrans %} {% trans %}Created{% endtrans %} @@ -36,6 +38,8 @@ {{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }} {{ fetch.username }} {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} + {% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} + {% for folder in fetch.folders %}{{ folder }},{% endfor %} {{ fetch.last_check | format_datetime or '-' }} {{ fetch.error or '-' }} {{ fetch.created_at | format_date }} diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index ec208af1..3c03a351 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -26,6 +26,8 @@ def fetch_create(user_email): if form.validate_on_submit(): fetch = models.Fetch(user=user) form.populate_obj(fetch) + if form.folders.data: + fetch.folders = form.folders.data.replace(' ','').split(',') models.db.session.add(fetch) models.db.session.commit() flask.flash('Fetch configuration created') @@ -43,6 +45,8 @@ def fetch_edit(fetch_id): if not form.password.data: form.password.data = fetch.password form.populate_obj(fetch) + if form.folders.data: + fetch.folders = form.folders.data.replace(' ','').split(',') models.db.session.commit() flask.flash('Fetch configuration updated') return flask.redirect( diff --git a/core/admin/migrations/versions/f4f0f89e0047_.py b/core/admin/migrations/versions/f4f0f89e0047_.py new file mode 100644 index 00000000..5843e9d2 --- /dev/null +++ b/core/admin/migrations/versions/f4f0f89e0047_.py @@ -0,0 +1,25 @@ +"""empty message + +Revision ID: f4f0f89e0047 +Revises: 8f9ea78776f4 +Create Date: 2022-11-13 16:29:01.246509 + +""" + +# revision identifiers, used by Alembic. +revision = 'f4f0f89e0047' +down_revision = '8f9ea78776f4' + +from alembic import op +import sqlalchemy as sa +import mailu + +def upgrade(): + with op.batch_alter_table('fetch') as batch: + batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false())) + batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True)) + +def downgrade(): + with op.batch_alter_table('fetch') as batch: + batch.drop_column('fetch', 'folders') + batch.drop_column('fetch', 'scan') diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 32751ed7..9dc4a14c 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -2,11 +2,14 @@ import time import os +from pathlib import Path +from pwd import getpwnam import tempfile import shlex import subprocess import re import requests +from socrate import system import sys import traceback @@ -14,6 +17,7 @@ import traceback FETCHMAIL = """ fetchmail -N \ --idfile /data/fetchids --uidl \ + --pidfile /dev/shm/fetchmail.pid \ --sslcertck --sslcertpath /etc/ssl/certs \ -f {} """ @@ -24,7 +28,9 @@ poll "{host}" proto {protocol} port {port} user "{username}" password "{password}" is "{user_email}" smtphost "{smtphost}" + {folders} {options} + {lmtp} """ @@ -48,26 +54,37 @@ def fetchmail(fetchmailrc): def run(debug): try: - fetches = requests.get("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch").json() - smtphost, smtpport = extract_host_port(os.environ.get("HOST_SMTP", "smtp"), None) + os.environ["SMTP_ADDRESS"] = system.get_host_address_from_environment("SMTP", "smtp") + os.environ["ADMIN_ADDRESS"] = system.get_host_address_from_environment("ADMIN", "admin") + fetches = requests.get(f"http://{os.environ['ADMIN_ADDRESS']}/internal/fetch").json() + smtphost, smtpport = extract_host_port(os.environ["SMTP_ADDRESS"], None) if smtpport is None: smtphostport = smtphost else: smtphostport = "%s/%d" % (smtphost, smtpport) + os.environ["LMTP_ADDRESS"] = system.get_host_address_from_environment("LMTP", "imap:2525") + lmtphost, lmtpport = extract_host_port(os.environ["LMTP_ADDRESS"], None) + if lmtpport is None: + lmtphostport = lmtphost + else: + lmtphostport = "%s/%d" % (lmtphost, lmtpport) for fetch in fetches: fetchmailrc = "" options = "options antispam 501, 504, 550, 553, 554" options += " ssl" if fetch["tls"] else "" options += " keep" if fetch["keep"] else " fetchall" + folders = "folders %s" % ((','.join('"' + item + '"' for item in fetch['folders'])) if fetch['folders'] else '"INBOX"') fetchmailrc += RC_LINE.format( user_email=escape_rc_string(fetch["user_email"]), protocol=fetch["protocol"], host=escape_rc_string(fetch["host"]), port=fetch["port"], - smtphost=smtphostport, + smtphost=smtphostport if fetch['scan'] else lmtphostport, username=escape_rc_string(fetch["username"]), password=escape_rc_string(fetch["password"]), - options=options + options=options, + folders=folders, + lmtp='' if fetch['scan'] else 'lmtp', ) if debug: print(fetchmailrc) @@ -86,7 +103,7 @@ def run(debug): user_info in error_message): print(error_message) finally: - requests.post("http://" + os.environ.get("HOST_ADMIN", "admin") + "/internal/fetch/{}".format(fetch["id"]), + requests.post("http://" + os.environ["ADMIN_ADDRESS"] + "/internal/fetch/{}".format(fetch["id"]), json=error_message.split("\n")[0] ) except Exception: @@ -94,6 +111,13 @@ def run(debug): if __name__ == "__main__": + id_fetchmail = getpwnam('fetchmail') + Path('/data/fetchids').touch() + os.chown("/data/fetchids", id_fetchmail.pw_uid, id_fetchmail.pw_gid) + os.chown("/data/", id_fetchmail.pw_uid, id_fetchmail.pw_gid) + os.chmod("/data/fetchids", 0o700) + os.setgid(id_fetchmail.pw_gid) + os.setuid(id_fetchmail.pw_uid) while True: time.sleep(int(os.environ.get("FETCHMAIL_DELAY", 60))) run(os.environ.get("DEBUG", None) == "True") diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 6dac166b..11596729 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -157,8 +157,11 @@ services: env_file: {{ env }} volumes: - "{{ root }}/data/fetchmail:/data" - {% if resolver_enabled %} depends_on: + - admin + - smtp + - imap + {% if resolver_enabled %} - resolver dns: - {{ dns }} diff --git a/towncrier/newsfragments/1231.bugfix b/towncrier/newsfragments/1231.bugfix new file mode 100644 index 00000000..333ae35f --- /dev/null +++ b/towncrier/newsfragments/1231.bugfix @@ -0,0 +1 @@ +Add an option so that emails fetched with fetchmail don't go through the filters (closes #1231) diff --git a/towncrier/newsfragments/2246.bugfix b/towncrier/newsfragments/2246.bugfix new file mode 100644 index 00000000..92e90ac6 --- /dev/null +++ b/towncrier/newsfragments/2246.bugfix @@ -0,0 +1 @@ +Fetchmail: Missing support for '*_ADDRESS' env vars diff --git a/towncrier/newsfragments/711.feature b/towncrier/newsfragments/711.feature new file mode 100644 index 00000000..aa605aa2 --- /dev/null +++ b/towncrier/newsfragments/711.feature @@ -0,0 +1 @@ +Allow other folders to be synced by fetchmail From 9c7dfbeb24efc14c8fb7d21a1cd4dce241d42ecd Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 14 Nov 2022 13:43:42 +0100 Subject: [PATCH 27/68] Doc --- docs/webadministration.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/webadministration.rst b/docs/webadministration.rst index e17d12f0..247b66a7 100644 --- a/docs/webadministration.rst +++ b/docs/webadministration.rst @@ -157,7 +157,11 @@ You can add a fetched account by clicking on the `Add an account` button on the * Keep emails on the server. When ticked, retains the email message in the email account after retrieving it. -Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after 10 minutes. +* Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...). + +* Folders. A comma separated list of folders to fetch from the server. + +Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``. Authentication tokens From c0c91691fd6364e6f4e82cf9c36a61f65c4eed06 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 14 Nov 2022 16:24:09 +0100 Subject: [PATCH 28/68] Fix the issue on /admin/fetch/edit --- core/admin/mailu/ui/templates/fetch/create.html | 2 ++ core/admin/mailu/ui/templates/fetch/edit.html | 12 ++++++++++++ core/admin/run_dev.sh | 6 +++--- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/ui/templates/fetch/create.html b/core/admin/mailu/ui/templates/fetch/create.html index 69584d15..a48c1a48 100644 --- a/core/admin/mailu/ui/templates/fetch/create.html +++ b/core/admin/mailu/ui/templates/fetch/create.html @@ -25,7 +25,9 @@ {%- call macros.card(title="Settings") %} {{ macros.form_field(form.keep) }} {{ macros.form_field(form.scan) }} +{%- block folders %} {{ macros.form_field(form.folders) }} +{%- endblock %} {%- endcall %} {{ macros.form_field(form.submit) }} diff --git a/core/admin/mailu/ui/templates/fetch/edit.html b/core/admin/mailu/ui/templates/fetch/edit.html index 62af51a9..c7ff5a87 100644 --- a/core/admin/mailu/ui/templates/fetch/edit.html +++ b/core/admin/mailu/ui/templates/fetch/edit.html @@ -7,3 +7,15 @@ {%- block subtitle %} {{ user }} {%- endblock %} + +{%- block folders %} +
+
+
+ + + +
+
+
+{%- endblock %} diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 4ab76e74..c12bb9ee 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -48,7 +48,7 @@ sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Docker # assets cp "${assets}/package.json" . -cp -r "${assets}/assets/" . +cp -r "${assets}/assets" ./assets awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile @@ -65,7 +65,7 @@ RUN set -euxo pipefail \ ; ln -s /app/start.py / ENV \ - FLASK_ENV="development" \ + FLASK_DEBUG="true" \ MEMORY_SESSIONS="true" \ RATELIMIT_STORAGE_URL="memory://" \ SESSION_COOKIE_SECURE="false" \ @@ -82,7 +82,7 @@ ENV \ REDIS_ADDRESS="127.0.0.1" \ WEBMAIL_ADDRESS="127.0.0.1" -CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null && flask run --host=0.0.0.0 --port=8080"] +CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask run --debugger --host=0.0.0.0 --port=8080"] EOF # build From 647410805692ae538f8a63e848ebffa6001711cd Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 14 Nov 2022 16:36:37 +0100 Subject: [PATCH 29/68] Use a join() instead --- core/admin/mailu/ui/templates/fetch/list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html index 63968e96..e502d96a 100644 --- a/core/admin/mailu/ui/templates/fetch/list.html +++ b/core/admin/mailu/ui/templates/fetch/list.html @@ -39,7 +39,7 @@ {{ fetch.username }} {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} {% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} - {% for folder in fetch.folders %}{{ folder }},{% endfor %} + {{ fetch.folders.data | join(',') }} {{ fetch.last_check | format_datetime or '-' }} {{ fetch.error or '-' }} {{ fetch.created_at | format_date }} From 385b6ac85d3fcdee97d5743c4e5da8b6c0c9f728 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 14 Nov 2022 16:47:43 +0100 Subject: [PATCH 30/68] Use string formatting --- optional/fetchmail/fetchmail.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optional/fetchmail/fetchmail.py b/optional/fetchmail/fetchmail.py index 9dc4a14c..137a061a 100755 --- a/optional/fetchmail/fetchmail.py +++ b/optional/fetchmail/fetchmail.py @@ -103,8 +103,8 @@ def run(debug): user_info in error_message): print(error_message) finally: - requests.post("http://" + os.environ["ADMIN_ADDRESS"] + "/internal/fetch/{}".format(fetch["id"]), - json=error_message.split("\n")[0] + requests.post("http://{}/internal/fetch/{}".format(os.environ['ADMIN_ADDRESS'],fetch['id']), + json=error_message.split('\n')[0] ) except Exception: traceback.print_exc() From 19af2944d7ac2c660a2b1aabcc35dd63464443cb Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 14 Nov 2022 19:05:41 +0100 Subject: [PATCH 31/68] Refactor as requested --- core/admin/mailu/ui/forms.py | 2 +- core/admin/mailu/ui/templates/fetch/create.html | 2 -- core/admin/mailu/ui/templates/fetch/edit.html | 12 ------------ core/admin/mailu/ui/views/fetches.py | 4 +++- core/admin/mailu/ui/views/users.py | 6 +----- core/admin/mailu/utils.py | 7 +++++++ 6 files changed, 12 insertions(+), 21 deletions(-) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index f6d03fc2..fa81adc3 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -46,7 +46,7 @@ class MultipleFoldersVerify(object): self.message = message def __call__(self, form, field): - pattern = re.compile(r'^\w+(,\w+)*$') + pattern = re.compile(r'^\w+(\s*,\s*\w+)*$') if not pattern.match(field.data.replace(" ", "")): raise validators.ValidationError(self.message) diff --git a/core/admin/mailu/ui/templates/fetch/create.html b/core/admin/mailu/ui/templates/fetch/create.html index a48c1a48..69584d15 100644 --- a/core/admin/mailu/ui/templates/fetch/create.html +++ b/core/admin/mailu/ui/templates/fetch/create.html @@ -25,9 +25,7 @@ {%- call macros.card(title="Settings") %} {{ macros.form_field(form.keep) }} {{ macros.form_field(form.scan) }} -{%- block folders %} {{ macros.form_field(form.folders) }} -{%- endblock %} {%- endcall %} {{ macros.form_field(form.submit) }} diff --git a/core/admin/mailu/ui/templates/fetch/edit.html b/core/admin/mailu/ui/templates/fetch/edit.html index c7ff5a87..62af51a9 100644 --- a/core/admin/mailu/ui/templates/fetch/edit.html +++ b/core/admin/mailu/ui/templates/fetch/edit.html @@ -7,15 +7,3 @@ {%- block subtitle %} {{ user }} {%- endblock %} - -{%- block folders %} -
-
-
- - - -
-
-
-{%- endblock %} diff --git a/core/admin/mailu/ui/views/fetches.py b/core/admin/mailu/ui/views/fetches.py index 3c03a351..69018ba9 100644 --- a/core/admin/mailu/ui/views/fetches.py +++ b/core/admin/mailu/ui/views/fetches.py @@ -1,4 +1,4 @@ -from mailu import models +from mailu import models, utils from mailu.ui import ui, forms, access import flask @@ -23,6 +23,7 @@ def fetch_create(user_email): user = models.User.query.get(user_email) or flask.abort(404) form = forms.FetchForm() form.password.validators = [wtforms.validators.DataRequired()] + utils.formatCSVField(form.folders) if form.validate_on_submit(): fetch = models.Fetch(user=user) form.populate_obj(fetch) @@ -41,6 +42,7 @@ def fetch_create(user_email): def fetch_edit(fetch_id): fetch = models.Fetch.query.get(fetch_id) or flask.abort(404) form = forms.FetchForm(obj=fetch) + utils.formatCSVField(form.folders) if form.validate_on_submit(): if not form.password.data: form.password.data = fetch.password diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index 85a5c2db..b5e7304c 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -99,11 +99,7 @@ def user_settings(user_email): user_email_or_current = user_email or flask_login.current_user.email user = models.User.query.get(user_email_or_current) or flask.abort(404) form = forms.UserSettingsForm(obj=user) - if isinstance(form.forward_destination.data,str): - data = form.forward_destination.data.replace(" ","").split(",") - else: - data = form.forward_destination.data - form.forward_destination.data = ", ".join(data) + utils.formatCSVField(form.forward_destination) if form.validate_on_submit(): form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",") form.populate_obj(user) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index f160fe3f..b432192d 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -518,3 +518,10 @@ def isBadOrPwned(form): if breaches > 0: return f"This password appears in {breaches} data breaches! It is not unique; please change it." return None + +def formatCSVField(field): + if isinstance(field.data,str): + data = field.data.replace(" ","").split(",") + else: + data = field.data + field.data = ", ".join(data) From b9564c0bc9beab80e4a64f6f2871843bfa524db1 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 14 Nov 2022 19:37:04 +0100 Subject: [PATCH 32/68] This shouldn't have been commited --- core/admin/run_dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index c12bb9ee..dbe3368a 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -82,7 +82,7 @@ ENV \ REDIS_ADDRESS="127.0.0.1" \ WEBMAIL_ADDRESS="127.0.0.1" -CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask run --debugger --host=0.0.0.0 --port=8080"] +CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"] EOF # build From 15b889fac812ce4da29d29b572b9f3bd39f56381 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 16 Nov 2022 14:17:56 +0100 Subject: [PATCH 33/68] Specify that this is optional --- docs/webadministration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/webadministration.rst b/docs/webadministration.rst index 247b66a7..fde4a271 100644 --- a/docs/webadministration.rst +++ b/docs/webadministration.rst @@ -159,7 +159,7 @@ You can add a fetched account by clicking on the `Add an account` button on the * Scan emails. When ticked, all the fetched emails will go through the local filters (rspamd, clamav, ...). -* Folders. A comma separated list of folders to fetch from the server. +* Folders. A comma separated list of folders to fetch from the server. This is optional, by default only the INBOX will be pulled. Click the submit button to apply settings. With the default polling interval, fetchmail will start polling the email account after ``FETCHMAIL_DELAY``. From 86637f025969800521ad7bbd4c933c087c4b1e72 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 14:18:51 +0100 Subject: [PATCH 34/68] Make setup use the base image --- core/none/Dockerfile | 2 +- setup/Dockerfile | 26 +++++++++++--------------- setup/requirements.txt | 12 ------------ tests/build.hcl | 3 +++ 4 files changed, 15 insertions(+), 28 deletions(-) delete mode 100644 setup/requirements.txt diff --git a/core/none/Dockerfile b/core/none/Dockerfile index f06cc31c..d605cb07 100644 --- a/core/none/Dockerfile +++ b/core/none/Dockerfile @@ -10,5 +10,5 @@ RUN echo $VERSION >/version HEALTHCHECK CMD true -USER app +USER mailu CMD ["/bin/bash", "-c", "sleep infinity"] diff --git a/setup/Dockerfile b/setup/Dockerfile index 85e5f55b..7be8f1b0 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -1,24 +1,20 @@ -ARG DISTRO=alpine:3.14.5 -FROM $DISTRO -ARG VERSION -ENV TZ Etc/UTC +# syntax=docker/dockerfile-upstream:1.4.3 + +# setup image +FROM base + +ARG VERSION=local LABEL version=$VERSION - -RUN mkdir -p /app -WORKDIR /app - -COPY requirements.txt requirements.txt -RUN apk add --no-cache curl python3 py3-pip \ - && pip3 install -r requirements.txt - -COPY server.py ./server.py -COPY main.py ./main.py COPY flavors /data/flavors COPY templates /data/templates COPY static ./static +COPY server.py ./server.py +COPY main.py ./main.py + +RUN echo $VERSION >> /version EXPOSE 80/tcp +HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost/ CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app -RUN echo $VERSION >> /version diff --git a/setup/requirements.txt b/setup/requirements.txt deleted file mode 100644 index b6f9f713..00000000 --- a/setup/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -Flask==1.0.2 -Flask-Bootstrap==3.3.7.1 -gunicorn==19.9.0 -redis==3.2.1 -Jinja2==3.0.3 -MarkupSafe==2.1.0 -Werkzeug==2.0.3 -click==8.0.3 -dominate==2.6.0 -itsdangerous==2.0.1 -redis==3.2.1 -visitor==0.1.3 diff --git a/tests/build.hcl b/tests/build.hcl index 34955270..0f6226c8 100644 --- a/tests/build.hcl +++ b/tests/build.hcl @@ -107,6 +107,9 @@ target "docs" { target "setup" { inherits = ["defaults"] context = "setup/" + contexts = { + base = "target:base" + } tags = tag("setup") } From e5a1a353dbeb8f884e84b0b205a13927d9e173be Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 14:19:22 +0100 Subject: [PATCH 35/68] Upgrade to alpine 3.16.3 This has PHP fixes and a new rspamd --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 6f31e21c..af99ccc3 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile-upstream:1.4.3 # base system image (intermediate) -ARG DISTRO=alpine:3.16.2 +ARG DISTRO=alpine:3.16.3 FROM $DISTRO as system ENV TZ=Etc/UTC LANG=C.UTF-8 From 21b9f76ebced4ad11e586216b9328b2b5b74c9a4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 14:34:55 +0100 Subject: [PATCH 36/68] setup doesn't need root --- setup/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup/Dockerfile b/setup/Dockerfile index 7be8f1b0..d6a5f083 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -12,9 +12,14 @@ COPY static ./static COPY server.py ./server.py COPY main.py ./main.py +RUN set -euxo pipefail \ + ; apk add --no-cache libcap \ + ; setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn + RUN echo $VERSION >> /version EXPOSE 80/tcp HEALTHCHECK --start-period=350s CMD curl -skfLo /dev/null http://localhost/ +USER mailu CMD gunicorn -w 4 -b :80 --access-logfile - --error-logfile - --preload main:app From 80559ecb71112d38c6f6ec38fc5eb8e1489c0439 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 14:43:31 +0100 Subject: [PATCH 37/68] optimize caching --- setup/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup/Dockerfile b/setup/Dockerfile index d6a5f083..7e04931a 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -6,16 +6,16 @@ FROM base ARG VERSION=local LABEL version=$VERSION +RUN set -euxo pipefail \ + ; apk add --no-cache libcap \ + ; setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn + COPY flavors /data/flavors COPY templates /data/templates COPY static ./static COPY server.py ./server.py COPY main.py ./main.py -RUN set -euxo pipefail \ - ; apk add --no-cache libcap \ - ; setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn - RUN echo $VERSION >> /version EXPOSE 80/tcp From 42cd5bf2dc43b67551edeb0d7a171138b29f3dee Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 15:17:24 +0100 Subject: [PATCH 38/68] Move it to base since admin will also use it --- core/base/Dockerfile | 3 ++- setup/Dockerfile | 4 ---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index af99ccc3..814a946a 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -13,7 +13,7 @@ ARG TARGETPLATFORM RUN set -euxo pipefail \ ; addgroup -Sg ${MAILU_GID} mailu \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ - ; apk add --no-cache bash ca-certificates curl python3 tzdata \ + ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \ ; machine="$(uname -m)" \ ; ! [[ "${TARGETPLATFORM}" != linux/arm/v7 && \( "${machine}" == x86_64 || "${machine}" == armv8* || "${machine}" == aarch64 \) ]] \ || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc @@ -72,6 +72,7 @@ RUN set -euxo pipefail \ FROM system COPY --from=build /app/venv/ /app/venv/ +RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn ENV VIRTUAL_ENV=/app/venv ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" diff --git a/setup/Dockerfile b/setup/Dockerfile index 7e04931a..a410871d 100644 --- a/setup/Dockerfile +++ b/setup/Dockerfile @@ -6,10 +6,6 @@ FROM base ARG VERSION=local LABEL version=$VERSION -RUN set -euxo pipefail \ - ; apk add --no-cache libcap \ - ; setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn - COPY flavors /data/flavors COPY templates /data/templates COPY static ./static From 699be6f9fa62ef538fdac0f09003b2c54838eda4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 16:03:37 +0100 Subject: [PATCH 39/68] Drop privs when running admin too --- core/admin/Dockerfile | 18 +++++++++--------- core/admin/start.py | 6 ++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 600c3e9f..b43e46b6 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -9,23 +9,23 @@ LABEL version=$VERSION RUN set -euxo pipefail \ ; apk add --no-cache libressl mariadb-connector-c postgresql-libs -COPY --from=assets /work/static/ ./mailu/static/ +EXPOSE 80/tcp +HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login + +VOLUME ["/data","/dkim"] + +ENV FLASK_APP=mailu + +COPY --from=assets /work/static/ ./mailu/static/ COPY audit.py / COPY start.py / - COPY migrations/ ./migrations/ - COPY mailu/ ./mailu/ + RUN set -euxo pipefail \ ; venv/bin/pybabel compile -d mailu/translations RUN echo $VERSION >/version -EXPOSE 80/tcp -HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index - -VOLUME ["/data","/dkim"] - -ENV FLASK_APP=mailu CMD /start.py diff --git a/core/admin/start.py b/core/admin/start.py index 3cb5c422..99b34a01 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -2,8 +2,14 @@ import os import logging as log +from pwd import getpwnam import sys +os.system("chown mailu:mailu -R /data /dkim") +mailu_id = getpwnam('mailu') +os.setgid(mailu_id.pw_gid) +os.setuid(mailu_id.pw_uid) + log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO")) os.system("flask mailu advertise") From e79d7fed550f2c546de2535e6ca4b993a66b1a44 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 16:21:52 +0100 Subject: [PATCH 40/68] Reduce the number of warnings on the CI --- .github/workflows/build_test_deploy.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index d1395bec..6512ccd5 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -126,7 +126,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -182,7 +182,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -244,7 +244,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -307,7 +307,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Build all docker images @@ -370,7 +370,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Install python packages @@ -416,7 +416,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Push image to Docker @@ -461,7 +461,7 @@ jobs: password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string - uses: ASzc/change-string-case-action@v2 + uses: ASzc/change-string-case-action@v5 with: string: ${{ github.repository_owner }} - name: Push image to Docker From 3b5b00d87d4a701661efaa3bea785b8f88e93796 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 16:37:17 +0100 Subject: [PATCH 41/68] towncrier --- towncrier/newsfragments/2539.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/2539.misc diff --git a/towncrier/newsfragments/2539.misc b/towncrier/newsfragments/2539.misc new file mode 100644 index 00000000..9dbf4e4d --- /dev/null +++ b/towncrier/newsfragments/2539.misc @@ -0,0 +1 @@ +Upgrade to Alpine 3.16.3; Make both setup and admin run without root privs. From e3b875aa6b46d674731c6677d32908c7a3c51410 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:09:00 +0100 Subject: [PATCH 42/68] Well, -i stands for --insecure --- core/rspamd/start.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 537d996d..0f351057 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -33,4 +33,6 @@ while True: log.warning("Admin is not up just yet, retrying in 1 second") # Run rspamd -os.execv("/usr/sbin/rspamd", ["rspamd", "-i", "-f"]) +os.system("mkdir -m 755 -p /run/rspamd") +os.system("chown rspamd:rspamd /run/rspamd") +os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"]) From 3cb87b6e49352e97a1a578b3b22ce5199eaee295 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:10:53 +0100 Subject: [PATCH 43/68] Update entry --- towncrier/newsfragments/2539.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/towncrier/newsfragments/2539.misc b/towncrier/newsfragments/2539.misc index 9dbf4e4d..0d9907ac 100644 --- a/towncrier/newsfragments/2539.misc +++ b/towncrier/newsfragments/2539.misc @@ -1 +1 @@ -Upgrade to Alpine 3.16.3; Make both setup and admin run without root privs. +Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. From 6137f93d23265d839cca09d0b5da1018751635e0 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:17:41 +0100 Subject: [PATCH 44/68] add a GTUBE test to check the antispam --- tests/compose/filters/02_email_antispam.sh | 6 ++++++ tests/compose/filters/gtube.txt | 1 + 2 files changed, 7 insertions(+) create mode 100755 tests/compose/filters/02_email_antispam.sh create mode 100644 tests/compose/filters/gtube.txt diff --git a/tests/compose/filters/02_email_antispam.sh b/tests/compose/filters/02_email_antispam.sh new file mode 100755 index 00000000..009a13a2 --- /dev/null +++ b/tests/compose/filters/02_email_antispam.sh @@ -0,0 +1,6 @@ +python3 tests/email_test.py message-virus "tests/compose/filters/gtube.txt" +if [ $? -eq 99 ]; then + exit 0 +else + exit 1 +fi diff --git a/tests/compose/filters/gtube.txt b/tests/compose/filters/gtube.txt new file mode 100644 index 00000000..728c7cc6 --- /dev/null +++ b/tests/compose/filters/gtube.txt @@ -0,0 +1 @@ +XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X From 1bfab1dbfaf7b139f27fc30e35cf22fb2592c74c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:32:39 +0100 Subject: [PATCH 45/68] Maybe fix the test? --- tests/compose/filters/02_email_antispam.sh | 3 ++- tests/compose/filters/gtube.txt | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 tests/compose/filters/gtube.txt diff --git a/tests/compose/filters/02_email_antispam.sh b/tests/compose/filters/02_email_antispam.sh index 009a13a2..923f1475 100755 --- a/tests/compose/filters/02_email_antispam.sh +++ b/tests/compose/filters/02_email_antispam.sh @@ -1,4 +1,5 @@ -python3 tests/email_test.py message-virus "tests/compose/filters/gtube.txt" +# GTUBE should be blocked, see https://rspamd.com/doc/gtube_patterns.html +python3 tests/email_test.py "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" if [ $? -eq 99 ]; then exit 0 else diff --git a/tests/compose/filters/gtube.txt b/tests/compose/filters/gtube.txt deleted file mode 100644 index 728c7cc6..00000000 --- a/tests/compose/filters/gtube.txt +++ /dev/null @@ -1 +0,0 @@ -XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X From b28798c74fed5dafc5da7d8d1fa599fc1c1a5b51 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 17 Nov 2022 18:46:04 +0100 Subject: [PATCH 46/68] doh --- tests/compose/filters/02_email_antispam.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compose/filters/02_email_antispam.sh b/tests/compose/filters/02_email_antispam.sh index 923f1475..ac2653a3 100755 --- a/tests/compose/filters/02_email_antispam.sh +++ b/tests/compose/filters/02_email_antispam.sh @@ -1,6 +1,6 @@ # GTUBE should be blocked, see https://rspamd.com/doc/gtube_patterns.html python3 tests/email_test.py "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" -if [ $? -eq 99 ]; then +if [ $? -eq 25 ]; then exit 0 else exit 1 From bdc085048d009154ea4bf04264fe982cdebe4fef Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 18 Nov 2022 10:40:42 +0100 Subject: [PATCH 47/68] Restore the Dockerfile like it was --- core/admin/Dockerfile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index b43e46b6..600c3e9f 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -9,23 +9,23 @@ LABEL version=$VERSION RUN set -euxo pipefail \ ; apk add --no-cache libressl mariadb-connector-c postgresql-libs -EXPOSE 80/tcp - -HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login - -VOLUME ["/data","/dkim"] - -ENV FLASK_APP=mailu - COPY --from=assets /work/static/ ./mailu/static/ + COPY audit.py / COPY start.py / -COPY migrations/ ./migrations/ -COPY mailu/ ./mailu/ +COPY migrations/ ./migrations/ + +COPY mailu/ ./mailu/ RUN set -euxo pipefail \ ; venv/bin/pybabel compile -d mailu/translations RUN echo $VERSION >/version +EXPOSE 80/tcp +HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index + +VOLUME ["/data","/dkim"] + +ENV FLASK_APP=mailu CMD /start.py From e5ab9821f9c7b3d80224222d9928526f5d3743bb Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 18 Nov 2022 13:25:02 +0100 Subject: [PATCH 48/68] Add snuffleupagus This seems to work in my limited testing. --- core/base/Dockerfile | 18 +++- webmails/Dockerfile | 6 +- webmails/php.ini | 2 + webmails/roundcube/config/config.inc.php | 2 +- webmails/snuffleupagus.rules | 126 +++++++++++++++++++++++ webmails/start.py | 6 +- 6 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 webmails/snuffleupagus.rules diff --git a/core/base/Dockerfile b/core/base/Dockerfile index d5be6a90..ca62e773 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -52,16 +52,32 @@ RUN set -euxo pipefail \ mkdir -p /root/.cargo/registry/index && \ git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ ; pip install -r requirements-${MAILU_DEPS}.txt \ - ; apk del -r .build-deps \ ; rm -rf /root/.cargo /tmp/*.pem \ ; } \ ; rm -rf /root/.cache +ARG SNUFFLEUPAGUS_VERSION=0.8.3 +ENV SNUFFLEUPAGUS_URL https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v$SNUFFLEUPAGUS_VERSION.tar.gz + +RUN set -euxo pipefail \ + ; curl -sL ${SNUFFLEUPAGUS_URL} | tar xz \ + ; cd snuffleupagus-$SNUFFLEUPAGUS_VERSION \ + ; rm -rf src/tests/*php7*/ src/tests/*session*/ src/tests/broken_configuration/ src/tests/*cookie* src/tests/upload_validation/ \ + ; apk add --virtual .build-deps php81-dev php81-cgi php81-simplexml php81-xml pcre-dev build-base php81-pear php81-openssl re2c \ + ; ln -s /usr/bin/phpize81 /usr/bin/phpize \ + ; ln -s /usr/bin/pecl81 /usr/bin/pecl \ + ; ln -s /usr/bin/php-config81 /usr/bin/php-config \ + ; ln -s /usr/bin/php81 /usr/bin/php \ + ; pecl install vld-beta \ + ; make -j $(grep -c processor /proc/cpuinfo) release \ + ; cp src/.libs/snuffleupagus.so /app \ + ; apk del -r .build-deps # base mailu image FROM system COPY --from=build /app/venv/ /app/venv/ +COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/ ENV VIRTUAL_ENV=/app/venv ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 376399bf..0ec12384 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -21,6 +21,8 @@ RUN set -euxo pipefail \ ; ln -s /usr/bin/php81 /usr/bin/php \ ; gpg --import /tmp/snappymail.asc \ ; gpg --import /tmp/roundcube.asc \ + ; echo extension=snuffleupagus > /etc/php81/conf.d/snuffleupagus.ini \ + ; rm -f /tmp/*asc \ ; mkdir -p /run/nginx \ ; mkdir -p /conf @@ -41,7 +43,8 @@ RUN set -euxo pipefail \ ; cd roundcube \ ; rm -rf CHANGELOG.md SECURITY.md INSTALL LICENSE README.md UPGRADING composer.json-dist installer composer.* \ ; ln -sf index.php /var/www/roundcube/public_html/sso.php \ - ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query} + ; rm -rf plugins/{autologon,example_addressbook,http_authentication,krb_authentication,new_user_identity,password,redundant_attachments,squirrelmail_usercopy,userinfo,virtuser_file,virtuser_query} \ + ; sed -i '/suhosin.session.encrypt/d;/mbstring\.func_overload/d' program/lib/Roundcube/bootstrap.php COPY roundcube/config/config.inc.php /conf/ COPY roundcube/login/mailu.php /var/www/roundcube/plugins/mailu/ @@ -81,6 +84,7 @@ COPY start.py / COPY php.ini /defaults/ COPY php-webmail.conf /etc/php81/php-fpm.d/php-webmail.conf COPY nginx-webmail.conf /conf/ +COPY snuffleupagus.rules /etc/snuffleupagus.rules.tpl EXPOSE 80/tcp VOLUME /data diff --git a/webmails/php.ini b/webmails/php.ini index d9ba892c..87660288 100644 --- a/webmails/php.ini +++ b/webmails/php.ini @@ -11,3 +11,5 @@ log_errors=On zlib.output_compression=Off access.log = /dev/fd/2 error_log = /dev/fd/2 +module=snuffleupagus.so +sp.configuration_file=/etc/snuffleupagus.rules diff --git a/webmails/roundcube/config/config.inc.php b/webmails/roundcube/config/config.inc.php index 6e5ea0bd..e8aedeff 100644 --- a/webmails/roundcube/config/config.inc.php +++ b/webmails/roundcube/config/config.inc.php @@ -5,7 +5,7 @@ $config = array(); // Generals $config['db_dsnw'] = '{{ DB_DSNW }}'; $config['temp_dir'] = '/dev/shm/'; -$config['des_key'] = '{{ SECRET_KEY }}'; +$config['des_key'] = '{{ ROUNDCUBE_KEY }}'; $config['cipher_method'] = 'AES-256-CBC'; $config['identities_level'] = 0; $config['reply_all_mode'] = 1; diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules new file mode 100644 index 00000000..35f0b7ce --- /dev/null +++ b/webmails/snuffleupagus.rules @@ -0,0 +1,126 @@ +# This is based on default configuration file for Snuffleupagus (https://snuffleupagus.rtfd.io), +# for php8. +# It contains "reasonable" defaults that won't break your websites, +# and a lot of commented directives that you can enable if you want to +# have a better protection. + +# Harden the PRNG +sp.harden_random.enable(); + +# Disabled XXE +sp.xxe_protection.enable(); + +# Global configuration variables +sp.global.secret_key("{{ SNUFFLEPAGUS_KEY }}"); + +# Globally activate strict mode +# https://www.php.net/manual/en/language.types.declarations.php#language.types.declarations.strict +sp.global_strict.enable(); + +# Prevent unserialize-related exploits +# sp.unserialize_hmac.enable(); + +# Only allow execution of read-only files. This is a low-hanging fruit that you should enable. +sp.readonly_exec.enable(); + +# PHP has a lot of wrappers, most of them aren't usually useful, you should +# only enable the ones you're using. +sp.wrappers_whitelist.list("file,php,phar,mailsosubstreams"); + +# Prevent sloppy comparisons. +sp.sloppy_comparison.enable(); + +# Use SameSite on session cookie +# https://snuffleupagus.readthedocs.io/features.html#protection-against-cross-site-request-forgery +sp.cookie.name("PHPSESSID").samesite("lax"); + +# Harden the `chmod` function (0777 (oct = 511, 0666 = 438) +sp.disable_function.function("chmod").param("permissions").value("438").drop(); +sp.disable_function.function("chmod").param("permissions").value("511").drop(); + +# Prevent various `mail`-related vulnerabilities +sp.disable_function.function("mail").param("additional_parameters").value_r("\\-").drop(); + +# Since it's now burned, me might as well mitigate it publicly +sp.disable_function.function("putenv").param("assignment").value_r("LD_").drop() + +# This one was burned in Nov 2019 - https://gist.github.com/LoadLow/90b60bd5535d6c3927bb24d5f9955b80 +sp.disable_function.function("putenv").param("assignment").value_r("GCONV_").drop() + +# Since people are stupid enough to use `extract` on things like $_GET or $_POST, we might as well mitigate this vector +sp.disable_function.function("extract").param("array").value_r("^_").drop() +sp.disable_function.function("extract").param("flags").value("0").drop() + +# This is also burned: +# ini_set('open_basedir','..');chdir('..');…;chdir('..');ini_set('open_basedir','/');echo(file_get_contents('/etc/passwd')); +# Since we have no way of matching on two parameters at the same time, we're +# blocking calls to open_basedir altogether: nobody is using it via ini_set anyway. +# Moreover, there are non-public bypasses that are also using this vector ;) +sp.disable_function.function("ini_set").param("option").value_r("open_basedir").drop() + +# Prevent various `include`-related vulnerabilities +sp.disable_function.function("require_once").value_r("\.(inc|phtml|php)$").allow(); +sp.disable_function.function("include_once").value_r("\.(inc|phtml|php)$").allow(); +sp.disable_function.function("require").value_r("\.(inc|phtml|php)$").allow(); +sp.disable_function.function("include").value_r("\.(inc|phtml|php)$").allow(); +sp.disable_function.function("require_once").drop() +sp.disable_function.function("include_once").drop() +sp.disable_function.function("require").drop() +sp.disable_function.function("include").drop() + +# Prevent `system`-related injections +sp.disable_function.function("system").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop(); +sp.disable_function.function("shell_exec").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop(); +sp.disable_function.function("exec").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop(); +sp.disable_function.function("proc_open").param("command").value_r("^gpg ").allow(); +sp.disable_function.function("proc_open").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop(); + +# Prevent runtime modification of interesting things +sp.disable_function.function("ini_set").param("option").value("assert.active").drop(); +sp.disable_function.function("ini_set").param("option").value("zend.assertions").drop(); +sp.disable_function.function("ini_set").param("option").value("memory_limit").drop(); +sp.disable_function.function("ini_set").param("option").value("include_path").drop(); +sp.disable_function.function("ini_set").param("option").value("open_basedir").drop(); + +# Detect some backdoors via environment recon +sp.disable_function.function("ini_get").param("option").value("allow_url_fopen").drop(); +sp.disable_function.function("ini_get").param("option").value("open_basedir").drop(); +sp.disable_function.function("ini_get").param("option").value_r("suhosin").drop(); +sp.disable_function.function("function_exists").param("function").value("eval").drop(); +sp.disable_function.function("function_exists").param("function").value("exec").drop(); +sp.disable_function.function("function_exists").param("function").value("system").drop(); +sp.disable_function.function("function_exists").param("function").value("shell_exec").drop(); +sp.disable_function.function("function_exists").param("function").value("proc_open").drop(); +sp.disable_function.function("function_exists").param("function").value("passthru").drop(); +sp.disable_function.function("is_callable").param("value").value("eval").drop(); +sp.disable_function.function("is_callable").param("value").value("exec").drop(); +sp.disable_function.function("is_callable").param("value").value("system").drop(); +sp.disable_function.function("is_callable").param("value").value("shell_exec").drop(); +sp.disable_function.function("is_callable").filename_r("/app/libraries/snappymail/pgp/gpg\.php$").param("value").value("proc_open").allow(); +sp.disable_function.function("is_callable").param("value").value("proc_open").drop(); +sp.disable_function.function("is_callable").param("value").value("passthru").drop(); + +# Ghetto error-based sqli detection +#sp.disable_function.function("mysql_query").ret("FALSE").drop(); +#sp.disable_function.function("mysqli_query").ret("FALSE").drop(); +#sp.disable_function.function("PDO::query").ret("FALSE").drop(); + +# Ensure that certificates are properly verified +sp.disable_function.function("curl_setopt").param("value").value("1").allow(); +sp.disable_function.function("curl_setopt").param("value").value("2").allow(); +# `81` is SSL_VERIFYHOST and `64` SSL_VERIFYPEER +sp.disable_function.function("curl_setopt").param("option").value("64").drop().alias("Please don't turn CURLOPT_SSL_VERIFYCLIENT off."); +sp.disable_function.function("curl_setopt").param("option").value("81").drop().alias("Please don't turn CURLOPT_SSL_VERIFYHOST off."); + +# File upload +sp.disable_function.function("move_uploaded_file").param("to").value_r("\\.ph").drop(); +sp.disable_function.function("move_uploaded_file").param("to").value_r("\\.ht").drop(); + +# Logging lockdown +sp.disable_function.function("ini_set").param("option").value_r("error_log").drop() +sp.disable_function.function("ini_set").param("option").value_r("display_errors").drop() + +sp.auto_cookie_secure.enable(); +sp.cookie.name("roundcube_sessauth").samesite("strict"); +sp.cookie.name("roundcube_sessid").samesite("strict"); +sp.ini_protection.policy_silent_fail(); diff --git a/webmails/start.py b/webmails/start.py index 06b90351..bf4472cd 100755 --- a/webmails/start.py +++ b/webmails/start.py @@ -51,7 +51,9 @@ if not secret_key: print(f"Can't read SECRET_KEY from file: {exc}", file=sys.stderr) exit(2) -context['SECRET_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest() +context['ROUNDCUBE_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest() +context['SNUFFLEPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEPAGUS_KEY', 'utf-8'), 'sha256').hexdigest() +conf.jinja("/etc/snuffleupagus.rules.tpl", context, "/etc/snuffleupagus.rules") # roundcube plugins # (using "dict" because it is ordered and "set" is not) @@ -118,7 +120,7 @@ if os.path.exists("/var/run/nginx.pid"): os.system("nginx -s reload") # clean env -[env.pop(key, None) for key in env.keys() if key == "SECRET_KEY" or key.startswith("ROUNDCUBE_")] +[env.pop(key, None) for key in env.keys() if key == "SECRET_KEY" or key.endswith("_KEY")] # run nginx os.system("php-fpm81") From 2a4f6836cfa66257c9c2aad4c52a7d9984bfeffe Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 18 Nov 2022 15:39:32 +0100 Subject: [PATCH 49/68] protect unserialize() --- webmails/snuffleupagus.rules | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index 35f0b7ce..f85b7fe4 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -124,3 +124,5 @@ sp.auto_cookie_secure.enable(); sp.cookie.name("roundcube_sessauth").samesite("strict"); sp.cookie.name("roundcube_sessid").samesite("strict"); sp.ini_protection.policy_silent_fail(); + +sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:\"").drop(); From 017ea5298e75749a97c394201ba9ebeb2f468a40 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 18 Nov 2022 15:52:05 +0100 Subject: [PATCH 50/68] typo --- webmails/snuffleupagus.rules | 2 +- webmails/start.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index f85b7fe4..df9dadc4 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -11,7 +11,7 @@ sp.harden_random.enable(); sp.xxe_protection.enable(); # Global configuration variables -sp.global.secret_key("{{ SNUFFLEPAGUS_KEY }}"); +sp.global.secret_key("{{ SNUFFLEUPAGUS_KEY }}"); # Globally activate strict mode # https://www.php.net/manual/en/language.types.declarations.php#language.types.declarations.strict diff --git a/webmails/start.py b/webmails/start.py index bf4472cd..f87ac55f 100755 --- a/webmails/start.py +++ b/webmails/start.py @@ -52,7 +52,7 @@ if not secret_key: exit(2) context['ROUNDCUBE_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('ROUNDCUBE_KEY', 'utf-8'), 'sha256').hexdigest() -context['SNUFFLEPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEPAGUS_KEY', 'utf-8'), 'sha256').hexdigest() +context['SNUFFLEUPAGUS_KEY'] = hmac.new(bytearray(secret_key, 'utf-8'), bytearray('SNUFFLEUPAGUS_KEY', 'utf-8'), 'sha256').hexdigest() conf.jinja("/etc/snuffleupagus.rules.tpl", context, "/etc/snuffleupagus.rules") # roundcube plugins From 840b2bd9df8ac45c5fee36f449eebac8435a50d6 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 18 Nov 2022 16:00:31 +0100 Subject: [PATCH 51/68] block o:0:{} too --- webmails/snuffleupagus.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index df9dadc4..baa5ecf8 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -125,4 +125,4 @@ sp.cookie.name("roundcube_sessauth").samesite("strict"); sp.cookie.name("roundcube_sessid").samesite("strict"); sp.ini_protection.policy_silent_fail(); -sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:\"").drop(); +sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:[\"{]").drop(); From b20bf996ec10f3b5b6616b25558bcd9d4b083a64 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sat, 19 Nov 2022 18:44:30 +0100 Subject: [PATCH 52/68] Fix #2231: make public announcements work --- core/admin/mailu/models.py | 22 +++++++++++++--------- core/admin/mailu/ui/views/base.py | 5 +++-- towncrier/newsfragments/2231.bugfix | 1 + 3 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 towncrier/newsfragments/2231.bugfix diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 48ce8b33..90dbbbf9 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -2,7 +2,6 @@ """ import os -import smtplib import json from datetime import date @@ -420,14 +419,19 @@ class Email(object): def sendmail(self, subject, body): """ send an email to the address """ - f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' - with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp: - to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' - msg = text.MIMEText(body) - msg['Subject'] = subject - msg['From'] = f_addr - msg['To'] = to_address - smtp.sendmail(f_addr, [to_address], msg.as_string()) + try: + f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}' + ip, port = app.config['HOST_LMTP'].rsplit(':') + with smtplib.LMTP(ip, port=port) as lmtp: + to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}' + msg = text.MIMEText(body) + msg['Subject'] = subject + msg['From'] = f_addr + msg['To'] = to_address + lmtp.sendmail(f_addr, [to_address], msg.as_string()) + return True + except smtplib.SMTPException: + return False @classmethod def resolve_domain(cls, email): diff --git a/core/admin/mailu/ui/views/base.py b/core/admin/mailu/ui/views/base.py index 9b7614e1..1d06464a 100644 --- a/core/admin/mailu/ui/views/base.py +++ b/core/admin/mailu/ui/views/base.py @@ -21,8 +21,9 @@ def announcement(): form = forms.AnnouncementForm() if form.validate_on_submit(): for user in models.User.query.all(): - user.sendmail(form.announcement_subject.data, - form.announcement_body.data) + if not user.sendmail(form.announcement_subject.data, + form.announcement_body.data): + flask.flash('Failed to send to %s' % user.email, 'error') # Force-empty the form form.announcement_subject.data = '' form.announcement_body.data = '' diff --git a/towncrier/newsfragments/2231.bugfix b/towncrier/newsfragments/2231.bugfix new file mode 100644 index 00000000..e710ea6d --- /dev/null +++ b/towncrier/newsfragments/2231.bugfix @@ -0,0 +1 @@ +Make public announcement bypass the filters. They may still time-out before being sent if there is a large number of users. From f802601a08b6c2e399a4520a76ef8cb028238359 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 20 Nov 2022 15:00:04 +0100 Subject: [PATCH 53/68] Update f4f0f89e0047_.py --- core/admin/migrations/versions/f4f0f89e0047_.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/migrations/versions/f4f0f89e0047_.py b/core/admin/migrations/versions/f4f0f89e0047_.py index 5843e9d2..8d20063c 100644 --- a/core/admin/migrations/versions/f4f0f89e0047_.py +++ b/core/admin/migrations/versions/f4f0f89e0047_.py @@ -1,4 +1,4 @@ -"""empty message +""" Add fetch.scan and fetch.folders Revision ID: f4f0f89e0047 Revises: 8f9ea78776f4 From db9ed1fd594c419c492553f988cd75d705b2b58d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 20 Nov 2022 16:26:27 +0100 Subject: [PATCH 54/68] Disable libhardened-malloc for non x86. @see #2541 Support is going to be a nightmare if RPI4 is not working. --- core/base/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 6f31e21c..25c8bd81 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -8,14 +8,13 @@ ENV TZ=Etc/UTC LANG=C.UTF-8 ARG MAILU_UID=1000 ARG MAILU_GID=1000 -ARG TARGETPLATFORM RUN set -euxo pipefail \ ; addgroup -Sg ${MAILU_GID} mailu \ ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \ ; apk add --no-cache bash ca-certificates curl python3 tzdata \ ; machine="$(uname -m)" \ - ; ! [[ "${TARGETPLATFORM}" != linux/arm/v7 && \( "${machine}" == x86_64 || "${machine}" == armv8* || "${machine}" == aarch64 \) ]] \ + ; ! [[ "${machine}" == x86_64 ]] \ || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so From dcf11aea482f97ad1c5a0acf9a0e54c66ed314eb Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 20 Nov 2022 09:10:39 +0100 Subject: [PATCH 55/68] Don't force a password reset --- core/admin/mailu/ui/views/users.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/admin/mailu/ui/views/users.py b/core/admin/mailu/ui/views/users.py index 85a5c2db..b1b42c17 100644 --- a/core/admin/mailu/ui/views/users.py +++ b/core/admin/mailu/ui/views/users.py @@ -64,10 +64,11 @@ def user_edit(user_email): form.quota_bytes.validators = [ wtforms.validators.NumberRange(max=max_quota_bytes)] if form.validate_on_submit(): - if msg := utils.isBadOrPwned(form): - flask.flash(msg, "error") - return flask.render_template('user/edit.html', form=form, user=user, - domain=user.domain, max_quota_bytes=max_quota_bytes) + if form.pw.data: + if msg := utils.isBadOrPwned(form): + flask.flash(msg, "error") + return flask.render_template('user/edit.html', form=form, user=user, + domain=user.domain, max_quota_bytes=max_quota_bytes) form.populate_obj(user) if form.pw.data: user.set_password(form.pw.data) From c79e8d3852aac0c57f88bb2a930ffe68e761403e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 10:37:36 +0100 Subject: [PATCH 56/68] Fix display bug --- core/admin/mailu/ui/templates/fetch/list.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/ui/templates/fetch/list.html b/core/admin/mailu/ui/templates/fetch/list.html index e502d96a..74d3a02f 100644 --- a/core/admin/mailu/ui/templates/fetch/list.html +++ b/core/admin/mailu/ui/templates/fetch/list.html @@ -39,7 +39,7 @@ {{ fetch.username }} {% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} {% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %} - {{ fetch.folders.data | join(',') }} + {{ fetch.folders | join(',') }} {{ fetch.last_check | format_datetime or '-' }} {{ fetch.error or '-' }} {{ fetch.created_at | format_date }} From 4da2db1b0bd2052445c09353485b6cbf9c3e360b Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 10:38:44 +0100 Subject: [PATCH 57/68] add comment as requested --- core/admin/mailu/ui/forms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/mailu/ui/forms.py b/core/admin/mailu/ui/forms.py index fa81adc3..ec19bb0b 100644 --- a/core/admin/mailu/ui/forms.py +++ b/core/admin/mailu/ui/forms.py @@ -42,6 +42,7 @@ class MultipleEmailAddressesVerify(object): raise validators.ValidationError(self.message) class MultipleFoldersVerify(object): + """ Ensure that we have CSV formated data """ def __init__(self,message=_('Invalid list of folders.')): self.message = message From 45b01db9de9f7866bb2ca1a51ccf788117916628 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 11:01:01 +0100 Subject: [PATCH 58/68] Fix the language switcher --- core/admin/mailu/sso/views/languages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/sso/views/languages.py b/core/admin/mailu/sso/views/languages.py index ff65af45..19764519 100644 --- a/core/admin/mailu/sso/views/languages.py +++ b/core/admin/mailu/sso/views/languages.py @@ -1,7 +1,7 @@ from mailu.sso import sso import flask -@sso.route('/language/', methods=['POST']) +@sso.route('/language/', methods=['GET','POST']) def set_language(language=None): if language: flask.session['language'] = language From 28d720bbc978fbcbd7b8654c0d2b13f04480f173 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 14:54:36 +0100 Subject: [PATCH 59/68] As requested --- webmails/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 376399bf..72c2ee18 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -79,7 +79,7 @@ RUN set -euxo pipefail \ # common COPY start.py / COPY php.ini /defaults/ -COPY php-webmail.conf /etc/php81/php-fpm.d/php-webmail.conf +COPY php-webmail.conf /etc/php81/php-fpm.d/ COPY nginx-webmail.conf /conf/ EXPOSE 80/tcp From ab852772f91cdcf0ec1617cea8ea2659fdecb8a4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 16:04:00 +0100 Subject: [PATCH 60/68] Bump snappymail to 2.21.3 --- webmails/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 72c2ee18..5d0bc23b 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -49,7 +49,7 @@ COPY roundcube/config/config.inc.carddav.php /var/www/roundcube/plugins/carddav/ # snappymail -ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.0/snappymail-2.21.0.tar.gz +ENV SNAPPYMAIL_URL https://github.com/the-djmaze/snappymail/releases/download/v2.21.3/snappymail-2.21.3.tar.gz RUN set -euxo pipefail \ ; mkdir /var/www/snappymail \ From 44c47586eaa6798809cac77f6967e57c5173d2ab Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 17:50:57 +0100 Subject: [PATCH 61/68] Fix potential permission problems --- core/admin/start.py | 4 +++- towncrier/newsfragments/2539.misc | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/admin/start.py b/core/admin/start.py index 99b34a01..d9fa6bef 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -5,7 +5,9 @@ import logging as log from pwd import getpwnam import sys -os.system("chown mailu:mailu -R /data /dkim") +os.system("chown mailu:mailu -R /dkim") +os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu") +os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown mailu:mailu") mailu_id = getpwnam('mailu') os.setgid(mailu_id.pw_gid) os.setuid(mailu_id.pw_uid) diff --git a/towncrier/newsfragments/2539.misc b/towncrier/newsfragments/2539.misc index 0d9907ac..10e3954e 100644 --- a/towncrier/newsfragments/2539.misc +++ b/towncrier/newsfragments/2539.misc @@ -1 +1 @@ -Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. +Upgrade to Alpine 3.16.3; Make setup, admin and rspamd run without root privs. Please ensure that your folder overrides/rspamd is owned by 1000:1000 From f994c8687ed35402d6a1a7750f2a61ce8cfb8451 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 21 Nov 2022 18:12:11 +0100 Subject: [PATCH 62/68] doh --- core/admin/start.py | 1 - core/rspamd/start.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/start.py b/core/admin/start.py index d9fa6bef..e2163398 100755 --- a/core/admin/start.py +++ b/core/admin/start.py @@ -7,7 +7,6 @@ import sys os.system("chown mailu:mailu -R /dkim") os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu") -os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown mailu:mailu") mailu_id = getpwnam('mailu') os.setgid(mailu_id.pw_gid) os.setuid(mailu_id.pw_uid) diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 0f351057..37de1df9 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -35,4 +35,5 @@ while True: # Run rspamd os.system("mkdir -m 755 -p /run/rspamd") os.system("chown rspamd:rspamd /run/rspamd") +os.system("find /var/lib/rspamd | grep -v /filter | xargs -n1 chown rspamd:rspamd") os.execv("/usr/sbin/rspamd", ["rspamd", "-f", "-u", "rspamd", "-g", "rspamd"]) From e94f6eaf33f726e33bfab2571f81f58f409d1331 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 22 Nov 2022 10:11:23 +0100 Subject: [PATCH 63/68] towncrier --- towncrier/newsfragments/2550.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 towncrier/newsfragments/2550.misc diff --git a/towncrier/newsfragments/2550.misc b/towncrier/newsfragments/2550.misc new file mode 100644 index 00000000..fcd5dacf --- /dev/null +++ b/towncrier/newsfragments/2550.misc @@ -0,0 +1 @@ +Add Snuffleupagus to protect webmails (a Suhosin replacement) From 9fa3a3e0c7283da2d8fdb6a9e2de4d6decfef9b4 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 22 Nov 2022 10:14:15 +0100 Subject: [PATCH 64/68] doc --- README.md | 2 +- docs/index.rst | 2 +- webmails/snuffleupagus.rules | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0fd737b6..b6ed040b 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Main features include: - **Web access**, multiple Webmails and administration interface - **User features**, aliases, auto-reply, auto-forward, fetched accounts - **Admin features**, global admins, announcements, per-domain delegation, quotas -- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner +- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, [Snuffleupagus](https://github.com/jvoisin/snuffleupagus/) - **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing - **Freedom**, all FOSS components, no tracker included diff --git a/docs/index.rst b/docs/index.rst index 5c004dc1..0b37cf43 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -28,7 +28,7 @@ Main features include: - **Web access**, multiple Webmails and administration interface - **User features**, aliases, auto-reply, auto-forward, fetched accounts - **Admin features**, global admins, announcements, per-domain delegation, quotas -- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner +- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, Snuffleupagus - **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing - **Freedom**, all FOSS components, no tracker included diff --git a/webmails/snuffleupagus.rules b/webmails/snuffleupagus.rules index baa5ecf8..ec7bee13 100644 --- a/webmails/snuffleupagus.rules +++ b/webmails/snuffleupagus.rules @@ -72,6 +72,7 @@ sp.disable_function.function("include").drop() sp.disable_function.function("system").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop(); sp.disable_function.function("shell_exec").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop(); sp.disable_function.function("exec").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop(); +# This is **very** broad but doing better is non-straightforward sp.disable_function.function("proc_open").param("command").value_r("^gpg ").allow(); sp.disable_function.function("proc_open").param("command").value_r("[$|;&`\\n\\(\\)\\\\]").drop(); @@ -121,8 +122,12 @@ sp.disable_function.function("ini_set").param("option").value_r("error_log").dro sp.disable_function.function("ini_set").param("option").value_r("display_errors").drop() sp.auto_cookie_secure.enable(); +# TODO: consider encrypting the cookies? +# TODO: ensure this is up to date sp.cookie.name("roundcube_sessauth").samesite("strict"); sp.cookie.name("roundcube_sessid").samesite("strict"); sp.ini_protection.policy_silent_fail(); +# roundcube uses unserialize() everywhere. +# This should do the job until https://github.com/jvoisin/snuffleupagus/issues/438 is implemented. sp.disable_function.function("unserialize").param("data").value_r("[cCoO]:\d+:[\"{]").drop(); From adacf579fce9ce1ff3841c91585bcc457f7c3d7a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 23 Nov 2022 15:49:58 +0100 Subject: [PATCH 65/68] Rollback to mysql-connector-python==8.0.29 See #2553 --- core/base/requirements-prod.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/requirements-prod.txt b/core/base/requirements-prod.txt index 716f848e..4cf70cd0 100644 --- a/core/base/requirements-prod.txt +++ b/core/base/requirements-prod.txt @@ -41,7 +41,7 @@ MarkupSafe==2.1.1 marshmallow==3.18.0 marshmallow-sqlalchemy==0.28.1 multidict==6.0.2 -mysql-connector-python==8.0.31 +mysql-connector-python==8.0.29 packaging==21.3 passlib==1.7.4 podop @ file:///app/libs/podop From 4881e0db2a62d3d5fed701d107dcdeeecd764c7f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Wed, 23 Nov 2022 17:15:03 +0100 Subject: [PATCH 66/68] ghost is right, it should be pinned here too --- core/base/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/requirements-dev.txt b/core/base/requirements-dev.txt index 92d34fdb..ebcdde92 100644 --- a/core/base/requirements-dev.txt +++ b/core/base/requirements-dev.txt @@ -23,7 +23,7 @@ itsdangerous limits marshmallow marshmallow-sqlalchemy -mysql-connector-python +mysql-connector-python==8.0.29 passlib psycopg2-binary Pygments From 63a12d985758f824c58d1bce440a69f29f1b0a65 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 24 Nov 2022 10:00:00 +0100 Subject: [PATCH 67/68] changes requested by ghost --- core/base/Dockerfile | 27 ++++++++++----------------- webmails/Dockerfile | 5 ++--- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 20d17f30..125d576e 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -49,26 +49,19 @@ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" COPY requirements-${MAILU_DEPS}.txt ./ COPY libs/ libs/ -RUN set -euxo pipefail \ - ; pip install -r requirements-${MAILU_DEPS}.txt || \ - { \ - machine="$(uname -m)" \ - ; deps="build-base gcc libffi-dev python3-dev" \ - ; [[ "${machine}" != x86_64 ]] && \ - deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ - ; apk add --virtual .build-deps ${deps} \ - ; [[ "${machine}" == armv7* ]] && \ - mkdir -p /root/.cargo/registry/index && \ - git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ - ; pip install -r requirements-${MAILU_DEPS}.txt \ - ; rm -rf /root/.cargo /tmp/*.pem \ - ; } \ - ; rm -rf /root/.cache - ARG SNUFFLEUPAGUS_VERSION=0.8.3 ENV SNUFFLEUPAGUS_URL https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v$SNUFFLEUPAGUS_VERSION.tar.gz RUN set -euxo pipefail \ + ; machine="$(uname -m)" \ + ; deps="build-base gcc libffi-dev python3-dev" \ + ; [[ "${machine}" != x86_64 ]] && \ + deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ + ; apk add --virtual .build-deps ${deps} \ + ; [[ "${machine}" == armv7* ]] && \ + mkdir -p /root/.cargo/registry/index && \ + git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ + ; pip install -r requirements-${MAILU_DEPS}.txt \ ; curl -sL ${SNUFFLEUPAGUS_URL} | tar xz \ ; cd snuffleupagus-$SNUFFLEUPAGUS_VERSION \ ; rm -rf src/tests/*php7*/ src/tests/*session*/ src/tests/broken_configuration/ src/tests/*cookie* src/tests/upload_validation/ \ @@ -80,7 +73,7 @@ RUN set -euxo pipefail \ ; pecl install vld-beta \ ; make -j $(grep -c processor /proc/cpuinfo) release \ ; cp src/.libs/snuffleupagus.so /app \ - ; apk del -r .build-deps + ; rm -rf /root/.cargo /tmp/*.pem /root/.cache # base mailu image FROM system diff --git a/webmails/Dockerfile b/webmails/Dockerfile index 2ae8a8f0..77bc65f3 100644 --- a/webmails/Dockerfile +++ b/webmails/Dockerfile @@ -22,9 +22,8 @@ RUN set -euxo pipefail \ ; gpg --import /tmp/snappymail.asc \ ; gpg --import /tmp/roundcube.asc \ ; echo extension=snuffleupagus > /etc/php81/conf.d/snuffleupagus.ini \ - ; rm -f /tmp/*asc \ - ; mkdir -p /run/nginx \ - ; mkdir -p /conf + ; rm -f /tmp/roundcube.asc /tmp/snappymail.asc \ + ; mkdir -p /run/nginx /conf # roundcube ENV ROUNDCUBE_URL https://github.com/roundcube/roundcubemail/releases/download/1.5.3/roundcubemail-1.5.3-complete.tar.gz From 9fcff5e7452e53dab8d5f0c5f5db7eab216d8514 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Thu, 24 Nov 2022 10:13:04 +0100 Subject: [PATCH 68/68] Pin what we get from edge --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 125d576e..65e8be94 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -15,7 +15,7 @@ RUN set -euxo pipefail \ ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \ ; machine="$(uname -m)" \ ; ! [[ "${machine}" == x86_64 ]] \ - || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc + || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0 ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"