Merge #2690

2690: Change rspamd override system to use .include with lowest priority. r=mergify[bot] a=Diman0

## What does this PR do?

All override files are used as if they were placed in the rspamd local.d folder.

New override system for Rspamd. In the old system, all files were placed in the Rspamd overrides folder. These overrides would override everything, including the Mailu Rspamd config.

Now overrides are placed in /overrides.
If you use your own map files, change the location to /override/myMapFile.map in the corresponding conf file. It works as following.
* If the override file overrides a Mailu defined config file, it will be included in the Mailu config file with lowest priority. It will merge with existing sections.
* If the override file does not override a Mailu defined config file, then the file will be placed in the rspamd local.d folder. It will merge with existing sections.

For more information, see the description of the local.d folder on the rspamd website: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories

## What type of PR?

enhancement

### Related issue(s)
- closes #2555 

## Prerequisites
Before we can consider review and merge, please make sure the following list is done and checked.
If an entry in not applicable, you can check it or remove it from the list.

- [x] In case of feature or enhancement: documentation updated accordingly
- [x] Unless it's docs or a minor change: add [changelog](https://mailu.io/master/contributors/workflow.html#changelog) entry file.


Co-authored-by: Dimitri Huisman <diman@huisman.xyz>

core/rspamd/Dockerfile

@@ -8,7 +8,8 @@ LABEL version=$VERSION
 
 RUN set -euxo pipefail \
   ; apk add --no-cache rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \
-  ; mkdir /run/rspamd
+  ; mkdir /run/rspamd \
+  ; mkdir /overrides
 
 COPY conf/ /conf/
 COPY start.py /

core/rspamd/conf/antivirus.conf

@@ -9,3 +9,4 @@ clamav {
   {% endif %}
 }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/antivirus.conf"

core/rspamd/conf/arc.conf

@@ -4,3 +4,4 @@ allow_username_mismatch = true;
 use_vault = true;
 vault_url = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/vault";
 vault_token = "mailu";
+.include(try=true,priority=1,duplicate=merge) "/overrides/arc.conf"

core/rspamd/conf/classifier-bayes.conf

@@ -4,3 +4,4 @@ autolearn {
   check_balance = true; # Check spam and ham balance
   min_balance = 0.9; # Keep diff for spam/ham learns for at least this value
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/classifier-bayes.conf"

core/rspamd/conf/composites.conf

@@ -12,3 +12,4 @@ OLETOOLS_MACRO_SUSPICIOUS {
     score = 20.0;
 }
 {% endif %}
+.include(try=true; priority=1; duplicate=merge) "/overrides/composites.conf"

core/rspamd/conf/dkim_signing.conf

@@ -4,3 +4,4 @@ allow_username_mismatch = true;
 use_vault = true;
 vault_url = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/vault";
 vault_token = "mailu";
+.include(try=true,priority=1,duplicate=merge) "/overrides/dkim_signing.conf"

core/rspamd/conf/external_services.conf

@@ -62,3 +62,4 @@ oletools {
   }
 }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/external_services.conf"

core/rspamd/conf/external_services_group.conf

@@ -1,3 +1,4 @@
+
 {% if SCAN_MACROS == 'True' %}
 # local.d/external_services_group.conf
 
@@ -38,3 +39,4 @@ symbols = {
   },
 }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/external_services_group.conf"

core/rspamd/conf/force_actions.conf

@@ -15,3 +15,4 @@ rules {
     message = "Rejected (anti-spoofing: auth-failed)";
   }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/force_actions.conf"

core/rspamd/conf/fuzzy_check.conf

@@ -13,7 +13,6 @@ rule "local" {
     skip_unknown = yes;
     # Hash generation algorithm
     algorithm = "mumhash";
-
     # Map flags to symbols
     fuzzy_map = {
         LOCAL_FUZZY_DENIED {
@@ -32,3 +31,4 @@ rule "local" {
         }
     }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/fuzzy_check.conf"

core/rspamd/conf/headers_group.conf

@@ -5,3 +5,4 @@ symbols = {
         description = "Last hop did not use encrypted transports";
     }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/headers_group.conf"

core/rspamd/conf/history_redis.conf

@@ -1 +1,2 @@
 servers = "{{ REDIS_ADDRESS }}";
+.include(try=true,priority=1,duplicate=merge) "/overrides/history_redis.conf"

core/rspamd/conf/metrics.conf

@@ -17,3 +17,4 @@ group "fuzzy" {
         description = "Whitelisted fuzzy hash";
     }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/metrics.conf"

core/rspamd/conf/milter_headers.conf

@@ -1,9 +1,7 @@
 authenticated_headers = ["authentication-results"];
 skip_local = false;
 skip_authenticated = false;
-
 use = ["x-spamd-bar", "x-spam-level", "x-virus", "authentication-results"];
-
 routines {
   authentication-results {
     add_smtp_user = false;
@@ -12,3 +10,4 @@ routines {
     symbols = ["CLAM_VIRUS", "FPROT_VIRUS", "JUST_EICAR"];
   }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/milter_headers.conf"

core/rspamd/conf/multimap.conf

@@ -3,18 +3,15 @@ IS_LOCAL_DOMAIN_H {
   selector = "from('mime'):domain";
   map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains";
 }
-
 IS_LOCAL_DOMAIN_E {
   type = "selector"
   selector = "from('smtp'):domain";
   map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains";
 }
-
 IS_LOCALLY_GENERATED {
   type = "ip"
   map = ["/etc/rspamd/local.d/local_subnet.map"];
 }
-
 FORBIDDEN_FILE_EXTENSION {
     type = "filename";
     filter = "extension";
@@ -27,3 +24,4 @@ FORBIDDEN_FILE_EXTENSION {
     description = "List of forbidden file extensions";
     message = "Forbidden attachment extension";
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/multimap.conf"

core/rspamd/conf/whitelist.conf

@@ -1,3 +1,4 @@
+
 rules {
         BLACKLIST_ANTISPOOF = {
             valid_dmarc = true;
@@ -6,3 +7,4 @@ rules {
 	    score = 0.0;
         }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/whitelist.conf"

core/rspamd/start.py

@@ -4,6 +4,7 @@ import os
 import glob
 import logging as log
 import requests
+import shutil
 import sys
 import time
 from socrate import system,conf
@@ -13,8 +14,14 @@ system.set_env()
 
 # Actual startup script
 
+config_files = []
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
+    config_files.append(os.path.basename(rspamd_file))
+
+for override_file in glob.glob("/overrides/*"):
+    if os.path.basename(override_file) not in config_files:
+        shutil.copyfile(override_file, os.path.join("/etc/rspamd/local.d", os.path.basename(override_file)))
 
 # Admin may not be up just yet
 healthcheck = f'http://{os.environ["ADMIN_ADDRESS"]}/internal/rspamd/local_domains'

setup/flavors/compose/docker-compose.yml

@@ -133,7 +133,7 @@ services:
 {% endif %}
     volumes:
       - "{{ root }}/filter:/var/lib/rspamd"
-      - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d:ro"
+      - "{{ root }}/overrides/rspamd:/overrides:ro"
     depends_on:
       - front
       - redis

towncrier/newsfragments/2555.feature

@@ -0,0 +1,15 @@
+New override system for Rspamd. In the old system, all files were placed in the Rspamd overrides folder.
+These overrides would override everything, including the Mailu Rspamd config.
+
+Now overrides are placed in /overrides.
+If you use your own map files, change the location to /override/myMapFile.map in the corresponding conf file.
+It works as following.
+* If the override file overrides a Mailu defined config file,
+  it will be included in the Mailu config file with lowest priority.
+  It will merge with existing sections.
+* If the override file does not override a Mailu defined config file,
+  then the file will be placed in the rspamd local.d folder.
+  It will merge with existing sections.
+
+For more information, see the description of the local.d folder on the rspamd website:
+https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories