Change rspamd override system to use include with lowest priority.

All override files are used as if they were placed in the rspamd
local.d folder.

From the newsfragment:
New override system for Rspamd. In the old system, all files were placed in the Rspamd overrides folder.
These overrides would override everything, including the Mailu Rspamd config.

Now overrides are placed in /overrides.
If you use your own map files, change the location to /override/myMapFile.map in the corresponding conf file.
It works as following.
* If the override file overrides a Mailu defined config file,
  it will be included in the Mailu config file with lowest priority.
  It will merge with existing sections.
* If the override file does not override a Mailu defined config file,
  then the file will be placed in the rspamd local.d folder.
  It will merge with existing sections.

For more information, see the description of the local.d folder on the rspamd website:
https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories

core/rspamd/Dockerfile

@@ -8,7 +8,8 @@ LABEL version=$VERSION
 
 RUN set -euxo pipefail \
   ; apk add --no-cache rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \
-  ; mkdir /run/rspamd
+  ; mkdir /run/rspamd \
+  ; mkdir /overrides
 
 COPY conf/ /conf/
 COPY start.py /

core/rspamd/conf/antivirus.conf

@@ -3,9 +3,10 @@ clamav {
   scan_mime_parts = true;
   symbol = "CLAM_VIRUS";
   type = "clamav";
-  servers = "{{ ANTIVIRUS_ADDRESS }}:3310";
+  servers = "{{ ANTIVIRUS_ADDRESS }}";
   {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %}
   action = "reject"
   {% endif %}
 }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/antivirus.conf"

core/rspamd/conf/arc.conf

@@ -4,3 +4,4 @@ allow_username_mismatch = true;
 use_vault = true;
 vault_url = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/vault";
 vault_token = "mailu";
+.include(try=true,priority=1,duplicate=merge) "/overrides/arc.conf"

core/rspamd/conf/classifier-bayes.conf

@@ -4,3 +4,4 @@ autolearn {
   check_balance = true; # Check spam and ham balance
   min_balance = 0.9; # Keep diff for spam/ham learns for at least this value
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/classifier-bayes.conf"

core/rspamd/conf/composites.conf

@@ -12,3 +12,4 @@ OLETOOLS_MACRO_SUSPICIOUS {
     score = 20.0;
 }
 {% endif %}
+.include(try=true; priority=1; duplicate=merge) "/overrides/composites.conf"

core/rspamd/conf/dkim_signing.conf

@@ -4,3 +4,4 @@ allow_username_mismatch = true;
 use_vault = true;
 vault_url = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/vault";
 vault_token = "mailu";
+.include(try=true,priority=1,duplicate=merge) "/overrides/dkim_signing.conf"

core/rspamd/conf/external_services.conf

@@ -2,14 +2,12 @@
 oletools {
   # default olefy settings
   servers = "{{ OLETOOLS_ADDRESS }}:11343"
-
   # needs to be set explicitly for Rspamd < 1.9.5
   scan_mime_parts = true;
   extended = true;
   max_size = 3145728;
   timeout = 20.0;
   retransmits = 1;
-
   patterns {
     OLETOOLS_MACRO_FOUND= '^.....M..$';
     OLETOOLS_AUTOEXEC   = '^A....M..$';
@@ -20,7 +18,6 @@ oletools {
     OLETOOLS_W   = '(?i)\b(?:FileCopy|CopyFile|Kill|CreateTextFile|VirtualAlloc|RtlMoveMemory|URLDownloadToFileA?|AltStartupPath|WriteProcessMemory|ADODB\.Stream|WriteText|SaveToFile|SaveAs|SaveAsRTF|FileSaveAs|MkDir|RmDir|SaveSetting|SetAttr)\b|(?:\bOpen\b[^\n]+\b(?:Write|Append|Binary|Output|Random)\b)';
     OLETOOLS_X   = '(?i)\b(?:Shell|CreateObject|GetObject|SendKeys|RUN|CALL|MacScript|FollowHyperlink|CreateThread|ShellExecuteA?|ExecuteExcel4Macro|EXEC|REGISTER|SetTimer)\b|(?:\bDeclare\b[^\n]+\bLib\b)';
   }
-
   # mime-part regex matching in content-type or filename
   mime_parts_filter_regex {
     #UNKNOWN = "application\/octet-stream";
@@ -62,3 +59,4 @@ oletools {
   }
 }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/external_services.conf"

core/rspamd/conf/external_services_group.conf

@@ -1,8 +1,9 @@
+
 {% if SCAN_MACROS == 'True' %}
 # local.d/external_services_group.conf
 
-description = "Oletools content rules";
-symbols = {
+ description = "Oletools content rules";
+ symbols = {
    "OLETOOLS" {
      weight = 1.0;
      description = "OLETOOLS found a Macro";
@@ -36,5 +37,6 @@ symbols = {
      weight = 0.0;
      one_shot = true;
    },
-}
+ }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/external_services_group.conf"

core/rspamd/conf/force_actions.conf

@@ -15,3 +15,4 @@ rules {
     message = "Rejected (anti-spoofing: auth-failed)";
   }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/force_actions.conf"

core/rspamd/conf/fuzzy_check.conf

@@ -13,7 +13,6 @@ rule "local" {
     skip_unknown = yes;
     # Hash generation algorithm
     algorithm = "mumhash";
-
     # Map flags to symbols
     fuzzy_map = {
         LOCAL_FUZZY_DENIED {
@@ -32,3 +31,4 @@ rule "local" {
         }
     }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/fuzzy_check.conf"

core/rspamd/conf/headers_group.conf

@@ -5,3 +5,4 @@ symbols = {
         description = "Last hop did not use encrypted transports";
     }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/headers_group.conf"

core/rspamd/conf/history_redis.conf

@@ -1 +1,2 @@
 servers = "{{ REDIS_ADDRESS }}";
+.include(try=true,priority=1,duplicate=merge) "/overrides/history_redis.conf"

core/rspamd/conf/metrics.conf

@@ -17,3 +17,4 @@ group "fuzzy" {
         description = "Whitelisted fuzzy hash";
     }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/metrics.conf"

core/rspamd/conf/milter_headers.conf

@@ -1,9 +1,7 @@
 authenticated_headers = ["authentication-results"];
 skip_local = false;
 skip_authenticated = false;
-
 use = ["x-spamd-bar", "x-spam-level", "x-virus", "authentication-results"];
-
 routines {
   authentication-results {
     add_smtp_user = false;
@@ -12,3 +10,4 @@ routines {
     symbols = ["CLAM_VIRUS", "FPROT_VIRUS", "JUST_EICAR"];
   }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/milter_headers.conf"

core/rspamd/conf/multimap.conf

@@ -3,18 +3,15 @@ IS_LOCAL_DOMAIN_H {
   selector = "from('mime'):domain";
   map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains";
 }
-
 IS_LOCAL_DOMAIN_E {
   type = "selector"
   selector = "from('smtp'):domain";
   map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains";
 }
-
 IS_LOCALLY_GENERATED {
   type = "ip"
   map = ["/etc/rspamd/local.d/local_subnet.map"];
 }
-
 FORBIDDEN_FILE_EXTENSION {
     type = "filename";
     filter = "extension";
@@ -27,3 +24,4 @@ FORBIDDEN_FILE_EXTENSION {
     description = "List of forbidden file extensions";
     message = "Forbidden attachment extension";
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/multimap.conf"

core/rspamd/conf/whitelist.conf

@@ -1,3 +1,4 @@
+
 rules {
         BLACKLIST_ANTISPOOF = {
             valid_dmarc = true;
@@ -6,3 +7,4 @@ rules {
 	    score = 0.0;
         }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/whitelist.conf"

core/rspamd/start.py

@@ -4,6 +4,7 @@ import os
 import glob
 import logging as log
 import requests
+import shutil
 import sys
 import time
 from socrate import system,conf
@@ -13,8 +14,14 @@ system.set_env()
 
 # Actual startup script
 
+config_files = []
 for rspamd_file in glob.glob("/conf/*"):
     conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
+    config_files.append(os.path.basename(rspamd_file))
+
+for override_file in glob.glob("/overrides/*"):
+    if os.path.basename(override_file) not in config_files:
+        shutil.copyfile(override_file, os.path.join("/etc/rspamd/local.d", os.path.basename(override_file)))
 
 # Admin may not be up just yet
 healthcheck = f'http://{os.environ["ADMIN_ADDRESS"]}/internal/rspamd/local_domains'

setup/flavors/compose/docker-compose.yml

@@ -133,7 +133,7 @@ services:
 {% endif %}
     volumes:
       - "{{ root }}/filter:/var/lib/rspamd"
-      - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d:ro"
+      - "{{ root }}/overrides/rspamd:/overrides:ro"
     depends_on:
       - front
       - redis

towncrier/newsfragments/2555.feature

@@ -0,0 +1,15 @@
+New override system for Rspamd. In the old system, all files were placed in the Rspamd overrides folder.
+These overrides would override everything, including the Mailu Rspamd config.
+
+Now overrides are placed in /overrides.
+If you use your own map files, change the location to /override/myMapFile.map in the corresponding conf file.
+It works as following.
+* If the override file overrides a Mailu defined config file,
+  it will be included in the Mailu config file with lowest priority.
+  It will merge with existing sections.
+* If the override file does not override a Mailu defined config file,
+  then the file will be placed in the rspamd local.d folder.
+  It will merge with existing sections.
+
+For more information, see the description of the local.d folder on the rspamd website:
+https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories