Change rspamd override system to use include with lowest priority.

All override files are used as if they were placed in the rspamd local.d folder. From the newsfragment: New override system for Rspamd. In the old system, all files were placed in the Rspamd overrides folder. These overrides would override everything, including the Mailu Rspamd config. Now overrides are placed in /overrides. If you use your own map files, change the location to /override/myMapFile.map in the corresponding conf file. It works as following. * If the override file overrides a Mailu defined config file, it will be included in the Mailu config file with lowest priority. It will merge with existing sections. * If the override file does not override a Mailu defined config file, then the file will be placed in the rspamd local.d folder. It will merge with existing sections. For more information, see the description of the local.d folder on the rspamd website: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
2 years ago · 8861ce6edb
parent 0de2430868
commit 8861ce6edb
19 changed files with 80 additions and 49 deletions
--- a/core/rspamd/Dockerfile
+++ b/core/rspamd/Dockerfile
@ -8,7 +8,8 @@ LABEL version=$VERSION

 RUN set -euxo pipefail \
  ; apk add --no-cache rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \
-  ; mkdir /run/rspamd
+  ; mkdir /run/rspamd \
+  ; mkdir /overrides

 COPY conf/ /conf/
 COPY start.py /
--- a/core/rspamd/conf/antivirus.conf
+++ b/core/rspamd/conf/antivirus.conf
@ -3,9 +3,10 @@ clamav {
  scan_mime_parts = true;
  symbol = "CLAM_VIRUS";
  type = "clamav";
-  servers = "{{ ANTIVIRUS_ADDRESS }}:3310";
+  servers = "{{ ANTIVIRUS_ADDRESS }}";
  {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %}
  action = "reject"
  {% endif %}
 }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/antivirus.conf"
--- a/core/rspamd/conf/arc.conf
+++ b/core/rspamd/conf/arc.conf
@ -4,3 +4,4 @@ allow_username_mismatch = true;
 use_vault = true;
 vault_url = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/vault";
 vault_token = "mailu";
+.include(try=true,priority=1,duplicate=merge) "/overrides/arc.conf"
--- a/core/rspamd/conf/classifier-bayes.conf
+++ b/core/rspamd/conf/classifier-bayes.conf
@ -4,3 +4,4 @@ autolearn {
  check_balance = true; # Check spam and ham balance
  min_balance = 0.9; # Keep diff for spam/ham learns for at least this value
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/classifier-bayes.conf"
--- a/core/rspamd/conf/composites.conf
+++ b/core/rspamd/conf/composites.conf
@ -12,3 +12,4 @@ OLETOOLS_MACRO_SUSPICIOUS {
    score = 20.0;
 }
 {% endif %}
+.include(try=true; priority=1; duplicate=merge) "/overrides/composites.conf"
--- a/core/rspamd/conf/dkim_signing.conf
+++ b/core/rspamd/conf/dkim_signing.conf
@ -4,3 +4,4 @@ allow_username_mismatch = true;
 use_vault = true;
 vault_url = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/vault";
 vault_token = "mailu";
+.include(try=true,priority=1,duplicate=merge) "/overrides/dkim_signing.conf"
--- a/core/rspamd/conf/external_services.conf
+++ b/core/rspamd/conf/external_services.conf
@ -2,14 +2,12 @@
 oletools {
  # default olefy settings
  servers = "{{ OLETOOLS_ADDRESS }}:11343"
-
  # needs to be set explicitly for Rspamd < 1.9.5
  scan_mime_parts = true;
  extended = true;
  max_size = 3145728;
  timeout = 20.0;
  retransmits = 1;
-
  patterns {
    OLETOOLS_MACRO_FOUND= '^.....M..$';
    OLETOOLS_AUTOEXEC   = '^A....M..$';
@ -20,7 +18,6 @@ oletools {
    OLETOOLS_W   = '(?i)\b(?:FileCopy|CopyFile|Kill|CreateTextFile|VirtualAlloc|RtlMoveMemory|URLDownloadToFileA?|AltStartupPath|WriteProcessMemory|ADODB\.Stream|WriteText|SaveToFile|SaveAs|SaveAsRTF|FileSaveAs|MkDir|RmDir|SaveSetting|SetAttr)\b|(?:\bOpen\b[^\n]+\b(?:Write|Append|Binary|Output|Random)\b)';
    OLETOOLS_X   = '(?i)\b(?:Shell|CreateObject|GetObject|SendKeys|RUN|CALL|MacScript|FollowHyperlink|CreateThread|ShellExecuteA?|ExecuteExcel4Macro|EXEC|REGISTER|SetTimer)\b|(?:\bDeclare\b[^\n]+\bLib\b)';
  }
-
  # mime-part regex matching in content-type or filename
  mime_parts_filter_regex {
    #UNKNOWN = "application\/octet-stream";
@ -62,3 +59,4 @@ oletools {
  }
 }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/external_services.conf"
--- a/core/rspamd/conf/external_services_group.conf
+++ b/core/rspamd/conf/external_services_group.conf
@ -1,40 +1,42 @@
+
 {% if SCAN_MACROS == 'True' %}
 # local.d/external_services_group.conf

-description = "Oletools content rules";
-symbols = {
-  "OLETOOLS" {
-    weight = 1.0;
-    description = "OLETOOLS found a Macro";
-    one_shot = true;
-  },
-  "OLETOOLS_MACRO_FOUND" {
-    weight = 0.0;
-    one_shot = true;
-  },
-  "OLETOOLS_AUTOEXEC" {
-    weight = 0.0;
-    one_shot = true;
-  },
-  "OLETOOLS_FLAG" {
-    weight = 0.0;
-    one_shot = true;
-  },
-  "OLETOOLS_VBASTOMP" {
-    weight = 0.0;
-    one_shot = true;
-  },
-  "OLETOOLS_A" {
-    weight = 0.0;
-    one_shot = true;
-  },
-  "OLETOOLS_W" {
-    weight = 0.0;
-    one_shot = true;
-  },
-  "OLETOOLS_X" {
-    weight = 0.0;
-    one_shot = true;
-  },
-}
+ description = "Oletools content rules";
+ symbols = {
+   "OLETOOLS" {
+     weight = 1.0;
+     description = "OLETOOLS found a Macro";
+     one_shot = true;
+   },
+   "OLETOOLS_MACRO_FOUND" {
+     weight = 0.0;
+     one_shot = true;
+   },
+   "OLETOOLS_AUTOEXEC" {
+     weight = 0.0;
+     one_shot = true;
+   },
+   "OLETOOLS_FLAG" {
+     weight = 0.0;
+     one_shot = true;
+   },
+   "OLETOOLS_VBASTOMP" {
+     weight = 0.0;
+     one_shot = true;
+   },
+   "OLETOOLS_A" {
+     weight = 0.0;
+     one_shot = true;
+   },
+   "OLETOOLS_W" {
+     weight = 0.0;
+     one_shot = true;
+   },
+   "OLETOOLS_X" {
+     weight = 0.0;
+     one_shot = true;
+   },
+ }
 {% endif %}
+.include(try=true,priority=1,duplicate=merge) "/overrides/external_services_group.conf"
--- a/core/rspamd/conf/force_actions.conf
+++ b/core/rspamd/conf/force_actions.conf
@ -15,3 +15,4 @@ rules {
    message = "Rejected (anti-spoofing: auth-failed)";
  }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/force_actions.conf"
--- a/core/rspamd/conf/fuzzy_check.conf
+++ b/core/rspamd/conf/fuzzy_check.conf
@ -13,7 +13,6 @@ rule "local" {
    skip_unknown = yes;
    # Hash generation algorithm
    algorithm = "mumhash";
-
    # Map flags to symbols
    fuzzy_map = {
        LOCAL_FUZZY_DENIED {
@ -32,3 +31,4 @@ rule "local" {
        }
    }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/fuzzy_check.conf"
--- a/core/rspamd/conf/headers_group.conf
+++ b/core/rspamd/conf/headers_group.conf
@ -1,7 +1,8 @@
 symbols = {
    "RCVD_NO_TLS_LAST" {
-	# see https://github.com/Mailu/Mailu/issues/1705
+    # see https://github.com/Mailu/Mailu/issues/1705
        weight = 0.0;
        description = "Last hop did not use encrypted transports";
    }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/headers_group.conf"
--- a/core/rspamd/conf/history_redis.conf
+++ b/core/rspamd/conf/history_redis.conf
@ -1 +1,2 @@
 servers = "{{ REDIS_ADDRESS }}";
+.include(try=true,priority=1,duplicate=merge) "/overrides/history_redis.conf"
--- a/core/rspamd/conf/metrics.conf
+++ b/core/rspamd/conf/metrics.conf
@ -17,3 +17,4 @@ group "fuzzy" {
        description = "Whitelisted fuzzy hash";
    }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/metrics.conf"
--- a/core/rspamd/conf/milter_headers.conf
+++ b/core/rspamd/conf/milter_headers.conf
@ -1,9 +1,7 @@
 authenticated_headers = ["authentication-results"];
 skip_local = false;
 skip_authenticated = false;
-
 use = ["x-spamd-bar", "x-spam-level", "x-virus", "authentication-results"];
-
 routines {
  authentication-results {
    add_smtp_user = false;
@ -12,3 +10,4 @@ routines {
    symbols = ["CLAM_VIRUS", "FPROT_VIRUS", "JUST_EICAR"];
  }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/milter_headers.conf"
--- a/core/rspamd/conf/multimap.conf
+++ b/core/rspamd/conf/multimap.conf
@ -3,18 +3,15 @@ IS_LOCAL_DOMAIN_H {
  selector = "from('mime'):domain";
  map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains";
 }
-
 IS_LOCAL_DOMAIN_E {
  type = "selector"
  selector = "from('smtp'):domain";
  map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains";
 }
-
 IS_LOCALLY_GENERATED {
  type = "ip"
  map = ["/etc/rspamd/local.d/local_subnet.map"];
 }
-
 FORBIDDEN_FILE_EXTENSION {
    type = "filename";
    filter = "extension";
@ -27,3 +24,4 @@ FORBIDDEN_FILE_EXTENSION {
    description = "List of forbidden file extensions";
    message = "Forbidden attachment extension";
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/multimap.conf"
--- a/core/rspamd/conf/whitelist.conf
+++ b/core/rspamd/conf/whitelist.conf
@ -1,3 +1,4 @@
+
 rules {
        BLACKLIST_ANTISPOOF = {
            valid_dmarc = true;
@ -6,3 +7,4 @@ rules {
 	    score = 0.0;
        }
 }
+.include(try=true,priority=1,duplicate=merge) "/overrides/whitelist.conf"
--- a/core/rspamd/start.py
+++ b/core/rspamd/start.py
@ -4,6 +4,7 @@ import os
 import glob
 import logging as log
 import requests
+import shutil
 import sys
 import time
 from socrate import system,conf
@ -13,8 +14,14 @@ system.set_env()

 # Actual startup script

+config_files = []
 for rspamd_file in glob.glob("/conf/*"):
    conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file)))
+    config_files.append(os.path.basename(rspamd_file))
+
+for override_file in glob.glob("/overrides/*"):
+    if os.path.basename(override_file) not in config_files:
+        shutil.copyfile(override_file, os.path.join("/etc/rspamd/local.d", os.path.basename(override_file)))

 # Admin may not be up just yet
 healthcheck = f'http://{os.environ["ADMIN_ADDRESS"]}/internal/rspamd/local_domains'
--- a/setup/flavors/compose/docker-compose.yml
+++ b/setup/flavors/compose/docker-compose.yml
@ -133,7 +133,7 @@ services:
 {% endif %}
    volumes:
      - "{{ root }}/filter:/var/lib/rspamd"
-      - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d:ro"
+      - "{{ root }}/overrides/rspamd:/overrides:ro"
    depends_on:
      - front
      - redis
--- a/towncrier/newsfragments/2555.feature
+++ b/towncrier/newsfragments/2555.feature
@ -0,0 +1,15 @@
+New override system for Rspamd. In the old system, all files were placed in the Rspamd overrides folder.
+These overrides would override everything, including the Mailu Rspamd config.
+
+Now overrides are placed in /overrides.
+If you use your own map files, change the location to /override/myMapFile.map in the corresponding conf file.
+It works as following.
+* If the override file overrides a Mailu defined config file,
+  it will be included in the Mailu config file with lowest priority.
+  It will merge with existing sections.
+* If the override file does not override a Mailu defined config file,
+  then the file will be placed in the rspamd local.d folder.
+  It will merge with existing sections.
+
+For more information, see the description of the local.d folder on the rspamd website:
+https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories