From 8861ce6edbd538eecc3c7c80570d66f52f9f9686 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 9 Mar 2023 08:21:45 +0000 Subject: [PATCH 1/3] Change rspamd override system to use include with lowest priority. All override files are used as if they were placed in the rspamd local.d folder. From the newsfragment: New override system for Rspamd. In the old system, all files were placed in the Rspamd overrides folder. These overrides would override everything, including the Mailu Rspamd config. Now overrides are placed in /overrides. If you use your own map files, change the location to /override/myMapFile.map in the corresponding conf file. It works as following. * If the override file overrides a Mailu defined config file, it will be included in the Mailu config file with lowest priority. It will merge with existing sections. * If the override file does not override a Mailu defined config file, then the file will be placed in the rspamd local.d folder. It will merge with existing sections. For more information, see the description of the local.d folder on the rspamd website: https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories --- core/rspamd/Dockerfile | 3 +- core/rspamd/conf/antivirus.conf | 3 +- core/rspamd/conf/arc.conf | 1 + core/rspamd/conf/classifier-bayes.conf | 1 + core/rspamd/conf/composites.conf | 1 + core/rspamd/conf/dkim_signing.conf | 1 + core/rspamd/conf/external_services.conf | 4 +- core/rspamd/conf/external_services_group.conf | 74 ++++++++++--------- core/rspamd/conf/force_actions.conf | 1 + core/rspamd/conf/fuzzy_check.conf | 2 +- core/rspamd/conf/headers_group.conf | 3 +- core/rspamd/conf/history_redis.conf | 1 + core/rspamd/conf/metrics.conf | 1 + core/rspamd/conf/milter_headers.conf | 3 +- core/rspamd/conf/multimap.conf | 4 +- core/rspamd/conf/whitelist.conf | 2 + core/rspamd/start.py | 7 ++ setup/flavors/compose/docker-compose.yml | 2 +- towncrier/newsfragments/2555.feature | 15 ++++ 19 files changed, 80 insertions(+), 49 deletions(-) create mode 100644 towncrier/newsfragments/2555.feature diff --git a/core/rspamd/Dockerfile b/core/rspamd/Dockerfile index 08ac0871..9f61f7d0 100644 --- a/core/rspamd/Dockerfile +++ b/core/rspamd/Dockerfile @@ -8,7 +8,8 @@ LABEL version=$VERSION RUN set -euxo pipefail \ ; apk add --no-cache rspamd rspamd-controller rspamd-fuzzy rspamd-proxy \ - ; mkdir /run/rspamd + ; mkdir /run/rspamd \ + ; mkdir /overrides COPY conf/ /conf/ COPY start.py / diff --git a/core/rspamd/conf/antivirus.conf b/core/rspamd/conf/antivirus.conf index 53da0768..6964d17a 100644 --- a/core/rspamd/conf/antivirus.conf +++ b/core/rspamd/conf/antivirus.conf @@ -3,9 +3,10 @@ clamav { scan_mime_parts = true; symbol = "CLAM_VIRUS"; type = "clamav"; - servers = "{{ ANTIVIRUS_ADDRESS }}:3310"; + servers = "{{ ANTIVIRUS_ADDRESS }}"; {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %} action = "reject" {% endif %} } {% endif %} +.include(try=true,priority=1,duplicate=merge) "/overrides/antivirus.conf" diff --git a/core/rspamd/conf/arc.conf b/core/rspamd/conf/arc.conf index 93414a96..25c288d2 100644 --- a/core/rspamd/conf/arc.conf +++ b/core/rspamd/conf/arc.conf @@ -4,3 +4,4 @@ allow_username_mismatch = true; use_vault = true; vault_url = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/vault"; vault_token = "mailu"; +.include(try=true,priority=1,duplicate=merge) "/overrides/arc.conf" diff --git a/core/rspamd/conf/classifier-bayes.conf b/core/rspamd/conf/classifier-bayes.conf index 510b5814..7aa36e17 100644 --- a/core/rspamd/conf/classifier-bayes.conf +++ b/core/rspamd/conf/classifier-bayes.conf @@ -4,3 +4,4 @@ autolearn { check_balance = true; # Check spam and ham balance min_balance = 0.9; # Keep diff for spam/ham learns for at least this value } +.include(try=true,priority=1,duplicate=merge) "/overrides/classifier-bayes.conf" diff --git a/core/rspamd/conf/composites.conf b/core/rspamd/conf/composites.conf index 68e03073..d438c90e 100644 --- a/core/rspamd/conf/composites.conf +++ b/core/rspamd/conf/composites.conf @@ -12,3 +12,4 @@ OLETOOLS_MACRO_SUSPICIOUS { score = 20.0; } {% endif %} +.include(try=true; priority=1; duplicate=merge) "/overrides/composites.conf" diff --git a/core/rspamd/conf/dkim_signing.conf b/core/rspamd/conf/dkim_signing.conf index 93414a96..043133b7 100644 --- a/core/rspamd/conf/dkim_signing.conf +++ b/core/rspamd/conf/dkim_signing.conf @@ -4,3 +4,4 @@ allow_username_mismatch = true; use_vault = true; vault_url = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/vault"; vault_token = "mailu"; +.include(try=true,priority=1,duplicate=merge) "/overrides/dkim_signing.conf" diff --git a/core/rspamd/conf/external_services.conf b/core/rspamd/conf/external_services.conf index d299ed89..cb4c0ee7 100644 --- a/core/rspamd/conf/external_services.conf +++ b/core/rspamd/conf/external_services.conf @@ -2,14 +2,12 @@ oletools { # default olefy settings servers = "{{ OLETOOLS_ADDRESS }}:11343" - # needs to be set explicitly for Rspamd < 1.9.5 scan_mime_parts = true; extended = true; max_size = 3145728; timeout = 20.0; retransmits = 1; - patterns { OLETOOLS_MACRO_FOUND= '^.....M..$'; OLETOOLS_AUTOEXEC = '^A....M..$'; @@ -20,7 +18,6 @@ oletools { OLETOOLS_W = '(?i)\b(?:FileCopy|CopyFile|Kill|CreateTextFile|VirtualAlloc|RtlMoveMemory|URLDownloadToFileA?|AltStartupPath|WriteProcessMemory|ADODB\.Stream|WriteText|SaveToFile|SaveAs|SaveAsRTF|FileSaveAs|MkDir|RmDir|SaveSetting|SetAttr)\b|(?:\bOpen\b[^\n]+\b(?:Write|Append|Binary|Output|Random)\b)'; OLETOOLS_X = '(?i)\b(?:Shell|CreateObject|GetObject|SendKeys|RUN|CALL|MacScript|FollowHyperlink|CreateThread|ShellExecuteA?|ExecuteExcel4Macro|EXEC|REGISTER|SetTimer)\b|(?:\bDeclare\b[^\n]+\bLib\b)'; } - # mime-part regex matching in content-type or filename mime_parts_filter_regex { #UNKNOWN = "application\/octet-stream"; @@ -62,3 +59,4 @@ oletools { } } {% endif %} +.include(try=true,priority=1,duplicate=merge) "/overrides/external_services.conf" diff --git a/core/rspamd/conf/external_services_group.conf b/core/rspamd/conf/external_services_group.conf index 0b44b229..bd9f9b45 100644 --- a/core/rspamd/conf/external_services_group.conf +++ b/core/rspamd/conf/external_services_group.conf @@ -1,40 +1,42 @@ + {% if SCAN_MACROS == 'True' %} # local.d/external_services_group.conf -description = "Oletools content rules"; -symbols = { - "OLETOOLS" { - weight = 1.0; - description = "OLETOOLS found a Macro"; - one_shot = true; - }, - "OLETOOLS_MACRO_FOUND" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_AUTOEXEC" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_FLAG" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_VBASTOMP" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_A" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_W" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_X" { - weight = 0.0; - one_shot = true; - }, -} + description = "Oletools content rules"; + symbols = { + "OLETOOLS" { + weight = 1.0; + description = "OLETOOLS found a Macro"; + one_shot = true; + }, + "OLETOOLS_MACRO_FOUND" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_AUTOEXEC" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_FLAG" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_VBASTOMP" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_A" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_W" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_X" { + weight = 0.0; + one_shot = true; + }, + } {% endif %} +.include(try=true,priority=1,duplicate=merge) "/overrides/external_services_group.conf" diff --git a/core/rspamd/conf/force_actions.conf b/core/rspamd/conf/force_actions.conf index 9346fdeb..b72a8a0c 100644 --- a/core/rspamd/conf/force_actions.conf +++ b/core/rspamd/conf/force_actions.conf @@ -15,3 +15,4 @@ rules { message = "Rejected (anti-spoofing: auth-failed)"; } } +.include(try=true,priority=1,duplicate=merge) "/overrides/force_actions.conf" diff --git a/core/rspamd/conf/fuzzy_check.conf b/core/rspamd/conf/fuzzy_check.conf index 7c87e1c3..4371de0c 100644 --- a/core/rspamd/conf/fuzzy_check.conf +++ b/core/rspamd/conf/fuzzy_check.conf @@ -13,7 +13,6 @@ rule "local" { skip_unknown = yes; # Hash generation algorithm algorithm = "mumhash"; - # Map flags to symbols fuzzy_map = { LOCAL_FUZZY_DENIED { @@ -32,3 +31,4 @@ rule "local" { } } } +.include(try=true,priority=1,duplicate=merge) "/overrides/fuzzy_check.conf" diff --git a/core/rspamd/conf/headers_group.conf b/core/rspamd/conf/headers_group.conf index bae6f02b..459cffe8 100644 --- a/core/rspamd/conf/headers_group.conf +++ b/core/rspamd/conf/headers_group.conf @@ -1,7 +1,8 @@ symbols = { "RCVD_NO_TLS_LAST" { - # see https://github.com/Mailu/Mailu/issues/1705 + # see https://github.com/Mailu/Mailu/issues/1705 weight = 0.0; description = "Last hop did not use encrypted transports"; } } +.include(try=true,priority=1,duplicate=merge) "/overrides/headers_group.conf" diff --git a/core/rspamd/conf/history_redis.conf b/core/rspamd/conf/history_redis.conf index 053f9dd2..0d0eb954 100644 --- a/core/rspamd/conf/history_redis.conf +++ b/core/rspamd/conf/history_redis.conf @@ -1 +1,2 @@ servers = "{{ REDIS_ADDRESS }}"; +.include(try=true,priority=1,duplicate=merge) "/overrides/history_redis.conf" diff --git a/core/rspamd/conf/metrics.conf b/core/rspamd/conf/metrics.conf index 6a31964f..8ccaacf0 100644 --- a/core/rspamd/conf/metrics.conf +++ b/core/rspamd/conf/metrics.conf @@ -17,3 +17,4 @@ group "fuzzy" { description = "Whitelisted fuzzy hash"; } } +.include(try=true,priority=1,duplicate=merge) "/overrides/metrics.conf" diff --git a/core/rspamd/conf/milter_headers.conf b/core/rspamd/conf/milter_headers.conf index 7da200df..34d4f570 100644 --- a/core/rspamd/conf/milter_headers.conf +++ b/core/rspamd/conf/milter_headers.conf @@ -1,9 +1,7 @@ authenticated_headers = ["authentication-results"]; skip_local = false; skip_authenticated = false; - use = ["x-spamd-bar", "x-spam-level", "x-virus", "authentication-results"]; - routines { authentication-results { add_smtp_user = false; @@ -12,3 +10,4 @@ routines { symbols = ["CLAM_VIRUS", "FPROT_VIRUS", "JUST_EICAR"]; } } +.include(try=true,priority=1,duplicate=merge) "/overrides/milter_headers.conf" diff --git a/core/rspamd/conf/multimap.conf b/core/rspamd/conf/multimap.conf index cc407269..46fd4d46 100644 --- a/core/rspamd/conf/multimap.conf +++ b/core/rspamd/conf/multimap.conf @@ -3,18 +3,15 @@ IS_LOCAL_DOMAIN_H { selector = "from('mime'):domain"; map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; } - IS_LOCAL_DOMAIN_E { type = "selector" selector = "from('smtp'):domain"; map = "http://{{ ADMIN_ADDRESS }}/internal/rspamd/local_domains"; } - IS_LOCALLY_GENERATED { type = "ip" map = ["/etc/rspamd/local.d/local_subnet.map"]; } - FORBIDDEN_FILE_EXTENSION { type = "filename"; filter = "extension"; @@ -27,3 +24,4 @@ FORBIDDEN_FILE_EXTENSION { description = "List of forbidden file extensions"; message = "Forbidden attachment extension"; } +.include(try=true,priority=1,duplicate=merge) "/overrides/multimap.conf" diff --git a/core/rspamd/conf/whitelist.conf b/core/rspamd/conf/whitelist.conf index 56f8a83d..6d9ad698 100644 --- a/core/rspamd/conf/whitelist.conf +++ b/core/rspamd/conf/whitelist.conf @@ -1,3 +1,4 @@ + rules { BLACKLIST_ANTISPOOF = { valid_dmarc = true; @@ -6,3 +7,4 @@ rules { score = 0.0; } } +.include(try=true,priority=1,duplicate=merge) "/overrides/whitelist.conf" diff --git a/core/rspamd/start.py b/core/rspamd/start.py index 507da65d..71eeb531 100755 --- a/core/rspamd/start.py +++ b/core/rspamd/start.py @@ -4,6 +4,7 @@ import os import glob import logging as log import requests +import shutil import sys import time from socrate import system,conf @@ -13,8 +14,14 @@ system.set_env() # Actual startup script +config_files = [] for rspamd_file in glob.glob("/conf/*"): conf.jinja(rspamd_file, os.environ, os.path.join("/etc/rspamd/local.d", os.path.basename(rspamd_file))) + config_files.append(os.path.basename(rspamd_file)) + +for override_file in glob.glob("/overrides/*"): + if os.path.basename(override_file) not in config_files: + shutil.copyfile(override_file, os.path.join("/etc/rspamd/local.d", os.path.basename(override_file))) # Admin may not be up just yet healthcheck = f'http://{os.environ["ADMIN_ADDRESS"]}/internal/rspamd/local_domains' diff --git a/setup/flavors/compose/docker-compose.yml b/setup/flavors/compose/docker-compose.yml index 3ca8236a..c1604ac9 100644 --- a/setup/flavors/compose/docker-compose.yml +++ b/setup/flavors/compose/docker-compose.yml @@ -133,7 +133,7 @@ services: {% endif %} volumes: - "{{ root }}/filter:/var/lib/rspamd" - - "{{ root }}/overrides/rspamd:/etc/rspamd/override.d:ro" + - "{{ root }}/overrides/rspamd:/overrides:ro" depends_on: - front - redis diff --git a/towncrier/newsfragments/2555.feature b/towncrier/newsfragments/2555.feature new file mode 100644 index 00000000..003d39da --- /dev/null +++ b/towncrier/newsfragments/2555.feature @@ -0,0 +1,15 @@ +New override system for Rspamd. In the old system, all files were placed in the Rspamd overrides folder. +These overrides would override everything, including the Mailu Rspamd config. + +Now overrides are placed in /overrides. +If you use your own map files, change the location to /override/myMapFile.map in the corresponding conf file. +It works as following. +* If the override file overrides a Mailu defined config file, + it will be included in the Mailu config file with lowest priority. + It will merge with existing sections. +* If the override file does not override a Mailu defined config file, + then the file will be placed in the rspamd local.d folder. + It will merge with existing sections. + +For more information, see the description of the local.d folder on the rspamd website: +https://www.rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories From 7ce28bd6e90278133fd9facabd0c043af8ac2ed7 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 9 Mar 2023 08:28:18 +0000 Subject: [PATCH 2/3] Fix some small errors --- core/rspamd/conf/antivirus.conf | 2 +- core/rspamd/conf/external_services_group.conf | 72 +++++++++---------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/core/rspamd/conf/antivirus.conf b/core/rspamd/conf/antivirus.conf index 6964d17a..3c3a2887 100644 --- a/core/rspamd/conf/antivirus.conf +++ b/core/rspamd/conf/antivirus.conf @@ -3,7 +3,7 @@ clamav { scan_mime_parts = true; symbol = "CLAM_VIRUS"; type = "clamav"; - servers = "{{ ANTIVIRUS_ADDRESS }}"; + servers = "{{ ANTIVIRUS_ADDRESS }}:3310"; {% if ANTIVIRUS_ACTION|default('discard') == 'reject' %} action = "reject" {% endif %} diff --git a/core/rspamd/conf/external_services_group.conf b/core/rspamd/conf/external_services_group.conf index bd9f9b45..3b7f2d96 100644 --- a/core/rspamd/conf/external_services_group.conf +++ b/core/rspamd/conf/external_services_group.conf @@ -2,41 +2,41 @@ {% if SCAN_MACROS == 'True' %} # local.d/external_services_group.conf - description = "Oletools content rules"; - symbols = { - "OLETOOLS" { - weight = 1.0; - description = "OLETOOLS found a Macro"; - one_shot = true; - }, - "OLETOOLS_MACRO_FOUND" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_AUTOEXEC" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_FLAG" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_VBASTOMP" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_A" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_W" { - weight = 0.0; - one_shot = true; - }, - "OLETOOLS_X" { - weight = 0.0; - one_shot = true; - }, - } +description = "Oletools content rules"; +symbols = { + "OLETOOLS" { + weight = 1.0; + description = "OLETOOLS found a Macro"; + one_shot = true; + }, + "OLETOOLS_MACRO_FOUND" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_AUTOEXEC" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_FLAG" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_VBASTOMP" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_A" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_W" { + weight = 0.0; + one_shot = true; + }, + "OLETOOLS_X" { + weight = 0.0; + one_shot = true; + }, +} {% endif %} .include(try=true,priority=1,duplicate=merge) "/overrides/external_services_group.conf" From 45177bd25a1fde5942f650f3890d15f956672247 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Thu, 9 Mar 2023 08:30:58 +0000 Subject: [PATCH 3/3] bring back removed blank lines --- core/rspamd/conf/external_services.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/rspamd/conf/external_services.conf b/core/rspamd/conf/external_services.conf index cb4c0ee7..bcc6012d 100644 --- a/core/rspamd/conf/external_services.conf +++ b/core/rspamd/conf/external_services.conf @@ -2,12 +2,14 @@ oletools { # default olefy settings servers = "{{ OLETOOLS_ADDRESS }}:11343" + # needs to be set explicitly for Rspamd < 1.9.5 scan_mime_parts = true; extended = true; max_size = 3145728; timeout = 20.0; retransmits = 1; + patterns { OLETOOLS_MACRO_FOUND= '^.....M..$'; OLETOOLS_AUTOEXEC = '^A....M..$'; @@ -18,6 +20,7 @@ oletools { OLETOOLS_W = '(?i)\b(?:FileCopy|CopyFile|Kill|CreateTextFile|VirtualAlloc|RtlMoveMemory|URLDownloadToFileA?|AltStartupPath|WriteProcessMemory|ADODB\.Stream|WriteText|SaveToFile|SaveAs|SaveAsRTF|FileSaveAs|MkDir|RmDir|SaveSetting|SetAttr)\b|(?:\bOpen\b[^\n]+\b(?:Write|Append|Binary|Output|Random)\b)'; OLETOOLS_X = '(?i)\b(?:Shell|CreateObject|GetObject|SendKeys|RUN|CALL|MacScript|FollowHyperlink|CreateThread|ShellExecuteA?|ExecuteExcel4Macro|EXEC|REGISTER|SetTimer)\b|(?:\bDeclare\b[^\n]+\bLib\b)'; } + # mime-part regex matching in content-type or filename mime_parts_filter_regex { #UNKNOWN = "application\/octet-stream";