From 12294a6e5aa3c57c8c603d97a35fa443a962a8e2 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Wed, 6 Jun 2018 18:40:51 +0000 Subject: [PATCH 1/6] Trying to enable fuzzy hashes for rspamd --- services/rspamd/Dockerfile | 5 +--- services/rspamd/conf/fuzzy_check.conf | 34 ++++++++++++++++++++++ services/rspamd/conf/metrics.conf | 19 ++++++++++++ services/rspamd/conf/worker-controller.inc | 1 + services/rspamd/conf/worker-fuzzy.inc | 5 ++++ services/rspamd/conf/worker-normal.inc | 1 + 6 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 services/rspamd/conf/fuzzy_check.conf create mode 100644 services/rspamd/conf/metrics.conf create mode 100644 services/rspamd/conf/worker-fuzzy.inc diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index c6c2afdd..1b8d7e6b 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -1,15 +1,12 @@ FROM alpine:edge -RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates +RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates RUN mkdir /run/rspamd COPY conf/ /conf COPY start.py /start.py -# Temporary fix to remove references to rspamd-fuzzy for now -RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf - EXPOSE 11332/tcp 11334/tcp CMD /start.py diff --git a/services/rspamd/conf/fuzzy_check.conf b/services/rspamd/conf/fuzzy_check.conf new file mode 100644 index 00000000..7c87e1c3 --- /dev/null +++ b/services/rspamd/conf/fuzzy_check.conf @@ -0,0 +1,34 @@ +rule "local" { + # Fuzzy storage server list + servers = "localhost:11335"; + # Default symbol for unknown flags + symbol = "LOCAL_FUZZY_UNKNOWN"; + # Additional mime types to store/check + mime_types = ["application/*"]; + # Hash weight threshold for all maps + max_score = 20.0; + # Whether we can learn this storage + read_only = no; + # Ignore unknown flags + skip_unknown = yes; + # Hash generation algorithm + algorithm = "mumhash"; + + # Map flags to symbols + fuzzy_map = { + LOCAL_FUZZY_DENIED { + # Local threshold + max_score = 20.0; + # Flag to match + flag = 11; + } + LOCAL_FUZZY_PROB { + max_score = 10.0; + flag = 12; + } + LOCAL_FUZZY_WHITE { + max_score = 2.0; + flag = 13; + } + } +} diff --git a/services/rspamd/conf/metrics.conf b/services/rspamd/conf/metrics.conf new file mode 100644 index 00000000..6a31964f --- /dev/null +++ b/services/rspamd/conf/metrics.conf @@ -0,0 +1,19 @@ +group "fuzzy" { + max_score = 12.0; + symbol "LOCAL_FUZZY_UNKNOWN" { + weight = 5.0; + description = "Generic fuzzy hash match"; + } + symbol "LOCAL_FUZZY_DENIED" { + weight = 12.0; + description = "Denied fuzzy hash"; + } + symbol "LOCAL_FUZZY_PROB" { + weight = 5.0; + description = "Probable fuzzy hash"; + } + symbol "LOCAL_FUZZY_WHITE" { + weight = -2.1; + description = "Whitelisted fuzzy hash"; + } +} diff --git a/services/rspamd/conf/worker-controller.inc b/services/rspamd/conf/worker-controller.inc index 6a020672..dd143942 100644 --- a/services/rspamd/conf/worker-controller.inc +++ b/services/rspamd/conf/worker-controller.inc @@ -1,3 +1,4 @@ +type = "controller"; bind_socket = "*:11334"; password = "mailu"; secure_ip = "{{ FRONT_ADDRESS }}"; diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc new file mode 100644 index 00000000..a0021a03 --- /dev/null +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -0,0 +1,5 @@ +type = "fuzzy"; +count = 1; +backend = "redis"; +expire = 90d; +allow_update = ["127.0.0.1"]; diff --git a/services/rspamd/conf/worker-normal.inc b/services/rspamd/conf/worker-normal.inc index a6ee8317..ab996fb8 100644 --- a/services/rspamd/conf/worker-normal.inc +++ b/services/rspamd/conf/worker-normal.inc @@ -1 +1,2 @@ +type = "normal"; enabled = false; From 6b34b2728ece020918dcb10a901afdd9894a69e3 Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Sun, 7 Oct 2018 16:38:41 +0000 Subject: [PATCH 2/6] Declare fuzzy_worker port 11335 in EXPOSE section --- services/rspamd/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/rspamd/Dockerfile b/services/rspamd/Dockerfile index 7dff8c1f..cfb4d0eb 100644 --- a/services/rspamd/Dockerfile +++ b/services/rspamd/Dockerfile @@ -9,7 +9,7 @@ RUN mkdir /run/rspamd COPY conf/ /conf COPY start.py /start.py -EXPOSE 11332/tcp 11334/tcp +EXPOSE 11332/tcp 11334/tcp 11335/tcp VOLUME ["/var/lib/rspamd"] From b9b4a8cd77c4a483d7dbf70314db777f021f0443 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:47:55 +0200 Subject: [PATCH 3/6] Explicitely specify the fuzzy worker listen address --- services/rspamd/conf/worker-fuzzy.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc index a0021a03..0f71ba32 100644 --- a/services/rspamd/conf/worker-fuzzy.inc +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -1,4 +1,5 @@ type = "fuzzy"; +bind_socket = "*:11335"; count = 1; backend = "redis"; expire = 90d; From ce0bf3366d2193b01e34c04ede5f870c94d250c3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:48:28 +0200 Subject: [PATCH 4/6] Learn fuzzy hashes automatically --- core/dovecot/Dockerfile | 2 +- core/dovecot/conf/bin/ham | 4 ++++ core/dovecot/conf/bin/mailtrain | 3 --- core/dovecot/conf/bin/spam | 4 ++++ core/dovecot/conf/report-ham.sieve | 2 +- core/dovecot/conf/report-spam.sieve | 2 +- 6 files changed, 11 insertions(+), 6 deletions(-) create mode 100755 core/dovecot/conf/bin/ham delete mode 100755 core/dovecot/conf/bin/mailtrain create mode 100755 core/dovecot/conf/bin/spam diff --git a/core/dovecot/Dockerfile b/core/dovecot/Dockerfile index d8d4c55b..41437e23 100644 --- a/core/dovecot/Dockerfile +++ b/core/dovecot/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.8 RUN apk add --no-cache \ dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \ - python3 py3-pip \ + bash python3 py3-pip \ && pip3 install --upgrade pip \ && pip3 install jinja2 podop tenacity diff --git a/core/dovecot/conf/bin/ham b/core/dovecot/conf/bin/ham new file mode 100755 index 00000000..c74a97bd --- /dev/null +++ b/core/dovecot/conf/bin/ham @@ -0,0 +1,4 @@ +#!/bin/bash + +tee >(rspamc -h antispam:11334 -P mailu learn_ham /dev/stdin) \ + | rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /dev/stdin \ No newline at end of file diff --git a/core/dovecot/conf/bin/mailtrain b/core/dovecot/conf/bin/mailtrain deleted file mode 100755 index cfa36398..00000000 --- a/core/dovecot/conf/bin/mailtrain +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -rspamc -h antispam:11334 -P mailu "learn_$1" /dev/stdin <&0 diff --git a/core/dovecot/conf/bin/spam b/core/dovecot/conf/bin/spam new file mode 100755 index 00000000..e6a66f89 --- /dev/null +++ b/core/dovecot/conf/bin/spam @@ -0,0 +1,4 @@ +#!/bin/bash + +tee >(rspamc -h antispam:11334 -P mailu learn_spam /dev/stdin) \ + >(rspamc -h antispam:11334 -P mailu -f 11 fuzzy_add /dev/stdin) \ No newline at end of file diff --git a/core/dovecot/conf/report-ham.sieve b/core/dovecot/conf/report-ham.sieve index 1ad8abdf..0c69d67b 100644 --- a/core/dovecot/conf/report-ham.sieve +++ b/core/dovecot/conf/report-ham.sieve @@ -8,4 +8,4 @@ if string "${mailbox}" "Trash" { stop; } -execute :pipe "mailtrain" "ham"; +execute :pipe "ham"; diff --git a/core/dovecot/conf/report-spam.sieve b/core/dovecot/conf/report-spam.sieve index b2a544a6..108d6210 100644 --- a/core/dovecot/conf/report-spam.sieve +++ b/core/dovecot/conf/report-spam.sieve @@ -1,3 +1,3 @@ require "vnd.dovecot.execute"; -execute :pipe "mailtrain" "spam"; +execute :pipe "spam"; From d5162328eca5e2344c4ca1e432ec8f31bc5b7e1f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Wed, 17 Oct 2018 18:48:59 +0200 Subject: [PATCH 5/6] Allow dovecot to write the source configuration directory for compiling sieve scripts --- core/dovecot/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/dovecot/start.py b/core/dovecot/start.py index afd0513e..b65b4db9 100755 --- a/core/dovecot/start.py +++ b/core/dovecot/start.py @@ -36,5 +36,5 @@ for dovecot_file in glob.glob("/conf/*.conf"): # Run Podop, then postfix multiprocessing.Process(target=start_podop).start() -os.system("chown -R mail:mail /mail /var/lib/dovecot") +os.system("chown -R mail:mail /mail /var/lib/dovecot /conf") os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"]) From 86bdce840774f7c7170d4eccfc7dc9e9d3d9ea5e Mon Sep 17 00:00:00 2001 From: ofthesun9 Date: Wed, 17 Oct 2018 18:49:19 +0000 Subject: [PATCH 6/6] Explicitely specify the fuzzy worker listen address --- services/rspamd/conf/worker-fuzzy.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/services/rspamd/conf/worker-fuzzy.inc b/services/rspamd/conf/worker-fuzzy.inc index a0021a03..0f71ba32 100644 --- a/services/rspamd/conf/worker-fuzzy.inc +++ b/services/rspamd/conf/worker-fuzzy.inc @@ -1,4 +1,5 @@ type = "fuzzy"; +bind_socket = "*:11335"; count = 1; backend = "redis"; expire = 90d;