Merge pull request #527 from ofthesun9/feat-fuzzyhashes

Trying to enable fuzzy hashes for rspamd

core/dovecot/Dockerfile

@@ -2,7 +2,7 @@ FROM alpine:3.8
 
 RUN apk add --no-cache \
      dovecot dovecot-pigeonhole-plugin dovecot-fts-lucene rspamd-client \
-     python3 py3-pip \
+     bash python3 py3-pip \
  && pip3 install --upgrade pip \
  && pip3 install jinja2 podop tenacity
 

core/dovecot/conf/bin/ham

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+tee >(rspamc -h antispam:11334 -P mailu learn_ham /dev/stdin) \
+    | rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /dev/stdin

core/dovecot/conf/bin/mailtrain

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-rspamc -h antispam:11334 -P mailu "learn_$1" /dev/stdin <&0

core/dovecot/conf/bin/spam

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+tee >(rspamc -h antispam:11334 -P mailu learn_spam /dev/stdin) \
+    >(rspamc -h antispam:11334 -P mailu -f 11 fuzzy_add /dev/stdin)

core/dovecot/conf/report-ham.sieve

@@ -8,4 +8,4 @@ if string "${mailbox}" "Trash" {
   stop;
 }
 
-execute :pipe "mailtrain" "ham";
+execute :pipe "ham";

core/dovecot/conf/report-spam.sieve

@@ -1,3 +1,3 @@
 require "vnd.dovecot.execute";
 
-execute :pipe "mailtrain" "spam";
+execute :pipe "spam";

core/dovecot/start.py

@@ -36,5 +36,5 @@ for dovecot_file in glob.glob("/conf/*.conf"):
 
 # Run Podop, then postfix
 multiprocessing.Process(target=start_podop).start()
-os.system("chown -R mail:mail /mail /var/lib/dovecot")
+os.system("chown -R mail:mail /mail /var/lib/dovecot /conf")
 os.execv("/usr/sbin/dovecot", ["dovecot", "-c", "/etc/dovecot/dovecot.conf", "-F"])

services/rspamd/Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine:3.8
 
-RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy ca-certificates py-pip \
+RUN apk add --no-cache python py-jinja2 rspamd rspamd-controller rspamd-proxy rspamd-fuzzy ca-certificates py-pip \
  && pip install --upgrade pip \
  && pip install tenacity
 
@@ -9,10 +9,7 @@ RUN mkdir /run/rspamd
 COPY conf/ /conf
 COPY start.py /start.py
 
-# Temporary fix to remove references to rspamd-fuzzy for now
+EXPOSE 11332/tcp 11334/tcp 11335/tcp
-RUN sed -i '/fuzzy/,$d' /etc/rspamd/rspamd.conf
-
-EXPOSE 11332/tcp 11334/tcp
 
 VOLUME ["/var/lib/rspamd"]
 

services/rspamd/conf/fuzzy_check.conf

@@ -0,0 +1,34 @@
+rule "local" {
+    # Fuzzy storage server list
+    servers = "localhost:11335";
+    # Default symbol for unknown flags
+    symbol = "LOCAL_FUZZY_UNKNOWN";
+    # Additional mime types to store/check
+    mime_types = ["application/*"];
+    # Hash weight threshold for all maps
+    max_score = 20.0;
+    # Whether we can learn this storage
+    read_only = no;
+    # Ignore unknown flags
+    skip_unknown = yes;
+    # Hash generation algorithm
+    algorithm = "mumhash";
+
+    # Map flags to symbols
+    fuzzy_map = {
+        LOCAL_FUZZY_DENIED {
+            # Local threshold
+            max_score = 20.0;
+            # Flag to match
+            flag = 11;
+        }
+        LOCAL_FUZZY_PROB {
+            max_score = 10.0;
+            flag = 12;
+        }
+        LOCAL_FUZZY_WHITE {
+            max_score = 2.0;
+            flag = 13;
+        }
+    }
+}

services/rspamd/conf/metrics.conf

@@ -0,0 +1,19 @@
+group "fuzzy" {
+    max_score = 12.0;
+    symbol "LOCAL_FUZZY_UNKNOWN" {
+        weight = 5.0;
+        description = "Generic fuzzy hash match";
+    }
+    symbol "LOCAL_FUZZY_DENIED" {
+        weight = 12.0;
+        description = "Denied fuzzy hash";
+    }
+    symbol "LOCAL_FUZZY_PROB" {
+        weight = 5.0;
+        description = "Probable fuzzy hash";
+    }
+    symbol "LOCAL_FUZZY_WHITE" {
+        weight = -2.1;
+        description = "Whitelisted fuzzy hash";
+    }
+}

services/rspamd/conf/worker-controller.inc

@@ -1,3 +1,4 @@
+type = "controller";
 bind_socket = "*:11334";
 password = "mailu";
 secure_ip = "{{ FRONT_ADDRESS }}";

services/rspamd/conf/worker-fuzzy.inc

@@ -0,0 +1,6 @@
+type = "fuzzy";
+bind_socket = "*:11335";
+count = 1;
+backend = "redis";
+expire = 90d;
+allow_update = ["127.0.0.1"];

services/rspamd/conf/worker-normal.inc

@@ -1 +1,2 @@
+type = "normal";
 enabled = false;