dovecot: use rspamd X-Spamd-Result percentage to evaluate spam

- configures dovecot to use the spamtest sieve plugins
- configures sieve to read the score from X-Spamd-Result: headers
- before.sieve applies the ${spam_threshold} to the spamtest percentage
- freeposte.db stores a percentage for ${spam_threshold}
- migrate freeposte.db spam_threshold from X/15 to percentages

the filter investigates the overall ratio of the `rspamd` header
`X-Spamd-Result` that looks something like this:

X-Spamd-Result: default: True [12.36 / 15.00]
 RBL_SPAMHAUS_XBL(4.00)[]
 BAYES_SPAM(3.06)[92.67%]
 RBL_SPAMHAUS_XBL_ANY(4.00)[]
 ONCE_RECEIVED_STRICT(4.00)[]
 HFILTER_HELO_BAREIP(3.00)[]
 RBL_SORBS_DUL(2.00)[]
 HFILTER_HOSTNAME_UNKNOWN(2.50)[]
 RBL_SPAMHAUS_PBL(2.00)[]
 RBL_SORBS_RECENT(1.50)[]
 MIME_UNKNOWN(0.10)[application/x-rar-compressed]
 RDNS_NONE(1.00)[]
 RBL_SORBS(0.00)[]
 R_SPF_NEUTRAL(0.00)[?all]
 ONCE_RECEIVED(0.10)[]
 RBL_SEM(1.00)[]
 MIME_HTML_ONLY(0.20)[]
 RBL_UCEPROTECT_LEVEL1(1.00)[]
 MIME_GOOD(-0.10)[multipart/mixed]

the sieve `spamtest :percent :value` in this case would be
   100*12.36/15 = 82.4%

admin/freeposte/admin/models.py

@@ -143,7 +143,7 @@ class User(Email):
     # Settings
     displayed_name = db.Column(db.String(160), nullable=False, default="")
     spam_enabled = db.Column(db.Boolean(), nullable=False, default=True)
-    spam_threshold = db.Column(db.Numeric(), nullable=False, default=10.0)
+    spam_threshold = db.Column(db.Numeric(), nullable=False, default=80.0)
 
     # Flask-login attributes
     is_authenticated = True

admin/freeposte/admin/templates/user/settings.html

@@ -13,8 +13,8 @@ User settings
   {{ form.hidden_tag() }}
   {{ macros.form_field(form.displayed_name) }}
   {{ macros.form_field(form.spam_enabled) }}
-  {{ macros.form_field(form.spam_threshold, step=1, max=15,
-      prepend='<span class="input-group-addon"><span id="threshold">'+(form.spam_threshold.data).__int__().__str__()+'</span>&nbsp;/&nbsp;15</span>',
+  {{ macros.form_field(form.spam_threshold, step=1, max=100,
+      prepend='<span class="input-group-addon"><span id="threshold">'+(form.spam_threshold.data).__int__().__str__()+'</span>&nbsp;/&nbsp;100</span>',
       oninput='$("#threshold").text(this.value);') }}
   {{ macros.form_field(form.submit) }}
 </form>

admin/migrations/versions/27ae2f102682_.py

@@ -0,0 +1,37 @@
+"""spam_threshold in percent
+
+Revision ID: 27ae2f102682
+Revises: dc8c25cf5b98
+Create Date: 2016-09-30 08:06:15.025190
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '27ae2f102682'
+down_revision = 'dc8c25cf5b98'
+
+from alembic import op
+import sqlalchemy as sa
+
+from freeposte.admin.models import User
+from freeposte import db
+
+def upgrade():
+    # spam_threshold is a X/15 based value, we're converting it to percent.
+    for user in User.query.all():
+         user.spam_threshold = int(100. * float(user.spam_threshold or 0.) / 15.)
+    db.session.commit()
+
+    # set default to 80%
+    with op.batch_alter_table('user') as batch:
+        batch.alter_column('spam_threshold', default=80.)
+
+def downgrade():
+    # spam_threshold is a X/15 based value, we're converting it from percent.
+    for user in User.query.all():
+         user.spam_threshold = int(15. * float(user.spam_threshold or 0.) / 100.)
+    db.session.commit()
+
+    # set default to 10/15
+    with op.batch_alter_table('user') as batch:
+        batch.alter_column('spam_threshold', default=10.)

dovecot/conf/dovecot.conf

@@ -149,7 +149,7 @@ service managesieve-login {
 plugin {
 	sieve_dir = ~/sieve
   sieve_plugins = sieve_extdata
-  sieve_extensions = +vnd.dovecot.extdata
+  sieve_extensions = +vnd.dovecot.extdata +spamtest +spamtestplus
 	sieve_before = /var/lib/dovecot/before.sieve
 	sieve_default = /var/lib/dovecot/default.sieve
 	sieve_after = /var/lib/dovecot/after.sieve
@@ -162,8 +162,13 @@ plugin {
   antispam_mail_spam = learn_spam
   antispam_mail_notspam = learn_ham
   antispam_mail_sendmail_args = -h;antispam:11334;-P;q1
-}
 
+  # extract spam score from
+  # X-Spam-Result: .... [<value> / <max_value] ...
+  sieve_spamtest_status_type = score
+  sieve_spamtest_status_header = X-Spamd-Result: .*\[(-?[[:digit:]]+\.[[:digit:]]+) .*\]
+  sieve_spamtest_max_header = X-Spamd-Result: .*\[.* ([[:digit:]]+\.[[:digit:]]+)\]
+}
 ###############
 # Filtering
 ###############

dovecot/sieve/before.sieve

@@ -8,16 +8,14 @@ require "regex";
 require "relational";
 require "comparator-i;ascii-numeric";
 require "vnd.dovecot.extdata";
+require "spamtestplus";
 
 if allof (string :is "${extdata.spam_enabled}" "1",
-          not header :matches "X-Spam-Status" "* score=-*",
-          header :matches "X-Spam-Status" "* score=*")
+          spamtest :percent :value "gt" :comparator "i;ascii-numeric"  "${extdata.spam_threshold}")
 {
-  if string :value "ge" :comparator "i;ascii-numeric" "${2}" "${extdata.spam_threshold}" {
   setflag "\\seen";
   fileinto :create "Junk";
   stop;
-  }
 }
 
 if string :is "${extdata.reply_enabled}" "1" {