From 6c83d253123debfc86b0746da70f1d5f6ca2385c Mon Sep 17 00:00:00 2001 From: enginefeeder101 Date: Tue, 26 Apr 2022 21:54:25 +0200 Subject: [PATCH 1/5] Configurable default spam threshold used for new users This commit adds functionality to set a custom default spam threshold for new users. The environment variable ``DEFAULT_SPAM_THRESHOLD`` can be used for this purpose. When not set, it defaults back to 80%, as the default value was before If ``DEFAULT_SPAM_THRESHOLD`` is set to a value that Python cannot parse as an integer, a ValueError is thrown. There is no error handling for that case built-in. --- core/admin/mailu/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 08738600..6e277fe3 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -509,7 +509,8 @@ class User(Base, Email): displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean, nullable=False, default=True) spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True) - spam_threshold = db.Column(db.Integer, nullable=False, default=80) + spam_threshold = db.Column(db.Integer, nullable=False, + default=int(os.environ.get('DEFAULT_SPAM_THRESHOLD', 80))) # Flask-login attributes is_authenticated = True From 4da0ff1856ff8a39b308305ad9257710a39ffb46 Mon Sep 17 00:00:00 2001 From: enginefeeder101 Date: Tue, 26 Apr 2022 22:07:50 +0200 Subject: [PATCH 2/5] Documentation for configurable default spam threshold --- docs/antispam.rst | 2 ++ docs/configuration.rst | 4 ++++ setup/flavors/compose/mailu.env | 3 +++ towncrier/newsfragments/2328.feature | 1 + 4 files changed, 10 insertions(+) create mode 100644 towncrier/newsfragments/2328.feature diff --git a/docs/antispam.rst b/docs/antispam.rst index a95109e8..7971200c 100644 --- a/docs/antispam.rst +++ b/docs/antispam.rst @@ -43,6 +43,8 @@ Rspamd rejects non-compliant email messages and email messages that contain viru 66% (10/15) is less than 80%, so the email is classified as ham. This email message will go to the inbox folder. If the user wants email messages with a score of 10 (66%) to be classified as spam, then the user defined spam filter tolerance can be lowered to 65% in the administration web interface. + The default spam filter tolerance used for new users can be configured using the environment variable ``DEFAULT_SPAM_THRESHOLD``. See also: :ref:`_advanced_cfg`. + .. image:: assets/screenshots/SpamFiltering.png The location in the administration web interface where the spam filter and spam filter tolerance can be configured. diff --git a/docs/configuration.rst b/docs/configuration.rst index 198315c6..0df8633f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -181,6 +181,8 @@ An example: Depending on your particular deployment you most probably will want to change the default. +.. _advanced_cfg: + Advanced settings ----------------- @@ -209,6 +211,8 @@ The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usua .. _`TZ database name`: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones +The ``DEFAULT_SPAM_THRESHOLD`` (default: 80) setting is the default spam tolerance used when creating a new user. + Antivirus settings ------------------ diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index ed7ecfdd..675b80ef 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -173,6 +173,9 @@ LOG_LEVEL=WARNING # Timezone for the Mailu containers. See this link for all possible values https://en.wikipedia.org/wiki/List_of_tz_database_time_zones TZ=Etc/UTC +# Default spam threshold used for new users +DEFAULT_SPAM_THRESHOLD=80 + ################################### # Database settings ################################### diff --git a/towncrier/newsfragments/2328.feature b/towncrier/newsfragments/2328.feature new file mode 100644 index 00000000..f0d6eea7 --- /dev/null +++ b/towncrier/newsfragments/2328.feature @@ -0,0 +1 @@ +Configurable default spam threshold used for new users From 82860d0f8013742ae3d1d8be9d17b78dc55c59b9 Mon Sep 17 00:00:00 2001 From: enginefeeder101 Date: Wed, 8 Jun 2022 17:13:38 +0200 Subject: [PATCH 3/5] Moved parsing environment variable to global application config dictionary Per requested changes added the ``DEFAULT_SPAM_THRESHOLD`` to the main application configuration dictionary in ``configuration.py`` and updated ``models.py`` accordingly. No error handling is added, as that was not required. --- core/admin/mailu/configuration.py | 2 ++ core/admin/mailu/models.py | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 912a0204..f1bb2283 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -75,6 +75,7 @@ DEFAULT_CONFIG = { 'SESSION_COOKIE_SECURE': True, 'CREDENTIAL_ROUNDS': 12, 'TZ': 'Etc/UTC', + 'DEFAULT_SPAM_THRESHOLD': 80, # Host settings 'HOST_IMAP': 'imap', 'HOST_LMTP': 'imap:2525', @@ -163,6 +164,7 @@ class ConfigManager: self.config['MESSAGE_RATELIMIT_EXEMPTION'] = set([s for s in self.config['MESSAGE_RATELIMIT_EXEMPTION'].lower().replace(' ', '').split(',') if s]) self.config['HOSTNAMES'] = ','.join(hostnames) self.config['HOSTNAME'] = hostnames[0] + self.config['DEFAULT_SPAM_THRESHOLD'] = int(self.config['DEFAULT_SPAM_THRESHOLD']) # update the app config app.config.update(self.config) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 6e277fe3..3722003c 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -509,8 +509,7 @@ class User(Base, Email): displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean, nullable=False, default=True) spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True) - spam_threshold = db.Column(db.Integer, nullable=False, - default=int(os.environ.get('DEFAULT_SPAM_THRESHOLD', 80))) + spam_threshold = db.Column(db.Integer, nullable=False, default=app.config['DEFAULT_SPAM_THRESHOLD']) # Flask-login attributes is_authenticated = True From 81c9e01d2487e813cb86652667e15b1fcb172d64 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 19 Aug 2022 17:58:33 +0000 Subject: [PATCH 4/5] finishing touches for PR# 2328 Antispam.rst contained a syntax error. Move config description to common section which is more fitting. Fixed wrong assignment of default value for DEFAULT_SPAM_THRESHOLD in models.py. --- core/admin/mailu/models.py | 7 ++++++- docs/antispam.rst | 32 ++++++++++++++++---------------- docs/configuration.rst | 6 +++--- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 3722003c..1e703e1a 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -509,7 +509,12 @@ class User(Base, Email): displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean, nullable=False, default=True) spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True) - spam_threshold = db.Column(db.Integer, nullable=False, default=app.config['DEFAULT_SPAM_THRESHOLD']) + def default_spam_threshold(cls): + if app: + return app.config['DEFAULT_SPAM_THRESHOLD'] + else: + return 80 + spam_threshold = db.Column(db.Integer, nullable=False, default=default_spam_threshold) # Flask-login attributes is_authenticated = True diff --git a/docs/antispam.rst b/docs/antispam.rst index 7971200c..9293653b 100644 --- a/docs/antispam.rst +++ b/docs/antispam.rst @@ -21,29 +21,29 @@ Rspamd rejects non-compliant email messages and email messages that contain viru 1. When an email message is received or send by Postfix, it is scanned by Rspamd. If the email message receives a spam score between 6 and 15, a header is added to the email message with the spam score. This is an example of an email header for a spam score of 14:: X-Spamd-Bar: ++++++++++++++ - X-Spam-Level: ************** + X-Spam-Level: ************** Authentication-Results: test.mailu.io; dkim=pass header.d=example.com header.s=mailing header.b=ABCDE; dkim=pass header.d=example.com header.s=mailing header.b=ABCDE; dmarc=pass (policy=none) header.from=eventim.de; spf=pass (test.mailu.io: domain of return@example.com designates 11.22.33.44 as permitted sender) smtp.mailfrom=return@example.com - X-Spam: Yes - + X-Spam: Yes + 2. Dovecot is then responsible for classifying the email message to the Junk folder based on user preferences. It works as following: * In the administration web interface, under settings under Antispam 'Enable spam filter' must be ticked. If this option is disabled, then all email messages will automatically go to the inbox folder. Except for email messages with a score of 15 or higher, as these email messages are rejected by Rspamd. - + * In the administration web interface, under settings under Antispam, the user defined spam filter tolerance must be configured. The default value is 80%. The lower the spam filter tolerance, the more false positives (ham classified as spam). The user can change this setting to finetune when an email message is classified as spam. * Dovecot extracts the X-Spam-Level email header from the email message and converts the spam score (0 - 15) to a 0 - 100 percent scale. This spam score is compared with the user defined spam filter tolerance. If the spam score is lower than the user defined spam filter tolerance, then the email message is accepted. In logic: - + If is greater than , then move the email message to the spam folder and mark the email message as read. - + For example if the user defined spam filter tolerance is set to 80%(default) and the spam score of an email message is 10: - + 66% (10/15) is less than 80%, so the email is classified as ham. This email message will go to the inbox folder. If the user wants email messages with a score of 10 (66%) to be classified as spam, then the user defined spam filter tolerance can be lowered to 65% in the administration web interface. - - The default spam filter tolerance used for new users can be configured using the environment variable ``DEFAULT_SPAM_THRESHOLD``. See also: :ref:`_advanced_cfg`. + + The default spam filter tolerance used for new users can be configured using the environment variable ``DEFAULT_SPAM_THRESHOLD``. See also :ref:`common_cfg` in the configuration reference. .. image:: assets/screenshots/SpamFiltering.png @@ -62,7 +62,7 @@ If you already have an existing mailbox and want Mailu to learn them all as ham rspamc -h antispam:11334 -P mailu -f 13 fuzzy_add /mail/user\@example.com/.Ham_Learn/cur/ -This should learn every file located in the ``Ham_Learn`` folder from user@example.com +This should learn every file located in the ``Ham_Learn`` folder from user@example.com Likewise, to learn all messages within the folder ``Spam_Learn`` as spam messages : @@ -77,7 +77,7 @@ Likewise, to learn all messages within the folder ``Spam_Learn`` as spam message How can I block emails from a domain? ------------------------------------- -Via the multimap filter it is possible to block emails from a sender domain. See the `official rspamd documentation`_ for more information. A local blacklist that contains the domains to be blocked can be configured via the :ref:`Rspamd overrides folder `. +Via the multimap filter it is possible to block emails from a sender domain. See the `official rspamd documentation`_ for more information. A local blacklist that contains the domains to be blocked can be configured via the :ref:`Rspamd overrides folder `. The following steps have to be taken to configure an additional symbol (rule) that uses the multimap filter to block emails from sender domain. @@ -99,7 +99,7 @@ The following steps have to be taken to configure an additional symbol (rule) th } Note the "action = "reject";" line. This is a so-called pre-filter. No further filters/rules are processed when a pre-filter is used. If you omit this line, then the configured score will be added to the total score of the email message. Depending on the end-score after processing all rules, a verdict is made. To override this, you can add the action line. When this symbol (rule) is fired, then this action is immediately taken and no further processing occurs. You can use the following actions: - + * discard: drop an email message, but return success for sender (should be used merely in special cases) * reject: reject the email message. This enables the actual blocking of mails from the domain. @@ -112,9 +112,9 @@ The following steps have to be taken to configure an additional symbol (rule) th To move an email message to the Junk (Spam) folder, a score of 15 can be used in combination with the action "add header". The above example configuration will reject all emails send from domains that are listed in '/etc/rspamd/override.d/blacklist.inc'. - -2. In the Rspamd overrides folder create a map that contains the domains to be blocked. You can use # to add comments. + +2. In the Rspamd overrides folder create a map that contains the domains to be blocked. You can use # to add comments. Create the file /mailu/overrides/rspamd/blacklist.inc with the following contents: .. code-block:: bash @@ -130,12 +130,12 @@ The following steps have to be taken to configure an additional symbol (rule) th docker-compose scale antispam=0 docker-compose scale antispam=1 -4. (Optional) Check if the custom symbol is loaded. To access the Rspamd webgui, log in the Mailu administration web interface with a user that is an administrator and go to Antispam. In Rspamd webgui go to tab Symbols. Change the group drop-down box to local_bl. The following additional rule will be listed. +4. (Optional) Check if the custom symbol is loaded. To access the Rspamd webgui, log in the Mailu administration web interface with a user that is an administrator and go to Antispam. In Rspamd webgui go to tab Symbols. Change the group drop-down box to local_bl. The following additional rule will be listed. .. image:: assets/screenshots/RspamdSymbolBlacklist.png The symbol is only displayed if the symbol has no pre-filter (action= line) configured. Changes made in this screen are not saved to the configuration file. - + 5. Check if the map is available. In rspamd webgui to to configuration. A map is available with the path: /etc/rspamd/override.d/blacklist.inc Senders domain part is on the local blacklist diff --git a/docs/configuration.rst b/docs/configuration.rst index 0df8633f..3bbc42f0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -62,6 +62,8 @@ there is a good way to disable rate limiting altogether. The ``TLS_FLAVOR`` sets how Mailu handles TLS connections. Setting this value to ``notls`` will cause Mailu not to server any web content! More on :ref:`tls_flavor`. +The ``DEFAULT_SPAM_THRESHOLD`` (default: 80) is the default spam tolerance used when creating a new user. + Mail settings ------------- @@ -163,7 +165,7 @@ To have the account created automatically, you just need to define a few environ - ``INITIAL_ADMIN_DOMAIN``: the domain appendix: Most probably identical to the ``DOMAIN`` variable. - ``INITIAL_ADMIN_PW``: the admin password. - ``INITIAL_ADMIN_MODE``: use one of the options below for configuring how the admin account must be created: - + - ``create``: (default) creates a new admin account and raises an exception when it already exists. - ``ifmissing``: creates a new admin account when the admin account does not exist. - ``update``: creates a new admin account when it does not exist, or update the password of an existing admin account. @@ -211,8 +213,6 @@ The ``TZ`` sets the timezone Mailu will use. The timezone naming convention usua .. _`TZ database name`: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones -The ``DEFAULT_SPAM_THRESHOLD`` (default: 80) setting is the default spam tolerance used when creating a new user. - Antivirus settings ------------------ From 06b784da5705b0bf7a471da6d73c46ef27ac1f46 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sat, 29 Oct 2022 08:57:48 +0000 Subject: [PATCH 5/5] Shorten default function by using lambda --- core/admin/mailu/models.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 1e703e1a..2d276a8a 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -509,12 +509,7 @@ class User(Base, Email): displayed_name = db.Column(db.String(160), nullable=False, default='') spam_enabled = db.Column(db.Boolean, nullable=False, default=True) spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True) - def default_spam_threshold(cls): - if app: - return app.config['DEFAULT_SPAM_THRESHOLD'] - else: - return 80 - spam_threshold = db.Column(db.Integer, nullable=False, default=default_spam_threshold) + spam_threshold = db.Column(db.Integer, nullable=False, default=lambda:int(app.config.get("DEFAULT_SPAM_THRESHOLD", 80))) # Flask-login attributes is_authenticated = True