From 83b1fbb9d67b6752a7d8bb4f8e41e3d35e56ed4d Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 14 Mar 2021 18:09:21 +0100 Subject: [PATCH 1/9] Lazy loading of KVSessionExtension - call cleanup_sessions on first kvstore access this allows to run cmdline actions without redis (and makes it faster) - Allow development using DictStore by setting REDIS_ADDRESS to the empty string in env - don't sign 64bit random session id as suggested by nextgens --- core/admin/mailu/__init__.py | 12 +++--- core/admin/mailu/utils.py | 65 +++++++++++++++++++++++++++++++- core/admin/requirements-prod.txt | 1 + 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index f9ca2466..40cc9cff 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -1,8 +1,8 @@ +""" Mailu admin app +""" + import flask import flask_bootstrap -import redis -from flask_kvsession import KVSessionExtension -from simplekv.memory.redisstore import RedisStore from mailu import utils, debug, models, manage, configuration @@ -20,7 +20,8 @@ def create_app_from_config(config): # Initialize application extensions config.init_app(app) models.db.init_app(app) - KVSessionExtension(RedisStore(redis.StrictRedis().from_url('redis://{0}/3'.format(config['REDIS_ADDRESS']))), app).cleanup_sessions(app) + utils.kvsession.init_kvstore(config) + utils.kvsession.init_app(app) utils.limiter.init_app(app) utils.babel.init_app(app) utils.login.init_app(app) @@ -53,8 +54,7 @@ def create_app_from_config(config): def create_app(): - """ Create a new application based on the config module + """ Create a new application based on the config module """ config = configuration.ConfigManager() return create_app_from_config(config) - diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index ce12a09a..852fe8ad 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -1,11 +1,18 @@ -from mailu import models, limiter +""" Mailu admin app utilities +""" + +from mailu import limiter import flask import flask_login -import flask_script import flask_migrate import flask_babel +import flask_kvsession +import redis +from simplekv.memory import DictStore +from simplekv.memory.redisstore import RedisStore +from itsdangerous.encoding import want_bytes from werkzeug.contrib import fixers @@ -33,6 +40,10 @@ def get_locale(): # Proxy fixer class PrefixMiddleware(object): + """ fix proxy headers """ + def __init__(self): + self.app = None + def __call__(self, environ, start_response): prefix = environ.get('HTTP_X_FORWARDED_PREFIX', '') if prefix: @@ -48,3 +59,53 @@ proxy = PrefixMiddleware() # Data migrate migrate = flask_migrate.Migrate() + + +# session store +class NullSigner(object): + """NullSigner does not sign nor unsign""" + def __init__(self, *args, **kwargs): + pass + def sign(self, value): + """Signs the given string.""" + return want_bytes(value) + def unsign(self, signed_value): + """Unsigns the given string.""" + return want_bytes(signed_value) + +class KVSessionIntf(flask_kvsession.KVSessionInterface): + """ KVSession interface allowing to run int function on first access """ + def __init__(self, app, init_fn=None): + if init_fn: + app.kvsession_init = init_fn + else: + self._first_run(None) + def _first_run(self, app): + if app: + app.kvsession_init() + self.open_session = super().open_session + self.save_session = super().save_session + def open_session(self, app, request): + self._first_run(app) + return super().open_session(app, request) + def save_session(self, app, session, response): + self._first_run(app) + return super().save_session(app, session, response) + +class KVSessionExt(flask_kvsession.KVSessionExtension): + """ Activates Flask-KVSession for an application. """ + def init_kvstore(self, config): + """ Initialize kvstore - fallback to DictStore without REDIS_ADDRESS """ + if addr := config.get('REDIS_ADDRESS'): + self.default_kvstore = RedisStore(redis.StrictRedis().from_url(f'redis://{addr}/3')) + else: + self.default_kvstore = DictStore() + + def init_app(self, app, session_kvstore=None): + """ Initialize application and KVSession. """ + super().init_app(app, session_kvstore) + app.session_interface = KVSessionIntf(app, self.cleanup_sessions) + +kvsession = KVSessionExt() + +flask_kvsession.Signer = NullSigner diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index 54cf9a14..cd084684 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -39,6 +39,7 @@ python-editor==1.0.4 pytz==2019.1 PyYAML==5.1 redis==3.2.1 +simplekv==0.14.1 #alpine3:12 provides six==1.15.0 #six==1.12.0 socrate==0.1.1 From f0f79b23a328064a787d99b3755d43a946a87d9b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 14 Mar 2021 21:38:16 +0100 Subject: [PATCH 2/9] Allow cleanup of sessions by key&value in data This can be used to delete all sessions belonging to a user/login. For no it just iterates over all sessions. This could be enhanced by using a prefix for and deleting by prefix. --- core/admin/mailu/utils.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 852fe8ad..9394c38f 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -1,6 +1,8 @@ """ Mailu admin app utilities """ +from datetime import datetime + from mailu import limiter import flask @@ -22,6 +24,7 @@ login.login_view = "ui.login" @login.unauthorized_handler def handle_needs_login(): + """ redirect unauthorized requests to login page """ return flask.redirect( flask.url_for('ui.login', next=flask.request.endpoint) ) @@ -34,7 +37,8 @@ babel = flask_babel.Babel() @babel.localeselector def get_locale(): - translations = list(map(str, babel.list_translations())) + """ selects locale for translation """ + translations = [str(translation) for translation in babel.list_translations()] return flask.request.accept_languages.best_match(translations) @@ -101,6 +105,32 @@ class KVSessionExt(flask_kvsession.KVSessionExtension): else: self.default_kvstore = DictStore() + def cleanup_sessions(self, app=None, dkey=None, dvalue=None): + """ Remove sessions from the store. """ + if not app: + app = flask.current_app + if dkey is None and dvalue is None: + now = datetime.utcnow() + for key in app.kvsession_store.keys(): + try: + sid = flask_kvsession.SessionID.unserialize(key) + except ValueError: + pass + else: + if sid.has_expired( + app.config['PERMANENT_SESSION_LIFETIME'], + now + ): + app.kvsession_store.delete(key) + elif dkey is not None and dvalue is not None: + for key in app.kvsession_store.keys(): + if app.session_interface.serialization_method.loads( + app.kvsession_store.get(key) + ).get(dkey, None) == dvalue: + app.kvsession_store.delete(key) + else: + raise ValueError('Need dkey and dvalue.') + def init_app(self, app, session_kvstore=None): """ Initialize application and KVSession. """ super().init_app(app, session_kvstore) From 4b71bd56c49dceef38f5cc4267d22936ecaf0bc4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 4 Apr 2021 14:35:31 +0200 Subject: [PATCH 3/9] replace flask_kvsession with mailu's own storage --- core/admin/mailu/__init__.py | 3 +- core/admin/mailu/configuration.py | 5 +- core/admin/mailu/utils.py | 408 +++++++++++++++++++++++++----- core/admin/requirements-prod.txt | 2 - core/admin/requirements.txt | 1 - 5 files changed, 343 insertions(+), 76 deletions(-) diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index 40cc9cff..690837ea 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -20,8 +20,7 @@ def create_app_from_config(config): # Initialize application extensions config.init_app(app) models.db.init_app(app) - utils.kvsession.init_kvstore(config) - utils.kvsession.init_app(app) + utils.session.init_app(app) utils.limiter.init_app(app) utils.babel.init_app(app) utils.login.init_app(app) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 6f65d17d..3d1b4fb5 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -14,6 +14,7 @@ DEFAULT_CONFIG = { 'DEBUG': False, 'DOMAIN_REGISTRATION': False, 'TEMPLATES_AUTO_RELOAD': True, + 'MEMORY_SESSIONS': False, # Database settings 'DB_FLAVOR': None, 'DB_USER': 'mailu', @@ -55,6 +56,7 @@ DEFAULT_CONFIG = { 'RECAPTCHA_PRIVATE_KEY': '', # Advanced settings 'LOG_LEVEL': 'WARNING', + 'SESSION_KEY_BITS': 128, 'SESSION_LIFETIME': 24, 'SESSION_COOKIE_SECURE': True, 'CREDENTIAL_ROUNDS': 12, @@ -65,7 +67,6 @@ DEFAULT_CONFIG = { 'HOST_SMTP': 'smtp', 'HOST_AUTHSMTP': 'smtp', 'HOST_ADMIN': 'admin', - 'WEBMAIL': 'none', 'HOST_WEBMAIL': 'webmail', 'HOST_WEBDAV': 'webdav:5232', 'HOST_REDIS': 'redis', @@ -136,9 +137,9 @@ class ConfigManager(dict): self.config['RATELIMIT_STORAGE_URL'] = 'redis://{0}/2'.format(self.config['REDIS_ADDRESS']) self.config['QUOTA_STORAGE_URL'] = 'redis://{0}/1'.format(self.config['REDIS_ADDRESS']) + self.config['SESSION_STORAGE_URL'] = 'redis://{0}/3'.format(self.config['REDIS_ADDRESS']) self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True - self.config['SESSION_KEY_BITS'] = 128 self.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=int(self.config['SESSION_LIFETIME'])) # update the app config itself app.config = self diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 9394c38f..1deaa4ae 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -1,7 +1,14 @@ """ Mailu admin app utilities """ -from datetime import datetime +try: + import cPickle as pickle +except ImportError: + import pickle + +import hashlib +import secrets +import time from mailu import limiter @@ -9,12 +16,11 @@ import flask import flask_login import flask_migrate import flask_babel -import flask_kvsession import redis -from simplekv.memory import DictStore -from simplekv.memory.redisstore import RedisStore +from flask.sessions import SessionMixin, SessionInterface from itsdangerous.encoding import want_bytes +from werkzeug.datastructures import CallbackDict from werkzeug.contrib import fixers @@ -65,77 +71,341 @@ proxy = PrefixMiddleware() migrate = flask_migrate.Migrate() -# session store -class NullSigner(object): - """NullSigner does not sign nor unsign""" - def __init__(self, *args, **kwargs): - pass - def sign(self, value): - """Signs the given string.""" - return want_bytes(value) - def unsign(self, signed_value): - """Unsigns the given string.""" - return want_bytes(signed_value) +# session store (inspired by https://github.com/mbr/flask-kvsession) +class RedisStore: + """ Stores session data in a redis db. """ -class KVSessionIntf(flask_kvsession.KVSessionInterface): - """ KVSession interface allowing to run int function on first access """ - def __init__(self, app, init_fn=None): - if init_fn: - app.kvsession_init = init_fn + has_ttl = True + + def __init__(self, redisstore): + self.redis = redisstore + + def get(self, key): + """ load item from store. """ + value = self.redis.get(key) + if value is None: + raise KeyError(key) + return value + + def put(self, key, value, ttl_secs=None): + """ save item to store. """ + if ttl_secs: + self.redis.setex(key, int(ttl_secs), value) else: - self._first_run(None) - def _first_run(self, app): - if app: - app.kvsession_init() - self.open_session = super().open_session - self.save_session = super().save_session - def open_session(self, app, request): - self._first_run(app) - return super().open_session(app, request) - def save_session(self, app, session, response): - self._first_run(app) - return super().save_session(app, session, response) + self.redis.set(key, value) -class KVSessionExt(flask_kvsession.KVSessionExtension): - """ Activates Flask-KVSession for an application. """ - def init_kvstore(self, config): - """ Initialize kvstore - fallback to DictStore without REDIS_ADDRESS """ - if addr := config.get('REDIS_ADDRESS'): - self.default_kvstore = RedisStore(redis.StrictRedis().from_url(f'redis://{addr}/3')) - else: - self.default_kvstore = DictStore() + def delete(self, key): + """ delete item from store. """ + self.redis.delete(key) - def cleanup_sessions(self, app=None, dkey=None, dvalue=None): - """ Remove sessions from the store. """ - if not app: + def list(self, prefix=None): + """ return list of keys starting with prefix """ + if prefix: + prefix += b'*' + return list(self.redis.scan_iter(match=prefix)) + +class DictStore: + """ Stores session data in a python dict. """ + + has_ttl = False + + def __init__(self): + self.dict = {} + + def get(self, key): + """ load item from store. """ + return self.dict[key] + + def put(self, key, value, ttl_secs=None): + """ save item to store. """ + self.dict[key] = value + + def delete(self, key): + """ delete item from store. """ + try: + del self.dict[key] + except KeyError: + pass + + def list(self, prefix=None): + """ return list of keys starting with prefix """ + if prefix is None: + return list(self.dict.keys()) + return [key for key in self.dict if key.startswith(prefix)] + +class MailuSession(CallbackDict, SessionMixin): + """ Custom flask session storage. """ + + # default modified to false + modified = False + + def __init__(self, key=None, app=None): + + self.app = app or flask.current_app + + initial = None + + key = want_bytes(key) + if parsed := self.app.session_config.parse_key(key, self.app): + try: + initial = pickle.loads(app.session_store.get(key)) + except (KeyError, EOFError, pickle.UnpicklingError): + # either the cookie was manipulated or we did not find the + # session in the backend or the pickled data is invalid. + # => start new session + pass + else: + (self._uid, self._sid, self._created) = parsed + self._key = key + + if initial is None: + # start new session + self.new = True + self._uid = None + self._sid = None + self._created = self.app.session_config.gen_created() + self._key = None + + def _on_update(obj): + obj.modified = True + + CallbackDict.__init__(self, initial, _on_update) + + @property + def sid(self): + """ this reflects the session's id. """ + if self._sid is None or self._uid is None or self._created is None: + return None + return b''.join([self._uid, self._sid, self._created]) + + def destroy(self): + """ destroy session for security reasons. """ + + if self._key is not None: + self.app.session_store.delete(self._key) + self._key = None + + self._uid = None + self._sid = None + self._created = None + + self.clear() + + self.modified = False + self.new = False + + def regenerate(self): + """ generate new id for session to avoid `session fixation`. """ + + if self._key is not None: + self.app.session_store.delete(self._key) + self._key = None + + self._sid = None + self._created = self.app.session_config.gen_created() + + self.modified = True + + def save(self): + """ Save session to store. """ + + # don't save if session was destroyed or is not modified + if self._created is None or not self.modified: + return False + + # set uid from dict data + if self._uid is None: + self._uid = self.app.session_config.gen_uid(self.get('user_id', '')) + + # create new session id for new or regenerated sessions + if self._sid is None: + self._sid = self.app.session_config.gen_sid() + + # set created if permanent state changed + if self.permanent: + if self._created: + self._created = b'' + elif not self._created: + self._created = self.app.session_config.gen_created() + + # get new session key + key = self.sid + + # delete old session if key has changed + if key != self._key and self._key is not None: + self.app.session_store.delete(self._key) + + # save session + self.app.session_store.put( + key, + pickle.dumps(dict(self)), + None if self.permanent else self.app.permanent_session_lifetime.total_seconds() + ) + + self._key = key + + self.new = False + self.modified = False + + return True + +class MailuSessionConfig: + """ Stores sessions crypto config """ + + def __init__(self, app=None): + + if app is None: app = flask.current_app - if dkey is None and dvalue is None: - now = datetime.utcnow() - for key in app.kvsession_store.keys(): - try: - sid = flask_kvsession.SessionID.unserialize(key) - except ValueError: - pass - else: - if sid.has_expired( - app.config['PERMANENT_SESSION_LIFETIME'], - now - ): - app.kvsession_store.delete(key) - elif dkey is not None and dvalue is not None: - for key in app.kvsession_store.keys(): - if app.session_interface.serialization_method.loads( - app.kvsession_store.get(key) - ).get(dkey, None) == dvalue: - app.kvsession_store.delete(key) + + bits = app.config.get('SESSION_KEY_BITS', 64) + + if bits < 64: + raise ValueError('Session id entropy must not be less than 64 bits!') + + hash_bytes = bits//8 + (bits%8>0) + time_bytes = 4 # 32 bit timestamp for now + + self._shake_fn = hashlib.shake_256 if bits>128 else hashlib.shake_128 + self._hash_len = hash_bytes + self._hash_b64 = len(self._encode(bytes(hash_bytes))) + self._key_min = 2*self._hash_b64 + self._key_max = self._key_min + len(self._encode(bytes(time_bytes))) + + def gen_sid(self): + """ Generate random session id. """ + return self._encode(secrets.token_bytes(self._hash_len)) + + def gen_uid(self, uid): + """ Generate hashed user id part of session key. """ + return self._encode(self._shake_fn(want_bytes(uid)).digest(self._hash_len)) + + def gen_created(self, now=None): + """ Generate base64 representation of creation time. """ + return self._encode(int(now or time.time()).to_bytes(8, byteorder='big').lstrip(b'\0')) + + def parse_key(self, key, app=None, now=None): + """ Split key into sid, uid and creation time. """ + + if not (isinstance(key, bytes) and self._key_min <= len(key) <= self._key_max): + return None + + uid = key[:self._hash_b64] + sid = key[self._hash_b64:self._key_min] + crt = key[self._key_min:] + + # validate if parts are decodeable + created = self._decode(crt) + if created is None or self._decode(uid) is None or self._decode(sid) is None: + return None + + # validate creation time when requested or store does not support ttl + if now is not None or not app.session_store.has_ttl: + created = int.from_bytes(created, byteorder='big') + if created > 0: + if now is None: + now = int(time.time()) + if created < now < created + app.permanent_session_lifetime.total_seconds(): + return None + + return (uid, sid, crt) + + def _encode(self, value): + return secrets.base64.urlsafe_b64encode(value).rstrip(b'=') + + def _decode(self, value): + try: + return secrets.base64.urlsafe_b64decode(value + b'='*(4-len(value)%4)) + except secrets.binascii.Error: + return None + +class MailuSessionInterface(SessionInterface): + """ Custom flask session interface. """ + + def open_session(self, app, request): + """ Load or create session. """ + return MailuSession(request.cookies.get(app.config['SESSION_COOKIE_NAME'], None), app) + + def save_session(self, app, session, response): + """ Save modified session. """ + + if session.save(): + # session saved. update cookie + response.set_cookie( + key=app.config['SESSION_COOKIE_NAME'], + value=session.sid, + expires=self.get_expiration_time(app, session), + path=self.get_cookie_path(app), + domain=self.get_cookie_domain(app), + secure=app.config['SESSION_COOKIE_SECURE'], + httponly=app.config['SESSION_COOKIE_HTTPONLY'] + ) + +class MailuSessionExtension: + """ Server side session handling """ + + @staticmethod + def cleanup_sessions(app=None): + """ Remove invalid or expired sessions. """ + + app = app or flask.current_app + now = int(time.time()) + + count = 0 + for key in app.session_store.list(): + if not app.session_config.parse_key(key, app, now): + app.session_store.delete(key) + count += 1 + + return count + + @staticmethod + def prune_sessions(uid=None, keep_permanent=False, keep=None, app=None): + """ Remove sessions + uid: remove all sessions (NONE) or sessions belonging to a specific user + keep_permanent: also delete permanent sessions? + keep: keep listed sessions + """ + + keep = keep or set() + app = app or flask.current_app + now = int(time.time()) + + prefix = None if uid is None else app.session_config.gen_uid(uid) + + count = 0 + for key in app.session_store.list(prefix): + if key in keep: + continue + if keep_permanent: + if parsed := app.session_config.parse_key(key, app, now): + if not parsed[2]: + continue + app.session_store.delete(key) + count += 1 + + return count + + def init_app(self, app): + """ Replace session management of application. """ + + if app.config.get('MEMORY_SESSIONS'): + # in-memory session store for use in development + app.session_store = DictStore() + else: - raise ValueError('Need dkey and dvalue.') + # redis-based session store for use in production + app.session_store = RedisStore( + redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL']) + ) - def init_app(self, app, session_kvstore=None): - """ Initialize application and KVSession. """ - super().init_app(app, session_kvstore) - app.session_interface = KVSessionIntf(app, self.cleanup_sessions) + # clean expired sessions on first use in case lifetime was changed + def cleaner(): + MailuSessionExtension.cleanup_sessions(app) -kvsession = KVSessionExt() + # TODO: hmm. this will clean once per gunicorn worker + app.before_first_request(cleaner) -flask_kvsession.Signer = NullSigner + app.session_config = MailuSessionConfig(app) + app.session_interface = MailuSessionInterface() + +session = MailuSessionExtension() diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index cd084684..f767f431 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -13,7 +13,6 @@ Flask==1.0.2 Flask-Babel==0.12.2 Flask-Bootstrap==3.3.7.1 Flask-DebugToolbar==0.10.1 -Flask-KVSession==0.6.2 Flask-Limiter==1.0.1 Flask-Login==0.4.1 Flask-Migrate==2.4.0 @@ -39,7 +38,6 @@ python-editor==1.0.4 pytz==2019.1 PyYAML==5.1 redis==3.2.1 -simplekv==0.14.1 #alpine3:12 provides six==1.15.0 #six==1.12.0 socrate==0.1.1 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index abb37234..9739ed3f 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -3,7 +3,6 @@ Flask-Login Flask-SQLAlchemy Flask-bootstrap Flask-Babel -Flask-KVSession Flask-migrate Flask-script Flask-wtf From 4b8bbf760b423657a564b478cf72cbbc0125e7c3 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 4 Apr 2021 14:40:49 +0200 Subject: [PATCH 4/9] default to 128 bits --- core/admin/mailu/configuration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 3d1b4fb5..679c6c7e 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -140,6 +140,7 @@ class ConfigManager(dict): self.config['SESSION_STORAGE_URL'] = 'redis://{0}/3'.format(self.config['REDIS_ADDRESS']) self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True + self.config['SESSION_KEY_BITS'] = 128 self.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=int(self.config['SESSION_LIFETIME'])) # update the app config itself app.config = self From 731ce8ede91a7c3e37c8efc0161c0e6b06d330fc Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sun, 4 Apr 2021 18:02:43 +0200 Subject: [PATCH 5/9] fix permanent sessions. hash uid using SECRET_KEY clean session in redis only once when starting --- core/admin/mailu/utils.py | 140 +++++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 61 deletions(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 1deaa4ae..30725ff7 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -10,6 +10,8 @@ import hashlib import secrets import time +from multiprocessing import Value + from mailu import limiter import flask @@ -87,10 +89,10 @@ class RedisStore: raise KeyError(key) return value - def put(self, key, value, ttl_secs=None): + def put(self, key, value, ttl=None): """ save item to store. """ - if ttl_secs: - self.redis.setex(key, int(ttl_secs), value) + if ttl: + self.redis.setex(key, int(ttl), value) else: self.redis.set(key, value) @@ -171,6 +173,11 @@ class MailuSession(CallbackDict, SessionMixin): CallbackDict.__init__(self, initial, _on_update) + @property + def saved(self): + """ this reflects if the session was saved. """ + return self._key is not None + @property def sid(self): """ this reflects the session's id. """ @@ -181,9 +188,7 @@ class MailuSession(CallbackDict, SessionMixin): def destroy(self): """ destroy session for security reasons. """ - if self._key is not None: - self.app.session_store.delete(self._key) - self._key = None + self.delete() self._uid = None self._sid = None @@ -191,28 +196,28 @@ class MailuSession(CallbackDict, SessionMixin): self.clear() - self.modified = False + self.modified = True self.new = False def regenerate(self): """ generate new id for session to avoid `session fixation`. """ - if self._key is not None: - self.app.session_store.delete(self._key) - self._key = None + self.delete() self._sid = None self._created = self.app.session_config.gen_created() self.modified = True + def delete(self): + """ Delete stored session. """ + if self.saved: + self.app.session_store.delete(self._key) + self._key = None + def save(self): """ Save session to store. """ - # don't save if session was destroyed or is not modified - if self._created is None or not self.modified: - return False - # set uid from dict data if self._uid is None: self._uid = self.app.session_config.gen_uid(self.get('user_id', '')) @@ -221,25 +226,18 @@ class MailuSession(CallbackDict, SessionMixin): if self._sid is None: self._sid = self.app.session_config.gen_sid() - # set created if permanent state changed - if self.permanent: - if self._created: - self._created = b'' - elif not self._created: - self._created = self.app.session_config.gen_created() - # get new session key key = self.sid # delete old session if key has changed - if key != self._key and self._key is not None: - self.app.session_store.delete(self._key) + if key != self._key: + self.delete() # save session self.app.session_store.put( key, pickle.dumps(dict(self)), - None if self.permanent else self.app.permanent_session_lifetime.total_seconds() + self.app.permanent_session_lifetime.total_seconds() ) self._key = key @@ -247,8 +245,6 @@ class MailuSession(CallbackDict, SessionMixin): self.new = False self.modified = False - return True - class MailuSessionConfig: """ Stores sessions crypto config """ @@ -264,8 +260,9 @@ class MailuSessionConfig: hash_bytes = bits//8 + (bits%8>0) time_bytes = 4 # 32 bit timestamp for now + shaker = hashlib.shake_256 if bits>128 else hashlib.shake_128 - self._shake_fn = hashlib.shake_256 if bits>128 else hashlib.shake_128 + self._shaker = shaker(want_bytes(app.config.get('SECRET_KEY', ''))) self._hash_len = hash_bytes self._hash_b64 = len(self._encode(bytes(hash_bytes))) self._key_min = 2*self._hash_b64 @@ -277,13 +274,15 @@ class MailuSessionConfig: def gen_uid(self, uid): """ Generate hashed user id part of session key. """ - return self._encode(self._shake_fn(want_bytes(uid)).digest(self._hash_len)) + shaker = self._shaker.copy() + shaker.update(want_bytes(uid)) + return self._encode(shaker.digest(self._hash_len)) def gen_created(self, now=None): """ Generate base64 representation of creation time. """ return self._encode(int(now or time.time()).to_bytes(8, byteorder='big').lstrip(b'\0')) - def parse_key(self, key, app=None, now=None): + def parse_key(self, key, app=None, validate=False, now=None): """ Split key into sid, uid and creation time. """ if not (isinstance(key, bytes) and self._key_min <= len(key) <= self._key_max): @@ -299,13 +298,12 @@ class MailuSessionConfig: return None # validate creation time when requested or store does not support ttl - if now is not None or not app.session_store.has_ttl: + if validate or not app.session_store.has_ttl: + if now is None: + now = int(time.time()) created = int.from_bytes(created, byteorder='big') - if created > 0: - if now is None: - now = int(time.time()) - if created < now < created + app.permanent_session_lifetime.total_seconds(): - return None + if not (created < now < created + app.permanent_session_lifetime.total_seconds()): + return None return (uid, sid, crt) @@ -328,17 +326,40 @@ class MailuSessionInterface(SessionInterface): def save_session(self, app, session, response): """ Save modified session. """ - if session.save(): - # session saved. update cookie - response.set_cookie( - key=app.config['SESSION_COOKIE_NAME'], - value=session.sid, - expires=self.get_expiration_time(app, session), - path=self.get_cookie_path(app), - domain=self.get_cookie_domain(app), - secure=app.config['SESSION_COOKIE_SECURE'], - httponly=app.config['SESSION_COOKIE_HTTPONLY'] - ) + # If the session is modified to be empty, remove the cookie. + # If the session is empty, return without setting the cookie. + if not session: + if session.modified: + session.delete() + response.delete_cookie( + app.session_cookie_name, + domain=self.get_cookie_domain(app), + path=self.get_cookie_path(app), + ) + return + + # Add a "Vary: Cookie" header if the session was accessed + if session.accessed: + response.vary.add('Cookie') + + # TODO: set cookie from time to time to prevent expiration in browser + # also update expire in redis + + if not self.should_set_cookie(app, session): + return + + # save session and update cookie + session.save() + response.set_cookie( + app.session_cookie_name, + session.sid, + expires=self.get_expiration_time(app, session), + httponly=self.get_cookie_httponly(app), + domain=self.get_cookie_domain(app), + path=self.get_cookie_path(app), + secure=self.get_cookie_secure(app), + samesite=self.get_cookie_samesite(app) + ) class MailuSessionExtension: """ Server side session handling """ @@ -352,36 +373,29 @@ class MailuSessionExtension: count = 0 for key in app.session_store.list(): - if not app.session_config.parse_key(key, app, now): + if not app.session_config.parse_key(key, app, validate=True, now=now): app.session_store.delete(key) count += 1 return count @staticmethod - def prune_sessions(uid=None, keep_permanent=False, keep=None, app=None): + def prune_sessions(uid=None, keep=None, app=None): """ Remove sessions uid: remove all sessions (NONE) or sessions belonging to a specific user - keep_permanent: also delete permanent sessions? keep: keep listed sessions """ keep = keep or set() app = app or flask.current_app - now = int(time.time()) prefix = None if uid is None else app.session_config.gen_uid(uid) count = 0 for key in app.session_store.list(prefix): - if key in keep: - continue - if keep_permanent: - if parsed := app.session_config.parse_key(key, app, now): - if not parsed[2]: - continue - app.session_store.delete(key) - count += 1 + if key not in keep: + app.session_store.delete(key) + count += 1 return count @@ -398,14 +412,18 @@ class MailuSessionExtension: redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL']) ) - # clean expired sessions on first use in case lifetime was changed + # clean expired sessions oonce on first use in case lifetime was changed def cleaner(): - MailuSessionExtension.cleanup_sessions(app) + with cleaned.get_lock(): + if not cleaned.value: + cleaned.value = True + flask.current_app.logger.error('cleaning') + MailuSessionExtension.cleanup_sessions(app) - # TODO: hmm. this will clean once per gunicorn worker app.before_first_request(cleaner) app.session_config = MailuSessionConfig(app) app.session_interface = MailuSessionInterface() +cleaned = Value('i', False) session = MailuSessionExtension() From 9ef8aaf6989376480b0ecd6c4c6fc2f98a7af408 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Wed, 16 Jun 2021 22:06:28 +0200 Subject: [PATCH 6/9] removed double confiog and fixed shaker --- core/admin/mailu/configuration.py | 1 - core/admin/mailu/utils.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 679c6c7e..3d1b4fb5 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -140,7 +140,6 @@ class ConfigManager(dict): self.config['SESSION_STORAGE_URL'] = 'redis://{0}/3'.format(self.config['REDIS_ADDRESS']) self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True - self.config['SESSION_KEY_BITS'] = 128 self.config['PERMANENT_SESSION_LIFETIME'] = timedelta(hours=int(self.config['SESSION_LIFETIME'])) # update the app config itself app.config = self diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 30725ff7..214a9a2d 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -260,9 +260,8 @@ class MailuSessionConfig: hash_bytes = bits//8 + (bits%8>0) time_bytes = 4 # 32 bit timestamp for now - shaker = hashlib.shake_256 if bits>128 else hashlib.shake_128 - self._shaker = shaker(want_bytes(app.config.get('SECRET_KEY', ''))) + self._shaker = hashlib.shake_128(want_bytes(app.config.get('SECRET_KEY', ''))) self._hash_len = hash_bytes self._hash_b64 = len(self._encode(bytes(hash_bytes))) self._key_min = 2*self._hash_b64 From 3f23e199f6b64bb4960d8368a4c670f16c5f705b Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 17 Jun 2021 17:53:15 +0200 Subject: [PATCH 7/9] modified generation of session key and added refresh - the session key is now generated using - a hash of the uid seeded by the apps secret_key (size: SESSION_KEY_BITS) - a random token (size: 128 bits) - the session's creation time (size: 32 bits) - redis server side sessions are now refreshed after 1/2 the session lifetime even if not modified - the cookie is also updated if necessary --- core/admin/mailu/utils.py | 96 +++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 35 deletions(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index 214a9a2d..c7e1f73c 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -6,7 +6,7 @@ try: except ImportError: import pickle -import hashlib +import hmac import secrets import time @@ -218,13 +218,16 @@ class MailuSession(CallbackDict, SessionMixin): def save(self): """ Save session to store. """ + set_cookie = False + # set uid from dict data if self._uid is None: self._uid = self.app.session_config.gen_uid(self.get('user_id', '')) - # create new session id for new or regenerated sessions + # create new session id for new or regenerated sessions and force setting the cookie if self._sid is None: self._sid = self.app.session_config.gen_sid() + set_cookie = True # get new session key key = self.sid @@ -233,6 +236,9 @@ class MailuSession(CallbackDict, SessionMixin): if key != self._key: self.delete() + # remember time to refresh + self['_refresh'] = int(time.time()) + self.app.permanent_session_lifetime.total_seconds()/2 + # save session self.app.session_store.put( key, @@ -245,37 +251,52 @@ class MailuSession(CallbackDict, SessionMixin): self.new = False self.modified = False + return set_cookie + + def needs_refresh(self): + """ Checks if server side session needs to be refreshed. """ + + return int(time.time()) > self.get('_refresh', 0) + class MailuSessionConfig: """ Stores sessions crypto config """ + # default size of session key parts + uid_bits = 64 # default if SESSION_KEY_BITS is not set in config + sid_bits = 128 # for now. must be multiple of 8! + time_bits = 32 # for now. must be multiple of 8! + def __init__(self, app=None): if app is None: app = flask.current_app - bits = app.config.get('SESSION_KEY_BITS', 64) + bits = app.config.get('SESSION_KEY_BITS', self.uid_bits) + if not 64 <= bits <= 256: + raise ValueError('SESSION_KEY_BITS must be between 64 and 256!') - if bits < 64: - raise ValueError('Session id entropy must not be less than 64 bits!') + uid_bytes = bits//8 + (bits%8>0) + sid_bytes = self.sid_bits//8 - hash_bytes = bits//8 + (bits%8>0) - time_bytes = 4 # 32 bit timestamp for now + key = want_bytes(app.secret_key) - self._shaker = hashlib.shake_128(want_bytes(app.config.get('SECRET_KEY', ''))) - self._hash_len = hash_bytes - self._hash_b64 = len(self._encode(bytes(hash_bytes))) - self._key_min = 2*self._hash_b64 - self._key_max = self._key_min + len(self._encode(bytes(time_bytes))) + self._hmac = hmac.new(hmac.digest(key, key, digest='sha256'), digestmod='sha256') + self._uid_len = uid_bytes + self._uid_b64 = len(self._encode(bytes(uid_bytes))) + self._sid_len = sid_bytes + self._sid_b64 = len(self._encode(bytes(sid_bytes))) + self._key_min = self._uid_b64 + self._sid_b64 + self._key_max = self._key_min + len(self._encode(bytes(self.time_bits//8))) def gen_sid(self): """ Generate random session id. """ - return self._encode(secrets.token_bytes(self._hash_len)) + return self._encode(secrets.token_bytes(self._sid_len)) def gen_uid(self, uid): """ Generate hashed user id part of session key. """ - shaker = self._shaker.copy() - shaker.update(want_bytes(uid)) - return self._encode(shaker.digest(self._hash_len)) + _hmac = self._hmac.copy() + _hmac.update(want_bytes(uid)) + return self._encode(_hmac.digest()[:self._uid_len]) def gen_created(self, now=None): """ Generate base64 representation of creation time. """ @@ -287,8 +308,8 @@ class MailuSessionConfig: if not (isinstance(key, bytes) and self._key_min <= len(key) <= self._key_max): return None - uid = key[:self._hash_b64] - sid = key[self._hash_b64:self._key_min] + uid = key[:self._uid_b64] + sid = key[self._uid_b64:self._key_min] crt = key[self._key_min:] # validate if parts are decodeable @@ -301,7 +322,7 @@ class MailuSessionConfig: if now is None: now = int(time.time()) created = int.from_bytes(created, byteorder='big') - if not (created < now < created + app.permanent_session_lifetime.total_seconds()): + if not created < now < created + app.permanent_session_lifetime.total_seconds(): return None return (uid, sid, crt) @@ -341,24 +362,29 @@ class MailuSessionInterface(SessionInterface): if session.accessed: response.vary.add('Cookie') - # TODO: set cookie from time to time to prevent expiration in browser - # also update expire in redis + set_cookie = session.permanent and app.config['SESSION_REFRESH_EACH_REQUEST'] + need_refresh = session.needs_refresh() - if not self.should_set_cookie(app, session): - return + # save modified session or refresh unmodified session + if session.modified or need_refresh: + set_cookie |= session.save() - # save session and update cookie - session.save() - response.set_cookie( - app.session_cookie_name, - session.sid, - expires=self.get_expiration_time(app, session), - httponly=self.get_cookie_httponly(app), - domain=self.get_cookie_domain(app), - path=self.get_cookie_path(app), - secure=self.get_cookie_secure(app), - samesite=self.get_cookie_samesite(app) - ) + # set cookie on refreshed permanent sessions + if need_refresh and session.permanent: + set_cookie = True + + # set or update cookie if necessary + if set_cookie: + response.set_cookie( + app.session_cookie_name, + session.sid, + expires=self.get_expiration_time(app, session), + httponly=self.get_cookie_httponly(app), + domain=self.get_cookie_domain(app), + path=self.get_cookie_path(app), + secure=self.get_cookie_secure(app), + samesite=self.get_cookie_samesite(app) + ) class MailuSessionExtension: """ Server side session handling """ From 6740c77e4383b8bfb9713ec1b34280af0feb6cbc Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 2 Jul 2021 18:44:21 +0200 Subject: [PATCH 8/9] small bugfix for exception --- core/admin/mailu/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/schemas.py b/core/admin/mailu/schemas.py index 2742edf1..191d01ac 100644 --- a/core/admin/mailu/schemas.py +++ b/core/admin/mailu/schemas.py @@ -590,7 +590,7 @@ class DkimKeyField(fields.String): value = value[:pos] else: footer = '-----END PRIVATE KEY-----' - except ValueError: + except ValueError as exc: raise ValidationError(f'invalid dkim key {bad_key!r}') from exc # remove whitespace from key data From 8b71a92219bd9e477eaa13536ff298a989858dcd Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Sat, 3 Jul 2021 22:32:47 +0200 Subject: [PATCH 9/9] use fixed msg for key derivation --- core/admin/mailu/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/utils.py b/core/admin/mailu/utils.py index c7e1f73c..02150754 100644 --- a/core/admin/mailu/utils.py +++ b/core/admin/mailu/utils.py @@ -280,7 +280,7 @@ class MailuSessionConfig: key = want_bytes(app.secret_key) - self._hmac = hmac.new(hmac.digest(key, key, digest='sha256'), digestmod='sha256') + self._hmac = hmac.new(hmac.digest(key, b'SESSION_UID_HASH', digest='sha256'), digestmod='sha256') self._uid_len = uid_bytes self._uid_b64 = len(self._encode(bytes(uid_bytes))) self._sid_len = sid_bytes