From cf34be967cf45b06d1d98387ce3c80547671c5fb Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 20:15:10 +0100 Subject: [PATCH 01/15] Implement ITERATE --- core/admin/mailu/internal/views/dovecot.py | 5 ++++ core/base/libs/podop/podop/dovecot.py | 33 +++++++++++++++++++--- core/dovecot/conf/auth.conf | 1 + towncrier/newsfragments/2498.feature | 1 + 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 towncrier/newsfragments/2498.feature diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 0d56950b..10f3d5fb 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -19,6 +19,11 @@ def dovecot_passdb_dict(user_email): "allow_nets": ",".join(allow_nets) }) +@internal.route("/dovecot/userdb/") +def dovecot_userdb_dict_list(): + return flask.jsonify([ + user[0] for user in models.User.query.filter(models.User.enabled.is_(True)).with_entities(models.User.email).all() + ]) @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 40b48eef..18956dd4 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -77,7 +77,7 @@ class DictProtocol(asyncio.Protocol): logging.debug("Client {}.{} type {}, user {}, dict {}".format( self.major, self.minor, self.value_type, self.user, dict_name)) - async def process_lookup(self, key, user=None): + async def process_lookup(self, key, user=None, is_iter=False): """ Process a dict lookup message """ logging.debug("Looking up {} for {}".format(key, user)) @@ -93,10 +93,33 @@ class DictProtocol(asyncio.Protocol): response = result else: response = json.dumps(result).encode("ascii") - return self.reply(b"O", response) + logging.debug("Replying {}".format(key)) + return self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response) except KeyError: return self.reply(b"N") + async def process_iterate(self, flags, max_rows, path, user=None): + """ Process an iterate command + """ + logging.debug("Iterate flags {} max_rows {} on {} for {}".format(flags, max_rows, path, user)) + # Priv and shared keys are handled slighlty differently + key_type, key = path.decode("utf8").split("/", 1) + max_rows = int(max_rows.decode("utf-8")) + flags = int(flags.decode("utf-8")) + if flags != 0: # not implemented + return self.reply(b"F") + try: + result = await self.dict.iter(key) + logging.debug("Found {} entries: {}".format(len(result), result)) + returned_results = 0 + for k in result: + if max_rows == 0 or returned_results < max_rows: + await self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True) + returned_results += 1 + return self.reply(b"\n") # ITER_FINISHED + except KeyError: + return self.reply(b"F") + def process_begin(self, transaction_id, user=None): """ Process a dict begin message """ @@ -126,11 +149,12 @@ class DictProtocol(asyncio.Protocol): del self.transactions_user[transaction_id] return self.reply(b"O", transaction_id) - def reply(self, command, *args): + def reply(self, command, *args, end=True): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) - self.transport.write(b"\n") + if end: + self.transport.write(b"\n") @classmethod def factory(cls, table_map): @@ -141,6 +165,7 @@ class DictProtocol(asyncio.Protocol): COMMANDS = { ord("H"): process_hello, ord("L"): process_lookup, + ord("I"): process_iterate, ord("B"): process_begin, ord("C"): process_commit, ord("S"): process_set diff --git a/core/dovecot/conf/auth.conf b/core/dovecot/conf/auth.conf index 44a874ba..1f122416 100644 --- a/core/dovecot/conf/auth.conf +++ b/core/dovecot/conf/auth.conf @@ -1,5 +1,6 @@ uri = proxy:/tmp/podop.socket:auth iterate_disable = yes +iterate_prefix = 'userdb/' default_pass_scheme = plain password_key = passdb/%u user_key = userdb/%u diff --git a/towncrier/newsfragments/2498.feature b/towncrier/newsfragments/2498.feature new file mode 100644 index 00000000..961b6a84 --- /dev/null +++ b/towncrier/newsfragments/2498.feature @@ -0,0 +1 @@ +Implement the required glue to make "doveadm -A" work From 5ec4277e1ed838c6645efae7335457a6a0aa464f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:11:45 +0100 Subject: [PATCH 02/15] Make it async. I'm not sure it's a good idea --- core/base/libs/podop/podop/dovecot.py | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 18956dd4..6dc25af5 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -40,6 +40,7 @@ class DictProtocol(asyncio.Protocol): def connection_made(self, transport): logging.info('Connect {}'.format(transport.get_extra_info('peername'))) self.transport = transport + self.transport_lock = asyncio.Lock() def data_received(self, data): logging.debug("Received {}".format(data)) @@ -94,9 +95,9 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response) + return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response)) except KeyError: - return self.reply(b"N") + return await self.reply(b"N") async def process_iterate(self, flags, max_rows, path, user=None): """ Process an iterate command @@ -107,18 +108,24 @@ class DictProtocol(asyncio.Protocol): max_rows = int(max_rows.decode("utf-8")) flags = int(flags.decode("utf-8")) if flags != 0: # not implemented - return self.reply(b"F") + return await self.reply(b"F") + rows = [] try: result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) returned_results = 0 for k in result: if max_rows == 0 or returned_results < max_rows: - await self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True) + rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) returned_results += 1 - return self.reply(b"\n") # ITER_FINISHED + await asyncio.gather(*rows) + return await self.reply(b"\n") # ITER_FINISHED except KeyError: - return self.reply(b"F") + return await self.reply(b"F") + except Exception as e: + logging.error(f"Got {e}, cancelling remaining tasks") + for task in rows: + task.cancel() def process_begin(self, transaction_id, user=None): """ Process a dict begin message @@ -147,14 +154,15 @@ class DictProtocol(asyncio.Protocol): # Remove stored transaction del self.transactions[transaction_id] del self.transactions_user[transaction_id] - return self.reply(b"O", transaction_id) + return await self.reply(b"O", transaction_id) - def reply(self, command, *args, end=True): + async def reply(self, command, *args, end=True): logging.debug("Replying {} with {}".format(command, args)) - self.transport.write(command) - self.transport.write(b"\t".join(map(tabescape, args))) - if end: - self.transport.write(b"\n") + async with self.transport_lock: + self.transport.write(command) + self.transport.write(b"\t".join(map(tabescape, args))) + if end: + self.transport.write(b"\n") @classmethod def factory(cls, table_map): From 1ae4c37cb9351828b2d1687f5c59c9407c11fb1f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:25:34 +0100 Subject: [PATCH 03/15] Don't do fancy, just re-raise it --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 6dc25af5..fcf01fab 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -123,9 +123,9 @@ class DictProtocol(asyncio.Protocol): except KeyError: return await self.reply(b"F") except Exception as e: - logging.error(f"Got {e}, cancelling remaining tasks") for task in rows: task.cancel() + raise e def process_begin(self, transaction_id, user=None): """ Process a dict begin message From e10527a4bf6afde33154c4486324a640eda7f4de Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:33:10 +0100 Subject: [PATCH 04/15] This is not used anymore --- core/base/libs/podop/podop/dovecot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index fcf01fab..94385a75 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -95,7 +95,7 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response)) + return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") @@ -156,7 +156,7 @@ class DictProtocol(asyncio.Protocol): del self.transactions_user[transaction_id] return await self.reply(b"O", transaction_id) - async def reply(self, command, *args, end=True): + async def reply(self, command, *args): logging.debug("Replying {} with {}".format(command, args)) async with self.transport_lock: self.transport.write(command) From 1ce889b91b83f444bbdcf755afd6a17b62295218 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:43:34 +0100 Subject: [PATCH 05/15] Do it the pythonic way --- core/base/libs/podop/podop/dovecot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 94385a75..8c074186 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -82,6 +82,7 @@ class DictProtocol(asyncio.Protocol): """ Process a dict lookup message """ logging.debug("Looking up {} for {}".format(key, user)) + orig_key = key # Priv and shared keys are handled slighlty differently key_type, key = key.decode("utf8").split("/", 1) try: @@ -95,7 +96,7 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response) if is_iter else self.reply(b"O", response)) + return await (self.reply(b"O", orig_key, response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") @@ -113,11 +114,10 @@ class DictProtocol(asyncio.Protocol): try: result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) - returned_results = 0 - for k in result: - if max_rows == 0 or returned_results < max_rows: - rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) - returned_results += 1 + for i,k in enumerate(result): + if max_rows > 0 and max_rows >= i: + break + rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) return await self.reply(b"\n") # ITER_FINISHED except KeyError: From 2a417dbfc23e5c00d193a889c35fef8bd7a80d70 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:51:29 +0100 Subject: [PATCH 06/15] doesn't belong here --- core/base/libs/podop/podop/dovecot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 8c074186..7b27fa86 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -95,7 +95,6 @@ class DictProtocol(asyncio.Protocol): response = result else: response = json.dumps(result).encode("ascii") - logging.debug("Replying {}".format(key)) return await (self.reply(b"O", orig_key, response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") From cdc9b63a46561beabceed0bbb40a21ccc8b508dc Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:54:03 +0100 Subject: [PATCH 07/15] Guard the message logging too --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 7b27fa86..8fc06d6d 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -156,8 +156,8 @@ class DictProtocol(asyncio.Protocol): return await self.reply(b"O", transaction_id) async def reply(self, command, *args): - logging.debug("Replying {} with {}".format(command, args)) async with self.transport_lock: + logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) if end: From 96d928963085aae311a20c84ac74578879b3632d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 22:12:15 +0100 Subject: [PATCH 08/15] No need to send an extra \n --- core/base/libs/podop/podop/dovecot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 8fc06d6d..1206b5ef 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -118,7 +118,9 @@ class DictProtocol(asyncio.Protocol): break rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) - return await self.reply(b"\n") # ITER_FINISHED + async with self.transport_lock: + self.transport.write(b"\n") # ITER_FINISHED + return except KeyError: return await self.reply(b"F") except Exception as e: @@ -160,8 +162,7 @@ class DictProtocol(asyncio.Protocol): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) - if end: - self.transport.write(b"\n") + self.transport.write(b"\n") @classmethod def factory(cls, table_map): From c1f571a4c342bc45714123a08a479feeac72655a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 08:48:55 +0100 Subject: [PATCH 09/15] Speed things up. If we want to go further than this we should change podop's incr(), pass the flags and make admin process the results. --- core/admin/mailu/internal/views/dovecot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 10f3d5fb..783a14f4 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,9 +27,9 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - user = models.User.query.get(user_email) or flask.abort(404) + quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes={}".format(user.quota_bytes) + "quota_rule": "*:bytes="+quota[0]) }) From 6def1b555b086bde18d3aa6604fe8c8e20ffb008 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 10:06:55 +0100 Subject: [PATCH 10/15] doh --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 1206b5ef..0afa3dd4 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -114,7 +114,7 @@ class DictProtocol(asyncio.Protocol): result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) for i,k in enumerate(result): - if max_rows > 0 and max_rows >= i: + if max_rows > 0 and i >= max_rows: break rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) From dec5309ef97ce0bc02dfb9b70f796c8a22223f84 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 16:39:29 +0100 Subject: [PATCH 11/15] Fix typo --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 783a14f4..7aeeeb6b 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -29,7 +29,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes="+quota[0]) + "quota_rule": "*:bytes="+quota[0] }) From bec0b1c3b221fc4771583f41ce6a7fc06350a511 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:26:27 +0100 Subject: [PATCH 12/15] Fix variable name --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 7aeeeb6b..2cb2d526 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,7 +27,7 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ "quota_rule": "*:bytes="+quota[0] }) From 595b32cf97445e60f9b6fbad3528908b1c478b48 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:37:21 +0100 Subject: [PATCH 13/15] Fix quota return value --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 2cb2d526..ab33b220 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -29,7 +29,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes="+quota[0] + "quota_rule": f"*:bytes={quota[0]}" }) From 46773f639b153d07d03f014483ff60cc3847cef8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:45:21 +0100 Subject: [PATCH 14/15] Return 404 is user-id cannot be parsed --- core/admin/mailu/internal/views/dovecot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index ab33b220..3df3839e 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,7 +27,10 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + try: + quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + except ValueError: + flask.abort(404) return flask.jsonify({ "quota_rule": f"*:bytes={quota[0]}" }) From c57706ad279cc32da3e7f2140d6f3cc67d84cb3a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:50:39 +0100 Subject: [PATCH 15/15] Duh --- core/admin/mailu/internal/views/dovecot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 3df3839e..07fce5b2 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -5,6 +5,7 @@ from flask import current_app as app import flask import socket import os +import sqlalchemy.exc @internal.route("/dovecot/passdb/") def dovecot_passdb_dict(user_email): @@ -29,7 +30,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): try: quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) - except ValueError: + except sqlalchemy.exc.StatementError as exc: flask.abort(404) return flask.jsonify({ "quota_rule": f"*:bytes={quota[0]}"