From cf34be967cf45b06d1d98387ce3c80547671c5fb Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 20:15:10 +0100 Subject: [PATCH 01/40] Implement ITERATE --- core/admin/mailu/internal/views/dovecot.py | 5 ++++ core/base/libs/podop/podop/dovecot.py | 33 +++++++++++++++++++--- core/dovecot/conf/auth.conf | 1 + towncrier/newsfragments/2498.feature | 1 + 4 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 towncrier/newsfragments/2498.feature diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 0d56950b..10f3d5fb 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -19,6 +19,11 @@ def dovecot_passdb_dict(user_email): "allow_nets": ",".join(allow_nets) }) +@internal.route("/dovecot/userdb/") +def dovecot_userdb_dict_list(): + return flask.jsonify([ + user[0] for user in models.User.query.filter(models.User.enabled.is_(True)).with_entities(models.User.email).all() + ]) @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 40b48eef..18956dd4 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -77,7 +77,7 @@ class DictProtocol(asyncio.Protocol): logging.debug("Client {}.{} type {}, user {}, dict {}".format( self.major, self.minor, self.value_type, self.user, dict_name)) - async def process_lookup(self, key, user=None): + async def process_lookup(self, key, user=None, is_iter=False): """ Process a dict lookup message """ logging.debug("Looking up {} for {}".format(key, user)) @@ -93,10 +93,33 @@ class DictProtocol(asyncio.Protocol): response = result else: response = json.dumps(result).encode("ascii") - return self.reply(b"O", response) + logging.debug("Replying {}".format(key)) + return self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response) except KeyError: return self.reply(b"N") + async def process_iterate(self, flags, max_rows, path, user=None): + """ Process an iterate command + """ + logging.debug("Iterate flags {} max_rows {} on {} for {}".format(flags, max_rows, path, user)) + # Priv and shared keys are handled slighlty differently + key_type, key = path.decode("utf8").split("/", 1) + max_rows = int(max_rows.decode("utf-8")) + flags = int(flags.decode("utf-8")) + if flags != 0: # not implemented + return self.reply(b"F") + try: + result = await self.dict.iter(key) + logging.debug("Found {} entries: {}".format(len(result), result)) + returned_results = 0 + for k in result: + if max_rows == 0 or returned_results < max_rows: + await self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True) + returned_results += 1 + return self.reply(b"\n") # ITER_FINISHED + except KeyError: + return self.reply(b"F") + def process_begin(self, transaction_id, user=None): """ Process a dict begin message """ @@ -126,11 +149,12 @@ class DictProtocol(asyncio.Protocol): del self.transactions_user[transaction_id] return self.reply(b"O", transaction_id) - def reply(self, command, *args): + def reply(self, command, *args, end=True): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) - self.transport.write(b"\n") + if end: + self.transport.write(b"\n") @classmethod def factory(cls, table_map): @@ -141,6 +165,7 @@ class DictProtocol(asyncio.Protocol): COMMANDS = { ord("H"): process_hello, ord("L"): process_lookup, + ord("I"): process_iterate, ord("B"): process_begin, ord("C"): process_commit, ord("S"): process_set diff --git a/core/dovecot/conf/auth.conf b/core/dovecot/conf/auth.conf index 44a874ba..1f122416 100644 --- a/core/dovecot/conf/auth.conf +++ b/core/dovecot/conf/auth.conf @@ -1,5 +1,6 @@ uri = proxy:/tmp/podop.socket:auth iterate_disable = yes +iterate_prefix = 'userdb/' default_pass_scheme = plain password_key = passdb/%u user_key = userdb/%u diff --git a/towncrier/newsfragments/2498.feature b/towncrier/newsfragments/2498.feature new file mode 100644 index 00000000..961b6a84 --- /dev/null +++ b/towncrier/newsfragments/2498.feature @@ -0,0 +1 @@ +Implement the required glue to make "doveadm -A" work From 5ec4277e1ed838c6645efae7335457a6a0aa464f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:11:45 +0100 Subject: [PATCH 02/40] Make it async. I'm not sure it's a good idea --- core/base/libs/podop/podop/dovecot.py | 32 +++++++++++++++++---------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 18956dd4..6dc25af5 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -40,6 +40,7 @@ class DictProtocol(asyncio.Protocol): def connection_made(self, transport): logging.info('Connect {}'.format(transport.get_extra_info('peername'))) self.transport = transport + self.transport_lock = asyncio.Lock() def data_received(self, data): logging.debug("Received {}".format(data)) @@ -94,9 +95,9 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response) + return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response)) except KeyError: - return self.reply(b"N") + return await self.reply(b"N") async def process_iterate(self, flags, max_rows, path, user=None): """ Process an iterate command @@ -107,18 +108,24 @@ class DictProtocol(asyncio.Protocol): max_rows = int(max_rows.decode("utf-8")) flags = int(flags.decode("utf-8")) if flags != 0: # not implemented - return self.reply(b"F") + return await self.reply(b"F") + rows = [] try: result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) returned_results = 0 for k in result: if max_rows == 0 or returned_results < max_rows: - await self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True) + rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) returned_results += 1 - return self.reply(b"\n") # ITER_FINISHED + await asyncio.gather(*rows) + return await self.reply(b"\n") # ITER_FINISHED except KeyError: - return self.reply(b"F") + return await self.reply(b"F") + except Exception as e: + logging.error(f"Got {e}, cancelling remaining tasks") + for task in rows: + task.cancel() def process_begin(self, transaction_id, user=None): """ Process a dict begin message @@ -147,14 +154,15 @@ class DictProtocol(asyncio.Protocol): # Remove stored transaction del self.transactions[transaction_id] del self.transactions_user[transaction_id] - return self.reply(b"O", transaction_id) + return await self.reply(b"O", transaction_id) - def reply(self, command, *args, end=True): + async def reply(self, command, *args, end=True): logging.debug("Replying {} with {}".format(command, args)) - self.transport.write(command) - self.transport.write(b"\t".join(map(tabescape, args))) - if end: - self.transport.write(b"\n") + async with self.transport_lock: + self.transport.write(command) + self.transport.write(b"\t".join(map(tabescape, args))) + if end: + self.transport.write(b"\n") @classmethod def factory(cls, table_map): From 1ae4c37cb9351828b2d1687f5c59c9407c11fb1f Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:25:34 +0100 Subject: [PATCH 03/40] Don't do fancy, just re-raise it --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 6dc25af5..fcf01fab 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -123,9 +123,9 @@ class DictProtocol(asyncio.Protocol): except KeyError: return await self.reply(b"F") except Exception as e: - logging.error(f"Got {e}, cancelling remaining tasks") for task in rows: task.cancel() + raise e def process_begin(self, transaction_id, user=None): """ Process a dict begin message From e10527a4bf6afde33154c4486324a640eda7f4de Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:33:10 +0100 Subject: [PATCH 04/40] This is not used anymore --- core/base/libs/podop/podop/dovecot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index fcf01fab..94385a75 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -95,7 +95,7 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response, end=True) if is_iter else self.reply(b"O", response)) + return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") @@ -156,7 +156,7 @@ class DictProtocol(asyncio.Protocol): del self.transactions_user[transaction_id] return await self.reply(b"O", transaction_id) - async def reply(self, command, *args, end=True): + async def reply(self, command, *args): logging.debug("Replying {} with {}".format(command, args)) async with self.transport_lock: self.transport.write(command) From 1ce889b91b83f444bbdcf755afd6a17b62295218 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:43:34 +0100 Subject: [PATCH 05/40] Do it the pythonic way --- core/base/libs/podop/podop/dovecot.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 94385a75..8c074186 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -82,6 +82,7 @@ class DictProtocol(asyncio.Protocol): """ Process a dict lookup message """ logging.debug("Looking up {} for {}".format(key, user)) + orig_key = key # Priv and shared keys are handled slighlty differently key_type, key = key.decode("utf8").split("/", 1) try: @@ -95,7 +96,7 @@ class DictProtocol(asyncio.Protocol): else: response = json.dumps(result).encode("ascii") logging.debug("Replying {}".format(key)) - return await (self.reply(b"O", (key_type+'/'+key).encode("utf8"), response) if is_iter else self.reply(b"O", response)) + return await (self.reply(b"O", orig_key, response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") @@ -113,11 +114,10 @@ class DictProtocol(asyncio.Protocol): try: result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) - returned_results = 0 - for k in result: - if max_rows == 0 or returned_results < max_rows: - rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) - returned_results += 1 + for i,k in enumerate(result): + if max_rows > 0 and max_rows >= i: + break + rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) return await self.reply(b"\n") # ITER_FINISHED except KeyError: From 2a417dbfc23e5c00d193a889c35fef8bd7a80d70 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:51:29 +0100 Subject: [PATCH 06/40] doesn't belong here --- core/base/libs/podop/podop/dovecot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 8c074186..7b27fa86 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -95,7 +95,6 @@ class DictProtocol(asyncio.Protocol): response = result else: response = json.dumps(result).encode("ascii") - logging.debug("Replying {}".format(key)) return await (self.reply(b"O", orig_key, response) if is_iter else self.reply(b"O", response)) except KeyError: return await self.reply(b"N") From cdc9b63a46561beabceed0bbb40a21ccc8b508dc Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 21:54:03 +0100 Subject: [PATCH 07/40] Guard the message logging too --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 7b27fa86..8fc06d6d 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -156,8 +156,8 @@ class DictProtocol(asyncio.Protocol): return await self.reply(b"O", transaction_id) async def reply(self, command, *args): - logging.debug("Replying {} with {}".format(command, args)) async with self.transport_lock: + logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) if end: From 96d928963085aae311a20c84ac74578879b3632d Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Sun, 30 Oct 2022 22:12:15 +0100 Subject: [PATCH 08/40] No need to send an extra \n --- core/base/libs/podop/podop/dovecot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 8fc06d6d..1206b5ef 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -118,7 +118,9 @@ class DictProtocol(asyncio.Protocol): break rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) - return await self.reply(b"\n") # ITER_FINISHED + async with self.transport_lock: + self.transport.write(b"\n") # ITER_FINISHED + return except KeyError: return await self.reply(b"F") except Exception as e: @@ -160,8 +162,7 @@ class DictProtocol(asyncio.Protocol): logging.debug("Replying {} with {}".format(command, args)) self.transport.write(command) self.transport.write(b"\t".join(map(tabescape, args))) - if end: - self.transport.write(b"\n") + self.transport.write(b"\n") @classmethod def factory(cls, table_map): From c1f571a4c342bc45714123a08a479feeac72655a Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 08:48:55 +0100 Subject: [PATCH 09/40] Speed things up. If we want to go further than this we should change podop's incr(), pass the flags and make admin process the results. --- core/admin/mailu/internal/views/dovecot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 10f3d5fb..783a14f4 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,9 +27,9 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - user = models.User.query.get(user_email) or flask.abort(404) + quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes={}".format(user.quota_bytes) + "quota_rule": "*:bytes="+quota[0]) }) From 6def1b555b086bde18d3aa6604fe8c8e20ffb008 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Mon, 31 Oct 2022 10:06:55 +0100 Subject: [PATCH 10/40] doh --- core/base/libs/podop/podop/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/libs/podop/podop/dovecot.py b/core/base/libs/podop/podop/dovecot.py index 1206b5ef..0afa3dd4 100644 --- a/core/base/libs/podop/podop/dovecot.py +++ b/core/base/libs/podop/podop/dovecot.py @@ -114,7 +114,7 @@ class DictProtocol(asyncio.Protocol): result = await self.dict.iter(key) logging.debug("Found {} entries: {}".format(len(result), result)) for i,k in enumerate(result): - if max_rows > 0 and max_rows >= i: + if max_rows > 0 and i >= max_rows: break rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True)) await asyncio.gather(*rows) From bba98b320e0d10ac8cc8333d991d215a0c6b5eb1 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 31 Oct 2022 23:40:51 +0100 Subject: [PATCH 11/40] Fix armv7 build by manually downloading crates.io index --- core/base/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 721b5db1..6f6cb7a2 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -32,7 +32,8 @@ RUN set -euxo pipefail \ ; apk add --no-cache py3-pip \ ; python3 -m venv ${VIRTUAL_ENV} \ ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt \ - ; apk del -r py3-pip + ; apk del -r py3-pip \ + ; rm -f /tmp/*.pem ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" @@ -42,11 +43,12 @@ COPY libs/ libs/ RUN set -euxo pipefail \ ; machine="$(uname -m)"; deps="" \ ; [[ "${machine}" == arm* || "${machine}" == aarch64 ]] && deps="${deps} build-base gcc libffi-dev python3-dev" \ - ; [[ "${machine}" == armv7* ]] && deps="${deps} cargo libressl-dev mariadb-connector-c-dev postgresql-dev" \ + ; [[ "${machine}" == armv7* ]] && deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ ; [[ "${deps}" ]] && apk add --virtual .build-deps ${deps} \ + ; [[ "${machine}" == armv7* ]] && mkdir -p /root/.cargo/registry/index && git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ ; pip install -r requirements-${MAILU_ENV}.txt \ ; apk -e info -q .build-deps && apk del -r .build-deps \ - ; rm -rf /root/.cache /tmp/*.pem + ; rm -rf /root/.cargo /root/.cache /tmp/*.pem # base mailu image From 91f86a4c2ae64ea05e890862ad49f5d6e8094c89 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 31 Oct 2022 23:57:51 +0100 Subject: [PATCH 12/40] Resolve using socrate function --- core/admin/mailu/internal/nginx.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/core/admin/mailu/internal/nginx.py b/core/admin/mailu/internal/nginx.py index 870cc76d..43e4dd6a 100644 --- a/core/admin/mailu/internal/nginx.py +++ b/core/admin/mailu/internal/nginx.py @@ -1,12 +1,11 @@ from mailu import models, utils from flask import current_app as app +from socrate import system import re import urllib import ipaddress -import socket import sqlalchemy.exc -import tenacity SUPPORTED_AUTH_METHODS = ["none", "plain"] @@ -146,13 +145,5 @@ def get_server(protocol, authenticated=False): ipaddress.ip_address(hostname) except: # hostname is not an ip address - so we need to resolve it - hostname = resolve_hostname(hostname) + hostname = system.resolve_hostname(hostname) return hostname, port - -@tenacity.retry(stop=tenacity.stop_after_attempt(100), - wait=tenacity.wait_random(min=2, max=5)) -def resolve_hostname(hostname): - """ This function uses system DNS to resolve a hostname. - It is capable of retrying in case the host is not immediately available - """ - return socket.gethostbyname(hostname) From e915e444e95be5b702e345e7942cea4f1533405b Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 09:38:14 +0000 Subject: [PATCH 13/40] Remove superfluous cache export entry for arm --- .github/workflows/build_test_deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index e8bb1a0a..487470b1 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -253,7 +253,6 @@ jobs: *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm - *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max *.platform=${{ inputs.architecture }} # This job runs all the tests. From a2d43be6def679d5f9afbb90217bfd8248e201e4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Tue, 1 Nov 2022 11:02:21 +0100 Subject: [PATCH 14/40] Fix building wheels when deps need to compile --- core/base/Dockerfile | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 6f6cb7a2..a661877b 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -22,7 +22,7 @@ CMD /bin/bash # build virtual env (intermediate) FROM system as build -ARG MAILU_ENV=prod +ARG MAILU_DEPS=prod ENV VIRTUAL_ENV=/app/venv @@ -37,18 +37,25 @@ RUN set -euxo pipefail \ ENV PATH="${VIRTUAL_ENV}/bin:${PATH}" -COPY requirements-${MAILU_ENV}.txt ./ +COPY requirements-${MAILU_DEPS}.txt ./ COPY libs/ libs/ RUN set -euxo pipefail \ - ; machine="$(uname -m)"; deps="" \ - ; [[ "${machine}" == arm* || "${machine}" == aarch64 ]] && deps="${deps} build-base gcc libffi-dev python3-dev" \ - ; [[ "${machine}" == armv7* ]] && deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ - ; [[ "${deps}" ]] && apk add --virtual .build-deps ${deps} \ - ; [[ "${machine}" == armv7* ]] && mkdir -p /root/.cargo/registry/index && git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ - ; pip install -r requirements-${MAILU_ENV}.txt \ - ; apk -e info -q .build-deps && apk del -r .build-deps \ - ; rm -rf /root/.cargo /root/.cache /tmp/*.pem + ; pip install -r requirements-${MAILU_DEPS}.txt || \ + { \ + machine="$(uname -m)" \ + ; deps="build-base gcc libffi-dev python3-dev" \ + ; [[ "${machine}" == armv7 ]] && \ + deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ + ; apk add --virtual .build-deps ${deps} \ + ; [[ "${machine}" == armv7 ]] && \ + mkdir -p /root/.cargo/registry/index && \ + git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ + ; pip install -r requirements-${MAILU_DEPS}.txt \ + ; apk del -r .build-deps \ + ; rm -rf /root/.cargo /tmp/*.pem \ + ; } \ + ; rm -rf /root/.cache # base mailu image From c7cba1b07553570122cc835d39994393a3b4c344 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 10:49:44 +0000 Subject: [PATCH 15/40] Finishing touches for fixing arm builds - Use self-hosted runners for arm base image - Use seperate docker image cache for arm build - Remove unneeded needs items. --- .github/workflows/build_test_deploy.yml | 64 +++++++++++++++++++++---- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 487470b1..c01f67fa 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -89,9 +89,10 @@ jobs: run: | echo ${{ steps.targets.outputs.matrix }} -## This job buils the base image. The base image is used by all other images. - build-base-image: +## This job builds the base image. The base image is used by all other images. + build-base-image-x64: name: Build base image + if: inputs.architecture == 'linux/amd64' needs: - targets runs-on: ubuntu-latest @@ -139,14 +140,64 @@ jobs: *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }},mode=max *.platform=${{ inputs.architecture }} +## This job builds the base image. The base image is used by all other images. + build-base-image-arm: + name: Build base image + if: inputs.architecture != 'linux/amd64' + needs: + - targets + runs-on: self-hosted + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v3 + - name: Retrieve global variables + shell: bash + run: | + echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV + echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV + echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV + echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - uses: crazy-max/ghaction-github-runtime@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Helper to convert docker org to lowercase + id: string + uses: ASzc/change-string-case-action@v2 + with: + string: ${{ github.repository_owner }} + - name: Build all docker images + env: + DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }} + MAILU_VERSION: ${{ env.MAILU_VERSION }} + PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }} + uses: docker/bake-action@v2 + with: + files: ${{env.HCL_FILE}} + targets: base + load: false + push: false + set: | + *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm + *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max + *.platform=${{ inputs.architecture }} + # This job builds all the images. The build cache is stored in the github actions cache. # In further jobs, this cache is used to quickly rebuild the images. build: name: Build images for linux/amd64 if: inputs.architecture == 'linux/amd64' needs: - - targets - - build-base-image + - build-base-image-x64 strategy: fail-fast: false matrix: @@ -203,8 +254,7 @@ jobs: name: Build images for ARM64 & ARM/V7 if: inputs.architecture != 'linux/amd64' needs: - - targets - - build-base-image + - build-base-image-arm strategy: fail-fast: false matrix: @@ -264,7 +314,6 @@ jobs: contents: read packages: read needs: - - targets - build strategy: fail-fast: false @@ -319,7 +368,6 @@ jobs: if: inputs.deploy == 'true' runs-on: ubuntu-latest needs: - - build - tests strategy: fail-fast: false From 6549dbf24788e8ec949eac5bb22b57c8424ce6f4 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 10:56:33 +0000 Subject: [PATCH 16/40] Sigh. needs.* context is only available if you include it in needs: --- .github/workflows/build_test_deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index c01f67fa..f3977e1f 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -197,6 +197,7 @@ jobs: name: Build images for linux/amd64 if: inputs.architecture == 'linux/amd64' needs: + - targets - build-base-image-x64 strategy: fail-fast: false @@ -254,6 +255,7 @@ jobs: name: Build images for ARM64 & ARM/V7 if: inputs.architecture != 'linux/amd64' needs: + - targets - build-base-image-arm strategy: fail-fast: false From 5137b235e9df6d5a1a44b4d1a04f701e209b2cd8 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 1 Nov 2022 13:47:21 +0100 Subject: [PATCH 17/40] whitelist what we know works If other people use other arch and want their builds to go faster we can whitelist them too after they have confirmed it works. --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index a661877b..29223182 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -45,7 +45,7 @@ RUN set -euxo pipefail \ { \ machine="$(uname -m)" \ ; deps="build-base gcc libffi-dev python3-dev" \ - ; [[ "${machine}" == armv7 ]] && \ + ; [[ "${machine}" != x86_64 ]] && \ deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ ; apk add --virtual .build-deps ${deps} \ ; [[ "${machine}" == armv7 ]] && \ From ff9f152a529e47be7bd723dada8937f1ec1957e5 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Tue, 1 Nov 2022 14:11:59 +0100 Subject: [PATCH 18/40] This may be helpful too --- core/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/base/Dockerfile b/core/base/Dockerfile index 29223182..d5be6a90 100644 --- a/core/base/Dockerfile +++ b/core/base/Dockerfile @@ -48,7 +48,7 @@ RUN set -euxo pipefail \ ; [[ "${machine}" != x86_64 ]] && \ deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \ ; apk add --virtual .build-deps ${deps} \ - ; [[ "${machine}" == armv7 ]] && \ + ; [[ "${machine}" == armv7* ]] && \ mkdir -p /root/.cargo/registry/index && \ git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \ ; pip install -r requirements-${MAILU_DEPS}.txt \ From b3151e99047fd3b529117833eb4450ca81abb0e2 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 14:36:17 +0000 Subject: [PATCH 19/40] Actually push the build arm images to ghcr.io --- .github/workflows/build_test_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index f3977e1f..2f710f4e 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -300,7 +300,7 @@ jobs: files: ${{env.HCL_FILE}} targets: ${{ matrix.target }} load: false - push: false + push: true set: | *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max From db7ce8c83e083bd0a086dbc9855f344eae656306 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Tue, 1 Nov 2022 15:18:03 +0000 Subject: [PATCH 20/40] Login docker.io to prevent rate limiting for pulling images --- .github/workflows/build_test_deploy.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 2f710f4e..93e1214d 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -91,7 +91,7 @@ jobs: ## This job builds the base image. The base image is used by all other images. build-base-image-x64: - name: Build base image + name: Build base image x64 if: inputs.architecture == 'linux/amd64' needs: - targets @@ -119,6 +119,11 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v2 @@ -142,7 +147,7 @@ jobs: ## This job builds the base image. The base image is used by all other images. build-base-image-arm: - name: Build base image + name: Build base image arm if: inputs.architecture != 'linux/amd64' needs: - targets @@ -170,6 +175,11 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v2 @@ -227,6 +237,11 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v2 @@ -285,6 +300,11 @@ jobs: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.Docker_Login }} + password: ${{ secrets.Docker_Password }} - name: Helper to convert docker org to lowercase id: string uses: ASzc/change-string-case-action@v2 From d920b3d0370e9a54a24a02244c8de46e05bb38c3 Mon Sep 17 00:00:00 2001 From: wkr Date: Wed, 2 Nov 2022 17:48:22 +0100 Subject: [PATCH 21/40] fix(auto-reply): include start and end dates in the auto-reply period; issue #2512 --- core/admin/mailu/models.py | 4 ++-- towncrier/newsfragments/2512.bugfix | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 towncrier/newsfragments/2512.bugfix diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index f30ef387..48ce8b33 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -546,8 +546,8 @@ class User(Base, Email): now = date.today() return ( self.reply_enabled and - self.reply_startdate < now and - self.reply_enddate > now + self.reply_startdate <= now and + self.reply_enddate >= now ) @property diff --git a/towncrier/newsfragments/2512.bugfix b/towncrier/newsfragments/2512.bugfix new file mode 100644 index 00000000..b1b6aa99 --- /dev/null +++ b/towncrier/newsfragments/2512.bugfix @@ -0,0 +1 @@ +Fix: include start and end dates in the auto-reply period \ No newline at end of file From dec5309ef97ce0bc02dfb9b70f796c8a22223f84 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 16:39:29 +0100 Subject: [PATCH 22/40] Fix typo --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 783a14f4..7aeeeb6b 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -29,7 +29,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes="+quota[0]) + "quota_rule": "*:bytes="+quota[0] }) From bec0b1c3b221fc4771583f41ce6a7fc06350a511 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:26:27 +0100 Subject: [PATCH 23/40] Fix variable name --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 7aeeeb6b..2cb2d526 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,7 +27,7 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - quota = models.User.query.filter(models.User.email==email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ "quota_rule": "*:bytes="+quota[0] }) From 595b32cf97445e60f9b6fbad3528908b1c478b48 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:37:21 +0100 Subject: [PATCH 24/40] Fix quota return value --- core/admin/mailu/internal/views/dovecot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 2cb2d526..ab33b220 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -29,7 +29,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) return flask.jsonify({ - "quota_rule": "*:bytes="+quota[0] + "quota_rule": f"*:bytes={quota[0]}" }) From 46773f639b153d07d03f014483ff60cc3847cef8 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:45:21 +0100 Subject: [PATCH 25/40] Return 404 is user-id cannot be parsed --- core/admin/mailu/internal/views/dovecot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index ab33b220..3df3839e 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -27,7 +27,10 @@ def dovecot_userdb_dict_list(): @internal.route("/dovecot/userdb/") def dovecot_userdb_dict(user_email): - quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + try: + quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) + except ValueError: + flask.abort(404) return flask.jsonify({ "quota_rule": f"*:bytes={quota[0]}" }) From c57706ad279cc32da3e7f2140d6f3cc67d84cb3a Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Thu, 3 Nov 2022 17:50:39 +0100 Subject: [PATCH 26/40] Duh --- core/admin/mailu/internal/views/dovecot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/mailu/internal/views/dovecot.py b/core/admin/mailu/internal/views/dovecot.py index 3df3839e..07fce5b2 100644 --- a/core/admin/mailu/internal/views/dovecot.py +++ b/core/admin/mailu/internal/views/dovecot.py @@ -5,6 +5,7 @@ from flask import current_app as app import flask import socket import os +import sqlalchemy.exc @internal.route("/dovecot/passdb/") def dovecot_passdb_dict(user_email): @@ -29,7 +30,7 @@ def dovecot_userdb_dict_list(): def dovecot_userdb_dict(user_email): try: quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404) - except ValueError: + except sqlalchemy.exc.StatementError as exc: flask.abort(404) return flask.jsonify({ "quota_rule": f"*:bytes={quota[0]}" From 3e9def6cd9530843e325ac84b130b27c6732911e Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 4 Nov 2022 10:46:45 +0100 Subject: [PATCH 27/40] Use the new notation: arm64/v8 instead of arm64 --- .github/workflows/arm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/arm.yml b/.github/workflows/arm.yml index 7125737c..1240c630 100644 --- a/.github/workflows/arm.yml +++ b/.github/workflows/arm.yml @@ -76,7 +76,7 @@ jobs: - derive-variables uses: ./.github/workflows/build_test_deploy.yml with: - architecture: 'linux/arm64,linux/arm/v7' + architecture: 'linux/arm64/v8,linux/arm/v7' mailu_version: ${{needs.derive-variables.outputs.MAILU_VERSION}}-arm pinned_mailu_version: ${{needs.derive-variables.outputs.PINNED_MAILU_VERSION}}-arm docker_org: ${{needs.derive-variables.outputs.DOCKER_ORG}} From b2e47642f7b547e76a9bc8363a63027b5ba0ce26 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 4 Nov 2022 13:49:05 +0100 Subject: [PATCH 28/40] Tag the images with latest tag as well. --- .github/workflows/build_test_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 93e1214d..3e883432 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -470,6 +470,7 @@ jobs: docker buildx imagetools create \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \ ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} #This job creates a tagged release. A tag is created for the pinned version x.y.z. The GH release refers to this tag. From 2a3266b6b8061e5a1a070078319798a11e3e1fb3 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 4 Nov 2022 14:13:06 +0100 Subject: [PATCH 29/40] Forgot to update both deploy jobs --- .github/workflows/build_test_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index 3e883432..d1395bec 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -426,6 +426,7 @@ jobs: docker buildx imagetools create \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \ --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \ + --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \ ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} deploy-arm: From 8732b70b309de278478b0c9a7393d3b12f4a3c78 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 15:15:19 +0100 Subject: [PATCH 30/40] Add shell script to run admin dev environment --- core/admin/.gitignore | 1 + core/admin/mailu/__init__.py | 6 ++- core/admin/mailu/configuration.py | 2 + core/admin/run_dev.sh | 83 +++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100755 core/admin/run_dev.sh diff --git a/core/admin/.gitignore b/core/admin/.gitignore index 5bb3bd8e..10ac2b76 100644 --- a/core/admin/.gitignore +++ b/core/admin/.gitignore @@ -2,3 +2,4 @@ lib64 .vscode tags +dev diff --git a/core/admin/mailu/__init__.py b/core/admin/mailu/__init__.py index fe1f376c..5bb40447 100644 --- a/core/admin/mailu/__init__.py +++ b/core/admin/mailu/__init__.py @@ -44,8 +44,10 @@ def create_app_from_config(config): # Initialize debugging tools if app.config.get("DEBUG"): debug.toolbar.init_app(app) - # TODO: add a specific configuration variable for profiling - # debug.profiler.init_app(app) + if app.config.get("DEBUG_PROFILER"): + debug.profiler.init_app(app) + if assets := app.config.get('DEBUG_ASSETS'): + app.static_folder = assets # Inject the default variables in the Jinja parser # TODO: move this to blueprints when needed diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 081c4ee0..a4c8c537 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -13,6 +13,8 @@ DEFAULT_CONFIG = { 'RATELIMIT_STORAGE_URL': '', 'QUOTA_STORAGE_URL': '', 'DEBUG': False, + 'DEBUG_PROFILER': False, + 'DEBUG_ASSETS': '', 'DOMAIN_REGISTRATION': False, 'TEMPLATES_AUTO_RELOAD': True, 'MEMORY_SESSIONS': False, diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh new file mode 100755 index 00000000..74bce531 --- /dev/null +++ b/core/admin/run_dev.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +### CONFIG + +DEV_PORT="${DEV_PORT:-8080}" +DEV_NAME="${DEV_NAME:-mailu-dev}" +DEV_PROFILE="${DEV_PROFILE:-false}" + + +### MAIN + +here="$(realpath "$(pwd)/${0%/*}")" +cd "${here}" + +docker="$(command -v podman || command -v docker || echo echo docker)" + +[[ -d dev ]] && rm -rf dev +mkdir -p dev/data || exit 1 + +# base +cp ../base/requirements-* dev/ +cp -r ../base/libs dev/ +sed -E '/^#/d;s:^FROM system$:FROM system AS base:' ../base/Dockerfile > dev/Dockerfile + +# assets +cp -r assets/content dev/ +sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' assets/Dockerfile >> dev/Dockerfile + +cat >> dev/Dockerfile <> dev/Dockerfile + +cat >> dev/Dockerfile < Date: Fri, 4 Nov 2022 18:29:45 +0100 Subject: [PATCH 31/40] Improve dev runner --- core/admin/run_dev.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 74bce531..fc553cc4 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -11,11 +11,16 @@ DEV_PROFILE="${DEV_PROFILE:-false}" ### MAIN +docker="$(command -v podman || command -v docker || echo false)" +[[ "${docker}" == "false" ]] && { + echo "Sorry, you'll need podman or docker to run this." + exit 1 +} + here="$(realpath "$(pwd)/${0%/*}")" cd "${here}" -docker="$(command -v podman || command -v docker || echo echo docker)" - +# TODO: use /tmp/... folder [[ -d dev ]] && rm -rf dev mkdir -p dev/data || exit 1 @@ -36,6 +41,12 @@ EOF # admin sed -E '/^#/d;/^(COPY|EXPOSE|HEALTHCHECK|VOLUME|CMD) /d; s:^(.* )[^ ]*pybabel[^\\]*(.*):\1true \2:' Dockerfile >> dev/Dockerfile +DEV_URI="http://" +[[ "${DEV_PORT}" == *:* ]] || DEV_URI="${DEV_URI}localhost:" +DEV_URI="${DEV_URI}${DEV_PORT}/admin/ui/" + +MSG="\\n======================================================================\\nUI is found here: ${DEV_URI}\\nLog in with: admin@example.com and password admin if this is a new DB.\\n======================================================================\\n" + cat >> dev/Dockerfile </dev/null && flask mailu admin admin example.com admin --mode ifmissing >/dev/null && echo -e '${MSG}' 1>&2 && flask run --host=0.0.0.0 --port=8080"] EOF # TODO: re-compile assets on change? # TODO: re-run babel on change? # build -${docker} build --tag "${DEV_NAME}:latest" dev/ +chmod -R u+rwX,go+rX dev/ +"${docker}" build --tag "${DEV_NAME}:latest" dev/ # run args=( --rm -it --name "${DEV_NAME}" --publish "${DEV_PORT}:8080" --volume "${here}/dev/data/:/data/" ) @@ -78,6 +91,6 @@ for file in "${here}"/assets/content/assets/*; do args+=( --volume "${file}:/app/static/${file/*\//}" ) done -${docker} run "${args[@]}" "${DEV_NAME}" +"${docker}" run "${args[@]}" "${DEV_NAME}" # TODO: remove dev folder? From 1d90dc3ea382ec9f45f7d06a38ef905141f53ba4 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 18:54:59 +0100 Subject: [PATCH 32/40] Allow running without redis --- core/admin/mailu/configuration.py | 5 +++-- core/admin/run_dev.sh | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index a4c8c537..e18eb0b2 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -151,8 +151,9 @@ class ConfigManager: template = self.DB_TEMPLATES[self.config['DB_FLAVOR']] self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config) - self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' - self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' +# remove? self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' + if not self.config.get('RATELIMIT_STORAGE_URL'): + self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3' self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index fc553cc4..62919f6f 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -56,6 +56,7 @@ RUN set -euxo pipefail \ ENV FLASK_ENV=development ENV MEMORY_SESSIONS=true +ENV RATELIMIT_STORAGE_URL="memory://" ENV SESSION_COOKIE_SECURE=false ENV DEBUG=true From bbeb211d7262ea52f4501facb4c7b9d7fa0e1970 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 21:41:31 +0100 Subject: [PATCH 33/40] Listen to localhost by default --- core/admin/run_dev.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 62919f6f..03003348 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -4,10 +4,10 @@ set -euxo pipefail ### CONFIG -DEV_PORT="${DEV_PORT:-8080}" DEV_NAME="${DEV_NAME:-mailu-dev}" -DEV_PROFILE="${DEV_PROFILE:-false}" - +DEV_PROFILER="${DEV_PROFILER:-false}" +DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}" +[[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}" ### MAIN @@ -41,11 +41,7 @@ EOF # admin sed -E '/^#/d;/^(COPY|EXPOSE|HEALTHCHECK|VOLUME|CMD) /d; s:^(.* )[^ ]*pybabel[^\\]*(.*):\1true \2:' Dockerfile >> dev/Dockerfile -DEV_URI="http://" -[[ "${DEV_PORT}" == *:* ]] || DEV_URI="${DEV_URI}localhost:" -DEV_URI="${DEV_URI}${DEV_PORT}/admin/ui/" - -MSG="\\n======================================================================\\nUI is found here: ${DEV_URI}\\nLog in with: admin@example.com and password admin if this is a new DB.\\n======================================================================\\n" +MSG="\\n======================================================================\\nUI can be found here: http://${DEV_LISTEN}/sso/login\\nLog in with: admin@example.com and password admin if this is a new DB.\\n======================================================================\\n" cat >> dev/Dockerfile < Date: Fri, 4 Nov 2022 22:20:08 +0100 Subject: [PATCH 34/40] Remove unused QUOTA_STORAGE_URL --- core/admin/mailu/configuration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index e18eb0b2..b941e95c 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -11,7 +11,6 @@ DEFAULT_CONFIG = { 'BABEL_DEFAULT_TIMEZONE': 'UTC', 'BOOTSTRAP_SERVE_LOCAL': True, 'RATELIMIT_STORAGE_URL': '', - 'QUOTA_STORAGE_URL': '', 'DEBUG': False, 'DEBUG_PROFILER': False, 'DEBUG_ASSETS': '', @@ -151,9 +150,9 @@ class ConfigManager: template = self.DB_TEMPLATES[self.config['DB_FLAVOR']] self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config) -# remove? self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1' if not self.config.get('RATELIMIT_STORAGE_URL'): self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2' + self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3' self.config['SESSION_COOKIE_SAMESITE'] = 'Strict' self.config['SESSION_COOKIE_HTTPONLY'] = True @@ -162,9 +161,9 @@ class ConfigManager: self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME']) self.config['AUTH_RATELIMIT_IP_V4_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V4_MASK']) self.config['AUTH_RATELIMIT_IP_V6_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V6_MASK']) - hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')] self.config['AUTH_RATELIMIT_EXEMPTION'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['AUTH_RATELIMIT_EXEMPTION'].split(',')) if cidr) self.config['MESSAGE_RATELIMIT_EXEMPTION'] = set([s for s in self.config['MESSAGE_RATELIMIT_EXEMPTION'].lower().replace(' ', '').split(',') if s]) + hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')] self.config['HOSTNAMES'] = ','.join(hostnames) self.config['HOSTNAME'] = hostnames[0] self.config['DEFAULT_SPAM_THRESHOLD'] = int(self.config['DEFAULT_SPAM_THRESHOLD']) From 71263f1a8cd687cfad4b29f75f9f5d5aa03b62db Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 23:21:11 +0100 Subject: [PATCH 35/40] Add more env variables and restyle code --- core/admin/run_dev.sh | 154 +++++++++++++++++++++++++++--------------- 1 file changed, 100 insertions(+), 54 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 03003348..a2e32912 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -1,93 +1,139 @@ #!/usr/bin/env bash -set -euxo pipefail +set -euo pipefail ### CONFIG DEV_NAME="${DEV_NAME:-mailu-dev}" +DEV_DB="${DEV_DB:-}" DEV_PROFILER="${DEV_PROFILER:-false}" DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}" [[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}" +DEV_ADMIN="${DEV_ADMIN:-admin@example.com}" +DEV_PASSWORD="${DEV_PASSWORD:-admin}" ### MAIN -docker="$(command -v podman || command -v docker || echo false)" -[[ "${docker}" == "false" ]] && { - echo "Sorry, you'll need podman or docker to run this." - exit 1 +[[ -n "${DEV_DB}" ]] && { + [[ -f "${DEV_DB}" ]] || { + echo "Sorry, can't find DEV_DB: '${DEV_DB}'" + exit 1 + } + DEV_DB="$(realpath "${DEV_DB}")" } -here="$(realpath "$(pwd)/${0%/*}")" -cd "${here}" +docker="$(command -v podman || command -v docker || echo false)" +[[ "${docker}" == "false" ]] && { + echo "Sorry, you'll need podman or docker to run this." + exit 1 +} -# TODO: use /tmp/... folder -[[ -d dev ]] && rm -rf dev -mkdir -p dev/data || exit 1 +tmp="$(mktemp -d)" +[[ -n "${tmp}" && -d "${tmp}" ]] || { + echo "Sorry, can't create temporary folder." + exit 1 +} +trap "rm -rf '${tmp}'" INT TERM EXIT + +admin="$(realpath "$(pwd)/${0%/*}")" +base="${admin}/../base" +assets="${admin}/assets" + +cd "${tmp}" # base -cp ../base/requirements-* dev/ -cp -r ../base/libs dev/ -sed -E '/^#/d;s:^FROM system$:FROM system AS base:' ../base/Dockerfile > dev/Dockerfile +cp "${base}"/requirements-* . +cp -r "${base}"/libs . +sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Dockerfile # assets -cp -r assets/content dev/ -sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' assets/Dockerfile >> dev/Dockerfile +cp -r "${assets}/content" . +sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile -cat >> dev/Dockerfile <>Dockerfile <> dev/Dockerfile +sed -E '/^#/d;/^(COPY|EXPOSE|HEALTHCHECK|VOLUME|CMD) /d; s:^(.* )[^ ]*pybabel[^\\]*(.*):\1true \2:' "${admin}/Dockerfile" >>Dockerfile -MSG="\\n======================================================================\\nUI can be found here: http://${DEV_LISTEN}/sso/login\\nLog in with: admin@example.com and password admin if this is a new DB.\\n======================================================================\\n" - -cat >> dev/Dockerfile <>Dockerfile </dev/null && flask mailu admin admin example.com admin --mode ifmissing >/dev/null && echo -e '${MSG}' 1>&2 && flask run --host=0.0.0.0 --port=8080"] +CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null && flask run --host=0.0.0.0 --port=8080"] EOF -# TODO: re-compile assets on change? -# TODO: re-run babel on change? - # build -chmod -R u+rwX,go+rX dev/ -"${docker}" build --tag "${DEV_NAME}:latest" dev/ +chmod -R u+rwX,go+rX . +cat Dockerfile +"${docker}" build --tag "${DEV_NAME}:latest" . + +# gather volumes to map into container +volumes=() + +if [[ -n "${DEV_DB}" ]]; then + volumes+=( --volume "${DEV_DB}:/data/main.db" ) +else + mkdir -p "data" + volumes=( --volume "${tmp}/data/:/data/" ) +fi + +for vol in audit.py start.py mailu/ migrations/; do + volumes+=( --volume "${admin}/${vol}:/app/${vol}" ) +done + +for file in "${assets}/content/assets"/*; do + [[ "${file}" == */vendor.js ]] && continue + volumes+=( --volume "${file}:/app/static/${file/*\//}" ) +done + +# show configuration +cat < Date: Fri, 4 Nov 2022 23:39:39 +0100 Subject: [PATCH 36/40] Speed up asset building when developing --- core/admin/assets/Dockerfile | 20 ++++++++++++------ .../admin/assets/{content => }/assets/app.css | 0 core/admin/assets/{content => }/assets/app.js | 0 .../assets/{content => }/assets/mailu.png | Bin .../assets/{content => }/assets/vendor.js | 0 core/admin/assets/{content => }/package.json | 0 .../assets/{content => }/webpack.config.js | 0 core/admin/run_dev.sh | 13 +++++------- 8 files changed, 18 insertions(+), 15 deletions(-) rename core/admin/assets/{content => }/assets/app.css (100%) rename core/admin/assets/{content => }/assets/app.js (100%) rename core/admin/assets/{content => }/assets/mailu.png (100%) rename core/admin/assets/{content => }/assets/vendor.js (100%) rename core/admin/assets/{content => }/package.json (100%) rename core/admin/assets/{content => }/webpack.config.js (100%) diff --git a/core/admin/assets/Dockerfile b/core/admin/assets/Dockerfile index c8556f47..613aa6c0 100644 --- a/core/admin/assets/Dockerfile +++ b/core/admin/assets/Dockerfile @@ -4,13 +4,19 @@ FROM node:16-alpine3.16 WORKDIR /work -COPY content/ ./ +COPY package.json ./ +COPY webpack.config.js ./ RUN set -euxo pipefail \ - && npm config set update-notifier false \ - && npm install --no-audit --no-fund \ - && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ - && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ + ; npm config set update-notifier false \ + ; npm install --no-audit --no-fund \ + ; sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \ + ; mkdir assets \ + ; for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \ cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \ - done \ - && node_modules/.bin/webpack-cli --color + done + +COPY assets/ ./assets/ + +RUN set -euxo pipefail \ + ; node_modules/.bin/webpack-cli --color diff --git a/core/admin/assets/content/assets/app.css b/core/admin/assets/assets/app.css similarity index 100% rename from core/admin/assets/content/assets/app.css rename to core/admin/assets/assets/app.css diff --git a/core/admin/assets/content/assets/app.js b/core/admin/assets/assets/app.js similarity index 100% rename from core/admin/assets/content/assets/app.js rename to core/admin/assets/assets/app.js diff --git a/core/admin/assets/content/assets/mailu.png b/core/admin/assets/assets/mailu.png similarity index 100% rename from core/admin/assets/content/assets/mailu.png rename to core/admin/assets/assets/mailu.png diff --git a/core/admin/assets/content/assets/vendor.js b/core/admin/assets/assets/vendor.js similarity index 100% rename from core/admin/assets/content/assets/vendor.js rename to core/admin/assets/assets/vendor.js diff --git a/core/admin/assets/content/package.json b/core/admin/assets/package.json similarity index 100% rename from core/admin/assets/content/package.json rename to core/admin/assets/package.json diff --git a/core/admin/assets/content/webpack.config.js b/core/admin/assets/webpack.config.js similarity index 100% rename from core/admin/assets/content/webpack.config.js rename to core/admin/assets/webpack.config.js diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index a2e32912..a682252c 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -47,14 +47,11 @@ cp -r "${base}"/libs . sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Dockerfile # assets -cp -r "${assets}/content" . +cp "${assets}/package.json" . +cp -r "${assets}/assets/" . +awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile -cat >>Dockerfile <>Dockerfile @@ -107,8 +104,8 @@ for vol in audit.py start.py mailu/ migrations/; do volumes+=( --volume "${admin}/${vol}:/app/${vol}" ) done -for file in "${assets}/content/assets"/*; do - [[ "${file}" == */vendor.js ]] && continue +for file in "${assets}/assets"/*; do + [[ ! -f "${file}" || "${file}" == */vendor.js ]] && continue volumes+=( --volume "${file}:/app/static/${file/*\//}" ) done From db87a0f3a101ba3b4aaed5237824e82f95fc14c1 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Fri, 4 Nov 2022 23:51:32 +0100 Subject: [PATCH 37/40] Move temporary db into container and show docker run command --- core/admin/run_dev.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index a682252c..4a0120c9 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -93,12 +93,7 @@ cat Dockerfile # gather volumes to map into container volumes=() -if [[ -n "${DEV_DB}" ]]; then - volumes+=( --volume "${DEV_DB}:/data/main.db" ) -else - mkdir -p "data" - volumes=( --volume "${tmp}/data/:/data/" ) -fi +[[ -n "${DEV_DB}" ]] && volumes+=( --volume "${DEV_DB}:/data/main.db" ) for vol in audit.py start.py mailu/ migrations/; do volumes+=( --volume "${admin}/${vol}:/app/${vol}" ) @@ -113,6 +108,8 @@ done cat < Date: Mon, 7 Nov 2022 16:35:01 +0100 Subject: [PATCH 38/40] Use default password used everywhere else --- core/admin/run_dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 4a0120c9..372c7390 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -10,7 +10,7 @@ DEV_PROFILER="${DEV_PROFILER:-false}" DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}" [[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}" DEV_ADMIN="${DEV_ADMIN:-admin@example.com}" -DEV_PASSWORD="${DEV_PASSWORD:-admin}" +DEV_PASSWORD="${DEV_PASSWORD:-letmein}" ### MAIN From dd3cd1263e76559ac1418252b0e797eb6beffd18 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 7 Nov 2022 16:47:13 +0100 Subject: [PATCH 39/40] Add development documentation again --- docs/contributors/environment.rst | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/contributors/environment.rst b/docs/contributors/environment.rst index 1b53afe6..5949fbe5 100644 --- a/docs/contributors/environment.rst +++ b/docs/contributors/environment.rst @@ -313,6 +313,48 @@ If git opens a editor for a commit message just save and exit as-is. If you have see above and do the complete procedure from ``git fetch`` onward again. +Web administration development +------------------------------ + +The administration web interface requires a proper dev environment that can easily +be setup using the ``run_dev.sh`` shell script. You need ``docker`` or ``podman`` +to run it. It will create a local webserver listening at port 8080: + +.. code-block:: bash + + cd core/admin + ./run_dev.sh + pip install -r requirements.txt + [...] + ============================================================================= + The "mailu-dev" container was built using this configuration: + + DEV_NAME="mailu-dev" + DEV_DB="" + DEV_PROFILER="false" + DEV_LISTEN="127.0.0.1:8080" + DEV_ADMIN="admin@example.com" + DEV_PASSWORD="letmein" + ============================================================================= + [...] + ============================================================================= + The Mailu UI can be found here: http://127.0.0.1:8080/sso/login + You can log in with user admin@example.com and password letmein + ============================================================================= + +The container will use an empty database and a default user/password unless you +specify a database file to use by setting ``$DEV_DB``. + +.. code-block:: bash + + DEV_DB="/path/to/dev.db" ./run_dev.sh + +Any change to the files will automatically restart the Web server and reload the files. + +When using the development environment, a debugging toolbar is displayed on the right +side of the screen, where you can access query details, internal variables, etc. + + Documentation ------------- From 36019a8ce9b5bad62f176fb9ab5cf0d3afc65298 Mon Sep 17 00:00:00 2001 From: Alexander Graf Date: Mon, 7 Nov 2022 16:48:58 +0100 Subject: [PATCH 40/40] Don't show Dockerfile before building --- core/admin/run_dev.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/core/admin/run_dev.sh b/core/admin/run_dev.sh index 372c7390..4ab76e74 100755 --- a/core/admin/run_dev.sh +++ b/core/admin/run_dev.sh @@ -87,7 +87,6 @@ EOF # build chmod -R u+rwX,go+rX . -cat Dockerfile "${docker}" build --tag "${DEV_NAME}:latest" . # gather volumes to map into container