From 19eda03a49dd27f11a3b46c6e37e9841a0b66a4c Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 19 Aug 2022 18:47:33 +0200 Subject: [PATCH 01/11] Build wheels only if we have to. --- core/admin/Dockerfile | 7 ++++--- core/admin/requirements-prod.txt | 8 +++++--- core/admin/requirements.txt | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index a9cc61c2..01be460b 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -36,11 +36,12 @@ WORKDIR /app COPY requirements-prod.txt requirements.txt RUN set -eu \ - && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ - && apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ && pip install --upgrade pip \ + && pip install --extra-index-url=https://www.piwheels.org/simple -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ + || ( apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ + && apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ && pip install -r requirements.txt \ - && apk del --no-cache build-dep + && apk del --no-cache build-dep ) COPY --from=assets static ./mailu/static COPY mailu ./mailu diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index f14e484c..297c6902 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -5,7 +5,7 @@ bcrypt==3.2.0 blinker==1.4 CacheControl==0.12.9 certifi==2021.10.8 -cffi==1.15.0 +# cffi==1.15.0 chardet==4.0.0 click==8.0.3 colorama==0.4.4 @@ -43,13 +43,15 @@ MarkupSafe==2.0.1 marshmallow==3.14.0 marshmallow-sqlalchemy==0.26.1 msgpack==1.0.2 -mysqlclient==2.0.3 +# mysqlclient==2.0.3 +mysql-connector-python==8.0.25 ordered-set==4.0.2 # packaging==20.9 passlib==1.7.4 # pep517==0.10.0 progress==1.6 -psycopg2==2.9.1 +#psycopg2==2.9.1 +psycopg2-binary==2.9.3 pycparser==2.20 Pygments==2.10.0 pyOpenSSL==21.0.0 diff --git a/core/admin/requirements.txt b/core/admin/requirements.txt index 61fbd92c..5bf57874 100644 --- a/core/admin/requirements.txt +++ b/core/admin/requirements.txt @@ -19,7 +19,7 @@ PyOpenSSL Pygments dnspython tenacity -mysqlclient +mysql-connector-python idna srslib marshmallow From f760024812e87dfcba04a9f72d02859bced0f6bc Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 19 Aug 2022 19:08:10 +0200 Subject: [PATCH 02/11] These are required for the healthcheck --- core/admin/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 01be460b..fad27574 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -36,9 +36,10 @@ WORKDIR /app COPY requirements-prod.txt requirements.txt RUN set -eu \ + && apk add --no-cache libressl curl \ && pip install --upgrade pip \ && pip install --extra-index-url=https://www.piwheels.org/simple -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ - || ( apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ + || ( apk add --no-cache postgresql-libs mariadb-connector-c \ && apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ && pip install -r requirements.txt \ && apk del --no-cache build-dep ) From e0643cf45ce9a4fb1448c2a25c8d86162b24e085 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 19 Aug 2022 19:23:29 +0200 Subject: [PATCH 03/11] Disable the cache; don't upgrade pip if not req --- core/admin/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index fad27574..5b02f07f 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -37,10 +37,10 @@ WORKDIR /app COPY requirements-prod.txt requirements.txt RUN set -eu \ && apk add --no-cache libressl curl \ - && pip install --upgrade pip \ - && pip install --extra-index-url=https://www.piwheels.org/simple -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ + && pip install --no-cache --extra-index-url=https://www.piwheels.org/simple -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ || ( apk add --no-cache postgresql-libs mariadb-connector-c \ && apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ + && pip install --upgrade pip \ && pip install -r requirements.txt \ && apk del --no-cache build-dep ) From bd5fd9536d31822f260530d2c90ce3f453f16012 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 19 Aug 2022 19:25:49 +0200 Subject: [PATCH 04/11] doh --- core/admin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 5b02f07f..05ab88ff 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -37,7 +37,7 @@ WORKDIR /app COPY requirements-prod.txt requirements.txt RUN set -eu \ && apk add --no-cache libressl curl \ - && pip install --no-cache --extra-index-url=https://www.piwheels.org/simple -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ + && pip install --no-cache-dir --extra-index-url=https://www.piwheels.org/simple -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ || ( apk add --no-cache postgresql-libs mariadb-connector-c \ && apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ && pip install --upgrade pip \ From 1500936232dc5bd6d89e173c684cd3fd7d6d05b2 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 19 Aug 2022 19:32:42 +0200 Subject: [PATCH 05/11] Some people will need this at runtime --- core/admin/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/admin/Dockerfile b/core/admin/Dockerfile index 05ab88ff..4d6aa75a 100644 --- a/core/admin/Dockerfile +++ b/core/admin/Dockerfile @@ -36,10 +36,9 @@ WORKDIR /app COPY requirements-prod.txt requirements.txt RUN set -eu \ - && apk add --no-cache libressl curl \ + && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \ && pip install --no-cache-dir --extra-index-url=https://www.piwheels.org/simple -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \ - || ( apk add --no-cache postgresql-libs mariadb-connector-c \ - && apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ + || ( apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \ && pip install --upgrade pip \ && pip install -r requirements.txt \ && apk del --no-cache build-dep ) From 630a23da3c2dc59baf07d97dfa189217e4fe7ad2 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 19 Aug 2022 20:04:45 +0200 Subject: [PATCH 06/11] Don't wait for the timeout in the tests If the containers are ready use them. --- tests/compose/test.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/compose/test.py b/tests/compose/test.py index 690855d4..4be823b4 100755 --- a/tests/compose/test.py +++ b/tests/compose/test.py @@ -1,6 +1,7 @@ import sys import os import time +import datetime import docker from colorama import Fore, Style import subprocess @@ -22,12 +23,7 @@ def stop(exit_code): print(subprocess.check_output("docker-compose -f " + compose_file + " down", shell=True).decode()) sys.exit(exit_code) -# Sleep for a defined amount of time -def sleep(): - print(Fore.LIGHTMAGENTA_EX + "Sleeping for " + str(timeout) + "m" + Style.RESET_ALL) - time.sleep(timeout*60) - -def health_checks(): +def health_checks(deadline): exit_code = 0 #Iterating trough all containers dictionary for container in client.containers(all=True): @@ -58,7 +54,9 @@ def health_checks(): #Adding the generated dictionary to a list containers.append(containers_dict) - if exit_code != 0: + if exit_code == 0: + return True + else if exit_code != 0 and deadline > datetime.now(): stop(exit_code) def print_logs(): @@ -86,14 +84,19 @@ def hooks(): # Start up containers sys.stdout.flush() +deadline=datetime.now()+datetime.timedelta(minutes=timeout) print(subprocess.check_output("docker-compose -f " + compose_file + " up -d", shell=True).decode()) print() -sleep() +print(Fore.LIGHTMAGENTA_EX + "Sleeping for 10s" + Style.RESET_ALL) +time.sleep(10) print() sys.stdout.flush() print(subprocess.check_output("docker ps -a", shell=True).decode()) print() -health_checks() +while not health_checks(deadline): + print(Fore.LIGHTMAGENTA_EX + "Sleeping for 10s" + Style.RESET_ALL) + sys.stdout.flush() + time.sleep(10) print() hooks() print() From 6fb554bc2e9144f986003d3be6c35e2ecc7fa2c6 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 19 Aug 2022 20:21:00 +0200 Subject: [PATCH 07/11] Do the same for the postfix container --- core/postfix/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/postfix/Dockerfile b/core/postfix/Dockerfile index f9285ebf..b048a15e 100644 --- a/core/postfix/Dockerfile +++ b/core/postfix/Dockerfile @@ -22,9 +22,9 @@ RUN apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev \ # Image specific layers under this line # Building pycares from source requires py3-wheel and libffi-dev packages -RUN apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev py3-wheel libffi-dev \ - && pip3 install --no-binary :all: postfix-mta-sts-resolver==1.0.1 \ - && apk del .build-deps +RUN pip install --no-cache-dir --extra-index-url=https://www.piwheels.org/simple --only-binary=:all: postfix-mta-sts-resolver==1.0.1 || (apk add --no-cache --virtual .build-deps gcc musl-dev python3-dev py3-wheel libffi-dev \ + && pip3 install postfix-mta-sts-resolver==1.0.1 \ + && apk del .build-deps ) RUN apk add --no-cache postfix postfix-pcre cyrus-sasl-login rsyslog logrotate @@ -37,4 +37,4 @@ VOLUME ["/queue"] CMD /start.py HEALTHCHECK --start-period=350s CMD echo QUIT|nc localhost 25|grep "220 .* ESMTP Postfix" -RUN echo $VERSION >> /version \ No newline at end of file +RUN echo $VERSION >> /version From 72b8939ad7c43b3adab6cd8d322a163cb9a17859 Mon Sep 17 00:00:00 2001 From: Florent Daigniere Date: Fri, 19 Aug 2022 20:25:34 +0200 Subject: [PATCH 08/11] doh --- tests/compose/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compose/test.py b/tests/compose/test.py index 4be823b4..50e9b9ad 100755 --- a/tests/compose/test.py +++ b/tests/compose/test.py @@ -56,7 +56,7 @@ def health_checks(deadline): if exit_code == 0: return True - else if exit_code != 0 and deadline > datetime.now(): + elif exit_code != 0 and deadline > datetime.now(): stop(exit_code) def print_logs(): From 9339ce78e83c156c292918eecd75ce7243e1b68f Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 19 Aug 2022 18:57:58 +0000 Subject: [PATCH 09/11] Fix datetime usage in test.py --- tests/compose/test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/compose/test.py b/tests/compose/test.py index 50e9b9ad..0b63e522 100755 --- a/tests/compose/test.py +++ b/tests/compose/test.py @@ -2,6 +2,7 @@ import sys import os import time import datetime +from xmlrpc.client import DateTime import docker from colorama import Fore, Style import subprocess @@ -56,7 +57,7 @@ def health_checks(deadline): if exit_code == 0: return True - elif exit_code != 0 and deadline > datetime.now(): + elif exit_code != 0 and deadline > datetime.datetime.now(): stop(exit_code) def print_logs(): @@ -84,7 +85,7 @@ def hooks(): # Start up containers sys.stdout.flush() -deadline=datetime.now()+datetime.timedelta(minutes=timeout) +deadline=datetime.datetime.now()+datetime.timedelta(minutes=timeout) print(subprocess.check_output("docker-compose -f " + compose_file + " up -d", shell=True).decode()) print() print(Fore.LIGHTMAGENTA_EX + "Sleeping for 10s" + Style.RESET_ALL) From 867c71ca83b67656afe89a6fe1056b4403fa4a40 Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 19 Aug 2022 20:24:07 +0000 Subject: [PATCH 10/11] Fix date (deadline) calculation and comparison in test.py --- tests/compose/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/compose/test.py b/tests/compose/test.py index 0b63e522..7ac16d73 100755 --- a/tests/compose/test.py +++ b/tests/compose/test.py @@ -6,6 +6,7 @@ from xmlrpc.client import DateTime import docker from colorama import Fore, Style import subprocess +import calendar # Declare variables for service name and sleep time test_name=sys.argv[1] @@ -57,7 +58,7 @@ def health_checks(deadline): if exit_code == 0: return True - elif exit_code != 0 and deadline > datetime.datetime.now(): + elif exit_code != 0 and deadline > datetime.datetime.now().timestamp(): stop(exit_code) def print_logs(): @@ -86,6 +87,7 @@ def hooks(): # Start up containers sys.stdout.flush() deadline=datetime.datetime.now()+datetime.timedelta(minutes=timeout) +deadline=calendar.timegm(deadline.timetuple()) print(subprocess.check_output("docker-compose -f " + compose_file + " up -d", shell=True).decode()) print() print(Fore.LIGHTMAGENTA_EX + "Sleeping for 10s" + Style.RESET_ALL) From 3493e9ffa94fa652a01aa327a964d98e93e884fa Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Fri, 19 Aug 2022 20:32:53 +0000 Subject: [PATCH 11/11] Doh! --- tests/compose/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compose/test.py b/tests/compose/test.py index 7ac16d73..02575535 100755 --- a/tests/compose/test.py +++ b/tests/compose/test.py @@ -58,7 +58,7 @@ def health_checks(deadline): if exit_code == 0: return True - elif exit_code != 0 and deadline > datetime.datetime.now().timestamp(): + elif exit_code != 0 and deadline < datetime.datetime.now().timestamp(): stop(exit_code) def print_logs():