From 709edb522b85e46fcb8cd8e14e367c1d69bc161a Mon Sep 17 00:00:00 2001 From: Dimitri Huisman Date: Sun, 26 Mar 2023 12:21:00 +0000 Subject: [PATCH] Introduce connection string (database url) for roundcube. Remove database choice from setup. Remove the old *DB_* database env variables from the documentation. The env vars are deprecated now. They will be removed after the upcoming Mailu release. --- .github/workflows/build_test_deploy.yml | 4 +++ core/admin/mailu/configuration.py | 1 + core/base/requirements-prod.txt | 2 +- docs/configuration.rst | 26 ++++++-------- docs/database.rst | 19 +++++------ setup/flavors/compose/mailu.env | 18 ---------- setup/templates/steps/database.html | 45 ------------------------- setup/templates/wizard.html | 1 - towncrier/newsfragments/2533.misc | 17 ++++++++++ webmails/start.py | 5 ++- 10 files changed, 47 insertions(+), 91 deletions(-) delete mode 100644 setup/templates/steps/database.html create mode 100644 towncrier/newsfragments/2533.misc diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index a30d0d2b..9ec997b6 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -154,6 +154,7 @@ jobs: shell: bash command: | set -euxo pipefail \ + ; /usr/bin/docker info \ ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ ; echo "$DOCKER_PASSW" | docker login --username "$DOCKER_LOGIN" --password-stdin \ ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ @@ -220,6 +221,7 @@ jobs: shell: bash command: | set -euxo pipefail \ + ; /usr/bin/docker info \ ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ ; echo "$DOCKER_PASSW2" | docker login --username "$DOCKER_LOGIN2" --password-stdin \ ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ @@ -295,6 +297,7 @@ jobs: shell: bash command: | set -euxo pipefail \ + ; /usr/bin/docker info \ ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ ; echo "$DOCKER_PASSW" | docker login --username "$DOCKER_LOGIN" --password-stdin \ ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ @@ -368,6 +371,7 @@ jobs: shell: bash command: | set -euxo pipefail \ + ; /usr/bin/docker info \ ; echo "${{ github.token }}" | docker login --username "${{ github.repository_owner }}" --password-stdin ghcr.io \ ; echo "$DOCKER_PASSW2" | docker login --username "$DOCKER_LOGIN2" --password-stdin \ ; /usr/bin/docker buildx rm builder-${{ env.BUILDER }} \ diff --git a/core/admin/mailu/configuration.py b/core/admin/mailu/configuration.py index 4f3f8e84..3adf9a6c 100644 --- a/core/admin/mailu/configuration.py +++ b/core/admin/mailu/configuration.py @@ -27,6 +27,7 @@ DEFAULT_CONFIG = { 'DB_NAME': 'mailu', 'SQLITE_DATABASE_FILE': 'data/main.db', 'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db', + 'SQLALCHEMY_DATABASE_URI_ROUNDCUBE': 'sqlite:////data/roundcube.db', 'SQLALCHEMY_TRACK_MODIFICATIONS': False, # Statistics management 'INSTANCE_ID_PATH': '/data/instance', diff --git a/core/base/requirements-prod.txt b/core/base/requirements-prod.txt index 5119520e..33739ee1 100644 --- a/core/base/requirements-prod.txt +++ b/core/base/requirements-prod.txt @@ -45,7 +45,7 @@ marshmallow==3.18.0 marshmallow-sqlalchemy==0.28.1 msoffcrypto-tool==5.0.0 multidict==6.0.2 -mysql-connector-python==8.0.29 +mysql-connector-python==8.0.32 olefile==0.46 oletools==0.60.1 packaging==21.3 diff --git a/docs/configuration.rst b/docs/configuration.rst index 645cd85d..af085f87 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -285,26 +285,22 @@ These are used for DNS based service discovery with possibly changing services I Database settings ----------------- +The admin service stores configurations in a database. The roundcube service also stores configurations in a database. +By default SQLite is used. SQLite is sufficient for practically any deployment. +The database contains static configuration. It is recommended to use SQLite. However, a different database back-end can be used via a +`DB URL`_. For Admin, only supported are SQLite, PostgreSQL and MariaDB/MySQL. -The admin service stores configurations in a database. +- ``SQLALCHEMY_DATABASE_URI`` (default: sqlite:////data/main.db): the SQLAlchemy database URL for accessing the database +- ``SQLALCHEMY_DATABASE_URI_ROUNDCUBE`` (default: sqlite:////data/roundcube.db): the Roundcube database URL for accessing the Roundcube database -- ``DB_FLAVOR``: the database type for mailu admin service. (``sqlite``, ``postgresql``, ``mysql``) -- ``DB_HOST``: the database host for mailu admin service. For non-default ports use the notation `host:port`. (when not ``sqlite``) -- ``DB_PW``: the database password for mailu admin service. (when not ``sqlite``) -- ``DB_USER``: the database user for mailu admin service. (when not ``sqlite``) -- ``DB_NAME``: the database name for mailu admin service. (when not ``sqlite``) +For PostgreSQL use driver postgresql (``SQLALCHEMY_DATABASE_URI=postgresql://mailu:mailu_secret_password@database/mailu``). -Alternatively, if you need more control, you can use a `DB URL`_ : do not set any of the ``DB_`` settings and set ``SQLALCHEMY_DATABASE_URI`` instead. +For MariaDB/MySQL use driver mysql+mysqlconnector (``SQLALCHEMY_DATABASE_URI= mysql+mysqlconnector://mailu:mailu_secret_password@database/mailu```). + +For Roundcube, refer to the `roundcube documentation`_ for the URL specification. .. _`DB URL`: https://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls - -The roundcube service stores configurations in a database. - -- ``ROUNDCUBE_DB_FLAVOR``: the database type for roundcube service. (``sqlite``, ``postgresql``, ``mysql``) -- ``ROUNDCUBE_DB_HOST``: the database host for roundcube service. For non-default ports use the notation `host:port`. (when not ``sqlite``) -- ``ROUNDCUBE_DB_PW``: the database password for roundcube service. (when not ``sqlite``) -- ``ROUNDCUBE_DB_USER``: the database user for roundcube service. (when not ``sqlite``) -- ``ROUNDCUBE_DB_NAME``: the database name for roundcube service. (when not ``sqlite``) +.. _`roundcube documentation`: https://github.com/roundcube/roundcubemail/blob/master/config/defaults.inc.php#L28 Webmail settings ---------------- diff --git a/docs/database.rst b/docs/database.rst index 31abc191..7163acaf 100644 --- a/docs/database.rst +++ b/docs/database.rst @@ -1,8 +1,10 @@ Changing the database back-end ============================== -By default Mailu uses a SQLite database. We have changed the internals of Mailu -to enable the support of alternative database solutions such as PostgreSQL and MySQL/MariaDB. +By default Mailu uses a SQLite database. It is possible to use an alternative database solutions such as PostgreSQL and MySQL/MariaDB. + +The Mailu database contains static data. SQLite is sufficient for any deployment scenario of Mailu. There is no need to use a different database system. +The Mailu development team recommends to use SQLite. Migrating to a different database back-end @@ -17,7 +19,7 @@ To switch to a different database back-end: 1. Run config-export to export the configuration. E.g. `docker compose exec admin flask mailu config-export --secrets --output mail-config.yml` 2. Set up your new database server. Refer to the subsequent sections for tips for creating the database. -3. Modify the database settings (DB_*) in mailu.env. Refer to the :ref:`configuration guide (link) ` for the exact settings. +3. Modify the database settings (SQLAlchemy database URL) in mailu.env. Refer to the :ref:`configuration guide (link) ` for the exact settings. 4. Start your Mailu deployment. 5. Run config-import to import the configuration. E.g. `docker exec -i $(docker compose ps -q admin) flask mailu config-import -v < mail-config.yml` @@ -80,7 +82,7 @@ In ``pg_hba.conf`` there should be a line like this: host mailu mailu /32 md5 -Note that this example is the bare-minimum to get Mailu working. Additional work needs to be +Note that this example is the bare-minimum to get Mailu working. Additional work needs to be done by the database admin to setup their own means of backups and TLS encrypted connections. Nowadays it is recommended to use the official PostgreSQL image from the PostgreSQL community. The repository is located `here `_. @@ -91,10 +93,10 @@ Mailu PostgreSQL ---------------- Mailu optionally came with a pre-configured PostgreSQL image which was deprecated in Mailu 1.8. -Since Mailu 1.9 it is removed from Mailu. The following section describes how to move to a different +Since Mailu 1.9 it is removed from Mailu. The following section describes how to move to a different PostgreSQL image for novice administrators. The official PostgreSQL image (Postgres) will be used. -A Mailu deployment with the Mailu PostgreSQL image, will only use PostgreSQL for the Admin container +A Mailu deployment with the Mailu PostgreSQL image, will only use PostgreSQL for the Admin container (Web administration interface). Roundcube uses SQLite as database back-end. Mailu uses the following configuration for connecting to the database: @@ -192,10 +194,7 @@ to .. code-block:: docker - DB_HOST=database - DB_PORT=5432 - DB_USER=mailu - DB_NAME=mailu + SQLALCHEMY_DATABASE_URI=postgresql://mailu:mailu@database/mailu Mailu is now configured to use the official PostgreSQL docker image. Bring your new deployment online diff --git a/setup/flavors/compose/mailu.env b/setup/flavors/compose/mailu.env index e6b40a61..dce1cb27 100644 --- a/setup/flavors/compose/mailu.env +++ b/setup/flavors/compose/mailu.env @@ -192,21 +192,3 @@ DEFAULT_SPAM_THRESHOLD=80 # This is a mandatory setting for using the RESTful API. API_TOKEN={{ api_token }} -################################### -# Database settings -################################### -DB_FLAVOR={{ db_flavor }} -{% if db_flavor == 'postgresql' or db_flavor == 'mysql' %} -DB_USER={{ db_user }} -DB_PW={{ db_pw }} -DB_HOST={{ db_url }} -DB_NAME={{ db_name }} -{% endif %} - -{% if ((db_flavor_rc == 'postgresql' or db_flavor_rc == 'mysql')) and webmail_type == 'roundcube' %} -ROUNDCUBE_DB_FLAVOR={{ db_flavor_rc }} -ROUNDCUBE_DB_USER={{ roundcube_db_user }} -ROUNDCUBE_DB_PW={{ roundcube_db_pw }} -ROUNDCUBE_DB_HOST={{ roundcube_db_url }} -ROUNDCUBE_DB_NAME={{ roundcube_db_name }} -{% endif %} diff --git a/setup/templates/steps/database.html b/setup/templates/steps/database.html deleted file mode 100644 index 03cb50b1..00000000 --- a/setup/templates/steps/database.html +++ /dev/null @@ -1,45 +0,0 @@ -{% call macros.panel("info", "Database preferences") %} - -
- -
- - -
- -{% endcall %} diff --git a/setup/templates/wizard.html b/setup/templates/wizard.html index c98672a6..74bbc432 100644 --- a/setup/templates/wizard.html +++ b/setup/templates/wizard.html @@ -14,7 +14,6 @@ {%for file in steps %} {% include "steps/" + flavor + "/" + file %} {% endfor %} - {% include "steps/database.html" %} {% endblock %} diff --git a/towncrier/newsfragments/2533.misc b/towncrier/newsfragments/2533.misc new file mode 100644 index 00000000..af1b88a2 --- /dev/null +++ b/towncrier/newsfragments/2533.misc @@ -0,0 +1,17 @@ +Introduce SQLAlchemy database uris for configuring the admin and roundcube database. +Remove the database configuration option from the setup utility. Using a different +database system than SQLite is not necessary for Mailu. The Mailu database generally +contains static data. + +The usage of the *DB_* environment variables is deprecated now. +They can still be used in the release after Mailu 1.9, but will be removed +after that version. This means it will be removed from master after the upcoming +Mailu release. + +To start using the new environment variables, all *DB_* environment variables must be changed to: +SQLALCHEMY_DATABASE_URI= +SQLALCHEMY_DATABASE_URI_ROUNDCUBE= + +If no URI is specified, SQLite is used with these settings: +SQLALCHEMY_DATABASE_URI=sqlite:////data/main.db +SQLALCHEMY_DATABASE_URI_ROUNDCUBE=sqlite:////data/roundcube.db diff --git a/webmails/start.py b/webmails/start.py index d872fe2d..09e1a362 100755 --- a/webmails/start.py +++ b/webmails/start.py @@ -26,7 +26,8 @@ with open("/etc/resolv.conf") as handle: resolver = content[content.index("nameserver") + 1] context["RESOLVER"] = f"[{resolver}]" if ":" in resolver else resolver -db_flavor = env.get("ROUNDCUBE_DB_FLAVOR", "sqlite") +db_uri = env.get("SQLALCHEMY_DATABASE_URI_ROUNDCUBE", "sqlite:////data/roundcube.db") +db_flavor = env.get("ROUNDCUBE_DB_FLAVOR") if db_flavor == "sqlite": context["DB_DSNW"] = "sqlite:////data/roundcube.db" elif db_flavor == "mysql": @@ -43,6 +44,8 @@ elif db_flavor == "postgresql": env.get("ROUNDCUBE_DB_HOST", "database"), env.get("ROUNDCUBE_DB_NAME", "roundcube") ) +elif db_uri: + context["DB_DSNW"] = db_uri else: print(f"Unknown ROUNDCUBE_DB_FLAVOR: {db_flavor}", file=sys.stderr) exit(1)