From 0f3c1b9d154ab3cd14b050385273ea1a7d438ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20M=C3=B6hlmann?= Date: Tue, 20 Nov 2018 14:41:17 +0200 Subject: [PATCH] Implement CIText as NOCASE alternative in postgresql --- core/admin/mailu/models.py | 8 +++++- .../migrations/versions/049fed905da7_.py | 12 +++++--- .../migrations/versions/9400a032eb1a_.py | 10 +++++-- .../migrations/versions/9c28df23f77e_.py | 28 +++++++++++++++---- .../migrations/versions/ff0417f4318f_.py | 2 +- core/admin/requirements-prod.txt | 1 + optional/postgresql/Dockerfile | 2 +- optional/postgresql/conf/queries.sql | 7 +++++ optional/postgresql/start.py | 6 +++- 9 files changed, 60 insertions(+), 16 deletions(-) diff --git a/core/admin/mailu/models.py b/core/admin/mailu/models.py index 15f95e97..b4f0efa0 100644 --- a/core/admin/mailu/models.py +++ b/core/admin/mailu/models.py @@ -1,10 +1,11 @@ -from mailu import dkim +from mailu import dkim, configuration from sqlalchemy.ext import declarative from passlib import context, hash from datetime import datetime, date from email.mime import text from flask import current_app as app +from citext import CIText import flask_sqlalchemy import sqlalchemy @@ -18,6 +19,7 @@ import dns db = flask_sqlalchemy.SQLAlchemy() +config = configuration.ConfigManager() class IdnaDomain(db.TypeDecorator): @@ -56,6 +58,10 @@ class IdnaEmail(db.TypeDecorator): idna.decode(domain_name), ) + def __init__(self): + if config['DB_FLAVOR'] == 'postgresql': + self.impl = CIText() + class CommaSeparatedList(db.TypeDecorator): """ Stores a list as a comma-separated string, compatible with Postfix. diff --git a/core/admin/migrations/versions/049fed905da7_.py b/core/admin/migrations/versions/049fed905da7_.py index feca8dbb..c30c9b9b 100644 --- a/core/admin/migrations/versions/049fed905da7_.py +++ b/core/admin/migrations/versions/049fed905da7_.py @@ -13,12 +13,16 @@ down_revision = '49d77a93118e' from alembic import op import sqlalchemy as sa from flask import current_app as app - +from citext import CIText def upgrade(): - if app.config['DB_FLAVOR'] == 'sqlite': - with op.batch_alter_table('user') as batch: - batch.alter_column('email', type_=sa.String(length=255, collation="NOCASE")) + if app.config['DB_FLAVOR'] == "postgresql": + email_type = CIText() + else: + email_type = sa.String(length=255, collation="NOCASE") + + with op.batch_alter_table('user') as batch: + batch.alter_column('email', type_=email_type) def downgrade(): diff --git a/core/admin/migrations/versions/9400a032eb1a_.py b/core/admin/migrations/versions/9400a032eb1a_.py index f629a7eb..cb9aeb27 100644 --- a/core/admin/migrations/versions/9400a032eb1a_.py +++ b/core/admin/migrations/versions/9400a032eb1a_.py @@ -12,15 +12,21 @@ down_revision = '9c28df23f77e' from alembic import op import sqlalchemy as sa - +from flask import current_app as app +from citext import CIText def upgrade(): + if app.config['DB_FLAVOR'] == "postgresql": + email_type = CIText() + else: + email_type = sa.String(length=255, collation="NOCASE") + op.create_table('token', sa.Column('created_at', sa.Date(), nullable=False), sa.Column('updated_at', sa.Date(), nullable=True), sa.Column('comment', sa.String(length=255), nullable=True), sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_email', sa.String(length=255), nullable=False), + sa.Column('user_email', email_type), sa.Column('password', sa.String(length=255), nullable=False), sa.Column('ip', sa.String(length=255), nullable=True), sa.ForeignKeyConstraint(['user_email'], ['user.email'], ), diff --git a/core/admin/migrations/versions/9c28df23f77e_.py b/core/admin/migrations/versions/9c28df23f77e_.py index 1d2d656f..bbff9329 100644 --- a/core/admin/migrations/versions/9c28df23f77e_.py +++ b/core/admin/migrations/versions/9c28df23f77e_.py @@ -13,14 +13,30 @@ down_revision = 'c162ac88012a' from alembic import op import sqlalchemy as sa from flask import current_app as app - +from citext import CIText def upgrade(): - if app.config['DB_FLAVOR'] == 'sqlite': - with op.batch_alter_table('user') as batch: - batch.alter_column('email', type_=sa.String(length=255, collation="NOCASE")) - with op.batch_alter_table('alias') as batch: - batch.alter_column('email', type_=sa.String(length=255, collation="NOCASE")) + if app.config['DB_FLAVOR'] == "postgresql": + email_type = CIText() + with op.batch_alter_table('fetch') as batch: + batch.drop_constraint('fetch_user_email_fkey') + with op.batch_alter_table('manager') as batch: + batch.drop_constraint('manager_user_email_fkey') + else: + email_type = sa.String(length=255, collation="NOCASE") + + with op.batch_alter_table('user') as batch: + batch.alter_column('email', type_=email_type) + with op.batch_alter_table('alias') as batch: + batch.alter_column('email', type_=email_type) + with op.batch_alter_table('fetch') as batch: + batch.alter_column('user_email', type_=email_type) + if app.config['DB_FLAVOR'] == "postgresql": + batch.create_foreign_key("fetch_user_email_fkey", "user", ["user_email"], ["email"]) + with op.batch_alter_table('manager') as batch: + batch.alter_column('user_email', type_=email_type) + if app.config['DB_FLAVOR'] == "postgresql": + batch.create_foreign_key("manager_user_email_fkey", "user", ["user_email"], ["email"]) def downgrade(): diff --git a/core/admin/migrations/versions/ff0417f4318f_.py b/core/admin/migrations/versions/ff0417f4318f_.py index ca3b6d5a..0ae7552c 100644 --- a/core/admin/migrations/versions/ff0417f4318f_.py +++ b/core/admin/migrations/versions/ff0417f4318f_.py @@ -64,7 +64,7 @@ def upgrade(): sa.Column('comment', sa.String(length=255), nullable=True), sa.Column('id', sa.Integer(), nullable=False), sa.Column('user_email', sa.String(length=255), nullable=False), - sa.Column('protocol', sa.Enum('imap', 'pop3'), nullable=False), + sa.Column('protocol', sa.Enum('imap', 'pop3', name='protocol'), nullable=False), sa.Column('host', sa.String(length=255), nullable=False), sa.Column('port', sa.Integer(), nullable=False), sa.Column('tls', sa.Boolean(), nullable=False), diff --git a/core/admin/requirements-prod.txt b/core/admin/requirements-prod.txt index 1281146a..0b9489cc 100644 --- a/core/admin/requirements-prod.txt +++ b/core/admin/requirements-prod.txt @@ -45,3 +45,4 @@ Werkzeug==0.14.1 WTForms==2.2.1 WTForms-Components==0.10.3 psycopg2 +sqlalchemy-citext diff --git a/optional/postgresql/Dockerfile b/optional/postgresql/Dockerfile index 14d11864..f8d5dced 100644 --- a/optional/postgresql/Dockerfile +++ b/optional/postgresql/Dockerfile @@ -7,7 +7,7 @@ RUN apk add --no-cache \ RUN pip3 install jinja2 # Image specific layers under this line RUN apk add --no-cache \ - postgresql postgresql-libs \ + postgresql postgresql-libs postgresql-contrib \ && apk add --virtual .build-deps gcc musl-dev postgresql-dev python3-dev \ && pip3 install psycopg2 anosql \ && apk --purge del .build-deps diff --git a/optional/postgresql/conf/queries.sql b/optional/postgresql/conf/queries.sql index 0aea23b3..3a058a94 100644 --- a/optional/postgresql/conf/queries.sql +++ b/optional/postgresql/conf/queries.sql @@ -34,3 +34,10 @@ select 1 create database mailu owner mailu; + +-- name: create_citext! +-- Install the CIText extension +create + extension + if not exists + citext; diff --git a/optional/postgresql/start.py b/optional/postgresql/start.py index 80c70904..58c94006 100755 --- a/optional/postgresql/start.py +++ b/optional/postgresql/start.py @@ -7,7 +7,7 @@ import glob import os def setup(): - conn = psycopg2.connect('user=postgres') + conn = psycopg2.connect(user = 'postgres') queries = anosql.load_queries('postgres', '/conf/queries.sql') # Mailu user queries.create_mailu_user(conn) @@ -21,6 +21,10 @@ def setup(): queries.create_db(conn) conn.set_isolation_level(1) conn.close() + conn = psycopg2.connect(user = 'postgres', database= 'mailu') + queries.create_citext(conn) + conn.commit() + conn.close() # Bootstrap the database if postgresql is running for the first time if not os.path.exists('/var/lib/postgresql/data/pg_wal'):