fixed data import via from_dict

- stabilized CommaSeparatedList by sorting values
- CommaSeparatedList can now handle list and set input

- from_dict now handles mapped keys
- from_dict now handles null values

- class Domain: handle dkim-key None correctly
- class User: delete obsolete keys after converting
- class Alias: now uses Email._dict_input

core/admin/mailu/models.py

@@ -69,12 +69,12 @@ class CommaSeparatedList(db.TypeDecorator):
     impl = db.String
 
     def process_bind_param(self, value, dialect):
-        if type(value) is not list:
+        if not isinstance(value, (list, set)):
             raise TypeError("Should be a list")
         for item in value:
             if "," in item:
                 raise ValueError("Item must not contain a comma")
-        return ",".join(value)
+        return ",".join(sorted(value))
 
     def process_result_value(self, value, dialect):
         return list(filter(bool, value.split(","))) if value else []
@@ -205,13 +205,13 @@ class Base(db.Model):
         for key, value in data.items():
 
             # check key
-            if not hasattr(model, key):
+            if not hasattr(model, key) and not key in model.__mapper__.relationships:
                 raise KeyError(f'unknown key {model.__table__}.{key}', model, key, data)
 
             # check value type
             col = model.__mapper__.columns.get(key)
             if col is not None:
-                if not type(value) is col.type.python_type:
+                if not ((value is None and col.nullable) or (type(value) is col.type.python_type)):
                     raise TypeError(f'{model.__table__}.{key} {value!r} has invalid type {type(value).__name__!r}', model, key, data)
             else:
                 rel = model.__mapper__.relationships.get(key)
@@ -229,25 +229,36 @@ class Base(db.Model):
                 if not isinstance(rel_model, sqlalchemy.orm.Mapper):
                     add = rel_model.from_dict(value, delete)
                     assert len(add) == 1
-                    item, updated = add[0]
+                    rel_item, updated = add[0]
-                    changed.append((item, updated))
+                    changed.append((rel_item, updated))
-                    data[key] = item
+                    data[key] = rel_item
 
-        # create or update item?
+        # create item if necessary
+        created = False
         item = model.query.get(data[pkey]) if pkey in data else None
         if item is None:
-            # create item
 
             # check for mandatory keys
             missing = getattr(model, '_dict_mandatory', set()) - set(data.keys())
             if missing:
                 raise ValueError(f'mandatory key(s) {", ".join(sorted(missing))} for {model.__table__} missing', model, missing, data)
 
-            changed.append((model(**data), True))
+            # remove mapped relationships from data
+            mapped = {}
+            for key in list(data.keys()):
+                if key in model.__mapper__.relationships:
+                    if isinstance(model.__mapper__.relationships[key].argument, sqlalchemy.orm.Mapper):
+                        mapped[key] = data[key]
+                        del data[key]
+
+            # create new item
+            item = model(**data)
+            created = True
+
+            # and update mapped relationships (below)
+            data = mapped
 
-        else:
         # update item
-
         updated = []
         for key, value in data.items():
 
@@ -315,13 +326,18 @@ class Base(db.Model):
             else:
                 # update key
                 old = getattr(item, key)
-                    if type(old) is list and not delete:
+                if type(old) is list:
-                        value = old + value
+                    # deduplicate list value
+                    assert type(value) is list
+                    value = set(value)
+                    old = set(old)
+                    if not delete:
+                        value = old | value
                 if value != old:
                     updated.append((key, old, value))
                     setattr(item, key, value)
 
-            changed.append((item, updated))
+        changed.append((item, created if created else updated))
 
         return changed
 
@@ -353,9 +369,11 @@ class Domain(Base):
     _dict_output = {'dkim_key': lambda v: v.decode('utf-8').strip().split('\n')[1:-1]}
     @staticmethod
     def _dict_input(data):
-        key = data.get('dkim_key')
+        if 'dkim_key' in data:
-        if key is not None:
             key = data['dkim_key']
+            if key is None:
+                del data['dkim_key']
+            else:
                 if type(key) is list:
                     key = ''.join(key)
                 if type(key) is str:
@@ -580,6 +598,8 @@ class User(Base, Email):
             if data['hash_scheme'] not in cls.scheme_dict:
                 raise ValueError(f'invalid password scheme {scheme!r}')
             data['password'] = '{'+data['hash_scheme']+'}'+ data['password_hash']
+            del data['hash_scheme']
+            del data['password_hash']
 
     domain = db.relationship(Domain,
         backref=db.backref('users', cascade='all, delete-orphan'))
@@ -709,6 +729,7 @@ class Alias(Base, Email):
     _dict_hide = {'domain_name', 'domain', 'localpart'}
     @staticmethod
     def _dict_input(data):
+        Email._dict_input(data)
         # handle comma delimited string for backwards compability
         dst = data.get('destination')
         if type(dst) is str: