Merge remote-tracking branch 'nextgens/dynamic-resolution' into dynamic-resolution

Get rid of all HOST_* variables, sanitize the environment in socrates

.github/workflows/arm.yml

@@ -61,7 +61,7 @@ jobs:
           echo "RELEASE=true" >> $GITHUB_ENV
           echo "DEPLOY=true" >> $GITHUB_ENV
           echo "RELEASE=true" >> $GITHUB_ENV
-      - name: Derive PINNED_MAILU_VERSION for staging for master
+      - name: Derive PINNED_MAILU_VERSION for master
         if: env.BRANCH == 'master'
         shell: bash
         env:
@@ -76,7 +76,7 @@ jobs:
       - derive-variables
     uses: ./.github/workflows/build_test_deploy.yml
     with:
-      architecture: 'linux/arm64,linux/arm/v7'
+      architecture: 'linux/arm64/v8,linux/arm/v7'
       mailu_version: ${{needs.derive-variables.outputs.MAILU_VERSION}}-arm
       pinned_mailu_version: ${{needs.derive-variables.outputs.PINNED_MAILU_VERSION}}-arm
       docker_org: ${{needs.derive-variables.outputs.DOCKER_ORG}}
@@ -98,4 +98,4 @@ jobs:
 #else
 #  pinned_version=$root_version.$(expr $patch_version + 1)
 #fi
-#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV
+#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV

.github/workflows/build_test_deploy.yml

@@ -58,15 +58,15 @@ on:
         required: true
         type: string
       deploy:
-        description: Deploy to docker hub. Happens for all branches but staging
+        description: Deploy to docker hub. Happens for all branches but staging. Use string true or false.
         default: true
         required: false
-        type: boolean
+        type: string
       release:
-        description: 'Tag and create the github release. Only happens for branch x.y (release branch)'
+        description: Tag and create the github release. Use string true or false.
         default: false
         required: false
-        type: boolean
+        type: string
 env:
   HCL_FILE: ./tests/build.hcl
 
@@ -84,11 +84,123 @@ jobs:
       - name: Create matrix
         id: targets
         run: |
-          echo ::set-output name=matrix::$(docker buildx bake -f ${{env.HCL_FILE}} --print | jq -cr '.group.default.targets')
+          echo matrix=$(docker buildx bake -f ${{env.HCL_FILE}} --print | jq -cr '.group.default.targets') >> $GITHUB_OUTPUT
       - name: Show matrix
         run: |
           echo ${{ steps.targets.outputs.matrix }}
 
+## This job builds the base image. The base image is used by all other images.
+  build-base-image-x64:
+    name: Build base image x64
+    if: inputs.architecture == 'linux/amd64'
+    needs:
+      - targets
+    runs-on: ubuntu-latest
+    permissions:
+     contents: read
+     packages: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Retrieve global variables
+        shell: bash
+        run: |
+          echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV
+          echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
+          echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
+          echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - uses: crazy-max/ghaction-github-runtime@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.Docker_Login }}
+          password: ${{ secrets.Docker_Password }}
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ github.repository_owner }}
+      - name: Build all docker images
+        env:
+          DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }}
+          MAILU_VERSION: ${{ env.MAILU_VERSION }}
+          PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
+        uses: docker/bake-action@v2
+        with:
+          files: ${{env.HCL_FILE}}
+          targets: base
+          load: false
+          push: false
+          set: |
+            *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}
+            *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }},mode=max
+            *.platform=${{ inputs.architecture }}
+
+## This job builds the base image. The base image is used by all other images.
+  build-base-image-arm:
+    name: Build base image arm
+    if: inputs.architecture != 'linux/amd64'
+    needs:
+      - targets
+    runs-on: self-hosted
+    permissions:
+     contents: read
+     packages: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Retrieve global variables
+        shell: bash
+        run: |
+          echo "BRANCH=${{ inputs.branch }}" >> $GITHUB_ENV
+          echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
+          echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
+          echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - uses: crazy-max/ghaction-github-runtime@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.Docker_Login }}
+          password: ${{ secrets.Docker_Password }}
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ github.repository_owner }}
+      - name: Build all docker images
+        env:
+          DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }}
+          MAILU_VERSION: ${{ env.MAILU_VERSION }}
+          PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
+        uses: docker/bake-action@v2
+        with:
+          files: ${{env.HCL_FILE}}
+          targets: base
+          load: false
+          push: false
+          set: |
+            *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm
+            *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm,mode=max
+            *.platform=${{ inputs.architecture }}
+
 # This job builds all the images. The build cache is stored in the github actions cache.
 # In further jobs, this cache is used to quickly rebuild the images.
   build:
@@ -96,6 +208,7 @@ jobs:
     if: inputs.architecture == 'linux/amd64'
     needs:
       - targets
+      - build-base-image-x64
     strategy:
       fail-fast: false
       matrix:
@@ -113,26 +226,30 @@ jobs:
           echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
           echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
           echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
-      - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/${{ matrix.target }}
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }}-${{ github.run_id }}
-          restore-keys: |
-            ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - uses: crazy-max/ghaction-github-runtime@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.Docker_Login }}
           password: ${{ secrets.Docker_Password }}
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ github.repository_owner }}
       - name: Build all docker images
         env:
-          DOCKER_ORG: ${{ env.DOCKER_ORG }}
+          DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }}
           MAILU_VERSION: ${{ env.MAILU_VERSION }}
           PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
         uses: docker/bake-action@v2
@@ -140,10 +257,11 @@ jobs:
           files: ${{env.HCL_FILE}}
           targets: ${{ matrix.target }}
           load: false
-          push: false
+          push: true
           set: |
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }}
-            *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max
+            *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache
+            *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache,mode=max
+            *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}
             *.platform=${{ inputs.architecture }}
 
 # This job builds all the images. The build cache is stored in the github actions cache.
@@ -153,6 +271,7 @@ jobs:
     if: inputs.architecture != 'linux/amd64'
     needs:
       - targets
+      - build-base-image-arm
     strategy:
       fail-fast: false
       matrix:
@@ -170,26 +289,30 @@ jobs:
           echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
           echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
           echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
-      - name: Configure actions/cache@v3 action for storing build cache in the ${{ runner.temp }}/cache folder
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/${{ matrix.target }}
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }}-${{ github.run_id }}
-          restore-keys: |
-            ${{ github.ref }}-${{ inputs.mailu_version }}-${{ matrix.target }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - uses: crazy-max/ghaction-github-runtime@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.Docker_Login }}
           password: ${{ secrets.Docker_Password }}
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ github.repository_owner }}
       - name: Build all docker images
         env:
-          DOCKER_ORG: ${{ env.DOCKER_ORG }}
+          DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }}
           MAILU_VERSION: ${{ env.MAILU_VERSION }}
           PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
         uses: docker/bake-action@v2
@@ -197,10 +320,11 @@ jobs:
           files: ${{env.HCL_FILE}}
           targets: ${{ matrix.target }}
           load: false
-          push: false
+          push: true
           set: |
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/${{ matrix.target }}
-            *.cache-to=type=local,dest=${{ runner.temp }}/cache/${{ matrix.target }},mode=max
+            *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm
+            *.cache-to=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:buildcache-arm,mode=max
+            *.cache-from=type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/base:${{ hashFiles('core/base/Dockerfile','core/base/requirements-prod.txt') }}-arm
             *.platform=${{ inputs.architecture }}
 
 # This job runs all the tests.
@@ -212,12 +336,11 @@ jobs:
      contents: read
      packages: read
     needs:
-      - targets
       - build
     strategy:
       fail-fast: false
       matrix:
-        target: ["core", "fetchmail", "filters", "snappymail", "roundcube", "webdav"]
+        target: ["core", "fetchmail", "filters", "webmail", "webdav"]
         time: ["2"]
         include:
           - target: "filters"
@@ -234,112 +357,22 @@ jobs:
           echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
           echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
           echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
-      - name: Configure /cache for image docs
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/docs
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }}
-      - name: Configure /cache for image setup
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/setup
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }}
-      - name: Configure /cache for image admin
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/admin
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }}
-      - name: Configure /cache for image antispam
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/antispam
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }}
-      - name: Configure /cache for image front
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/front
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }}
-      - name: Configure /cache for image imap
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/imap
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }}
-      - name: Configure /cache for image smtp
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/smtp
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }}
-      - name: Configure /cache for image snappymail
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/snappymail
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }}
-      - name: Configure /cache for image roundcube
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/roundcube
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }}
-      - name: Configure /cache for image antivirus
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/antivirus
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }}
-      - name: Configure /cache for image fetchmail
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/fetchmail
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }}
-      - name: Configure /cache for image resolver
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/resolver
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }}
-      - name: Configure /cache for image traefik-certdumper
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/traefik-certdumper
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }}
-      - name: Configure /cache for image webdav
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/webdav
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - uses: crazy-max/ghaction-github-runtime@v2
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      - name: Login to Docker Hub
+      - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
-          username: ${{ secrets.Docker_Login }}
-          password: ${{ secrets.Docker_Password }}
-      - name: Build docker images for testing from cache
-        env:
-          DOCKER_ORG: ${{ env.DOCKER_ORG }}
-          MAILU_VERSION: ${{ env.MAILU_VERSION }}
-          PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
-        uses: docker/bake-action@v2
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
         with:
-          files: ${{env.HCL_FILE}}
-          load: true
-          push: false
-          set: |
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/docs
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/setup
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/admin
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/front
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/imap
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav
-            *.platform=${{ inputs.architecture }}
+          string: ${{ github.repository_owner }}
       - name: Install python packages
         run: python3 -m pip install -r tests/requirements.txt
       - name: Copy all certs
@@ -347,20 +380,21 @@ jobs:
       - name: Test ${{ matrix.target }}
         run: python tests/compose/test.py ${{ matrix.target }} ${{ matrix.time }}
         env:
-          DOCKER_ORG: ${{ env.DOCKER_ORG }}
+          DOCKER_ORG: ghcr.io/${{ steps.string.outputs.lowercase }}
           MAILU_VERSION: ${{ env.MAILU_VERSION }}
           PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
 
-# This job deploys the docker images to the docker repository. The build.hcl file contains logic that determines what tags are pushed.
-# E.g. for master only the :master and :latest tags are pushed.
   deploy:
     name: Deploy images
     # Deploying is not required for staging
     if: inputs.deploy == 'true'
     runs-on: ubuntu-latest
     needs:
-      - build
       - tests
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx"]
     steps:
       - uses: actions/checkout@v3
       - name: Retrieve global variables
@@ -370,76 +404,6 @@ jobs:
           echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
           echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
           echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
-      - name: Configure /cache for image docs
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/docs
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }}
-      - name: Configure /cache for image setup
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/setup
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }}
-      - name: Configure /cache for image admin
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/admin
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }}
-      - name: Configure /cache for image antispam
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/antispam
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }}
-      - name: Configure /cache for image front
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/front
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }}
-      - name: Configure /cache for image imap
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/imap
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }}
-      - name: Configure /cache for image smtp
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/smtp
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }}
-      - name: Configure /cache for image snappymail
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/snappymail
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }}
-      - name: Configure /cache for image roundcube
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/roundcube
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }}
-      - name: Configure /cache for image antivirus
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/antivirus
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }}
-      - name: Configure /cache for image fetchmail
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/fetchmail
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }}
-      - name: Configure /cache for image resolver
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/resolver
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }}
-      - name: Configure /cache for image traefik-certdumper
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/traefik-certdumper
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }}
-      - name: Configure /cache for image webdav
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/webdav
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - uses: crazy-max/ghaction-github-runtime@v2
@@ -450,31 +414,20 @@ jobs:
         with:
           username: ${{ secrets.Docker_Login }}
           password: ${{ secrets.Docker_Password }}
-      - name: Deploy images to docker hub. Build.hcl contains the logic for the tags that are pushed.
-        env:
-          DOCKER_ORG: ${{ env.DOCKER_ORG }}
-          MAILU_VERSION: ${{ env.MAILU_VERSION }}
-          PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
-        uses: docker/bake-action@v2
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
         with:
-          files: ${{env.HCL_FILE}}
-          push: true
-          set: |
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/docs
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/setup
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/admin
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/front
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/imap
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav
-            *.platform=${{ inputs.architecture }}
+          string: ${{ github.repository_owner }}
+      - name: Push image to Docker
+        shell: bash
+        run: |
+          if [ '${{ env.MAILU_VERSION }}' == 'master' ]; then pinned_mailu_version='master'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi;
+          docker buildx imagetools create \
+            --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \
+            --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \
+            --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \
+            ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }}
 
   deploy-arm:
     name: Deploy images for arm
@@ -483,6 +436,10 @@ jobs:
     runs-on: self-hosted
     needs:
       - build-arm
+    strategy:
+      fail-fast: false
+      matrix:
+        target: ["setup", "docs", "fetchmail", "webmail", "admin", "traefik-certdumper", "radicale", "clamav", "rspamd", "postfix", "dovecot", "unbound", "nginx"]
     steps:
       - uses: actions/checkout@v3
       - name: Retrieve global variables
@@ -492,76 +449,6 @@ jobs:
           echo "MAILU_VERSION=${{ inputs.mailu_version }}" >> $GITHUB_ENV
           echo "PINNED_MAILU_VERSION=${{ inputs.pinned_mailu_version }}" >> $GITHUB_ENV
           echo "DOCKER_ORG=${{ inputs.docker_org }}" >> $GITHUB_ENV
-      - name: Configure /cache for image docs
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/docs
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-docs-${{ github.run_id }}
-      - name: Configure /cache for image setup
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/setup
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-setup-${{ github.run_id }}
-      - name: Configure /cache for image admin
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/admin
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-admin-${{ github.run_id }}
-      - name: Configure /cache for image antispam
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/antispam
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-antispam-${{ github.run_id }}
-      - name: Configure /cache for image front
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/front
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-front-${{ github.run_id }}
-      - name: Configure /cache for image imap
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/imap
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-imap-${{ github.run_id }}
-      - name: Configure /cache for image smtp
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/smtp
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-smtp-${{ github.run_id }}
-      - name: Configure /cache for image snappymail
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/snappymail
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-snappymail-${{ github.run_id }}
-      - name: Configure /cache for image roundcube
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/roundcube
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-roundcube-${{ github.run_id }}
-      - name: Configure /cache for image antivirus
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/antivirus
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-antivirus-${{ github.run_id }}
-      - name: Configure /cache for image fetchmail
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/fetchmail
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-fetchmail-${{ github.run_id }}
-      - name: Configure /cache for image resolver
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/resolver
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-resolver-${{ github.run_id }}
-      - name: Configure /cache for image traefik-certdumper
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/traefik-certdumper
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-traefik-certdumper-${{ github.run_id }}
-      - name: Configure /cache for image webdav
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/cache/webdav
-          key: ${{ github.ref }}-${{ inputs.mailu_version }}-webdav-${{ github.run_id }}
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - uses: crazy-max/ghaction-github-runtime@v2
@@ -572,31 +459,20 @@ jobs:
         with:
           username: ${{ secrets.Docker_Login }}
           password: ${{ secrets.Docker_Password }}
-      - name: Deploy images to docker hub. Build.hcl contains the logic for the tags that are pushed.
-        env:
-          DOCKER_ORG: ${{ env.DOCKER_ORG }}
-          MAILU_VERSION: ${{ env.MAILU_VERSION }}
-          PINNED_MAILU_VERSION: ${{ env.PINNED_MAILU_VERSION }}
-        uses: docker/bake-action@v2
+      - name: Helper to convert docker org to lowercase
+        id: string
+        uses: ASzc/change-string-case-action@v5
         with:
-          files: ${{env.HCL_FILE}}
-          push: true
-          set: |
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/docs
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/setup
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/admin
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/antispam
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/front
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/imap
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/smtp
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/snappymail
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/roundcube
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/antivirus
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/fetchmail
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/resolver
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/traefik-certdumper
-            *.cache-from=type=local,src=${{ runner.temp }}/cache/webdav
-            *.platform=${{ inputs.architecture }}
+          string: ${{ github.repository_owner }}
+      - name: Push image to Docker
+        shell: bash
+        run: |
+          if [ '${{ env.MAILU_VERSION }}' == 'master-arm' ]; then pinned_mailu_version='master-arm'; else pinned_mailu_version=${{ env.PINNED_MAILU_VERSION}}; fi;
+          docker buildx imagetools create \
+            --tag ${{ inputs.docker_org }}/${{ matrix.target }}:${{ env.MAILU_VERSION }} \
+            --tag ${{ inputs.docker_org }}/${{ matrix.target }}:$pinned_mailu_version \
+            --tag ${{ inputs.docker_org }}/${{ matrix.target }}:latest \
+            ghcr.io/${{ steps.string.outputs.lowercase }}/${{ matrix.target }}:${{ env.MAILU_VERSION }}
 
 #This job creates a tagged release. A tag is created for the pinned version x.y.z. The GH release refers to this tag.
   tag-release:

.github/workflows/x64.yml

@@ -82,7 +82,7 @@ jobs:
           echo "PINNED_MAILU_VERSION=staging" >> $GITHUB_ENV
           echo "DEPLOY=false" >> $GITHUB_ENV
           echo "RELEASE=false" >> $GITHUB_ENV
-      - name: Derive PINNED_MAILU_VERSION for staging for master
+      - name: Derive PINNED_MAILU_VERSION for master
         if: env.BRANCH == 'master'
         shell: bash
         env:
@@ -131,4 +131,4 @@ jobs:
 #else
 #  pinned_version=$root_version.$(expr $patch_version + 1)
 #fi
-#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV
+#echo "PINNED_MAILU_VERSION=$pinned_version" >> $GITHUB_ENV

.gitignore

@@ -1,5 +1,5 @@
-*.pyc
-*.mo
+**/*.pyc
+**/*.mo
 __pycache__
 pip-selfcheck.json
 /core/admin/lib*

CONTRIBUTING.md

@@ -4,4 +4,4 @@ This project is open source, and your contributions are all welcome. There are m
 2. contribute code and/or configuration to the repository (see [the development guidelines](https://mailu.io/master/contributors/workflow.html) for details);
 3. contribute localization to your native language (see [the localization docs](https://mailu.io/master/contributors/localization.html) for details);
 
-Either way, keep in mind that the code you write or the translation you produce muts be licensed under the same conditions as the project itself. Additionally, all contributors are considered equal co-authors of the project.
+Either way, keep in mind that the code you write or the translation you produce must be licensed under the same conditions as the project itself. Additionally, all contributors are considered equal co-authors of the project.

ISSUE_TEMPLATE.md

@@ -1,44 +1,71 @@
+<!--
+
 Thank you for opening an issue with Mailu. Please understand that issues are meant for bugs and enhancement-requests.
 For **user-support questions**, reach out to us  on [matrix](https://matrix.to/#/#mailu:tedomum.net).
 
 To be able to help you best, we need some more information.
 
-## Before you open your issue
-- [ ] Check if no issue or pull-request for this already exists.
-- [ ] Check [documentation](https://mailu.io/master/) and [FAQ](https://mailu.io/master/faq.html). (Tip, use the search function on the documentation page)
-- [ ] You understand `Mailu` is made by volunteers in their **free time** — be conscise, civil and accept that delays can occur.
-- [ ] The title of the issue should be short and simple. It should contain specific terms related to the actual issue. Be specific while writing the title.
+Before you open your issue
+- Check if no issue or pull-request for this already exists.
+- Check [documentation](https://mailu.io/master/) and [FAQ](https://mailu.io/master/faq.html). (Tip, use the search function on the documentation page)
+- You understand `Mailu` is made by volunteers in their **free time** — be concise, civil and accept that delays can occur.
+- The title of the issue should be short and simple. It should contain specific terms related to the actual issue. Be specific while writing the title.
+
+Please put your text outside of the comment blocks to be visible. You can use the button "Preview" above to check.
+
+-->
+
+## Environment & Version
 
-## Environment & Versions
 ### Environment
- - [ ] docker-compose
- - [ ] kubernetes
- - [ ] docker swarm
 
-### Versions
+- [ ] docker-compose
+- [ ] kubernetes
+- [ ] docker swarm
+
+### Version
+
+- Version: `master`
+
+<!--
 To find your version, get the image name of a mailu container and read  the version from the tag (example for version 1.7).
-```
+
 $> docker ps -a | grep mailu
 140b09d4b09c    mailu/roundcube:1.7    "docker-php-entrypoi…"    2 weeks ago    Up 2 days (healthy)    80/tcp
 $> grep MAILU_VERSION docker-compose.yml mailu.env
-```
+-->
 
 ## Description
+<!--
 Further explain the bug in a few words. It should be clear what the unexpected behaviour is.  Share it in an easy-to-understand language.
+-->
 
 ## Replication Steps
+<!--
 Steps for replicating your issue
+-->
+
+## Observed behaviour
+<!--
+Explain or paste the result you received.
+-->
 
 ## Expected behaviour
-Explain what results you expected - be as specific as possible. Just saying "it doesn’t work as expected" is not useful. It's also helpful to describe what you actually experienced.
+<!--
+Explain what results you expected - be as specific as possible.
+Just saying "it doesn’t work as expected" is not useful. It's also helpful to describe what you actually experienced.
+-->
 
 ## Logs
-Often it is very useful to include log fragments of the involved component. You can get the logs via `docker logs <container name> --tail 1000`. For example for the admin container:
-`docker logs mailu_admin_1 --tail 1000`
+<!--
+Often it is very useful to include log fragments of the involved component.
+You can get the logs via `docker logs <container name> --tail 1000`.
+For example for the admin container: `docker logs mailu_admin_1 --tail 1000`
 or using docker-compose `docker-compose -f /mailu/docker-compose.yml logs --tail 1000 admin`
+
 If you can find the relevant section, please share only the parts that seem relevant. If you have any logs, please enclose them in code tags, like so:
-````markdown
+
 ```
 Your logs here!
 ```
-````
+-->

README.md

@@ -22,8 +22,8 @@ Main features include:
 - **Web access**, multiple Webmails and administration interface
 - **User features**, aliases, auto-reply, auto-forward, fetched accounts
 - **Admin features**, global admins, announcements, per-domain delegation, quotas
-- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner
-- **Antispam**, auto-learn, greylisting, DMARC and SPF
+- **Security**, enforced TLS, DANE, MTA-STS, Letsencrypt!, outgoing DKIM, anti-virus scanner, [Snuffleupagus](https://github.com/jvoisin/snuffleupagus/)
+- **Antispam**, auto-learn, greylisting, DMARC and SPF, anti-spoofing
 - **Freedom**, all FOSS components, no tracker included
 
 ![Domains](docs/assets/screenshots/domains.png)

core/admin/.gitignore

@@ -2,3 +2,4 @@
 lib64
 .vscode
 tags
+dev

core/admin/Dockerfile

@@ -1,61 +1,31 @@
-# First stage to build assets
-ARG DISTRO=alpine:3.14.5
+# syntax=docker/dockerfile-upstream:1.4.3
 
-FROM node:16-alpine3.16 as assets
-
-COPY package.json ./
-RUN set -eu \
- && npm config set update-notifier false \
- && npm install --no-fund
-
-COPY webpack.config.js ./
-COPY assets ./assets
-RUN set -eu \
- && sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \
- && for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \
-      cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \
-    done \
- && node_modules/.bin/webpack-cli --color
-
-
-# Actual application
-FROM $DISTRO
-ARG VERSION
-
-ENV TZ Etc/UTC
+# admin image
+FROM base
 
+ARG VERSION=local
 LABEL version=$VERSION
 
-# python3 shared with most images
-RUN set -eu \
- && apk add --no-cache python3 py3-pip py3-wheel git bash tzdata \
- && pip3 install --upgrade pip
+RUN set -euxo pipefail \
+  ; apk add --no-cache libressl mariadb-connector-c postgresql-libs
 
-RUN mkdir -p /app
-WORKDIR /app
+COPY --from=assets /work/static/ ./mailu/static/
 
-COPY requirements-prod.txt requirements.txt
-RUN set -eu \
- && apk add --no-cache libressl curl postgresql-libs mariadb-connector-c \
- && pip install --no-cache-dir -r requirements.txt --only-binary=:all: --no-binary=Flask-bootstrap,PyYAML,SQLAlchemy \
- || ( apk add --no-cache --virtual build-dep libressl-dev libffi-dev python3-dev build-base postgresql-dev mariadb-connector-c-dev cargo \
- && pip install --upgrade pip \
- && pip install -r requirements.txt \
- && apk del --no-cache build-dep )
+COPY audit.py /
+COPY start.py /
 
-COPY --from=assets static ./mailu/static
-COPY mailu ./mailu
-COPY migrations ./migrations
-COPY start.py /start.py
-COPY audit.py /audit.py
+COPY migrations/ ./migrations/
 
-RUN pybabel compile -d mailu/translations
+COPY mailu/ ./mailu/
+RUN set -euxo pipefail \
+  ; venv/bin/pybabel compile -d mailu/translations
+
+RUN echo $VERSION >/version
 
 EXPOSE 80/tcp
+HEALTHCHECK CMD curl -skfLo /dev/null http://localhost/sso/login?next=ui.index
+
 VOLUME ["/data","/dkim"]
-ENV FLASK_APP mailu
 
+ENV FLASK_APP=mailu
 CMD /start.py
-
-HEALTHCHECK CMD curl -f -L http://localhost/sso/login?next=ui.index || exit 1
-RUN echo $VERSION >> /version

core/admin/assets/Dockerfile

@@ -0,0 +1,22 @@
+# syntax=docker/dockerfile-upstream:1.4.3
+
+FROM node:16-alpine3.16
+
+WORKDIR /work
+
+COPY package.json ./
+
+RUN set -euxo pipefail \
+  ; npm config set update-notifier false \
+  ; npm install --no-audit --no-fund \
+  ; sed -i 's/#007bff/#55a5d9/' node_modules/admin-lte/build/scss/_bootstrap-variables.scss \
+  ; mkdir assets \
+  ; for l in ca da de:de-DE en:en-GB es:es-ES eu fr:fr-FR he hu is it:it-IT ja nb_NO:no-NB nl:nl-NL pl pt:pt-PT ru sv:sv-SE zh; do \
+      cp node_modules/datatables.net-plugins/i18n/${l#*:}.json assets/${l%:*}.json; \
+    done
+
+COPY assets/ ./assets/
+COPY webpack.config.js ./
+
+RUN set -euxo pipefail \
+  ; node_modules/.bin/webpack-cli --color

core/admin/assets/app.js

@@ -1,79 +0,0 @@
-require('./app.css');
-
-import logo from './mailu.png';
-import modules from "./*.json";
-
-// TODO: conditionally (or lazy) load select2 and dataTable
-$('document').ready(function() {
-
-    // intercept anchors with data-clicked attribute and open alternate location instead
-    $('[data-clicked]').click(function(e) {
-        e.preventDefault();
-        window.location.href = $(this).data('clicked');
-    });
-
-    // use post for language selection
-    $('#mailu-languages > a').click(function(e) {
-        e.preventDefault();
-        $.post({
-            url: $(this).attr('href'),
-            success: function() {
-                window.location = window.location.href;
-            },
-        });
-    });
-
-    // allow en-/disabling of inputs in fieldset with checkbox in legend
-    $('fieldset legend input[type=checkbox]').change(function() {
-        var fieldset = $(this).parents('fieldset');
-        if (this.checked) {
-            fieldset.removeAttr('disabled');
-            fieldset.find('input,textarea').not(this).removeAttr('disabled');
-        } else {
-            fieldset.attr('disabled', '');
-            fieldset.find('input,textarea').not(this).attr('disabled', '');
-        }
-    });
-
-    // display of range input value
-    $('input[type=range]').each(function() {
-        var value_element = $('#'+this.id+'_value');
-        if (value_element.length) {
-            value_element = $(value_element[0]);
-            var infinity = $(this).data('infinity');
-            var step = $(this).attr('step');
-            $(this).on('input', function() {
-                var num = (infinity && this.value == 0) ? '∞' : (this.value/step).toFixed(2);
-                if (num.endsWith('.00')) num = num.substr(0, num.length - 3);
-                value_element.text(num);
-            }).trigger('input');
-        }
-    });
-
-    // init select2
-    $('.mailselect').select2({
-        tags: true,
-        tokenSeparators: [',', ' '],
-    });
-
-    // init dataTable
-    var d = $(document.documentElement);
-    $('.dataTable').DataTable({
-        'responsive': true,
-        language: {
-            url: d.data('static') + d.attr('lang') + '.json',
-        },
-    });
-
-    // init clipboard.js
-    new ClipboardJS('.btn-clip');
-
-    // disable login if not possible
-    var l = $('#login_needs_https');
-    if (l.length && window.location.protocol != 'https:') {
-        l.removeClass("d-none");
-        $('form :input').prop('disabled', true);
-    }
-
-});
-

core/admin/assets/assets/app.js

@@ -0,0 +1,136 @@
+// Inspired from https://github.com/mehdibo/hibp-js/blob/master/hibp.js
+function sha1(string) {
+    var buffer = new TextEncoder("utf-8").encode(string);
+    return crypto.subtle.digest("SHA-1", buffer).then(function (buffer) {
+        // Get the hex code
+        var hexCodes = [];
+        var view = new DataView(buffer);
+        for (var i = 0; i < view.byteLength; i += 4) {
+            // Using getUint32 reduces the number of iterations needed (we process 4 bytes each time)
+            var value = view.getUint32(i);
+            // toString(16) will give the hex representation of the number without padding
+            var stringValue = value.toString(16);
+            // We use concatenation and slice for padding
+            var padding = '00000000';
+            var paddedValue = (padding + stringValue).slice(-padding.length);
+            hexCodes.push(paddedValue);
+        }
+        // Join all the hex strings into one
+        return hexCodes.join("");
+    });
+}
+
+function hibpCheck(pwd) {
+    // We hash the pwd first
+    sha1(pwd).then(function(hash){
+        // We send the first 5 chars of the hash to hibp's API
+        const req = new XMLHttpRequest();
+        req.open('GET', 'https://api.pwnedpasswords.com/range/'+hash.substr(0, 5));
+        req.setRequestHeader('Add-Padding', 'true');
+        req.addEventListener("load", function(){
+            // When we get back a response from the server
+            // We create an array of lines and loop through them
+            const lines = this.responseText.split("\n");
+            const hashSub = hash.slice(5).toUpperCase();
+            for (var i in lines){
+                // Check if the line matches the rest of the hash
+                if (lines[i].substring(0, 35) == hashSub){
+                    const val = parseInt(lines[i].trimEnd("\r").split(":")[1]);
+                    if (val > 0) {
+                        $("#pwned").val(val);
+                    }
+                    return; // If found no need to continue the loop
+                }
+            }
+            $("#pwned").val(0);
+        });
+        req.send();
+    });
+}
+
+// TODO: conditionally (or lazy) load select2 and dataTable
+$('document').ready(function() {
+
+    // intercept anchors with data-clicked attribute and open alternate location instead
+    $('[data-clicked]').click(function(e) {
+        e.preventDefault();
+        window.location.href = $(this).data('clicked');
+    });
+
+    // use post for language selection
+    $('#mailu-languages > a').click(function(e) {
+        e.preventDefault();
+        $.post({
+            url: $(this).attr('href'),
+            success: function() {
+                window.location = window.location.href;
+            },
+        });
+    });
+
+    // allow en-/disabling of inputs in fieldset with checkbox in legend
+    $('fieldset legend input[type=checkbox]').change(function() {
+        var fieldset = $(this).parents('fieldset');
+        if (this.checked) {
+            fieldset.removeAttr('disabled');
+            fieldset.find('input,textarea').not(this).removeAttr('disabled');
+        } else {
+            fieldset.attr('disabled', '');
+            fieldset.find('input,textarea').not(this).attr('disabled', '');
+        }
+    });
+
+    // display of range input value
+    $('input[type=range]').each(function() {
+        var value_element = $('#'+this.id+'_value');
+        if (value_element.length) {
+            value_element = $(value_element[0]);
+            var infinity = $(this).data('infinity');
+            var step = $(this).attr('step');
+            $(this).on('input', function() {
+                var num = (infinity && this.value == 0) ? '∞' : (this.value/step).toFixed(2);
+                if (num.endsWith('.00')) num = num.substr(0, num.length - 3);
+                value_element.text(num);
+            }).trigger('input');
+        }
+    });
+
+    // init select2
+    $('.mailselect').select2({
+        tags: true,
+        tokenSeparators: [',', ' '],
+    });
+
+    // init dataTable
+    var d = $(document.documentElement);
+    $('.dataTable').DataTable({
+        'responsive': true,
+        language: {
+            url: d.data('static') + d.attr('lang') + '.json',
+        },
+    });
+
+    // init clipboard.js
+    new ClipboardJS('.btn-clip');
+
+    // disable login if not possible
+    var l = $('#login_needs_https');
+    if (l.length && window.location.protocol != 'https:') {
+        l.removeClass("d-none");
+        $('form :input').prop('disabled', true);
+    }
+
+    if (window.isSecureContext) {
+        $("#pw").on("change paste", function(){
+            hibpCheck($(this).val());
+            return true;
+        });
+        $("#pw").closest("form").submit(function(event){
+            if (parseInt($("#pwned").val()) < 0) {
+                hibpCheck($("#pw").val());
+            }
+        });
+    }
+
+});
+

core/admin/assets/assets/vendor.js

@@ -1,5 +1,5 @@
 // AdminLTE
-import 'admin-lte/plugins/jquery/jquery.min.js';
+window.$ = window.jQuery = require('admin-lte/plugins/jquery/jquery.min.js');
 import 'admin-lte/plugins/bootstrap/js/bootstrap.bundle.min.js';
 import 'admin-lte/build/scss/adminlte.scss';
 import 'admin-lte/build/js/AdminLTE.js';
@@ -18,7 +18,7 @@ import 'admin-lte/plugins/datatables/jquery.dataTables.min.js';
 import 'admin-lte/plugins/datatables-bs4/js/dataTables.bootstrap4.min.js';
 import 'admin-lte/plugins/datatables-responsive/js/dataTables.responsive.min.js';
 import 'admin-lte/plugins/datatables-responsive/js/responsive.bootstrap4.min.js';
+import modules from "./*.json";
 
 // clipboard.js
-import 'clipboard/dist/clipboard.min.js';
-
+window.ClipboardJS = require('clipboard/dist/clipboard.min.js');

core/admin/assets/webpack.config.js

@@ -9,7 +9,7 @@ module.exports = {
     mode: 'production',
     entry: {
         app: {
-            import: './assets/app.js',
+            import: ['./assets/app.css', './assets/mailu.png', './assets/app.js'],
             dependOn: 'vendor',
         },
         vendor: './assets/vendor.js',

core/admin/audit.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import sys
 import tabulate

core/admin/mailu/__init__.py

@@ -44,8 +44,10 @@ def create_app_from_config(config):
     # Initialize debugging tools
     if app.config.get("DEBUG"):
         debug.toolbar.init_app(app)
-        # TODO: add a specific configuration variable for profiling
-        # debug.profiler.init_app(app)
+    if app.config.get("DEBUG_PROFILER"):
+        debug.profiler.init_app(app)
+    if assets := app.config.get('DEBUG_ASSETS'):
+        app.static_folder = assets
 
     # Inject the default variables in the Jinja parser
     # TODO: move this to blueprints when needed
@@ -55,6 +57,7 @@ def create_app_from_config(config):
         return dict(
             signup_domains= signup_domains,
             config        = app.config,
+            get_locale    = utils.get_locale,
         )
 
     # Jinja filters

core/admin/mailu/configuration.py

@@ -1,7 +1,6 @@
 import os
 
 from datetime import timedelta
-from socrate import system
 import ipaddress
 
 DEFAULT_CONFIG = {
@@ -11,18 +10,21 @@ DEFAULT_CONFIG = {
     'BABEL_DEFAULT_TIMEZONE': 'UTC',
     'BOOTSTRAP_SERVE_LOCAL': True,
     'RATELIMIT_STORAGE_URL': '',
-    'QUOTA_STORAGE_URL': '',
     'DEBUG': False,
+    'DEBUG_PROFILER': False,
+    'DEBUG_TB_INTERCEPT_REDIRECTS': False,
+    'DEBUG_ASSETS': '',
     'DOMAIN_REGISTRATION': False,
     'TEMPLATES_AUTO_RELOAD': True,
     'MEMORY_SESSIONS': False,
+    'FETCHMAIL_ENABLED': False,
     # Database settings
     'DB_FLAVOR': None,
     'DB_USER': 'mailu',
     'DB_PW': None,
     'DB_HOST': 'database',
     'DB_NAME': 'mailu',
-    'SQLITE_DATABASE_FILE':'data/main.db',
+    'SQLITE_DATABASE_FILE': 'data/main.db',
     'SQLALCHEMY_DATABASE_URI': 'sqlite:////data/main.db',
     'SQLALCHEMY_TRACK_MODIFICATIONS': False,
     # Statistics management
@@ -59,7 +61,7 @@ DEFAULT_CONFIG = {
     # Web settings
     'SITENAME': 'Mailu',
     'WEBSITE': 'https://mailu.io',
-    'ADMIN' : 'none',
+    'ADMIN': 'none',
     'WEB_ADMIN': '/admin',
     'WEB_WEBMAIL': '/webmail',
     'WEBMAIL': 'none',
@@ -72,21 +74,14 @@ DEFAULT_CONFIG = {
     'SESSION_KEY_BITS': 128,
     'SESSION_TIMEOUT': 3600,
     'PERMANENT_SESSION_LIFETIME': 30*24*3600,
-    'SESSION_COOKIE_SECURE': True,
+    'SESSION_COOKIE_SECURE': None,
     'CREDENTIAL_ROUNDS': 12,
     'TLS_PERMISSIVE': True,
     'TZ': 'Etc/UTC',
-    # Host settings
-    'HOST_IMAP': 'imap',
-    'HOST_LMTP': 'imap:2525',
-    'HOST_POP3': 'imap',
-    'HOST_SMTP': 'smtp',
-    'HOST_AUTHSMTP': 'smtp',
-    'HOST_ADMIN': 'admin',
-    'HOST_WEBMAIL': 'webmail',
-    'HOST_WEBDAV': 'webdav:5232',
-    'HOST_REDIS': 'redis',
-    'HOST_FRONT': 'front',
+    'DEFAULT_SPAM_THRESHOLD': 80,
+    'PROXY_AUTH_WHITELIST': '',
+    'PROXY_AUTH_HEADER': 'X-Auth-Email',
+    'PROXY_AUTH_CREATE': False,
     'SUBNET': '192.168.203.0/24',
     'SUBNET6': None
 }
@@ -104,18 +99,9 @@ class ConfigManager:
     def __init__(self):
         self.config = dict()
 
-    def get_host_address(self, name):
-        # if MYSERVICE_ADDRESS is defined, use this
-        if f'{name}_ADDRESS' in os.environ:
-            return os.environ.get(f'{name}_ADDRESS')
-        # otherwise use the host name and resolve it
-        return system.resolve_address(self.config[f'HOST_{name}'])
-
     def resolve_hosts(self):
-        for key in ['IMAP', 'POP3', 'AUTHSMTP', 'SMTP', 'REDIS']:
-            self.config[f'{key}_ADDRESS'] = self.get_host_address(key)
-        if self.config['WEBMAIL'] != 'none':
-            self.config['WEBMAIL_ADDRESS'] = self.get_host_address('WEBMAIL')
+        for key in ['ADMIN', 'FRONT', 'SMTP', 'IMAP', 'REDIS', 'ANTIVIRUS:', 'ANTISPAM', 'WEBMAIL', 'WEBDAV']:
+            self.config[f'{key}_ADDRESS'] = os.environ.get(f'{key}_ADDRESS')
 
     def __get_env(self, key, value):
         key_file = key + "_FILE"
@@ -141,6 +127,7 @@ class ConfigManager:
             key: self.__coerce_value(self.__get_env(key, value))
             for key, value in DEFAULT_CONFIG.items()
         })
+
         self.resolve_hosts()
 
         # automatically set the sqlalchemy string
@@ -148,21 +135,26 @@ class ConfigManager:
             template = self.DB_TEMPLATES[self.config['DB_FLAVOR']]
             self.config['SQLALCHEMY_DATABASE_URI'] = template.format(**self.config)
 
-        self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2'
-        self.config['QUOTA_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/1'
+        if not self.config.get('RATELIMIT_STORAGE_URL'):
+            self.config['RATELIMIT_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/2'
+
         self.config['SESSION_STORAGE_URL'] = f'redis://{self.config["REDIS_ADDRESS"]}/3'
         self.config['SESSION_COOKIE_SAMESITE'] = 'Strict'
         self.config['SESSION_COOKIE_HTTPONLY'] = True
+        if self.config['SESSION_COOKIE_SECURE'] is None:
+            self.config['SESSION_COOKIE_SECURE'] = self.config['TLS_FLAVOR'] != 'notls'
         self.config['SESSION_PERMANENT'] = True
         self.config['SESSION_TIMEOUT'] = int(self.config['SESSION_TIMEOUT'])
         self.config['PERMANENT_SESSION_LIFETIME'] = int(self.config['PERMANENT_SESSION_LIFETIME'])
         self.config['AUTH_RATELIMIT_IP_V4_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V4_MASK'])
         self.config['AUTH_RATELIMIT_IP_V6_MASK'] = int(self.config['AUTH_RATELIMIT_IP_V6_MASK'])
-        hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')]
         self.config['AUTH_RATELIMIT_EXEMPTION'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['AUTH_RATELIMIT_EXEMPTION'].split(',')) if cidr)
         self.config['MESSAGE_RATELIMIT_EXEMPTION'] = set([s for s in self.config['MESSAGE_RATELIMIT_EXEMPTION'].lower().replace(' ', '').split(',') if s])
+        hostnames = [host.strip() for host in self.config['HOSTNAMES'].split(',')]
         self.config['HOSTNAMES'] = ','.join(hostnames)
         self.config['HOSTNAME'] = hostnames[0]
+        self.config['DEFAULT_SPAM_THRESHOLD'] = int(self.config['DEFAULT_SPAM_THRESHOLD'])
+        self.config['PROXY_AUTH_WHITELIST'] = set(ipaddress.ip_network(cidr, False) for cidr in (cidr.strip() for cidr in self.config['PROXY_AUTH_WHITELIST'].split(',')) if cidr)
 
         # update the app config
         app.config.update(self.config)

core/admin/mailu/internal/nginx.py

@@ -1,12 +1,10 @@
 from mailu import models, utils
 from flask import current_app as app
+from socrate import system
 
-import re
 import urllib
 import ipaddress
-import socket
 import sqlalchemy.exc
-import tenacity
 
 SUPPORTED_AUTH_METHODS = ["none", "plain"]
 
@@ -127,32 +125,20 @@ def get_status(protocol, status):
     status, codes = STATUSES[status]
     return status, codes[protocol]
 
-def extract_host_port(host_and_port, default_port):
-    host, _, port = re.match('^(.*?)(:([0-9]*))?$', host_and_port).groups()
-    return host, int(port) if port else default_port
-
 def get_server(protocol, authenticated=False):
     if protocol == "imap":
-        hostname, port = extract_host_port(app.config['IMAP_ADDRESS'], 143)
+        hostname, port = app.config['IMAP_ADDRESS'], 143
     elif protocol == "pop3":
-        hostname, port = extract_host_port(app.config['POP3_ADDRESS'], 110)
+        hostname, port = app.config['IMAP_ADDRESS'], 110
     elif protocol == "smtp":
         if authenticated:
-            hostname, port = extract_host_port(app.config['AUTHSMTP_ADDRESS'], 10025)
+            hostname, port = app.config['SMTP_ADDRESS'], 10025
         else:
-            hostname, port = extract_host_port(app.config['SMTP_ADDRESS'], 25)
+            hostname, port = app.config['SMTP_ADDRESS'], 25
     try:
-        # test if hostname is already resolved to an ip adddress
+        # test if hostname is already resolved to an ip address
         ipaddress.ip_address(hostname)
     except:
         # hostname is not an ip address - so we need to resolve it
-        hostname = resolve_hostname(hostname)
+        hostname = system.resolve_hostname(hostname)
     return hostname, port
-
-@tenacity.retry(stop=tenacity.stop_after_attempt(100),
-                wait=tenacity.wait_random(min=2, max=5))
-def resolve_hostname(hostname):
-    """ This function uses system DNS to resolve a hostname.
-    It is capable of retrying in case the host is not immediately available
-    """
-    return socket.gethostbyname(hostname)

core/admin/mailu/internal/views/dovecot.py

@@ -5,6 +5,7 @@ from flask import current_app as app
 import flask
 import socket
 import os
+import sqlalchemy.exc
 
 @internal.route("/dovecot/passdb/<path:user_email>")
 def dovecot_passdb_dict(user_email):
@@ -19,12 +20,20 @@ def dovecot_passdb_dict(user_email):
         "allow_nets": ",".join(allow_nets)
     })
 
+@internal.route("/dovecot/userdb/")
+def dovecot_userdb_dict_list():
+    return flask.jsonify([
+        user[0] for user in models.User.query.filter(models.User.enabled.is_(True)).with_entities(models.User.email).all()
+    ])
 
 @internal.route("/dovecot/userdb/<path:user_email>")
 def dovecot_userdb_dict(user_email):
-    user = models.User.query.get(user_email) or flask.abort(404)
+    try:
+        quota = models.User.query.filter(models.User.email==user_email).with_entities(models.User.quota_bytes).one_or_none() or flask.abort(404)
+    except sqlalchemy.exc.StatementError as exc:
+        flask.abort(404)
     return flask.jsonify({
-        "quota_rule": "*:bytes={}".format(user.quota_bytes)
+        "quota_rule": f"*:bytes={quota[0]}"
     })
 
 
@@ -33,6 +42,7 @@ def dovecot_quota(ns, user_email):
     user = models.User.query.get(user_email) or flask.abort(404)
     if ns == "storage":
         user.quota_bytes_used = flask.request.get_json()
+        user.dont_change_updated_at()
         models.db.session.commit()
     return flask.jsonify(None)
 

core/admin/mailu/internal/views/fetch.py

@@ -12,10 +12,12 @@ def fetch_list():
             "id": fetch.id,
             "tls": fetch.tls,
             "keep": fetch.keep,
+            "scan": fetch.scan,
             "user_email": fetch.user_email,
             "protocol": fetch.protocol,
             "host": fetch.host,
             "port": fetch.port,
+            "folders": fetch.folders,
             "username": fetch.username,
             "password": fetch.password
         } for fetch in models.Fetch.query.all()
@@ -27,6 +29,7 @@ def fetch_done(fetch_id):
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     fetch.last_check = datetime.datetime.now()
     fetch.error_message = str(flask.request.get_json())
+    fetch.dont_change_updated_at()
     models.db.session.add(fetch)
     models.db.session.commit()
     return ""

core/admin/mailu/internal/views/postfix.py

@@ -143,8 +143,9 @@ def postfix_sender_login(sender):
     if localpart is None:
         return flask.jsonify(",".join(wildcard_senders)) if wildcard_senders else flask.abort(404)
     localpart = localpart[:next((i for i, ch in enumerate(localpart) if ch in flask.current_app.config.get('RECIPIENT_DELIMITER')), None)]
-    destinations = models.Email.resolve_destination(localpart, domain_name, True) or []
-    destinations.extend(wildcard_senders)
+    destinations = set(models.Email.resolve_destination(localpart, domain_name, True) or [])
+    destinations.update(wildcard_senders)
+    destinations.update(i[0] for i in models.User.query.filter_by(allow_spoofing=True).with_entities(models.User.email).all())
     if destinations:
         return flask.jsonify(",".join(idna_encode(destinations)))
     return flask.abort(404)
@@ -158,21 +159,6 @@ def postfix_sender_rate(sender):
     user = models.User.get(sender) or flask.abort(404)
     return flask.abort(404) if user.sender_limiter.hit() else flask.jsonify("450 4.2.1 You are sending too many emails too fast.")
 
-@internal.route("/postfix/sender/access/<path:sender>")
-def postfix_sender_access(sender):
-    """ Simply reject any sender that pretends to be from a local domain
-    """
-    if '@' in sender:
-        if sender.startswith('<') and sender.endswith('>'):
-            sender = sender[1:-1]
-        try:
-            localpart, domain_name = models.Email.resolve_domain(sender)
-            if models.Domain.query.get(domain_name):
-                return flask.jsonify("REJECT")
-        except sqlalchemy.exc.StatementError:
-            pass
-    return flask.abort(404)
-
 # idna encode domain part of each address in list of addresses
 def idna_encode(addresses):
     return [

core/admin/mailu/internal/views/rspamd.py

@@ -25,3 +25,7 @@ def rspamd_dkim_key(domain_name):
                 }
             )
     return flask.jsonify({'data': {'selectors': selectors}})
+
+@internal.route("/rspamd/local_domains", methods=['GET'])
+def rspamd_local_domains():
+    return '\n'.join(domain[0] for domain in models.Domain.query.with_entities(models.Domain.name).all() + models.Alternative.query.with_entities(models.Alternative.name).all())

core/admin/mailu/models.py

@@ -2,7 +2,6 @@
 """
 
 import os
-import smtplib
 import json
 
 from datetime import date
@@ -25,6 +24,7 @@ from flask import current_app as app
 from sqlalchemy.ext import declarative
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.inspection import inspect
+from sqlalchemy.orm.attributes import flag_modified
 from werkzeug.utils import cached_property
 
 from mailu import dkim, utils
@@ -154,6 +154,10 @@ class Base(db.Model):
             self.__hashed = id(self) if primary is None else hash(primary)
         return self.__hashed
 
+    def dont_change_updated_at(self):
+        """ Mark updated_at as modified, but keep the old date when updating the model"""
+        flag_modified(self, 'updated_at')
+
 
 # Many-to-many association table for domain managers
 managers = db.Table('manager', Base.metadata,
@@ -415,14 +419,18 @@ class Email(object):
 
     def sendmail(self, subject, body):
         """ send an email to the address """
-        f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}'
-        with smtplib.SMTP(app.config['HOST_AUTHSMTP'], port=10025) as smtp:
-            to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}'
-            msg = text.MIMEText(body)
-            msg['Subject'] = subject
-            msg['From'] = f_addr
-            msg['To'] = to_address
-            smtp.sendmail(f_addr, [to_address], msg.as_string())
+        try:
+            f_addr = f'{app.config["POSTMASTER"]}@{idna.encode(app.config["DOMAIN"]).decode("ascii")}'
+            with smtplib.LMTP(ip=app.config['IMAP_ADDRESS'], port=2525) as lmtp:
+                to_address = f'{self.localpart}@{idna.encode(self.domain_name).decode("ascii")}'
+                msg = text.MIMEText(body)
+                msg['Subject'] = subject
+                msg['From'] = f_addr
+                msg['To'] = to_address
+                lmtp.sendmail(f_addr, [to_address], msg.as_string())
+            return True
+        except smtplib.SMTPException:
+            return False
 
     @classmethod
     def resolve_domain(cls, email):
@@ -496,6 +504,7 @@ class User(Base, Email):
     # Features
     enable_imap = db.Column(db.Boolean, nullable=False, default=True)
     enable_pop = db.Column(db.Boolean, nullable=False, default=True)
+    allow_spoofing = db.Column(db.Boolean, nullable=False, default=False)
 
     # Filters
     forward_enabled = db.Column(db.Boolean, nullable=False, default=False)
@@ -513,7 +522,7 @@ class User(Base, Email):
     displayed_name = db.Column(db.String(160), nullable=False, default='')
     spam_enabled = db.Column(db.Boolean, nullable=False, default=True)
     spam_mark_as_read = db.Column(db.Boolean, nullable=False, default=True)
-    spam_threshold = db.Column(db.Integer, nullable=False, default=80)
+    spam_threshold = db.Column(db.Integer, nullable=False, default=lambda:int(app.config.get("DEFAULT_SPAM_THRESHOLD", 80)))
 
     # Flask-login attributes
     is_authenticated = True
@@ -541,8 +550,8 @@ class User(Base, Email):
         now = date.today()
         return (
             self.reply_enabled and
-            self.reply_startdate < now and
-            self.reply_enddate > now
+            self.reply_startdate <= now and
+            self.reply_enddate >= now
         )
 
     @property
@@ -766,6 +775,8 @@ class Fetch(Base):
     username = db.Column(db.String(255), nullable=False)
     password = db.Column(db.String(255), nullable=False)
     keep = db.Column(db.Boolean, nullable=False, default=False)
+    scan = db.Column(db.Boolean, nullable=False, default=False)
+    folders = db.Column(CommaSeparatedList, nullable=True, default=list)
     last_check = db.Column(db.DateTime, nullable=True)
     error = db.Column(db.String(1023), nullable=True)
 

core/admin/mailu/schemas.py

@@ -968,7 +968,7 @@ class BaseSchema(ma.SQLAlchemyAutoSchema, Storage):
                                 ) from exc
                     # sort list of new values
                     data[key] = sorted(new_value)
-                    # log backref modification not catched by modify hook
+                    # log backref modification not caught by modify hook
                     if isinstance(self.fields[key], RelatedList):
                         if callback := self.context.get('callback'):
                             before = {str(v) for v in getattr(instance, key)}

core/admin/mailu/sso/forms.py

@@ -7,5 +7,6 @@ class LoginForm(flask_wtf.FlaskForm):
         csrf = False
     email = fields.StringField(_('E-mail'), [validators.Email(), validators.DataRequired()])
     pw = fields.PasswordField(_('Password'), [validators.DataRequired()])
+    pwned = fields.HiddenField(label='', default=-1)
     submitWebmail = fields.SubmitField(_('Sign in'))
     submitAdmin = fields.SubmitField(_('Sign in'))

core/admin/mailu/sso/templates/base_sso.html

@@ -1,7 +1,7 @@
 {%- import "macros.html" as macros %}
 {%- import "bootstrap/utils.html" as utils %}
 <!doctype html>
-<html lang="{{ session['language'] }}" data-static="/static/">
+<html lang="{{ get_locale() }}" data-static="/static/">
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
@@ -34,8 +34,8 @@
         <ul class="navbar-nav ml-auto">
           <li class="nav-item dropdown">
             <a class="nav-link" data-toggle="dropdown" href="#" aria-expanded="false">
-              <i class="fas fa-language text-xl" aria-hidden="true" title="{% trans %}change language{% endtrans %}"></i><span class="sr-only">Language</span>
-              <span class="badge badge-primary navbar-badge">{{ session['language'] }}</span></a>
+              <i class="fas fa-language text-xl" aria-hidden="true" title="{% trans %}change language{% endtrans %}"></i><span class="sr-only">{% trans %}change language{% endtrans %}</span>
+              <span class="badge badge-primary navbar-badge">{{ get_locale() }}</span></a>
             <div class="dropdown-menu dropdown-menu-right p-0" id="mailu-languages">
             {%- for locale in config.translations.values() %}
               <a class="dropdown-item{% if locale|string() == session['language'] %} active{% endif %}" href="{{ url_for('sso.set_language', language=locale) }}">{{ locale.get_language_name().title() }}</a>

core/admin/mailu/sso/templates/form_sso.html

@@ -3,6 +3,7 @@
 {%- block content %}
 {%- call macros.card() %}
 <form class="form" method="post" role="form">
+  {{ form.hidden_tag() }}
   {{ macros.form_field(form.email) }}
   {{ macros.form_field(form.pw) }}
   {{ macros.form_fields(fields, label=False, class="btn btn-default") }}

core/admin/mailu/sso/templates/sidebar_sso.html

@@ -36,6 +36,12 @@
         </a>
       </li>
     {%- endif %}
+      <li class="nav-item" role="none">
+        <a href="{{ url_for('sso.login') }}" class="nav-link" role="menuitem">
+          <i class="nav-icon fas fa-sign-in-alt"></i>
+          <p>{% trans %}Sign in{% endtrans %}</p>
+        </a>
+      </li>
     {#-
       User self-registration is only available when
        - Admin is available

core/admin/mailu/sso/views/base.py

@@ -6,6 +6,8 @@ from mailu.ui import access
 from flask import current_app as app
 import flask
 import flask_login
+import secrets
+import ipaddress
 
 @sso.route('/login', methods=['GET', 'POST'])
 def login():
@@ -40,7 +42,9 @@ def login():
             flask_login.login_user(user)
             response = flask.redirect(destination)
             response.set_cookie('rate_limit', utils.limiter.device_cookie(username), max_age=31536000, path=flask.url_for('sso.login'), secure=app.config['SESSION_COOKIE_SECURE'], httponly=True)
-            flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip}.')
+            flask.current_app.logger.info(f'Login succeeded for {username} from {client_ip} pwned={form.pwned.data}.')
+            if msg := utils.isBadOrPwned(form):
+                flask.flash(msg, "error")
             return response
         else:
             utils.limiter.rate_limit_user(username, client_ip, device_cookie, device_cookie_username) if models.User.get(username) else utils.limiter.rate_limit_ip(client_ip)
@@ -55,3 +59,41 @@ def logout():
     flask.session.destroy()
     return flask.redirect(flask.url_for('.login'))
 
+
+@sso.route('/proxy', methods=['GET'])
+@sso.route('/proxy/<target>', methods=['GET'])
+def proxy(target='webmail'):
+    ip = ipaddress.ip_address(flask.request.remote_addr)
+    if not any(ip in cidr for cidr in app.config['PROXY_AUTH_WHITELIST']):
+        return flask.abort(500, '%s is not on PROXY_AUTH_WHITELIST' % flask.request.remote_addr)
+
+    email = flask.request.headers.get(app.config['PROXY_AUTH_HEADER'])
+    if not email:
+        return flask.abort(500, 'No %s header' % app.config['PROXY_AUTH_HEADER'])
+
+    user = models.User.get(email)
+    if user:
+        flask.session.regenerate()
+        flask_login.login_user(user)
+        return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])
+
+    if not app.config['PROXY_AUTH_CREATE']:
+        return flask.abort(500, 'You don\'t exist. Go away! (%s)' % email)
+
+    client_ip = flask.request.headers.get('X-Real-IP', flask.request.remote_addr)
+    try:
+        localpart, desireddomain = email.rsplit('@')
+    except Exception as e:
+        flask.current_app.logger.error('Error creating a new user via proxy for %s from %s: %s' % (email, client_ip, str(e)), e)
+        return flask.abort(500, 'You don\'t exist. Go away! (%s)' % email)
+    domain = models.Domain.query.get(desireddomain) or flask.abort(500, 'You don\'t exist. Go away! (domain=%s)' % desireddomain)
+    if not domain.max_users == -1 and len(domain.users) >= domain.max_users:
+        flask.current_app.logger.warning('Too many users for domain %s' % domain)
+        return flask.abort(500, 'Too many users in (domain=%s)' % domain)
+    user = models.User(localpart=localpart, domain=domain)
+    user.set_password(secrets.token_urlsafe())
+    models.db.session.add(user)
+    models.db.session.commit()
+    user.send_welcome()
+    flask.current_app.logger.info(f'Login succeeded by proxy created user: {user} from {client_ip} through {flask.request.remote_addr}.')
+    return flask.redirect(app.config['WEB_ADMIN'] if target=='admin' else app.config['WEB_WEBMAIL'])

core/admin/mailu/sso/views/languages.py

@@ -1,7 +1,7 @@
 from mailu.sso import sso
 import flask
 
-@sso.route('/language/<language>', methods=['POST'])
+@sso.route('/language/<language>', methods=['GET','POST'])
 def set_language(language=None):
     if language:
         flask.session['language'] = language

core/admin/mailu/translations/ca/LC_MESSAGES/messages.po

@@ -660,7 +660,7 @@ msgid "New relay domain"
 msgstr "Nou domini llegat (relayed)"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Editeu domini llegat (relayed)"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/da/LC_MESSAGES/messages.po

@@ -649,7 +649,7 @@ msgid "New relay domain"
 msgstr ""
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr ""
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/de/LC_MESSAGES/messages.po

@@ -655,7 +655,7 @@ msgid "New relay domain"
 msgstr "Neue Relay-Domain"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Relay-Domain bearbeiten"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/en/LC_MESSAGES/messages.po

@@ -649,7 +649,7 @@ msgid "New relay domain"
 msgstr "New relay domain"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr ""
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/es/LC_MESSAGES/messages.po

@@ -651,7 +651,7 @@ msgid "New relay domain"
 msgstr "Nuevo dominio externo (relay)"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Editar dominio externo (relay)"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/eu/LC_MESSAGES/messages.po

@@ -649,7 +649,7 @@ msgid "New relay domain"
 msgstr "Igorritako domeinu berria"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Editatu igorritako domeinua"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/fr/LC_MESSAGES/messages.po

@@ -653,7 +653,7 @@ msgid "New relay domain"
 msgstr "Nouveau domaine relayé"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Modifier le domaine relayé"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/he/LC_MESSAGES/messages.po

@@ -658,7 +658,7 @@ msgid "New relay domain"
 msgstr "שם תחום מועבר"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "עריכת שמות תחום מועברים"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/hu/LC_MESSAGES/messages.po

@@ -652,7 +652,7 @@ msgid "New relay domain"
 msgstr "Új továbbító tartomány"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Továbbított tartomány szerkesztése"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/is/LC_MESSAGES/messages.po

@@ -648,7 +648,7 @@ msgid "New relay domain"
 msgstr ""
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr ""
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/it/LC_MESSAGES/messages.po

@@ -655,7 +655,7 @@ msgid "New relay domain"
 msgstr "Nuovo dominio affidato"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Editare dominio affidato"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/ja/LC_MESSAGES/messages.po

@@ -649,7 +649,7 @@ msgid "New relay domain"
 msgstr "リレードメイン名を追加"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "リレードメイン名を編集"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/nb_NO/LC_MESSAGES/messages.po

@@ -657,7 +657,7 @@ msgid "New relay domain"
 msgstr "Nytt videresendt domene"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Endre videresendt domene"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/nl/LC_MESSAGES/messages.po

@@ -650,7 +650,7 @@ msgid "New relay domain"
 msgstr "Nieuw relay domein"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Bewerk relay domein"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/pl/LC_MESSAGES/messages.po

@@ -659,7 +659,7 @@ msgstr "Nowa domena do przekierowania"
 
 #: mailu/ui/templates/relay/edit.html:4
 #, fuzzy
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Edycja domeny"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/pt/LC_MESSAGES/messages.po

@@ -650,7 +650,7 @@ msgid "New relay domain"
 msgstr "Novo domínio para encaminhamento"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Editar domínio de encaminhamento"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/ru/LC_MESSAGES/messages.po

@@ -656,7 +656,7 @@ msgid "New relay domain"
 msgstr "Новый релейный домен"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Изменить релейный домен"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/sv/LC_MESSAGES/messages.po

@@ -645,7 +645,7 @@ msgid "New relay domain"
 msgstr "Ny relä-domän"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "Redigera reläade domäner"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/translations/zh/LC_MESSAGES/messages.po

@@ -646,7 +646,7 @@ msgid "New relay domain"
 msgstr "新的中继域"
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr "编辑中继域"
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/mailu/ui/forms.py

@@ -41,6 +41,16 @@ class MultipleEmailAddressesVerify(object):
         if not pattern.match(field.data.replace(" ", "")):
             raise validators.ValidationError(self.message)
 
+class MultipleFoldersVerify(object):
+    """ Ensure that we have CSV formated data """
+    def __init__(self,message=_('Invalid list of folders.')):
+        self.message = message
+
+    def __call__(self, form, field):
+        pattern = re.compile(r'^\w+(\s*,\s*\w+)*$')
+        if not pattern.match(field.data.replace(" ", "")):
+            raise validators.ValidationError(self.message)
+
 class ConfirmationForm(flask_wtf.FlaskForm):
     submit = fields.SubmitField(_('Confirm'))
 
@@ -59,6 +69,7 @@ class DomainSignupForm(flask_wtf.FlaskForm):
     localpart = fields.StringField(_('Initial admin'), [validators.DataRequired()])
     pw = fields.PasswordField(_('Admin password'), [validators.DataRequired()])
     pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')])
+    pwned = fields.HiddenField(label='', default=-1)
     captcha = flask_wtf.RecaptchaField()
     submit = fields.SubmitField(_('Create'))
 
@@ -79,9 +90,11 @@ class UserForm(flask_wtf.FlaskForm):
     localpart = fields.StringField(_('E-mail'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)])
     pw = fields.PasswordField(_('Password'))
     pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')])
+    pwned = fields.HiddenField(label='', default=-1)
     quota_bytes = fields_.IntegerSliderField(_('Quota'), default=10**9)
     enable_imap = fields.BooleanField(_('Allow IMAP access'), default=True)
     enable_pop = fields.BooleanField(_('Allow POP3 access'), default=True)
+    allow_spoofing = fields.BooleanField(_('Allow the user to spoof the sender (send email as anyone)'), default=False)
     displayed_name = fields.StringField(_('Displayed name'))
     comment = fields.StringField(_('Comment'))
     enabled = fields.BooleanField(_('Enabled'), default=True)
@@ -92,6 +105,7 @@ class UserSignupForm(flask_wtf.FlaskForm):
     localpart = fields.StringField(_('Email address'), [validators.DataRequired(), validators.Regexp(LOCALPART_REGEX)])
     pw = fields.PasswordField(_('Password'), [validators.DataRequired()])
     pw2 = fields.PasswordField(_('Confirm password'), [validators.EqualTo('pw')])
+    pwned = fields.HiddenField(label='', default=-1)
     submit = fields.SubmitField(_('Sign up'))
 
 class UserSignupFormCaptcha(UserSignupForm):
@@ -111,6 +125,7 @@ class UserSettingsForm(flask_wtf.FlaskForm):
 class UserPasswordForm(flask_wtf.FlaskForm):
     pw = fields.PasswordField(_('Password'), [validators.DataRequired()])
     pw2 = fields.PasswordField(_('Password check'), [validators.DataRequired()])
+    pwned = fields.HiddenField(label='', default=-1)
     submit = fields.SubmitField(_('Update password'))
 
 
@@ -119,8 +134,8 @@ class UserReplyForm(flask_wtf.FlaskForm):
     reply_subject = fields.StringField(_('Reply subject'))
     reply_body = fields.StringField(_('Reply body'),
         widget=widgets.TextArea())
-    reply_startdate = fields.html5.DateField(_('Start of vacation'))
-    reply_enddate = fields.html5.DateField(_('End of vacation'))
+    reply_startdate = fields.DateField(_('Start of vacation'))
+    reply_enddate = fields.DateField(_('End of vacation'))
     submit = fields.SubmitField(_('Update'))
 
 
@@ -160,11 +175,13 @@ class FetchForm(flask_wtf.FlaskForm):
         ('imap', 'IMAP'), ('pop3', 'POP3')
     ])
     host = fields.StringField(_('Hostname or IP'), [validators.DataRequired()])
-    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)])
-    tls = fields.BooleanField(_('Enable TLS'))
+    port = fields.IntegerField(_('TCP port'), [validators.DataRequired(), validators.NumberRange(min=0, max=65535)], default=993)
+    tls = fields.BooleanField(_('Enable TLS'), default=True)
     username = fields.StringField(_('Username'), [validators.DataRequired()])
     password = fields.PasswordField(_('Password'))
     keep = fields.BooleanField(_('Keep emails on the server'))
+    scan = fields.BooleanField(_('Rescan emails locally'))
+    folders = fields.StringField(_('Folders to fetch on the server'), [validators.Optional(), MultipleFoldersVerify()], default='INBOX,Junk')
     submit = fields.SubmitField(_('Submit'))
 
 

core/admin/mailu/ui/templates/admin/list.html

@@ -14,7 +14,7 @@
 {%- call macros.table() %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
     <th>{% trans %}Email{% endtrans %}</th>
   </tr>
 </thead>

core/admin/mailu/ui/templates/alias/list.html

@@ -16,7 +16,7 @@
 {%- call macros.table() %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
     <th>{% trans %}Email{% endtrans %}</th>
     <th>{% trans %}Destination{% endtrans %}</th>
     <th>{% trans %}Comment{% endtrans %}</th>
@@ -34,8 +34,8 @@
     <td>{{ alias }}</td>
     <td>{{ alias.destination|join(', ') or '-' }}</td>
     <td>{{ alias.comment or '' }}</td>
-    <td>{{ alias.created_at | format_date }}</td>
-    <td>{{ alias.updated_at | format_date }}</td>
+    <td data-sort="{{ alias.created_at or '0000-00-00' }}">{{ alias.created_at | format_date }}</td>
+    <td data-sort="{{ alias.updated_at or '0000-00-00' }}">{{ alias.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/alternative/list.html

@@ -16,7 +16,7 @@
 {%- call macros.table() %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
     <th>{% trans %}Name{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
     <th>{% trans %}Last edit{% endtrans %}</th>
@@ -29,8 +29,8 @@
       <a href="{{ url_for('.alternative_delete', alternative=alternative.name) }}" title="{% trans %}Delete{% endtrans %}"><i class="fa fa-trash"></i></a>
     </td>
     <td>{{ alternative }}</td>
-    <td>{{ alternative.created_at | format_date }}</td>
-    <td>{{ alternative.updated_at | format_date }}</td>
+    <td data-sort="{{ alternative.created_at or '0000-00-00' }}">{{ alternative.created_at | format_date }}</td>
+    <td data-sort="{{ alternative.updated_at or '0000-00-00' }}">{{ alternative.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/base.html

@@ -1,7 +1,7 @@
 {%- import "macros.html" as macros %}
 {%- import "bootstrap/utils.html" as utils %}
 <!doctype html>
-<html lang="{{ session['language'] }}" data-static="/static/">
+<html lang="{{ get_locale() }}" data-static="/static/">
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
@@ -34,8 +34,8 @@
         <ul class="navbar-nav ml-auto">
           <li class="nav-item dropdown">
             <a class="nav-link" data-toggle="dropdown" href="#" aria-expanded="false">
-              <i class="fas fa-language text-xl" aria-hidden="true" title="{% trans %}change language{% endtrans %}"></i><span class="sr-only">Language</span>
-              <span class="badge badge-primary navbar-badge">{{ session['language'] }}</span></a>
+              <i class="fas fa-language text-xl" aria-hidden="true" title="{% trans %}change language{% endtrans %}"></i><span class="sr-only">{% trans %}change language{% endtrans %}</span>
+              <span class="badge badge-primary navbar-badge">{{ get_locale() }}</span></a>
             <div class="dropdown-menu dropdown-menu-right p-0" id="mailu-languages">
             {%- for locale in config.translations.values() %}
               <a class="dropdown-item{% if locale|string() == session['language'] %} active{% endif %}" href="{{ url_for('.set_language', language=locale) }}">{{ locale.get_language_name().title() }}</a>

core/admin/mailu/ui/templates/client.html

@@ -9,7 +9,6 @@
 {%- endblock %}
 
 {%- block content %}
-<div>If you use an Apple device, <a href="/apple.mobileconfig">click here to autoconfigure it.</a></div>
 {%- call macros.table(title=_("Incoming mail"), datatable=False) %}
   <tbody>
     <tr>
@@ -22,7 +21,7 @@
     </tr>
     <tr>
       <th>{% trans %}Server name{% endtrans %}</th>
-      <td><pre class="pre-config border bg-light">{{ config["HOSTNAMES"] }}</pre></td>
+      <td><pre class="pre-config border bg-light">{{ config["HOSTNAME"] }}</pre></td>
     </tr>
     <tr>
       <th>{% trans %}Username{% endtrans %}</th>
@@ -47,7 +46,7 @@
     </tr>
     <tr>
       <th>{% trans %}Server name{% endtrans %}</th>
-      <td><pre class="pre-config border bg-light">{{ config["HOSTNAMES"] }}</pre></td>
+      <td><pre class="pre-config border bg-light">{{ config["HOSTNAME"] }}</pre></td>
     </tr>
     <tr>
       <th>{% trans %}Username{% endtrans %}</th>
@@ -59,4 +58,8 @@
     </tr>
   </tbody>
 {%- endcall %}
+<blockquote>
+  {% trans %}If you use an Apple device,{% endtrans %}
+  <a href="/apple.mobileconfig">{% trans %}click here to autoconfigure it.{% endtrans %}</a>
+</blockquote>
 {%- endblock %}

core/admin/mailu/ui/templates/domain/create.html

@@ -10,7 +10,7 @@
   {{ form.hidden_tag() }}
   {{ macros.form_field(form.name) }}
   {{ macros.form_fields((form.max_users, form.max_aliases)) }}
-  {{ macros.form_field(form.max_quota_bytes, step=10**9, max=50*10**9, data_infinity="true",
+  {{ macros.form_field(form.max_quota_bytes, step=50*10**6, max=50*10**9, data_infinity="true",
       prepend='<span class="input-group-text"><span id="max_quota_bytes_value"></span>&nbsp;GB</span>') }}
   {{ macros.form_field(form.signup_enabled) }}
   {{ macros.form_field(form.comment) }}

core/admin/mailu/ui/templates/domain/list.html

@@ -11,15 +11,16 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[2,"asc"]]') %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
-    <th>{% trans %}Manage{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Manage{% endtrans %}</th>
     <th>{% trans %}Domain name{% endtrans %}</th>
     <th>{% trans %}Mailbox count{% endtrans %}</th>
     <th>{% trans %}Alias count{% endtrans %}</th>
     <th>{% trans %}Comment{% endtrans %}</th>
+    <th>{% trans %}Enable sign-up{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
     <th>{% trans %}Last edit{% endtrans %}</th>
   </tr>
@@ -43,11 +44,12 @@
       {%- endif %}
     </td>
     <td>{{ domain.name }}</td>
-    <td>{{ domain.users | count }} / {{ '∞' if domain.max_users == -1 else domain.max_users }}</td>
-    <td>{{ domain.aliases | count }} / {{ '∞' if domain.max_aliases == -1 else domain.max_aliases }}</td>
+    <td data-order="{{ domain.users | count }}">{{ domain.users | count }} / {{ '∞' if domain.max_users == -1 else domain.max_users }}</td>
+    <td data-order="{{ domain.aliases | count }}">{{ domain.aliases | count }} / {{ '∞' if domain.max_aliases == -1 else domain.max_aliases }}</td>
     <td>{{ domain.comment or '' }}</td>
-    <td>{{ domain.created_at | format_date }}</td>
-    <td>{{ domain.updated_at | format_date }}</td>
+    <td data-sort="{{ domain.signup_enabled }}">{% if domain.signup_enabled %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td data-order="{{ domain.created_at or '0000-00-00' }}">{{ domain.created_at | format_date }}</td>
+    <td data-order="{{ domain.updated_at or '0000-00-00' }}">{{ domain.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/fetch/create.html

@@ -24,6 +24,8 @@
 
   {%- call macros.card(title="Settings") %}
   {{ macros.form_field(form.keep) }}
+  {{ macros.form_field(form.scan) }}
+  {{ macros.form_field(form.folders) }}
   {%- endcall %}
 
   {{ macros.form_field(form.submit) }}

core/admin/mailu/ui/templates/fetch/list.html

@@ -16,10 +16,12 @@
 {%- call macros.table() %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
     <th>{% trans %}Endpoint{% endtrans %}</th>
     <th>{% trans %}Username{% endtrans %}</th>
     <th>{% trans %}Keep emails{% endtrans %}</th>
+    <th>{% trans %}Rescan emails{% endtrans %}</th>
+    <th>{% trans %}Folders{% endtrans %}</th>
     <th>{% trans %}Last check{% endtrans %}</th>
     <th>{% trans %}Status{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
@@ -35,11 +37,13 @@
     </td>
     <td>{{ fetch.protocol }}{{ 's' if fetch.tls else '' }}://{{ fetch.host }}:{{ fetch.port }}</td>
     <td>{{ fetch.username }}</td>
-    <td>{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td data-sort="{{ fetch.keep }}">{% if fetch.keep %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td data-sort="{{ fetch.scan }}">{% if fetch.scan %}{% trans %}yes{% endtrans %}{% else %}{% trans %}no{% endtrans %}{% endif %}</td>
+    <td>{{ fetch.folders | join(',') }}</td>
     <td>{{ fetch.last_check | format_datetime or '-' }}</td>
     <td>{{ fetch.error or '-' }}</td>
-    <td>{{ fetch.created_at | format_date }}</td>
-    <td>{{ fetch.updated_at | format_date }}</td>
+    <td data-sort="{{ fetch.created_at or '0000-00-00' }}">{{ fetch.created_at | format_date }}</td>
+    <td data-sort="{{ fetch.updated_at or '0000-00-00' }}">{{ fetch.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/macros.html

@@ -3,7 +3,7 @@
     {%- for fieldname, errors in form.errors.items() %}
       {%- if bootstrap_is_hidden_field(form[fieldname]) %}
         {%- for error in errors %}
-          <p class="error">{{error}}</p>
+          <p class="form-text text-danger">{{error}}</p>
         {%- endfor %}
       {%- endif %}
     {%- endfor %}
@@ -13,7 +13,7 @@
 {%- macro form_field_errors(field) %}
   {%- if field.errors %}
     {%- for error in field.errors %}
-      <p class="help-block inline">{{ error }}</p>
+      <p class="form-text text-danger">{{ error }}</p>
     {%- endfor %}
   {%- endif %}
 {%- endmacro %}
@@ -23,7 +23,7 @@
   <div class="form-group">
     <div class="row">
       {%- for field in fields %}
-      <div class="col-lg-{{ width }} col-xs-12 {{ 'has-error' if field.errors else '' }}">
+      <div class="col-lg-{{ width }} col-xs-12">
         {%- if field.__class__.__name__ == 'list' %}
           {%- for subfield in field %}
             {{ form_individual_field(subfield, prepend=prepend, append=append, label=label, **kwargs) }}
@@ -38,12 +38,13 @@
 {%- endmacro %}
 
 {%- macro form_individual_field(field, prepend='', append='', label=True, class_="") %}
+  {%- set fieldclass=" ".join(["form-control"] + ([class_] if class_ else []) + (["is-invalid"] if field.errors else [])) %}
   {%- if field.type == "BooleanField" %}
     {{ field(**kwargs) }}<span>&nbsp;&nbsp;</span>{{ field.label if label else '' }}
   {%- else %}
     {{ field.label if label else '' }}{{ form_field_errors(field) }}
     {%- if prepend %}<div class="input-group-prepend">{%- elif append %}<div class="input-group-append">{%- endif %}
-      {{ prepend|safe }}{{ field(class_=("form-control " + class_) if class_ else "form-control", **kwargs) }}{{ append|safe }}
+      {{ prepend|safe }}{{ field(class_=fieldclass, **kwargs) }}{{ append|safe }}
     {%- if prepend or append %}</div>{%- endif %}
   {%- endif %}
 {%- endmacro %}
@@ -60,9 +61,7 @@
 <form class="form" method="post" role="form">
   {{ form.hidden_tag() }}
   {%- for field in form %}
-    {%- if bootstrap_is_hidden_field(field) %}
-      {{ field() }}
-    {%- else %}
+    {%- if not bootstrap_is_hidden_field(field) %}
       {{ form_field(field) }}
     {%- endif %}
   {%- endfor %}
@@ -88,7 +87,7 @@
 </div>
 {%- endmacro %}
 
-{%- macro table(title=None, theme="primary", datatable=True) %}
+{%- macro table(title=None, theme="primary", datatable=True, order=None) %}
 <div class="row">
   <div class="col-lg-12">
     <div class="card card-outline card-{{ theme }}">
@@ -98,7 +97,7 @@
       </div>
       {%- endif %}
       <div class="card-body">
-        <table class="table table-bordered{% if datatable %} dataTable{% endif %}">
+        <table class="table table-bordered{% if datatable %} dataTable{% endif %}" data-order="{{ order or '[]' | e }}">
           {{- caller() }}
         </table>
       </div>

core/admin/mailu/ui/templates/manager/list.html

@@ -13,10 +13,10 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[2,"asc"]]') %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
     <th>{% trans %}Email{% endtrans %}</th>
   </tr>
 </thead>

core/admin/mailu/ui/templates/relay/edit.html

@@ -1,7 +1,7 @@
 {%- extends "form.html" %}
 
 {%- block title %}
-{% trans %}Edit relayd domain{% endtrans %}
+{% trans %}Edit relayed domain{% endtrans %}
 {%- endblock %}
 
 {%- block subtitle %}

core/admin/mailu/ui/templates/relay/list.html

@@ -11,10 +11,10 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
+{%- call macros.table(order='[[1,"asc"]]') %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
     <th>{% trans %}Domain name{% endtrans %}</th>
     <th>{% trans %}Remote host{% endtrans %}</th>
     <th>{% trans %}Comment{% endtrans %}</th>
@@ -32,8 +32,8 @@
     <td>{{ relay.name }}</td>
     <td>{{ relay.smtp or '-' }}</td>
     <td>{{ relay.comment or '' }}</td>
-    <td>{{ relay.created_at | format_date }}</td>
-    <td>{{ relay.updated_at | format_date }}</td>
+    <td data-sort="{{ relay.created_at or '0000-00-00' }}">{{ relay.created_at | format_date }}</td>
+    <td data-sort="{{ relay.updated_at or '0000-00-00' }}">{{ relay.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/sidebar.html

@@ -31,12 +31,14 @@
           <p>{% trans %}Auto-reply{% endtrans %}</p>
         </a>
       </li>
+      {%- if config["FETCHMAIL_ENABLED"] %}
       <li class="nav-item" role="none">
         <a href="{{ url_for('.fetch_list') }}" class="nav-link" role="menuitem">
           <i class="nav-icon fas fa-download"></i>
           <p>{% trans %}Fetched accounts{% endtrans %}</p>
         </a>
       </li>
+      {%- endif %}
       <li class="nav-item" role="none">
         <a href="{{ url_for('.token_list') }}" class="nav-link" role="menuitem">
           <i class="nav-icon fas fa-ticket-alt"></i>

core/admin/mailu/ui/templates/token/list.html

@@ -16,7 +16,7 @@
 {%- call macros.table() %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
     <th>{% trans %}Comment{% endtrans %}</th>
     <th>{% trans %}Authorized IP{% endtrans %}</th>
     <th>{% trans %}Created{% endtrans %}</th>
@@ -31,8 +31,8 @@
     </td>
     <td>{{ token.comment }}</td>
     <td>{{ token.ip or "any" }}</td>
-    <td>{{ token.created_at | format_date }}</td>
-    <td>{{ token.updated_at | format_date }}</td>
+    <td data-sort="{{ token.created_at or '0000-00-00' }}">{{ token.created_at | format_date }}</td>
+    <td data-sort="{{ token.updated_at or '0000-00-00' }}">{{ token.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/user/create.html

@@ -21,10 +21,11 @@
   {%- endcall %}
 
   {%- call macros.card(_("Features and quotas"), theme="success") %}
-  {{ macros.form_field(form.quota_bytes, step=1000000000, max=(max_quota_bytes or domain.max_quota_bytes or 50*10**9), data_infinity="true",
+  {{ macros.form_field(form.quota_bytes, step=50*10**6, max=(max_quota_bytes or domain.max_quota_bytes or 50*10**9), data_infinity="true",
       prepend='<span class="input-group-text"><span id="quota_bytes_value"></span>&nbsp;GB</span>') }}
   {{ macros.form_field(form.enable_imap) }}
   {{ macros.form_field(form.enable_pop) }}
+  {{ macros.form_field(form.allow_spoofing) }}
   {%- endcall %}
 
   {{ macros.form_field(form.submit) }}

core/admin/mailu/ui/templates/user/list.html

@@ -16,8 +16,8 @@
 {%- call macros.table() %}
 <thead>
   <tr>
-    <th>{% trans %}Actions{% endtrans %}</th>
-    <th>{% trans %}User settings{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}Actions{% endtrans %}</th>
+    <th data-orderable="false">{% trans %}User settings{% endtrans %}</th>
     <th>{% trans %}Email{% endtrans %}</th>
     <th>{% trans %}Features{% endtrans %}</th>
     <th>{% trans %}Quota{% endtrans %}</th>
@@ -39,14 +39,15 @@
       <a href="{{ url_for('.fetch_list', user_email=user.email) }}" title="{% trans %}Fetched accounts{% endtrans %}"><i class="fa fa-download"></i></a>&nbsp;
     </td>
     <td>{{ user }}</td>
-    <td>
-      {% if user.enable_imap %}<span class="badge bg-info">imap</span>{% endif %}
-      {% if user.enable_pop %}<span class="badge bg-info">pop3</span>{% endif %}
+    <td data-sort="{{ user.allow_spoofing*4 + user.enable_imap*2 + user.enable_pop }}">
+      {% if user.enable_imap %}<span class="badge bg-primary">imap</span>{% endif %}
+      {% if user.enable_pop %}<span class="badge bg-secondary">pop3</span>{% endif %}
+      {% if user.allow_spoofing %}<span class="badge bg-danger">allow-spoofing</span>{% endif %}
     </td>
-    <td>{{ user.quota_bytes_used | filesizeformat }} / {{ (user.quota_bytes | filesizeformat) if user.quota_bytes else '∞' }}</td>
+    <td data-sort="{{ user.quota_bytes_used }}">{{ user.quota_bytes_used | filesizeformat }} / {{ (user.quota_bytes | filesizeformat) if user.quota_bytes else '∞' }}</td>
     <td>{{ user.comment or '-' }}</td>
-    <td>{{ user.created_at | format_date }}</td>
-    <td>{{ user.updated_at | format_date }}</td>
+    <td data-sort="{{ user.created_at or '0000-00-00' }}">{{ user.created_at | format_date }}</td>
+    <td data-sort="{{ user.updated_at or '0000-00-00' }}">{{ user.updated_at | format_date }}</td>
   </tr>
   {%- endfor %}
 </tbody>

core/admin/mailu/ui/templates/user/signup_domain.html

@@ -9,18 +9,22 @@
 {%- endblock %}
 
 {%- block content %}
-{%- call macros.table() %}
-<tr>
-  <th>{% trans %}Domain{% endtrans %}</th>
-  <th>{% trans %}Available slots{% endtrans %}</th>
-  <th>{% trans %}Quota{% endtrans %}</th>
-</tr>
+{%- call macros.table(order='[[1,"asc"]]') %}
+<thead>
+  <tr>
+    <th>{% trans %}Domain{% endtrans %}</th>
+    <th>{% trans %}Available slots{% endtrans %}</th>
+    <th>{% trans %}Quota{% endtrans %}</th>
+  </tr>
+</thead>
+<tbody>
 {%- for domain_name, domain in available_domains.items() %}
 <tr>
   <td><a href="{{ url_for('.user_signup', domain_name=domain_name) }}">{{ domain_name }}</a></td>
-  <td>{{ '∞' if domain.max_users == -1 else domain.max_users - (domain.users | count)}}</td>
-  <td>{{ domain.max_quota_bytes or config['DEFAULT_QUOTA'] | filesizeformat }}</td>
+  <td data-sort="{{ -1 if domain.max_users == -1 else domain.max_users - (domain.users | count)}}">{{ '∞' if domain.max_users == -1 else domain.max_users - (domain.users | count)}}</td>
+  <td data-sort="{{ domain.max_quota_bytes or config['DEFAULT_QUOTA'] }}">{{ domain.max_quota_bytes or config['DEFAULT_QUOTA'] | filesizeformat }}</td>
 </tr>
 {%- endfor %}
+</tbody>
 {%- endcall %}
 {%- endblock %}

core/admin/mailu/ui/views/base.py

@@ -21,8 +21,9 @@ def announcement():
     form = forms.AnnouncementForm()
     if form.validate_on_submit():
         for user in models.User.query.all():
-            user.sendmail(form.announcement_subject.data,
-                form.announcement_body.data)
+            if not user.sendmail(form.announcement_subject.data,
+                    form.announcement_body.data):
+                flask.flash('Failed to send to %s' % user.email, 'error')
         # Force-empty the form
         form.announcement_subject.data = ''
         form.announcement_body.data = ''

core/admin/mailu/ui/views/domains.py

@@ -1,4 +1,4 @@
-from mailu import models
+from mailu import models, utils
 from mailu.ui import ui, forms, access
 from flask import current_app as app
 
@@ -93,6 +93,9 @@ def domain_signup(domain_name=None):
         del form.pw
         del form.pw2
     if form.validate_on_submit():
+        if msg := utils.isBadOrPwned(form):
+            flask.flash(msg, "error")
+            return flask.render_template('domain/signup.html', form=form)
         conflicting_domain = models.Domain.query.get(form.name.data)
         conflicting_alternative = models.Alternative.query.get(form.name.data)
         conflicting_relay = models.Relay.query.get(form.name.data)

core/admin/mailu/ui/views/fetches.py

@@ -1,5 +1,6 @@
-from mailu import models
+from mailu import models, utils
 from mailu.ui import ui, forms, access
+from flask import current_app as app
 
 import flask
 import flask_login
@@ -10,6 +11,8 @@ import wtforms
 @ui.route('/fetch/list/<path:user_email>', methods=['GET'])
 @access.owner(models.User, 'user_email')
 def fetch_list(user_email):
+    if not app.config['FETCHMAIL_ENABLED']:
+        flask.abort(404)
     user_email = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email) or flask.abort(404)
     return flask.render_template('fetch/list.html', user=user)
@@ -19,13 +22,18 @@ def fetch_list(user_email):
 @ui.route('/fetch/create/<path:user_email>', methods=['GET', 'POST'])
 @access.owner(models.User, 'user_email')
 def fetch_create(user_email):
+    if not app.config['FETCHMAIL_ENABLED']:
+        flask.abort(404)
     user_email = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email) or flask.abort(404)
     form = forms.FetchForm()
     form.password.validators = [wtforms.validators.DataRequired()]
+    utils.formatCSVField(form.folders)
     if form.validate_on_submit():
         fetch = models.Fetch(user=user)
         form.populate_obj(fetch)
+        if form.folders.data:
+            fetch.folders = form.folders.data.replace(' ','').split(',')
         models.db.session.add(fetch)
         models.db.session.commit()
         flask.flash('Fetch configuration created')
@@ -37,12 +45,17 @@ def fetch_create(user_email):
 @ui.route('/fetch/edit/<fetch_id>', methods=['GET', 'POST'])
 @access.owner(models.Fetch, 'fetch_id')
 def fetch_edit(fetch_id):
+    if not app.config['FETCHMAIL_ENABLED']:
+        flask.abort(404)
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     form = forms.FetchForm(obj=fetch)
+    utils.formatCSVField(form.folders)
     if form.validate_on_submit():
         if not form.password.data:
             form.password.data = fetch.password
         form.populate_obj(fetch)
+        if form.folders.data:
+            fetch.folders = form.folders.data.replace(' ','').split(',')
         models.db.session.commit()
         flask.flash('Fetch configuration updated')
         return flask.redirect(
@@ -55,6 +68,8 @@ def fetch_edit(fetch_id):
 @access.confirmation_required("delete a fetched account")
 @access.owner(models.Fetch, 'fetch_id')
 def fetch_delete(fetch_id):
+    if not app.config['FETCHMAIL_ENABLED']:
+        flask.abort(404)
     fetch = models.Fetch.query.get(fetch_id) or flask.abort(404)
     user = fetch.user
     models.db.session.delete(fetch)

core/admin/mailu/ui/views/users.py

@@ -1,4 +1,4 @@
-from mailu import models
+from mailu import models, utils
 from mailu.ui import ui, access, forms
 from flask import current_app as app
 
@@ -28,6 +28,10 @@ def user_create(domain_name):
         form.quota_bytes.validators = [
             wtforms.validators.NumberRange(max=domain.max_quota_bytes)]
     if form.validate_on_submit():
+        if msg := utils.isBadOrPwned(form):
+            flask.flash(msg, "error")
+            return flask.render_template('user/create.html',
+                domain=domain, form=form)
         if domain.has_email(form.localpart.data):
             flask.flash('Email is already used', 'error')
         else:
@@ -60,6 +64,11 @@ def user_edit(user_email):
         form.quota_bytes.validators = [
             wtforms.validators.NumberRange(max=max_quota_bytes)]
     if form.validate_on_submit():
+        if form.pw.data:
+            if msg := utils.isBadOrPwned(form):
+                flask.flash(msg, "error")
+                return flask.render_template('user/edit.html', form=form, user=user,
+                    domain=user.domain, max_quota_bytes=max_quota_bytes)
         form.populate_obj(user)
         if form.pw.data:
             user.set_password(form.pw.data)
@@ -91,11 +100,7 @@ def user_settings(user_email):
     user_email_or_current = user_email or flask_login.current_user.email
     user = models.User.query.get(user_email_or_current) or flask.abort(404)
     form = forms.UserSettingsForm(obj=user)
-    if isinstance(form.forward_destination.data,str):
-        data = form.forward_destination.data.replace(" ","").split(",")
-    else:
-        data = form.forward_destination.data
-    form.forward_destination.data = ", ".join(data)
+    utils.formatCSVField(form.forward_destination)
     if form.validate_on_submit():
         form.forward_destination.data = form.forward_destination.data.replace(" ","").split(",")
         form.populate_obj(user)
@@ -119,6 +124,9 @@ def user_password(user_email):
         if form.pw.data != form.pw2.data:
             flask.flash('Passwords do not match', 'error')
         else:
+            if msg := utils.isBadOrPwned(form):
+                flask.flash(msg, "error")
+                return flask.render_template('user/password.html', form=form, user=user)
             flask.session.regenerate()
             user.set_password(form.pw.data)
             models.db.session.commit()
@@ -170,6 +178,9 @@ def user_signup(domain_name=None):
         if domain.has_email(form.localpart.data) or models.Alias.resolve(form.localpart.data, domain_name):
             flask.flash('Email is already used', 'error')
         else:
+            if msg := utils.isBadOrPwned(form):
+                flask.flash(msg, "error")
+                return flask.render_template('user/signup.html', domain=domain, form=form)
             flask.session.regenerate()
             user = models.User(domain=domain)
             form.populate_obj(user)

core/admin/mailu/utils.py

@@ -472,7 +472,7 @@ class MailuSessionExtension:
                 redis.StrictRedis().from_url(app.config['SESSION_STORAGE_URL'])
             )
 
-            # clean expired sessions oonce on first use in case lifetime was changed
+            # clean expired sessions once on first use in case lifetime was changed
             def cleaner():
                 with cleaned.get_lock():
                     if not cleaned.value:
@@ -507,3 +507,21 @@ def gen_temp_token(email, session):
             app.config['PERMANENT_SESSION_LIFETIME'],
     )
     return token
+
+def isBadOrPwned(form):
+    try:
+        if len(form.pw.data) < 8:
+            return "This password is too short."
+        breaches = int(form.pwned.data)
+    except ValueError:
+        breaches = -1
+    if breaches > 0:
+        return f"This password appears in {breaches} data breaches! It is not unique; please change it."
+    return None
+
+def formatCSVField(field):
+    if isinstance(field.data,str):
+        data = field.data.replace(" ","").split(",")
+    else:
+        data = field.data
+    field.data = ", ".join(data)

core/admin/messages.pot

@@ -647,7 +647,7 @@ msgid "New relay domain"
 msgstr ""
 
 #: mailu/ui/templates/relay/edit.html:4
-msgid "Edit relayd domain"
+msgid "Edit relayed domain"
 msgstr ""
 
 #: mailu/ui/templates/relay/list.html:4

core/admin/migrations/versions/7ac252f2bbbf_.py

@@ -0,0 +1,22 @@
+""" Add user.allow_spoofing
+
+Revision ID: 7ac252f2bbbf
+Revises: 8f9ea78776f4
+Create Date: 2022-11-20 08:57:16.879152
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '7ac252f2bbbf'
+down_revision = 'f4f0f89e0047'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('user', sa.Column('allow_spoofing', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    op.drop_column('user', 'allow_spoofing')

core/admin/migrations/versions/f4f0f89e0047_.py

@@ -0,0 +1,25 @@
+""" Add fetch.scan and fetch.folders
+
+Revision ID: f4f0f89e0047
+Revises: 8f9ea78776f4
+Create Date: 2022-11-13 16:29:01.246509
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'f4f0f89e0047'
+down_revision = '8f9ea78776f4'
+
+from alembic import op
+import sqlalchemy as sa
+import mailu
+
+def upgrade():
+    with op.batch_alter_table('fetch') as batch:
+        batch.add_column(sa.Column('scan', sa.Boolean(), nullable=False, server_default=sa.sql.expression.false()))
+        batch.add_column(sa.Column('folders', mailu.models.CommaSeparatedList(), nullable=True))
+
+def downgrade():
+    with op.batch_alter_table('fetch') as batch:
+        batch.drop_column('fetch', 'folders')
+        batch.drop_column('fetch', 'scan')

core/admin/requirements-prod.txt

@@ -1,78 +0,0 @@
-alembic==1.7.4
-appdirs==1.4.4
-Babel==2.9.1
-bcrypt==3.2.0
-blinker==1.4
-CacheControl==0.12.9
-certifi==2021.10.8
-# cffi==1.15.0
-chardet==4.0.0
-click==8.0.3
-colorama==0.4.4
-contextlib2==21.6.0
-cryptography==35.0.0
-decorator==5.1.0
-# distlib==0.3.1
-# distro==1.5.0
-dnspython==2.1.0
-dominate==2.6.0
-email-validator==1.1.3
-Flask==2.0.2
-Flask-Babel==2.0.0
-Flask-Bootstrap==3.3.7.1
-Flask-DebugToolbar==0.11.0
-Flask-Limiter==1.4
-Flask-Login==0.5.0
-flask-marshmallow==0.14.0
-Flask-Migrate==3.1.0
-Flask-Script==2.0.6
-Flask-SQLAlchemy==2.5.1
-Flask-WTF==0.15.1
-greenlet==1.1.2
-gunicorn==20.1.0
-html5lib==1.1
-idna==3.3
-infinity==1.5
-intervals==0.9.2
-itsdangerous==2.0.1
-Jinja2==3.0.2
-limits==1.5.1
-lockfile==0.12.2
-Mako==1.1.5
-MarkupSafe==2.0.1
-marshmallow==3.14.0
-marshmallow-sqlalchemy==0.26.1
-msgpack==1.0.2
-# mysqlclient==2.0.3
-mysql-connector-python==8.0.25
-ordered-set==4.0.2
-# packaging==20.9
-passlib==1.7.4
-# pep517==0.10.0
-progress==1.6
-#psycopg2==2.9.1
-psycopg2-binary==2.9.3
-pycparser==2.20
-Pygments==2.10.0
-pyOpenSSL==21.0.0
-pyparsing==3.0.4
-pytz==2021.3
-PyYAML==6.0
-redis==3.5.3
-requests==2.26.0
-retrying==1.3.3
-# six==1.15.0
-socrate==0.2.0
-SQLAlchemy==1.4.26
-srslib==0.1.4
-tabulate==0.8.9
-tenacity==8.0.1
-toml==0.10.2
-urllib3==1.26.7
-validators==0.18.2
-visitor==0.1.3
-webencodings==0.5.1
-Werkzeug==2.0.2
-WTForms==2.3.3
-WTForms-Components==0.10.5
-xmltodict==0.12.0

core/admin/requirements.txt

@@ -1,28 +0,0 @@
-Flask
-Flask-Login
-Flask-SQLAlchemy
-Flask-bootstrap
-Flask-Babel
-Flask-migrate
-Flask-script
-Flask-wtf
-Flask-debugtoolbar
-limits
-redis
-WTForms-Components
-socrate
-passlib
-gunicorn
-tabulate
-PyYAML
-PyOpenSSL
-Pygments
-dnspython
-tenacity
-mysql-connector-python
-idna
-srslib
-marshmallow
-flask-marshmallow
-marshmallow-sqlalchemy
-xmltodict

core/admin/run_dev.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+### CONFIG
+
+DEV_NAME="${DEV_NAME:-mailu-dev}"
+DEV_DB="${DEV_DB:-}"
+DEV_PROFILER="${DEV_PROFILER:-false}"
+DEV_LISTEN="${DEV_LISTEN:-127.0.0.1:8080}"
+[[ "${DEV_LISTEN}" == *:* ]] || DEV_LISTEN="127.0.0.1:${DEV_LISTEN}"
+DEV_ADMIN="${DEV_ADMIN:-admin@example.com}"
+DEV_PASSWORD="${DEV_PASSWORD:-letmein}"
+
+### MAIN
+
+[[ -n "${DEV_DB}" ]] && {
+    [[ -f "${DEV_DB}" ]] || {
+        echo "Sorry, can't find DEV_DB: '${DEV_DB}'"
+        exit 1
+    }
+    DEV_DB="$(realpath "${DEV_DB}")"
+}
+
+docker="$(command -v podman || command -v docker || echo false)"
+[[ "${docker}" == "false" ]] && {
+    echo "Sorry, you'll need podman or docker to run this."
+    exit 1
+}
+
+tmp="$(mktemp -d)"
+[[ -n "${tmp}" && -d "${tmp}" ]] || {
+    echo "Sorry, can't create temporary folder."
+    exit 1
+}
+trap "rm -rf '${tmp}'" INT TERM EXIT
+
+admin="$(realpath "$(pwd)/${0%/*}")"
+base="${admin}/../base"
+assets="${admin}/assets"
+
+cd "${tmp}"
+
+# base
+cp "${base}"/requirements-* .
+cp -r "${base}"/libs .
+sed -E '/^#/d;s:^FROM system$:FROM system AS base:' "${base}/Dockerfile" >Dockerfile
+
+# assets
+cp "${assets}/package.json" .
+cp -r "${assets}/assets" ./assets
+awk '/new compress/{f=1}!f{print}/}),/{f=0}' <"${assets}/webpack.config.js" >webpack.config.js
+sed -E '/^#/d;s:^(FROM [^ ]+$):\1 AS assets:' "${assets}/Dockerfile" >>Dockerfile
+
+# admin
+sed -E '/^#/d;/^(COPY|EXPOSE|HEALTHCHECK|VOLUME|CMD) /d; s:^(.* )[^ ]*pybabel[^\\]*(.*):\1true \2:' "${admin}/Dockerfile" >>Dockerfile
+
+# development
+cat >>Dockerfile <<EOF
+COPY --from=assets /work/static/ ./static/
+
+RUN set -euxo pipefail \
+  ; mkdir /data \
+  ; ln -s /app/audit.py / \
+  ; ln -s /app/start.py /
+
+ENV \
+    FLASK_DEBUG="true" \
+    MEMORY_SESSIONS="true" \
+    RATELIMIT_STORAGE_URL="memory://" \
+    SESSION_COOKIE_SECURE="false" \
+    \
+    DEBUG="true" \
+    DEBUG_PROFILER="${DEV_PROFILER}" \
+    DEBUG_ASSETS="/app/static" \
+    DEBUG_TB_INTERCEPT_REDIRECTS=False \
+    \
+    ADMIN_ADDRESS="127.0.0.1" \
+    FRONT_ADDRESS="127.0.0.1" \
+    SMTP_ADDRESS="127.0.0.1" \
+    IMAP_ADDRESS="127.0.0.1" \
+    REDIS_ADDRESS="127.0.0.1" \
+    ANTIVIRUS_ADDRESS="127.0.0.1" \
+    ANTISPAM_ADDRESS="127.0.0.1" \
+    WEBMAIL_ADDRESS="127.0.0.1" \
+    WEBDAV_ADDRESS="127.0.0.1"
+
+CMD ["/bin/bash", "-c", "flask db upgrade &>/dev/null && flask mailu admin '${DEV_ADMIN/@*}' '${DEV_ADMIN#*@}' '${DEV_PASSWORD}' --mode ifmissing >/dev/null; flask --debug run --host=0.0.0.0 --port=8080"]
+EOF
+
+# build
+chmod -R u+rwX,go+rX .
+"${docker}" build --tag "${DEV_NAME}:latest" .
+
+# gather volumes to map into container
+volumes=()
+
+[[ -n "${DEV_DB}" ]] && volumes+=( --volume "${DEV_DB}:/data/main.db" )
+
+for vol in audit.py start.py mailu/ migrations/; do
+    volumes+=( --volume "${admin}/${vol}:/app/${vol}" )
+done
+
+for file in "${assets}/assets"/*; do
+    [[ ! -f "${file}" || "${file}" == */vendor.js ]] && continue
+    volumes+=( --volume "${file}:/app/static/${file/*\//}" )
+done
+
+# show configuration
+cat <<EOF
+
+=============================================================================
+The "${DEV_NAME}" container was built using this configuration:
+
+DEV_NAME="${DEV_NAME}"
+DEV_DB="${DEV_DB}"
+DEV_PROFILER="${DEV_PROFILER}"
+DEV_LISTEN="${DEV_LISTEN}"
+DEV_ADMIN="${DEV_ADMIN}"
+DEV_PASSWORD="${DEV_PASSWORD}"
+=============================================================================
+
+=============================================================================
+You can start the container later using this commandline:
+
+${docker/*\/} run --rm -it --name "${DEV_NAME}" --publish ${DEV_LISTEN}:8080$(printf " %q" "${volumes[@]}") "${DEV_NAME}"
+=============================================================================
+
+=============================================================================
+The Mailu UI can be found here: http://${DEV_LISTEN}/sso/login
+EOF
+[[ -z "${DEV_DB}" ]] && echo "You can log in with user ${DEV_ADMIN} and password ${DEV_PASSWORD}"
+cat <<EOF
+=============================================================================
+
+Starting mailu dev environment...
+EOF
+
+# run
+"${docker}" run --rm -it --name "${DEV_NAME}" --publish "${DEV_LISTEN}:8080" "${volumes[@]}" "${DEV_NAME}"
+

core/admin/start.py

@@ -1,10 +1,19 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import os
 import logging as log
+from pwd import getpwnam
 import sys
+from socrate import system
+
+os.system("chown mailu:mailu -R /dkim")
+os.system("find /data | grep -v /fetchmail | xargs -n1 chown mailu:mailu")
+mailu_id = getpwnam('mailu')
+os.setgid(mailu_id.pw_gid)
+os.setuid(mailu_id.pw_uid)
 
 log.basicConfig(stream=sys.stderr, level=os.environ.get("LOG_LEVEL", "INFO"))
+system.set_env(['SECRET'])
 
 os.system("flask mailu advertise")
 os.system("flask db upgrade")
@@ -15,7 +24,7 @@ password = os.environ.get("INITIAL_ADMIN_PW")
 
 if account is not None and domain is not None and password is not None:
     mode = os.environ.get("INITIAL_ADMIN_MODE", default="ifmissing")
-    log.info("Creating initial admin accout %s@%s with mode %s",account,domain,mode)
+    log.info("Creating initial admin account %s@%s with mode %s", account, domain, mode)
     os.system("flask mailu admin %s %s '%s' --mode %s" % (account, domain, password, mode))
 
 def test_DNS():
@@ -37,7 +46,7 @@ def test_DNS():
             try:
                 result = resolver.resolve('example.org', dns.rdatatype.A, dns.rdataclass.IN, lifetime=10)
             except Exception as e:
-                log.critical("Your DNS resolver at %s is not working (%s). Please see https://mailu.io/master/faq.html#the-admin-container-won-t-start-and-its-log-says-critical-your-dns-resolver-isn-t-doing-dnssec-validation", ns, e);
+                log.critical("Your DNS resolver at %s is not working (%s). Please see https://mailu.io/master/faq.html#the-admin-container-won-t-start-and-its-log-says-critical-your-dns-resolver-isn-t-doing-dnssec-validation", ns, e)
             else:
                 if result.response.flags & dns.flags.AD:
                     break

core/base/Dockerfile

@@ -0,0 +1,86 @@
+# syntax=docker/dockerfile-upstream:1.4.3
+
+# base system image (intermediate)
+ARG DISTRO=alpine:3.16.3
+FROM $DISTRO as system
+
+ENV TZ=Etc/UTC LANG=C.UTF-8
+
+ARG MAILU_UID=1000
+ARG MAILU_GID=1000
+
+RUN set -euxo pipefail \
+  ; addgroup -Sg ${MAILU_GID} mailu \
+  ; adduser -Sg ${MAILU_UID} -G mailu -h /app -g "mailu app" -s /bin/bash mailu \
+  ; apk add --no-cache bash ca-certificates curl python3 tzdata libcap \
+  ; machine="$(uname -m)" \
+  ; ! [[ "${machine}" == x86_64 ]] \
+    || apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing hardened-malloc==11-r0
+
+ENV LD_PRELOAD=/usr/lib/libhardened_malloc.so
+ENV CXXFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
+ENV CFLAGS="-g -O2 -fdebug-prefix-map=/app=. -fstack-protector-strong -Wformat -Werror=format-security -fstack-clash-protection -fexceptions"
+ENV CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2"
+ENV LDFLAGS="-Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now"
+
+WORKDIR /app
+
+CMD /bin/bash
+
+
+# build virtual env (intermediate)
+FROM system as build
+
+ARG MAILU_DEPS=prod
+
+ENV VIRTUAL_ENV=/app/venv
+
+COPY requirements-build.txt ./
+
+RUN set -euxo pipefail \
+  ; apk add --no-cache py3-pip \
+  ; python3 -m venv ${VIRTUAL_ENV} \
+  ; ${VIRTUAL_ENV}/bin/pip install --no-cache-dir -r requirements-build.txt \
+  ; apk del -r py3-pip \
+  ; rm -f /tmp/*.pem
+
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+
+COPY requirements-${MAILU_DEPS}.txt ./
+COPY libs/ libs/
+
+ARG SNUFFLEUPAGUS_VERSION=0.8.3
+ENV SNUFFLEUPAGUS_URL https://github.com/jvoisin/snuffleupagus/archive/refs/tags/v$SNUFFLEUPAGUS_VERSION.tar.gz
+
+RUN set -euxo pipefail \
+  ;  machine="$(uname -m)" \
+  ; deps="build-base gcc libffi-dev python3-dev" \
+  ; [[ "${machine}" != x86_64 ]] && \
+      deps="${deps} cargo git libressl-dev mariadb-connector-c-dev postgresql-dev" \
+  ; apk add --virtual .build-deps ${deps} \
+  ; [[ "${machine}" == armv7* ]] && \
+      mkdir -p /root/.cargo/registry/index && \
+      git clone --bare https://github.com/rust-lang/crates.io-index.git /root/.cargo/registry/index/github.com-1285ae84e5963aae \
+  ; pip install -r requirements-${MAILU_DEPS}.txt \
+  ; curl -sL  ${SNUFFLEUPAGUS_URL} | tar xz \
+  ; cd snuffleupagus-$SNUFFLEUPAGUS_VERSION \
+  ; rm -rf src/tests/*php7*/ src/tests/*session*/ src/tests/broken_configuration/ src/tests/*cookie* src/tests/upload_validation/ \
+  ; apk add --virtual .build-deps php81-dev php81-cgi php81-simplexml php81-xml pcre-dev build-base php81-pear php81-openssl re2c \
+  ; ln -s /usr/bin/phpize81 /usr/bin/phpize \
+  ; ln -s /usr/bin/pecl81 /usr/bin/pecl \
+  ; ln -s /usr/bin/php-config81 /usr/bin/php-config \
+  ; ln -s /usr/bin/php81 /usr/bin/php \
+  ; pecl install vld-beta \
+  ; make -j $(grep -c processor /proc/cpuinfo) release \
+  ; cp src/.libs/snuffleupagus.so /app \
+  ; rm -rf /root/.cargo /tmp/*.pem /root/.cache
+
+# base mailu image
+FROM system
+
+COPY --from=build /app/venv/ /app/venv/
+COPY --chown=root:root --from=build /app/snuffleupagus.so /usr/lib/php81/modules/
+RUN setcap 'cap_net_bind_service=+ep' /app/venv/bin/gunicorn
+
+ENV VIRTUAL_ENV=/app/venv
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"

core/base/libs/podop/.gitignore

@@ -0,0 +1,20 @@
+.DS_Store
+.idea
+tmp
+
+*.bak
+*~
+.*.swp
+
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+
+.build
+.env*
+.venv
+
+*.code-workspace
+
+build/

core/base/libs/podop/CONTRIBUTING.md

@@ -0,0 +1,7 @@
+This project is open source, and your contributions are all welcome. There are mostly three different ways one can contribute to the project:
+
+1. use Podop, either on test or on production servers, and report meaningful bugs when you find some;
+2. write and publish, or contribute to mail distributions based on Podop, like Mailu;
+2. contribute code and/or configuration to the repository (see [the development guidelines](https://mailu.io/contributors/guide.html) for details);
+
+Either way, keep in mind that the code you write must be licensed under the same conditions as the project itself. Additionally, all contributors are considered equal co-authors of the project.

core/base/libs/podop/LICENSE.md

@@ -0,0 +1,25 @@
+MIT License
+
+Copyright (c) 2018 All Podop contributors at the date
+
+This software consists of voluntary contributions made by multiple individuals.
+For exact contribution history, see the revision history available at
+https://github.com/Mailu/podop.git
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

core/base/libs/podop/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include LICENSE.md

core/base/libs/podop/README.md

@@ -0,0 +1,112 @@
+Podop is a piece of middleware designed to run between Postfix or Dovecot
+on one side, any Python implementation of a table lookup protocol on the
+other side.
+
+It is thus able to forward Postfix maps and Dovecot dicts to the same
+(or multiple) backends in order to write a single, more flexible backend
+for a mail distribution.
+
+Examples
+========
+
+- Connect Postfix to a DNS lookup so that every domain that has a proper MX
+  record to your Postfix is actually accepted as a local domain
+- Connect both Postfix and Dovecot to an HTTP microservice to run a high
+  availability microservice-based mail service
+- Use a single database server running any Python-compatible API for both
+  your Postfix and Dovecot servers
+
+Configure Podop tables
+======================
+
+Podop tables are configured through CLI arguments when running the server.
+You must provide a ``--name`` for the table, a ``--type`` for the table and
+a ``--param`` that parametrizes the map.
+
+URL table
+---------
+
+The URL table will initiate an HTTP GET request for read access and an HTTP
+POST request for write access to a table. The table is parametrized with
+a template URL containing ``§`` (or ``{}``) for inserting the table key.
+
+```
+--name test --type url --param http://microservice/api/v1/map/tests/§
+```
+
+GET requests should return ``200`` and a JSON-encoded object
+that will be passed either to Postfix or Dovecot. They should return ``4XX``
+for access issues that will result in lookup miss, and ``5XX`` for backend
+issues that will result in a temporary failure.
+
+POST requests will contain a JSON-encoded object in the request body, that
+will be saved in the table.
+
+Postfix usage
+=============
+
+In order to access Podop tables from Postfix, you should setup ``socketmap``
+Postfix maps. For instance, in order to access the ``test`` table on a Podop
+socket at ``/tmp/podop.socket``, use the following setup:
+
+```
+virtual_alias_maps = socketmap:unix:/tmp/podop.socket:test
+```
+
+Multiple maps or identical maps can be configured for various usages.
+
+```
+virtual_alias_maps = socketmap:unix:/tmp/podop.socket:alias
+virtual_mailbox_domains = socketmap:unix:/tmp/podop.socket:domain
+virtual_mailbox_maps = socketmap:unix:/tmp/podop.socket:alias
+```
+
+In order to simplify the configuration, you can setup a shortcut.
+
+```
+podop = socketmap:unix:/tmp/podop.socket
+virtual_alias_maps = ${podop}:alias
+virtual_mailbox_domains = ${podop}:domain
+virtual_mailbox_maps = ${podop}:alias
+```
+
+Dovecot usage
+=============
+
+In order to access Podop tables from Dovecot, you should setup a ``proxy``
+Dovecot dictionary. For instance, in order to access the ``test`` table on
+a Podop socket at ``/tmp/podop.socket``, use the following setup:
+
+```
+mail_attribute_dict = proxy:/tmp/podop.socket:test
+```
+
+Multiple maps or identical maps can be configured for various usages.
+
+```
+mail_attribute_dict = proxy:/tmp/podop.socket:meta
+
+passdb {
+  driver = dict
+  args = /etc/dovecot/auth.conf
+}
+
+userdb {
+  driver = dict
+  args = /etc/dovecot/auth.conf
+}
+
+# then in auth.conf
+uri = proxy:/tmp/podop.socket:auth
+iterate_disable = yes
+default_pass_scheme = plain
+password_key = passdb/%u
+user_key = userdb/%u
+```
+
+Contributing
+============
+
+Podop is free software, open to suggestions and contributions. All
+components are free software and compatible with the MIT license. All
+the code is placed under the MIT license.

core/base/libs/podop/podop/__init__.py

@@ -0,0 +1,46 @@
+""" Podop is a *Po*stfix and *Do*vecot proxy
+
+It is able to proxify postfix maps and dovecot dicts to any table
+"""
+
+import asyncio
+import logging
+import sys
+
+from podop import postfix, dovecot, table
+
+
+SERVER_TYPES = dict(
+    postfix=postfix.SocketmapProtocol,
+    dovecot=dovecot.DictProtocol
+)
+
+TABLE_TYPES = dict(
+    url=table.UrlTable
+)
+
+
+def run_server(verbosity, server_type, socket, tables):
+    """ Run the server, given its type, socket path and table list
+
+    The table list must be a list of tuples (name, type, param)
+    """
+    # Prepare the maps
+    table_map = {
+        name: TABLE_TYPES[table_type](param)
+        for name, table_type, param in tables
+    }
+    # Run the main loop
+    logging.basicConfig(stream=sys.stderr, level=max(3 - verbosity, 0) * 10,
+                        format='%(name)s (%(levelname)s): %(message)s')
+    loop = asyncio.get_event_loop()
+    server = loop.run_until_complete(loop.create_unix_server(
+        SERVER_TYPES[server_type].factory(table_map), socket
+    ))
+    try:
+        loop.run_forever()
+    except KeyboardInterrupt:
+        pass
+    server.close()
+    loop.run_until_complete(server.wait_closed())
+    loop.close()

core/base/libs/podop/podop/dovecot.py

@@ -0,0 +1,202 @@
+""" Dovecot dict proxy implementation
+"""
+
+import asyncio
+import logging
+import json
+
+
+class DictProtocol(asyncio.Protocol):
+    """ Protocol to answer Dovecot dict requests, as implemented in Dict proxy.
+
+    Only a subset of operations is handled properly by this proxy: hello,
+    lookup and transaction-based set.
+
+    There is very little documentation about the protocol, most of it was
+    reverse-engineered from :
+
+    https://github.com/dovecot/core/blob/master/src/dict/dict-connection.c
+    https://github.com/dovecot/core/blob/master/src/dict/dict-commands.c
+    https://github.com/dovecot/core/blob/master/src/lib-dict/dict-client.h
+    """
+
+    DATA_TYPES = {0: str, 1: int}
+
+    def __init__(self, table_map):
+        self.table_map = table_map
+        # Minor and major versions are not properly checked yet, but stored
+        # anyway
+        self.major_version = None
+        self.minor_version = None
+        # Every connection starts with specifying which table is used, dovecot
+        # tables are called dicts
+        self.dict = None
+        # Dictionary of active transaction lists per transaction id
+        self.transactions = {}
+        # Dictionary of user per transaction id
+        self.transactions_user = {}
+        super(DictProtocol, self).__init__()
+
+    def connection_made(self, transport):
+        logging.info('Connect {}'.format(transport.get_extra_info('peername')))
+        self.transport = transport
+        self.transport_lock = asyncio.Lock()
+
+    def data_received(self, data):
+        logging.debug("Received {}".format(data))
+        results = []
+        # Every command is separated by "\n"
+        for line in data.split(b"\n"):
+            # A command must at list have a type and one argument
+            if len(line) < 2:
+                continue
+            # The command function will handle the command itself
+            command = DictProtocol.COMMANDS.get(line[0])
+            if command is None:
+                logging.warning('Unknown command {}'.format(line[0]))
+                return self.transport.abort()
+            # Args are separated by "\t"
+            args = line[1:].strip().split(b"\t")
+            try:
+                future = command(self, *args)
+                if future:
+                    results.append(future)
+            except Exception:
+                logging.exception("Error when processing request")
+                return self.transport.abort()
+        # For asyncio consistency, wait for all results to fire before
+        # actually returning control
+        return asyncio.gather(*results)
+
+    def process_hello(self, major, minor, value_type, user, dict_name):
+        """ Process a dict protocol hello message
+        """
+        self.major, self.minor = int(major), int(minor)
+        self.value_type = DictProtocol.DATA_TYPES[int(value_type)]
+        self.user = user.decode("utf8")
+        self.dict = self.table_map[dict_name.decode("ascii")]
+        logging.debug("Client {}.{} type {}, user {}, dict {}".format(
+            self.major, self.minor, self.value_type, self.user, dict_name))
+
+    async def process_lookup(self, key, user=None, is_iter=False):
+        """ Process a dict lookup message
+        """
+        logging.debug("Looking up {} for {}".format(key, user))
+        orig_key = key
+        # Priv and shared keys are handled slighlty differently
+        key_type, key = key.decode("utf8").split("/", 1)
+        try:
+            result = await self.dict.get(
+                key, ns=((user.decode("utf8") if user else self.user) if key_type == "priv" else None)
+            )
+            if type(result) is str:
+                response = result.encode("utf8")
+            elif type(result) is bytes:
+                response = result
+            else:
+                response = json.dumps(result).encode("ascii")
+            return await (self.reply(b"O", orig_key, response) if is_iter else self.reply(b"O", response))
+        except KeyError:
+            return await self.reply(b"N")
+
+    async def process_iterate(self, flags, max_rows, path, user=None):
+        """ Process an iterate command
+        """
+        logging.debug("Iterate flags {} max_rows {} on {} for {}".format(flags, max_rows, path, user))
+        # Priv and shared keys are handled slighlty differently
+        key_type, key = path.decode("utf8").split("/", 1)
+        max_rows = int(max_rows.decode("utf-8"))
+        flags = int(flags.decode("utf-8"))
+        if flags != 0: # not implemented
+            return await self.reply(b"F")
+        rows = []
+        try:
+            result = await self.dict.iter(key)
+            logging.debug("Found {} entries: {}".format(len(result), result))
+            for i,k in enumerate(result):
+                if max_rows > 0 and i >= max_rows:
+                    break
+                rows.append(self.process_lookup((path.decode("utf8")+k).encode("utf8"), user, is_iter=True))
+            await asyncio.gather(*rows)
+            async with self.transport_lock:
+                self.transport.write(b"\n") # ITER_FINISHED
+            return
+        except KeyError:
+            return await self.reply(b"F")
+        except Exception as e:
+            for task in rows:
+                task.cancel()
+            raise e
+
+    def process_begin(self, transaction_id, user=None):
+        """ Process a dict begin message
+        """
+        self.transactions[transaction_id] = {}
+        self.transactions_user[transaction_id] = user.decode("utf8") if user else self.user
+
+    def process_set(self, transaction_id, key, value):
+        """ Process a dict set message
+        """
+        # Nothing is actually set until everything is commited
+        self.transactions[transaction_id][key] = value
+
+    async def process_commit(self, transaction_id):
+        """ Process a dict commit message
+        """
+        # Actually handle all set operations from the transaction store
+        results = []
+        for key, value in self.transactions[transaction_id].items():
+            logging.debug("Storing {}={}".format(key, value))
+            key_type, key = key.decode("utf8").split("/", 1)
+            result = await self.dict.set(
+                key, json.loads(value),
+                ns=(self.transactions_user[transaction_id] if key_type == "priv" else None)
+            )
+        # Remove stored transaction
+        del self.transactions[transaction_id]
+        del self.transactions_user[transaction_id]
+        return await self.reply(b"O", transaction_id)
+
+    async def reply(self, command, *args):
+        async with self.transport_lock:
+            logging.debug("Replying {} with {}".format(command, args))
+            self.transport.write(command)
+            self.transport.write(b"\t".join(map(tabescape, args)))
+            self.transport.write(b"\n")
+
+    @classmethod
+    def factory(cls, table_map):
+        """ Provide a protocol factory for a given map instance.
+        """
+        return lambda: cls(table_map)
+
+    COMMANDS = {
+        ord("H"): process_hello,
+        ord("L"): process_lookup,
+        ord("I"): process_iterate,
+        ord("B"): process_begin,
+        ord("C"): process_commit,
+        ord("S"): process_set
+    }
+
+
+def tabescape(unescaped):
+    """ Escape a string using the specific Dovecot tabescape
+    See: https://github.com/dovecot/core/blob/master/src/lib/strescape.c
+    """
+    return unescaped.replace(b"\x01", b"\x011")\
+                    .replace(b"\x00", b"\x010")\
+                    .replace(b"\t",   b"\x01t")\
+                    .replace(b"\n",   b"\x01n")\
+                    .replace(b"\r",   b"\x01r")
+
+
+def tabunescape(escaped):
+    """ Unescape a string using the specific Dovecot tabescape
+    See: https://github.com/dovecot/core/blob/master/src/lib/strescape.c
+    """
+    return escaped.replace(b"\x01r", b"\r")\
+                  .replace(b"\x01n", b"\n")\
+                  .replace(b"\x01t", b"\t")\
+                  .replace(b"\x010", b"\x00")\
+                  .replace(b"\x011", b"\x01")

core/base/libs/podop/podop/postfix.py

@@ -0,0 +1,116 @@
+""" Postfix map proxy implementation
+"""
+
+import asyncio
+import logging
+
+class NetstringProtocol(asyncio.Protocol):
+    """ Netstring asyncio protocol implementation.
+
+    For protocol details, see https://cr.yp.to/proto/netstrings.txt
+    """
+
+    # Length of the smallest allocated buffer, larger buffers will be
+    # allocated dynamically
+    BASE_BUFFER = 1024
+
+    # Maximum length of a buffer, will crash when exceeded
+    MAX_BUFFER = 65535
+
+    def __init__(self):
+        super(NetstringProtocol, self).__init__()
+        self.init_buffer()
+
+    def init_buffer(self):
+        self.len = None  # None when waiting for a length to be sent)
+        self.separator = -1  # -1 when not yet detected (str.find)
+        self.index = 0  # relative to the buffer
+        self.buffer = bytearray(NetstringProtocol.BASE_BUFFER)
+
+    def data_received(self, data):
+        # Manage the buffer
+        missing = len(data) - len(self.buffer) + self.index
+        if missing > 0:
+            if len(self.buffer) + missing > NetstringProtocol.MAX_BUFFER:
+                raise IOError("Not enough space when decoding netstring")
+            self.buffer.append(bytearray(missing + 1))
+        new_index = self.index + len(data)
+        self.buffer[self.index:new_index] = data
+        self.index = new_index
+        # Try to detect a length at the beginning of the string
+        if self.len is None:
+            self.separator = self.buffer.find(0x3a)
+            if self.separator != -1 and self.buffer[:self.separator].isdigit():
+                self.len = int(self.buffer[:self.separator], 10)
+        # Then get the complete string
+        if self.len is not None:
+            if self.index - self.separator == self.len + 2:
+                string = self.buffer[self.separator + 1:self.index - 1]
+                self.init_buffer()
+                self.string_received(string)
+
+    def string_received(self, string):
+        """ A new netstring was received
+        """
+        pass
+
+    def send_string(self, string):
+        """ Send a netstring
+        """
+        logging.debug("Replying {}".format(string))
+        self.transport.write(str(len(string)).encode('ascii'))
+        self.transport.write(b':')
+        self.transport.write(string)
+        self.transport.write(b',')
+
+
+class SocketmapProtocol(NetstringProtocol):
+    """ Protocol to answer Postfix socketmap and proxify lookups to
+    an outside object.
+
+    See http://www.postfix.org/socketmap_table.5.html for details on the
+    protocol.
+
+    A table map must be provided as a dictionary to lookup tables.
+    """
+
+    def __init__(self, table_map):
+        self.table_map = table_map
+        super(SocketmapProtocol, self).__init__()
+
+    def connection_made(self, transport):
+        logging.info('Connect {}'.format(transport.get_extra_info('peername')))
+        self.transport = transport
+
+    def string_received(self, string):
+        # The postfix format contains a space for separating the map name and
+        # the key
+        logging.debug("Received {}".format(string))
+        space = string.find(0x20)
+        if space != -1:
+            name = string[:space].decode('ascii')
+            key = string[space+1:].decode('utf8')
+            return asyncio.ensure_future(self.process_request(name, key))
+
+    async def process_request(self, name, key):
+        """ Process a request by querying the provided map.
+        """
+        logging.debug("Request {}/{}".format(name, key))
+        try:
+            table = self.table_map.get(name)
+        except KeyError:
+            return self.send_string(b'TEMP no such map')
+        try:
+            result = await table.get(key)
+            return self.send_string(b'OK ' + str(result).encode('utf8'))
+        except KeyError:
+            return self.send_string(b'NOTFOUND ')
+        except Exception:
+            logging.exception("Error when processing request")
+            return self.send_string(b'TEMP unknown error')
+
+    @classmethod
+    def factory(cls, table_map):
+        """ Provide a protocol factory for a given map instance.
+        """
+        return lambda: cls(table_map)

core/base/libs/podop/podop/table.py

@@ -0,0 +1,55 @@
+""" Table lookup backends for podop
+"""
+
+import aiohttp
+import logging
+from urllib.parse import quote
+
+class UrlTable(object):
+    """ Resolve an entry by querying a parametrized GET URL.
+    """
+
+    def __init__(self, url_pattern):
+        """ url_pattern must contain a format ``{}`` so the key is injected in
+        the url before the query, the ``§`` character will be replaced with
+        ``{}`` for easier setup.
+        """
+        self.url_pattern = url_pattern.replace('§', '{}')
+
+    async def get(self, key, ns=None):
+        """ Get the given key in the provided namespace
+        """
+        logging.debug("Table get {}".format(key))
+        if ns is not None:
+            key += "/" + ns
+        async with aiohttp.ClientSession() as session:
+            quoted_key = quote(key)
+            async with session.get(self.url_pattern.format(quoted_key)) as request:
+                if request.status == 200:
+                    result = await request.json()
+                    logging.debug("Table get {} is {}".format(key, result))
+                    return result
+                elif request.status == 404:
+                    raise KeyError()
+                else:
+                    raise Exception(request.status)
+
+    async def set(self, key, value, ns=None):
+        """ Set a value for the given key in the provided namespace
+        """
+        logging.debug("Table set {} to {}".format(key, value))
+        if ns is not None:
+            key += "/" + ns
+        async with aiohttp.ClientSession() as session:
+            quoted_key = quote(key)
+            await session.post(self.url_pattern.format(quoted_key), json=value)
+
+    async def iter(self, cat):
+        """ Iterate the given key (experimental)
+        """
+        logging.debug("Table iter {}".format(cat))
+        async with aiohttp.ClientSession() as session:
+            async with session.get(self.url_pattern.format(cat)) as request:
+                if request.status == 200:
+                    result = await request.json()
+                    return result

core/base/libs/podop/scripts/podop

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+import argparse
+
+from podop import run_server, SERVER_TYPES, TABLE_TYPES
+
+
+def main():
+    """ Run a podop server based on CLI arguments
+    """
+    parser = argparse.ArgumentParser("Postfix and Dovecot proxy")
+    parser.add_argument("--socket", required=True,
+                        help="path to the listening unix socket")
+    parser.add_argument("--mode", choices=SERVER_TYPES.keys(), required=True,
+                        help="select which server will connect to Podop")
+    parser.add_argument("--name", action="append",
+                        help="name of each configured table")
+    parser.add_argument("--type", choices=TABLE_TYPES.keys(), action="append",
+                        help="type of each configured table")
+    parser.add_argument("--param", action="append",
+                        help="mandatory param for each table configured")
+    parser.add_argument("-v", "--verbose", dest="verbosity",
+                        action="count", default=0,
+                        help="increases log verbosity for each occurence.")
+    args = parser.parse_args()
+    run_server(
+        args.verbosity, args.mode, args.socket,
+        zip(args.name, args.type, args.param) if args.name else []
+    )
+
+
+if __name__ == "__main__":
+    main()

core/base/libs/podop/setup.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setup(
+    name="podop",
+    version="0.2.5",
+    description="Postfix and Dovecot proxy",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    author="Pierre Jaury",
+    author_email="pierre@jaury.eu",
+    url="https://github.com/mailu/podop.git",
+    packages=["podop"],
+    include_package_data=True,
+    scripts=["scripts/podop"],
+    install_requires=[
+        "aiohttp"
+    ]
+)

core/base/libs/socrate/.gitignore

@@ -0,0 +1,22 @@
+.DS_Store
+.idea
+tmp
+
+*.bak
+*~
+.*.swp
+
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+
+.build
+.env*
+.venv
+
+*.code-workspace
+
+venv/
+build/
+dist/

core/base/libs/socrate/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Mailu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

core/base/libs/socrate/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include LICENSE.md

core/base/libs/socrate/README.md

@@ -0,0 +1,24 @@
+Socrate is a simple Python module providing a set of utility functions for
+Python daemon applications.
+
+The scope includes:
+- configuration utilities (configuration parsing, etc.)
+- system utilities (access to DNS, stats, etc.)
+
+Setup
+======
+
+Socrate is available on Pypi, simpy run:
+
+```
+pip install socrate
+```
+
+
+Contributing
+============
+
+Podop is free software, open to suggestions and contributions. All
+components are free software and compatible with the MIT license. All
+the code is placed under the MIT license.
+

core/base/libs/socrate/setup.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+import setuptools
+from distutils.core import setup
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+setup(
+    name="socrate",
+    version="0.2.0",
+    description="Socrate daemon utilities",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    author="Pierre Jaury",
+    author_email="pierre@jaury.eu",
+    url="https://github.com/mailu/socrate.git",
+    packages=["socrate"],
+    include_package_data=True,
+    install_requires=[
+        "jinja2",
+        "tenacity"
+    ]
+)