add initial (manual) failover script

failover.ps1

@@ -0,0 +1,66 @@
+$hetznerCredentials = Get-Credential
+
+$nodeList = docker node ls --format '{{json .}}'| ConvertFrom-Json | ForEach-Object {
+    docker node inspect $_.ID | ConvertFrom-Json
+}
+
+$onlineNodeList = $nodeList | Where-Object {$_.Spec.Availability -eq 'active'}
+$offlineNodeList = $nodeList | Where-Object {$_.Spec.Availability -ne 'active'}
+
+$failoverAddressList = (Invoke-RestMethod -Uri 'https://robot-ws.your-server.de/failover' -Authentication Basic -Credential $hetznerCredentials).failover
+
+$offlineAddressList = $failoverAddressList | Where-Object active_server_ip -notin $onlineNodeList.Status.Addr
+
+$maximumAddressCount = [Math]::Ceiling($failoverAddressList.Count / $onlineNodeList.Count)
+
+$i = 0
+$addressGroupList = $failoverAddressList `
+| Where-Object active_server_ip -in $onlineNodeList.Status.Addr `
+| Group-Object active_server_ip `
+| Sort-Object Count
+$distributionTarget = :distribution foreach ($addressGroup in $addressGroupList) {
+    $currentAddressCount = $addressGroup.Count
+
+    while ($currentAddressCount -lt $maximumAddressCount) {
+        if (!$offlineAddressList[$i]) {
+            # abort when all addresses are assigned
+            break distribution
+        }
+
+        @{
+            active_server_ip = $addressGroup.Name
+            ip = $offlineAddressList[$i++].ip
+            node = $nodeList | Where-Object {$_.Status.Addr -eq $addressGroup.Name}
+        }
+        $currentAddressCount++
+    }
+}
+
+#$offlineNodeList | Where-Object {$_. $distributionTarget.ip}
+#$offlineNodeList.Spec.Labels `
+#| Get-Member `
+#| Select-Object -ExpandProperty Name `
+#| Where-Object {$_ -like 'link.lubi.ip.*'} `
+#| ForEach-Object {
+#    $_.Replace('link.lubi.ip.','')
+#}
+
+# docker label updates must not run in parallel to prevent out-of-sequence errors
+$distributionTarget | ForEach-Object {
+    # first add the new labels and afterwards remove the old ones
+    # this prevents situations where no host has the label anymore
+    $labelName = 'link.lubi.ip.'+$_.ip
+
+    $filter = 'node.label='+$labelName
+    $cleanupNodeList = docker node ls --filter $filter --format '{{json .}}' | ConvertFrom-Json
+
+    $label = $labelName+'=true'
+    docker node update --label-add $label $($_.node.ID)
+
+    $cleanupNodeList | ForEach-Object {
+        docker node update --label-rm $labelName $($_.ID)
+    }
+}
+$distributionTarget | ForEach-Object -ThrottleLimit $failoverAddressList.Count -Parallel {
+    Invoke-RestMethod -Method Post -Uri ('https://robot-ws.your-server.de/failover/{0}' -f $_.ip) -Authentication Basic -Credential $using:hetznerCredentials -Body @{active_server_ip=$_.active_server_ip}
+}