systemd-networkd fix config

bookworm version is too old

authorized_keys/lub

@@ -1 +1,2 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCoCfs2IBe/6bzJs7eD6+a6jLpWt3PtfLiwd/nqsN9h4fc15qws2LlTg3rPwjxNJfGdYXB2CJHSMMIzvsNK3HBEagt/iBc8hV9qRHT6jLZqBJ0B2tXTQkawHizC0LP0gKGbH9aXPFaY29W/Xa+igNxsjDv9rZh0QaSa/3Z2nxzjJT4kwIXdwonREULtthsY5PnfyV/Qwe8pgbMPrSxq+rlsriGtU9Yr5qck5myK9/O+q8gQBTkbjGstPLRlDX37DDgHh9haQJ4yvGEy8p0akSdloQ9SgbISsZx+tdq9cB0TzRDY+YB2hVUrtBVhdcZ0HpDDv8J1GtkrUTfClnI6GNrkoBiJ6p2VgB0dzNQBIeuhvvzJPVmEAsi84FjwktdMx3IrrUJ1xgWftTOcZ2rxX4+vDCdr4NH1sAn9jqEUE28D1ouRwvMkyyPSictJ22CWaG4fSPfDlzylKRWFLkEPcKi06CAF8B9cGsoFGjEyiQ08lhHPHqQc1Gi2SBSXYE509wFNi385om61JY94izXAXAPx+n4OcpksEQangkBIYl0ZLyBRplQgqgzF/pCl59uosdGXjtEmOXxsYrdDmGnXCLInkA8N/bx6jZ6svGiaSXtKI/P6hY3MALuF37KzsMSNhnX6fG/a3l3WV4QYGgTtEPuYoYPpLnElMzM/MqHGEqcE/Q== lub
+sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAINQ0EPy3cbfX/wmelSsjRvrXpq9kZWo8tRh68r4Z3QhTAAAABHNzaDo= lub@primary
+sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIAUmJxFSwI95zf/NBKNMDXyiRViuKsWnVYl3Qd3DXdRWAAAABHNzaDo= lub@secondary

config/etc/apt/sources.list.d/debian-security.list

@@ -1 +1,2 @@
-deb https://deb.debian.org/debian-security buster/updates main contrib non-free
+deb https://security.debian.org/debian-security bookworm-security main contrib non-free non-free-firmware
+deb https://deb.debian.org/debian-security bookworm-security main contrib non-free non-free-firmware

config/etc/apt/sources.list.d/debian.list

@@ -1 +1 @@
-deb https://deb.debian.org/debian buster main contrib non-free
+deb https://deb.debian.org/debian bookworm main contrib non-free non-free-firmware

config/etc/apt/sources.list.d/docker.list

@@ -1 +0,0 @@
-deb [signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian buster stable

config/etc/crypttab

@@ -1,2 +1,2 @@
 # <target>      <source device>         <key file>      <options>
-md0-unlocked /dev/md0 none luks,discard
+root-unlocked UUID=%root_uuid% none luks,discard

config/etc/default/grub

@@ -0,0 +1,39 @@
+# If you change this file, run 'update-grub' afterwards to update
+# /boot/grub/grub.cfg.
+# For full documentation of the options in this file, see:
+#   info -f grub -n 'Simple configuration'
+
+GRUB_DEFAULT=0
+GRUB_TIMEOUT=0
+GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
+GRUB_CMDLINE_LINUX_DEFAULT="quiet nosmt"
+GRUB_CMDLINE_LINUX=""
+
+# If your computer has multiple operating systems installed, then you
+# probably want to run os-prober. However, if your computer is a host
+# for guest OSes installed via LVM or raw disk devices, running
+# os-prober can cause damage to those guest OSes as it mounts
+# filesystems to look for things.
+#GRUB_DISABLE_OS_PROBER=false
+
+# Uncomment to enable BadRAM filtering, modify to suit your needs
+# This works with Linux (no patch required) and with any kernel that obtains
+# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...)
+#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef"
+
+# Uncomment to disable graphical terminal
+#GRUB_TERMINAL=console
+
+# The resolution used on graphical terminal
+# note that you can use only modes which your graphic card supports via VBE
+# you can see them in real GRUB with the command `vbeinfo'
+#GRUB_GFXMODE=640x480
+
+# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux
+#GRUB_DISABLE_LINUX_UUID=true
+
+# Uncomment to disable generation of recovery mode menu entries
+#GRUB_DISABLE_RECOVERY="true"
+
+# Uncomment to get a beep at grub start
+#GRUB_INIT_TUNE="480 440 1"

config/etc/docker/daemon.json

@@ -0,0 +1,6 @@
+{
+    "experimental": true,
+    "ipv6": true,
+    "ip6tables": true,
+    "fixed-cidr-v6": "fd00::/48"
+}

config/etc/dropbear-initramfs/config

@@ -1 +0,0 @@
-DROPBEAR_OPTIONS="-p 222"

config/etc/dropbear/initramfs/dropbear.conf

@@ -0,0 +1,3 @@
+DROPBEAR_OPTIONS="-p 222"
+
+DROPBEAR_SK_ED25519=y

config/etc/fstab

@@ -3,6 +3,9 @@
 #
 # <file system> <dir>   <type>  <options>       <dump>  <pass>
 
-/dev/mapper/md0-unlocked / btrfs relatime 0 0
-/dev/md1 /boot btrfs relatime 0 0
-UUID=%esp_uuid% /boot/efi vfat relatime 0 0
+/dev/mapper/root-unlocked / ext4 relatime 0 1
+UUID=%boot_uuid% /boot ext4 relatime 0 2
+UUID=%esp_uuid% /boot/efi vfat relatime 0 2
+
+UUID=%disk1_uuid% /longhorn/01 ext4 relatime 0 2
+UUID=%disk2_uuid% /longhorn/02 ext4 relatime 0 2

config/etc/hosts

@@ -0,0 +1,4 @@
+127.0.0.1   localhost %fqdn%
+::1         localhost ip6-localhost ip6-loopback %fqdn%
+ff02::1     ip6-allnodes
+ff02::2     ip6-allrouters

config/etc/nftables.conf

@@ -1,75 +0,0 @@
-#!/usr/sbin/nft -f
-
-flush ruleset
-
-# don't use inet instead of ip&ip6,
-# because Docker doesn't support it, yet
-# see https://maximilianehlers.com/blog/nftables-and-docker/ for more details
-
-table ip filter {
-    chain INPUT {
-        type filter hook input priority 0;
-        policy drop;
-
-
-        # allow already established connections (e.g. initiated by this host)
-        ct state related,established accept
-
-        # allow ICMP
-        ip protocol icmp accept
-
-        # allow anything on localhost
-        iifname "lo" accept
-
-        # allow SSH for remote management
-        tcp dport 22 accept
-
-
-        ## docker
-
-        # cluster management communications
-        tcp dport 2377 accept
-
-        # communication among nodes
-        tcp dport 7946 accept
-        udp dport 7946 accept
-
-        # overlay network traffic
-        udp dport 4789 accept
-        
-        # allow IPSEC connections (encrypted overlay networks)
-        ip protocol esp accept
-    }
-    chain FORWARD {
-        type filter hook forward priority 0;
-        policy drop;
-    }
-    chain OUTPUT {
-        type filter hook output priority 0;
-        policy accept;
-    }
-}
-table ip6 filter {
-    chain INPUT {
-        type filter hook input priority 0;
-        policy drop;
-
-
-        # allow already established connections (e.g. initiated by this host)
-        ct state related,established accept
-
-        # allow ICMPv6
-        ip6 nexthdr icmpv6 accept
-
-        # allow anything on localhost
-        iifname "lo" accept
-    }
-    chain FORWARD {
-        type filter hook forward priority 0;
-        policy drop;
-    }
-    chain OUTPUT {
-        type filter hook output priority 0;
-        policy accept;
-    }
-}

config/etc/ssh/sshd_config

@@ -0,0 +1,2 @@
+PasswordAuthentication no
+PermitRootLogin no

config/etc/sysctl.d/90-kubelet.conf

@@ -0,0 +1,7 @@
+# https://docs.k3s.io/security/hardening-guide?_highlight=sysctl#ensure-protect-kernel-defaults-is-set
+
+vm.panic_on_oom=0
+vm.overcommit_memory=1
+kernel.panic=10
+kernel.panic_on_oops=1
+kernel.keys.root_maxbytes=25000000

hardware/hetzner_ax51-nvme/boot

@@ -0,0 +1 @@
+/dev/nvme0n1p2

hardware/hetzner_ax51-nvme/esp

@@ -0,0 +1 @@
+/dev/nvme0n1p3

hardware/hetzner_ax51-nvme/network.sh

@@ -0,0 +1,113 @@
+#!/bin/bash -e
+
+default_routev4=$(ip route list | grep -F 'default')
+
+interface=$(echo "$default_routev4" | perl -pe 's#^.* dev (\w+) .*$#$1#')
+macaddress=$(ip address show dev "$interface" | grep -F 'link/ether' | perl -pe 's#^.*link/ether ([a-z0-9:]*) brd .+$#$1#')
+
+addresses=$(ip address show dev "$interface" scope global)
+
+gatewayv4=$(echo "$default_routev4" | perl -pe 's#^.* via ([\d.]+) .*$#$1#')
+addressv4=$(echo "$addresses" | grep -F 'inet ' | perl -pe 's#^.* inet ([\d.]+).*#$1#')
+
+addressv6=$(echo "$addresses" | grep -F 'inet6 ' | perl -pe 's#^.* inet6 ([a-z0-9:]+)::[a-z0-9]+.*$#$1#')
+
+
+echo "[Match]
+MACAddress=${macaddress}
+Type=ether
+
+[Network]
+Gateway=fe80::1
+Gateway=${gatewayv4}
+
+[Address]
+Address=${addressv6}::1337/64
+
+[Address]
+Address=${addressv4}/32
+Peer=${gatewayv4}/32
+
+[Network]
+VLAN=ingress
+VLAN=kubernetes" > "/etc/systemd/network/egress.network"
+
+echo "[NetDev]
+Name=ingress
+Kind=vlan
+MTUBytes=1400
+
+[VLAN]
+Id=4000" > "/etc/systemd/network/ingress.netdev"
+
+echo "[Match]
+Name=ingress
+
+[Network]
+Description=\"ingress\"
+
+[Route]
+Gateway=2a01:4f8:fff0:a5::1
+Table=4000
+
+[Route]
+Gateway=157.90.103.81
+Table=4000
+
+
+[RoutingPolicyRule]
+From=2a01:4f8:fff0:a5::/64
+To=fd00:42::/31
+Priority=1000
+
+[RoutingPolicyRule]
+From=157.90.103.80/28
+To=10.42.0.0/15
+Priority=1000
+
+[RoutingPolicyRule]
+To=2a01:4f8:fff0:a5::/64
+From=fd00:42::/31
+Priority=1000
+
+[RoutingPolicyRule]
+To=157.90.103.80/28
+From=10.42.0.0/15
+Priority=1000
+
+
+[RoutingPolicyRule]
+From=2a01:4f8:fff0:a5::/64
+Table=4000
+Priority=1100
+
+[RoutingPolicyRule]
+From=157.90.103.80/28
+Table=4000
+Priority=1100
+
+[RoutingPolicyRule]
+To=2a01:4f8:fff0:a5::/64
+Table=4000
+Priority=1100
+
+[RoutingPolicyRule]
+To=157.90.103.80/28
+Table=4000
+Priority=1100" > "/etc/systemd/network/ingress.network"
+
+echo "[NetDev]
+Name=kubernetes
+Kind=vlan
+MTUBytes=1400
+
+[VLAN]
+Id=4010" > "/etc/systemd/network/kubernetes.netdev"
+
+echo "[Match]
+Name=kubernetes
+
+[Network]
+Description=\"kubernetes\"
+Address=10.73.19.fixme/24
+Address=fdad:73ce:19db::fixme/48" > "/etc/systemd/network/kubernetes.network"

hardware/hetzner_ax51-nvme/parted.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+
+# / 1020GB
+# /boot 1GB
+# /boot/efi 1GB
+# free 0GB
+
+parted -- /dev/nvme0n1 mklabel msdos
+parted -- /dev/nvme0n1 mkpart primary 1 1020GB
+parted -- /dev/nvme0n1 mkpart primary 1020GB 1022GB
+parted -- /dev/nvme0n1 mkpart primary 1022GB 1024GB
+# set flag for ESP
+parted -- /dev/nvme0n1 set 3 boot on
+
+parted -- /dev/nvme1n1 mklabel gpt
+parted -- /dev/nvme1n1 mkpart primary 1 -1
+parted -- /dev/nvme2n1 mklabel gpt
+parted -- /dev/nvme2n1 mkpart primary 1 -1
+
+sleep 1

hardware/hetzner_ax51-nvme/root

@@ -0,0 +1 @@
+/dev/nvme0n1p1

hardware/ovh_rise-1/boot

@@ -0,0 +1 @@
+/dev/md1

hardware/ovh_rise-1/root

@@ -0,0 +1 @@
+/dev/md0

post-debootstrap-installer.sh

@@ -7,7 +7,7 @@ cp -a /hardware-setup/config/* /
 
 # update apt because sources.list.d is also in config/*
 apt-get update
-apt-get dist-upgrade
+apt-get -y dist-upgrade
 
 
 # locales
@@ -23,52 +23,65 @@ apt-get -y install locales
 
 ### boot
 
-apt-get -y install mdadm cryptsetup systemd systemd-sysv btrfs-tools firmware-linux
+apt-get -y install mdadm cryptsetup systemd systemd-sysv firmware-linux
+systemctl enable fstrim.timer
 
-# --force-confold because we already provide /etc/dropbear-initramfs/config
+# --force-confold because we already provide /etc/dropbear/initramfs/dropbear.conf
 apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install dropbear-initramfs
 
 # generate minimal mdadm.conf
 mdadm --examine --scan | perl -pe 's/.*\/dev\/md\/?([0-9]+) .*UUID\=(.+?) .*/ARRAY \/dev\/md$1 UUID=$2/' > /etc/mdadm/mdadm.conf
 
 # concat user keys for cryptsetup unlocking at boot
-cat /hardware-setup/authorized_keys/* > /etc/dropbear-initramfs/authorized_keys
+cat /hardware-setup/authorized_keys/* > /etc/dropbear/initramfs/authorized_keys
+
+# install grub
+if [ -d /sys/firmware/efi ]; then
+    apt-get -y install grub-efi-amd64
+    grub-install
+else
+    apt-get -y install grub-pc
+
+    root_disk="/dev/$(lsblk -no pkname $(cryptsetup status root-unlocked | grep device | perl -pe 's#.*device.* (.*)#$1#') | sort | head -n1)"
+    grub-install "$root_disk"
+fi
+
+# edit uuids in fstab
+root_uuid=$(blkid --output value "$(cryptsetup status root-unlocked | grep device | perl -pe 's#.*device.* (.*)#$1#')" | head -n1)
+boot_uuid=$(blkid --output value "/hardware-setup/hardware/${1}/boot" | head -n1)
+esp_uuid=$(blkid --output value "/hardware-setup/hardware/${1}/esp" | head -n1)
+disk1_uuid=$(blkid --output value "/dev/nvme1n1p1" | head -n1)
+disk2_uuid=$(blkid --output value "/dev/nvme2n1p1" | head -n1)
+sed -i "s/%root_uuid%/${root_uuid}/" /etc/crypttab
+sed -i "s/%boot_uuid%/${boot_uuid}/" /etc/fstab
+sed -i "s/%esp_uuid%/${esp_uuid}/" /etc/fstab
+sed -i "s/%disk1_uuid%/${disk1_uuid}/" /etc/fstab
+sed -i "s/%disk2_uuid%/${disk2_uuid}/" /etc/fstab
+
+# install longhorn dependencies
+apt-get -y install open-iscsi nfs-common
+systemctl disable rpcbind.service # rpcbind is not used with NFS v4
 
 # after cryptsetup, mdadm, ... because of update-initramfs
 apt-get -y install linux-image-amd64
-
-# install grub
-apt-get -y install grub-efi-amd64
 update-grub
-grub-install
 
-# edit esp_uuid in fstab
-esp_uuid=$(blkid --output value "/hardware-setup/hardware/${1}/esp" | head -n1)
-sed -i "s/%esp_uuid%/${esp_uuid}/" /etc/fstab
 
 
 ### networking
 
 apt-get -y install iproute2
-# --force-confold because we already provide /etc/nftables.conf
-apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install nftables
 
 # generate configs for systemd-networkd.service
 # this is template specific
 "/hardware-setup/hardware/${1}/network.sh"
-
-systemctl enable systemd-networkd.service nftables.service
-
-# set DNS
-echo 'nameserver 1.1.1.1
-nameserver 8.8.8.8' > /etc/resolv.conf
+systemctl enable systemd-networkd.service
 
 
-### Docker
-# has to be executed before the users section,
-# because otherwise the docker group doesn't exist
+### ntp
 
-apt-get -y install docker-ce
+# automatically disables systemd-timesyncd.service
+apt-get -y install chrony
 
 
 ### users
@@ -85,7 +98,6 @@ for key in /hardware-setup/authorized_keys/*; do
     chown "$user": /home/"$user"/.ssh/authorized_keys
     
     adduser "$user" sudo
-    adduser "$user" docker
 done
 
 
@@ -93,10 +105,32 @@ done
 
 apt-get -y install \
     man-db \
+    byobu \
     vim \
     bash-completion \
     htop \
+    rsync \
+    curl \
     iputils-ping \
+    dnsutils \
     traceroute \
     tcpdump \
     openssh-server
+
+
+### clean up some things
+apt-get -y purge exim4-base
+apt-get -y --purge autoremove
+
+
+### dns
+
+apt-get -y install unbound
+
+# add fqdn to hosts file to mitigate nameserver failure
+sed -i "s/%fqdn%/$(hostname)/g" /etc/hosts
+
+# as last step set dns to local,
+# as unbound isn't running in the live/rescue system chroot,
+# which is where this script usually runs
+echo 'nameserver 127.0.0.1' > /etc/resolv.conf

reset.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 
 for device in /dev/md* /dev/nvme* /dev/sd* /dev/vd*; do
-    dd if=/dev/zero of=$device bs=10M count=100
+    dd if=/dev/zero of=$device bs=10M count=10 &
 done
-sync
+
+wait
+
+sync

setup.sh

@@ -5,45 +5,55 @@ set -e
 apt-get -y install mdadm cryptsetup debootstrap
 
 # returns /dev/md0 as root device
-# returns /dev/md1 as boot device
+# returns "$boot" as boot device
 "./hardware/${1}/parted.sh" "$2"
+root="hardware/${1}/root"
+boot="hardware/${1}/boot"
+esp="hardware/${1}/esp"
 
 # encrypt and unlock root device
 echo -n 'Enter luks password: '
-read -s md0pwd
+read -s root_pwd
 echo #to indicate progress after password prompt
-echo -n $md0pwd | cryptsetup -q luksFormat /dev/md0
-echo -n $md0pwd | cryptsetup open --type luks /dev/md0 md0-unlocked
-unset md0pwd
+echo -n $root_pwd | cryptsetup -q luksFormat "$root"
+echo -n $root_pwd | cryptsetup open --type luks "$root" root-unlocked
+unset root_pwd
 
 
 # format
 
-chroot=/mnt/md0-unlocked
+chroot=/mnt/root-unlocked
 
 # root device
-mkfs.btrfs /dev/mapper/md0-unlocked
-mkdir /mnt/md0-unlocked
-mount /dev/mapper/md0-unlocked /mnt/md0-unlocked
+mkfs.ext4 /dev/mapper/root-unlocked
+mkdir /mnt/root-unlocked
+mount /dev/mapper/root-unlocked /mnt/root-unlocked
 
 # boot device
-mkfs.btrfs /dev/md1
+mkfs.ext4 "$boot"
 mkdir "$chroot/boot"
-mount /dev/md1 "$chroot/boot"
+mount "$boot" "$chroot/boot"
 
 # esp device
-mkfs.fat "hardware/${1}/esp"
+mkfs.fat "$esp"
 mkdir "$chroot/boot/efi"
-mount "hardware/${1}/esp" "$chroot/boot/efi"
+mount "$esp" "$chroot/boot/efi"
+
+# additional data disks
+mkfs.ext4 /dev/nvme1n1p1
+mkdir --parents "$chroot/longhorn/01"
+mkfs.ext4 /dev/nvme2n1p1
+mkdir --parents "$chroot/longhorn/02"
 
 
 # debootstrap
 
-debootstrap --variant=minbase --arch=amd64 buster "$chroot" https://deb.debian.org/debian/
+debootstrap --variant=minbase --arch=amd64 bookworm "$chroot" https://deb.debian.org/debian/
 
 mount -t proc none "$chroot/proc"
 mount -t sysfs none "$chroot/sys"
-mount -o bind /dev "$chroot/dev"
+mount --bind /dev "$chroot/dev"
+mount --bind /run "$chroot/run"
 
 
 # set hostname
@@ -58,3 +68,9 @@ cp -a * "$chroot/hardware-setup"
 chroot "$chroot" /hardware-setup/post-debootstrap-installer.sh "$1"
 
 rm -r "$chroot/hardware-setup"
+
+echo "Don't forget to set a password with passwd ;-)"
+echo
+ls authorized_keys
+echo
+chroot "$chroot"