Fix LUKS on pindakaas

It took 3 days to investigate by myself and only get a partial idea of
the issue, but 5 minutes to ask for support and another 5 to get a
working solution.
Hopefully lesson learned.

build-pindakaas-disk.sh

@@ -1,71 +0,0 @@
-#!/usr/bin/env nix-shell
-#! nix-shell -i bash
-#! nix-shell -p bash nixos-install-tools
-
-set -euo pipefail
-
-# Parse arguments
-function help {
-    echo "Usage: $0 -e"
-    echo
-    echo "Options:"
-    echo "  -h: Display this help message."
-    echo "  -e: Erase the disk. This can be used in case the wanted partition scheme doesn't match."
-}
-
-disko_mode=mount
-while getopts "e" OPTION
-do
-    case "$OPTION" in
-        h)
-            help
-            exit 0
-            ;;
-        e)
-            disko_mode=disko
-            ;;
-        ?)
-            help
-            exit 2
-            ;;
-    esac
-done
-shift "$(($OPTIND -1))"
-
-if [ "$#" -gt 0 ]
-then
-    help
-    exit 2
-fi
-
-set -x
-
-# Add channels to root user, as nixos-install uses those.
-# Not great, but fixable with flakes I guess
-sudo ./add_channels.sh
-
-# Format or mount disk
-sudo nix --extra-experimental-features nix-command --extra-experimental-features flakes run github:nix-community/disko -- --mode $disko_mode ./pindakaas_disko.nix
-
-# Generate hardware-config.nix
-sudo nixos-generate-config --no-filesystems --root /mnt
-# --no-filesystems because they are imported via disko
-
-# Plug system configuration into this git repo
-sudo mkdir -p /mnt/etc/nixos
-echo "{ ... }: { imports = [ ./hardware-configuration.nix /home/geoffrey/.config/dotfiles/pindakaas.nix ]; }" | sudo tee /mnt/etc/nixos/configuration.nix > /dev/null
-# Everything there should be covered by (and conflicts with) the repo anyways.
-
-# Install NixOS! Or create a new generation.
-sudo nixos-install
-
-# Install dotfiles. Actually not needed by nixos-install since it doesn't rewrite global paths to /mnt.
-# Without it no nixos-rebuild from the system itself once installed though.
-# Should probably be replaced with something like git-sync
-# sudo mkdir -p /mnt/home/geoffrey/.config/
-# sudo cp -a ../dotfiles /mnt/home/geoffrey/.config/
-# sudo chown geoffrey:geoffrey /mnt/home/geoffrey -R
-
-# Signal we're done!
-# Although it might ask for passwords beforehand, so not sure it's really useful
-echo 

build-vm.sh

@@ -1,21 +0,0 @@
-#/usr/bin/env sh
-
-# Build a NixOS VM for testing
-#
-# For x86_64 hosts: use curacao config
-# For aarch64 hosts: use pindakaas config
-
-if ! command -v nom-build
-then
-    exec nix-shell -p nix-output-monitor --run $0
-fi
-
-if [ "$(uname -m)" == "x86_64" ]
-then
-    config=./curacao.nix
-elif [ "$(uname -m)" == "aarch64" ]
-then
-    config=./pindakaas.nix
-fi
-nom-build '<nixpkgs/nixos>' -A vm \
-    -I nixos-config=${config}

curacao.nix

@@ -1,14 +0,0 @@
-{ ... }:
-{
-  imports = [
-    "${builtins.fetchTarball "https://github.com/NixOS/nixos-hardware/archive/468a7a108108908c7a35d6549f1e1f0236a9448a.tar.gz"}/dell/g3/3779"
-    ./os/loader.nix
-  ];
-
-  boot.loader.efi.canTouchEfiVariables = true;
-
-  networking.hostName = "curacao";
-  frogeye.extra = true;
-  frogeye.desktop.xorg = true;
-  frogeye.dev.docker = true;
-}

curacao/hardware.nix

@@ -0,0 +1,8 @@
+{ lib, ... }:
+{
+  imports = [
+    "${builtins.fetchTarball "https://github.com/NixOS/nixos-hardware/archive/468a7a108108908c7a35d6549f1e1f0236a9448a.tar.gz"}/dell/g3/3779"
+  ];
+
+  boot.loader.efi.canTouchEfiVariables = lib.mkDefault true;
+}

curacao/options.nix

@@ -0,0 +1,10 @@
+{ ... }:
+{
+  frogeye = {
+    desktop.xorg = true;
+    dev = {
+      docker = true;
+    };
+    extra = true;
+  };
+}

curacao/os.nix

@@ -0,0 +1,11 @@
+{ ... }:
+{
+  imports = [
+    ../os
+    ./options.nix
+    ./hardware.nix
+    ./disko.nix
+  ];
+
+  networking.hostName = "curacao";
+}

curacao_test.nix

@@ -1,18 +0,0 @@
-{ ... }:
-{
-  imports = [
-    ./hm/loader.nix
-  ];
-
-  home.username = "gnix";
-  home.homeDirectory = "/home/gnix";
-
-  frogeye.extra = true;
-  frogeye.desktop.xorg = true;
-  frogeye.desktop.nixGLIntel = true;
-  frogeye.dev.ansible = true;
-  frogeye.dev.c = true;
-  frogeye.dev.docker = true;
-  frogeye.dev.fpga = true;
-  frogeye.dev.python = true;
-}

curacao_test/hm.nix

@@ -0,0 +1,12 @@
+{ ... }:
+{
+  imports = [
+    ../hm
+    ../curacao/options.nix
+  ];
+
+  home.username = "gnix";
+  home.homeDirectory = "/home/gnix";
+
+  frogeye.desktop.nixGLIntel = true;
+}

curacao_usb/disko-desired.nix

@@ -0,0 +1,64 @@
+{
+  disko.devices = {
+    disk = {
+      pindakaas_sd = {
+        type = "disk";
+        device = "/dev/disk/by-id/usb-Kingston_DataTraveler_3.0_E0D55EA57414F510489F0F1A-0:0";
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              # Needs enough to store multiple kernel generations
+              size = "512M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [
+                  "defaults"
+                ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "pindakaas_sd";
+                #passwordFile = "/tmp/secret.key"; # Commented out so asked interactively
+                settings = {
+                  # Not having SSDs die fast is more important than crypto
+                  # nerds that could potentially discover which filesystem I
+                  # use from TRIM patterns
+                  allowDiscards = true;
+                };
+                content = {
+                  type = "btrfs";
+                  extraArgs = [ "-f" ];
+                  subvolumes = {
+                    "/nixos" = {
+                      mountpoint = "/";
+                      mountOptions = [ "compress=zstd" "noatime" ];
+                    };
+                    "/home" = {
+                      mountpoint = "/home";
+                      mountOptions = [ "compress=zstd" "relatime" ];
+                    };
+                    "/nix" = {
+                      mountpoint = "/nix";
+                      mountOptions = [ "compress=zstd" "noatime" ];
+                    };
+                    # Maybe later
+                    # "/swap" = {
+                    #   mountpoint = "/.swapvol";
+                    #   swap.swapfile.size = "20M";
+                    # };
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+  }

curacao_usb/disko-hanging.nix

@@ -0,0 +1,38 @@
+{
+  disko.devices = {
+    disk = {
+      vdb = {
+        type = "disk";
+        device = "/dev/disk/by-id/usb-Kingston_DataTraveler_3.0_E0D55EA57414F510489F0F1A-0:0";
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              size = "500M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "crypted";
+                settings.allowDiscards = true;
+                passwordFile = "/tmp/secret.key";
+                content = {
+                  type = "filesystem";
+                  format = "ext4";
+                  mountpoint = "/";
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

curacao_usb/disko-working.nix

@@ -0,0 +1,32 @@
+{
+  disko.devices = {
+    disk = {
+      vdb = {
+        type = "disk";
+        device = "/dev/disk/by-id/usb-Kingston_DataTraveler_3.0_E0D55EA57414F510489F0F1A-0:0";
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              size = "500M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+              };
+            };
+            root = {
+              size = "100%";
+                content = {
+                  type = "filesystem";
+                  format = "ext4";
+                  mountpoint = "/";
+                };
+            };
+          };
+        };
+      };
+    };
+  };
+}

curacao_usb/disko.nix

@@ -0,0 +1 @@
+disko-hanging.nix

curacao_usb/os.nix

@@ -0,0 +1,30 @@
+{ pkgs, config, ... }:
+{
+  # This whole folder is DEBUG
+  imports = [
+    "${builtins.fetchTarball "https://github.com/nix-community/disko/archive/3cb78c93e6a02f494aaf6aeb37481c27a2e2ee22.tar.gz"}/module.nix"
+    ./disko.nix
+  ];
+
+  nixpkgs.config.allowUnfree = true;
+  networking.hostName = "curacao_usb";
+
+  boot = {
+    # nixos-hardware use latest kernel by default. It has been set a while ago, we maybe don't need it anymore?
+    kernelPackages = pkgs.linuxPackages;
+
+    # Pinebook supports UEFI, at least when tow-boot is installed on the SPI
+    loader = {
+      # EFI Variables don't work (no generation appears in systemd-boot)
+      efi.canTouchEfiVariables = false;
+
+      # systemd-boot crashes after booting, so GRUB it is
+      grub = {
+        enable = true;
+        efiSupport = true;
+        efiInstallAsRemovable = true;
+        device = "nodev";
+      };
+    };
+  };
+}

install_os.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash
+#! nix-shell -p bash nixos-install-tools
+
+set -euo pipefail
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# Parse arguments
+function help {
+    echo "Usage: $0 [-e] [-h] profile"
+    echo
+    echo "Arguments:"
+    echo "  file: OS/disk profile to use"
+    echo
+    echo "Options:"
+    echo "  -h: Display this help message."
+    echo "  -e: Erase the disk. This can be used in case the wanted partition scheme doesn't match."
+}
+
+disko_mode=mount
+while getopts "e" OPTION
+do
+    case "$OPTION" in
+        h)
+            help
+            exit 0
+            ;;
+        e)
+            disko_mode=disko
+            ;;
+        ?)
+            help
+            exit 2
+            ;;
+    esac
+done
+shift "$(($OPTIND -1))"
+
+if [ "$#" -ne 1 ]
+then
+    help
+    exit 2
+fi
+profile="$1"
+
+profile_dir="${SCRIPT_DIR}/${profile}"
+if [ ! -d "$profile_dir" ]
+then
+    echo "Profile not found."
+fi
+
+disko_config="${profile_dir}/disko.nix"
+if [ ! -f "$disko_config" ]
+then
+    echo "Disk configuration not found."
+fi
+
+nixos_config="${profile_dir}/os.nix"
+if [ ! -f "$nixos_config" ]
+then
+    echo "NixOS configuration not found."
+fi
+
+mountpoint="/mnt/nixos"
+nix_flakes_cmd="nix --extra-experimental-features nix-command --extra-experimental-features flakes"
+
+set -x
+
+sudo mkdir -p "$mountpoint"
+
+# Add channels to root user, as nixos-install uses those.
+# Not great, but fixable with flakes I guess
+sudo ./add_channels.sh
+
+# Format or mount disk
+sudo $nix_flakes_cmd run github:nix-community/disko -- --root-mountpoint "$mountpoint" --mode "$disko_mode" "$disko_config"
+
+# Generate hardware-config.nix
+sudo nixos-generate-config --no-filesystems --root "$mountpoint"
+# --no-filesystems because they are imported via disko
+
+# Plug system configuration into this git repo
+sudo mkdir -p "${mountpoint}/etc/nixos"
+echo "{ ... }: { imports = [ ./hardware-configuration.nix ${nixos_config} ]; }" | sudo tee "${mountpoint}/etc/nixos/configuration.nix" > /dev/null
+# Everything there should be covered by (and conflicts with) the repo anyways.
+
+# Install NixOS! Or create a new generation.
+sudo nixos-install --no-root-password --root "$mountpoint"
+
+# Install dotfiles. Actually not needed by nixos-install since it doesn't rewrite global paths to the mountpoint.
+# Without it no nixos-rebuild from the system itself once installed though.
+# Should probably be replaced with something like git-sync
+# sudo mkdir -p $mountpoint/home/geoffrey/.config/
+# sudo cp -a ../dotfiles $mountpoint/home/geoffrey/.config/
+# sudo chown geoffrey:geoffrey $mountpoint/home/geoffrey -R
+
+# Signal the installation is done!
+echo 

os/geoffrey.nix

@@ -22,7 +22,7 @@
   home-manager = {
     users.geoffrey = { pkgs, ... }: {
       imports = [
-        ../hm/loader.nix
+        ../hm
       ];
       frogeye = config.frogeye;
     };

pindakaas/hardware.nix

@@ -3,13 +3,15 @@
   imports = [
     # First commit before 23.05 release date:
     "${builtins.fetchTarball "https://github.com/NixOS/nixos-hardware/archive/468a7a108108908c7a35d6549f1e1f0236a9448a.tar.gz"}/pine64/pinebook-pro"
-    ./os/loader.nix
-    ./pindakaas_disko.nix
   ];
+
   boot = {
     # nixos-hardware use latest kernel by default. It has been set a while ago, we maybe don't need it anymore?
     kernelPackages = pkgs.linuxPackages;
 
+    # Otherwise it will not show stage1 echo and prompt
+    kernelParams = ["console=tty0"];
+
     # Pinebook supports UEFI, at least when tow-boot is installed on the SPI
     loader = {
       # EFI Variables don't work (no generation appears in systemd-boot)
@@ -25,14 +27,4 @@
     };
   };
 
-
-  networking.hostName = "pindakaas";
-
-  # DEBUG Barebones is faster
-  frogeye.dev.ansible = false;
-  frogeye.dev.python = false;
-
-  # frogeye.extra = false; # FIXME Not working yet
-  # frogeye.desktop.xorg = true;
-  # frogeye.dev.docker = true;
 }

pindakaas/options.nix

@@ -0,0 +1,6 @@
+{ ... }:
+{
+  frogeye.extra = false; # FIXME Not working yet
+  frogeye.desktop.xorg = true;
+  frogeye.dev.docker = true;
+}

pindakaas/os.nix

@@ -0,0 +1,11 @@
+{ ... }:
+{
+  imports = [
+    ../os
+    ./hardware.nix
+    ./disko.nix
+    ./options.nix
+  ];
+
+  networking.hostName = "pindakaas";
+}

pindakaas_disko.nix

@@ -1,78 +0,0 @@
-{
-  disko.devices = {
-    disk = {
-      pindakaas_sd = {
-        type = "disk";
-        device = "/dev/disk/by-id/mmc-SN32G_0xfb19ae99";
-        content = {
-          type = "gpt";
-          partitions = {
-            ESP = {
-              # Needs enough to store multiple kernel generations
-              size = "512M";
-              type = "EF00";
-              content = {
-                type = "filesystem";
-                format = "vfat";
-                mountpoint = "/boot";
-                mountOptions = [
-                  "defaults"
-                ];
-              };
-            };
-            root = {
-              size = "100%";
-              content = {
-                type = "filesystem";
-                format = "ext4";
-                mountpoint = "/";
-              };
-            };
-            # FIXME Hang on Linux boot, for some reason
-            # luks = {
-            #   size = "100%";
-            #   content = {
-            #     type = "luks";
-            #     name = "pindakaas_sd";
-            #     # disable settings.keyFile if you want to use interactive password entry
-            #     #passwordFile = "/tmp/secret.key"; # Interactive
-            #     settings = {
-            #       # Not having SSDs die fast is more important than crypto
-            #       # nerds that could potentially discover which filesystem I
-            #       # use from TRIM patterns
-            #       allowDiscards = true;
-            #       # keyFile = "/tmp/secret.key";
-            #       fallbackToPassword = true; # TEST
-            #     };
-            #     # additionalKeyFiles = [ "/tmp/additionalSecret.key" ];
-            #     content = {
-            #       type = "btrfs";
-            #       extraArgs = [ "-f" ];
-            #       subvolumes = {
-            #         "/nixos" = {
-            #           mountpoint = "/";
-            #           mountOptions = [ "compress=zstd" "noatime" ];
-            #         };
-            #         "/home" = {
-            #           mountpoint = "/home";
-            #           mountOptions = [ "compress=zstd" "relatime" ];
-            #         };
-            #         "/nix" = {
-            #           mountpoint = "/nix";
-            #           mountOptions = [ "compress=zstd" "noatime" ];
-            #         };
-            #         # Maybe later
-            #         # "/swap" = {
-            #         #   mountpoint = "/.swapvol";
-            #         #   swap.swapfile.size = "20M";
-            #         # };
-            #       };
-            #     };
-            #   };
-            # };
-          };
-        };
-      };
-    };
-  };
-}

pindakaas_sd/disko.nix

@@ -0,0 +1,65 @@
+{
+  disko.devices = {
+    disk = {
+      pindakaas_sd = {
+        type = "disk";
+        device = "/dev/disk/by-id/mmc-SN32G_0xfb19ae99";
+        content = {
+          type = "gpt";
+          partitions = {
+            ESP = {
+              # Needs enough to store multiple kernel generations
+              size = "512M";
+              type = "EF00";
+              content = {
+                type = "filesystem";
+                format = "vfat";
+                mountpoint = "/boot";
+                mountOptions = [
+                  "defaults"
+                ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "pindakaas_sd";
+                passwordFile = "/tmp/secret.key"; # TODO Generate this somehow
+                settings = {
+                  # Not having SSDs die fast is more important than crypto
+                  # nerds that could potentially discover which filesystem I
+                  # use from TRIM patterns
+                  allowDiscards = true;
+                };
+                content = {
+                  type = "btrfs";
+                  extraArgs = [ "-f" ];
+                  subvolumes = {
+                    "/nixos" = {
+                      mountpoint = "/";
+                      mountOptions = [ "compress=zstd" "noatime" ];
+                    };
+                    "/home" = {
+                      mountpoint = "/home";
+                      mountOptions = [ "compress=zstd" "relatime" ];
+                    };
+                    "/nix" = {
+                      mountpoint = "/nix";
+                      mountOptions = [ "compress=zstd" "noatime" ];
+                    };
+                    # Maybe later
+                    # "/swap" = {
+                    #   mountpoint = "/.swapvol";
+                    #   swap.swapfile.size = "20M";
+                    # };
+                  };
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

pindakaas_sd/hybrid

@@ -0,0 +1 @@
+/nix/store/bz6nqy0ky5l33rfrkb6k66bzkgw18k90-nixos-system-pindakaas_sd-23.05.4981.5b528f99f73c

pindakaas_sd/os.nix

@@ -0,0 +1,11 @@
+{ pkgs, config, ... }:
+{
+  imports = [
+    ../os
+    ../pindakaas/options.nix
+    ../pindakaas/hardware.nix
+    ./disko.nix
+  ];
+
+  networking.hostName = "pindakaas_sd";
+}

vm/build.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash
+#! nix-shell -p bash nix-output-monitor
+
+# Build a NixOS VM for testing
+
+set -euo pipefail
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+nom-build '<nixpkgs/nixos>' -A vm \
+    -I nixos-config=./os.nix

vm/options.nix

@@ -0,0 +1,14 @@
+{ ... }:
+{
+  frogeye = {
+    desktop.xorg = true;
+    dev = {
+      ansible = true;
+      c = true;
+      docker = true;
+      fpga = true;
+      python = true;
+    };
+    # extra = true; # FIXME Should support aarch64
+  };
+}

vm/os.nix

@@ -0,0 +1,10 @@
+{ ... }:
+{
+  imports = [
+    ../os
+    ./options.nix
+  ];
+
+  # Create a different disk image depending on the architecture
+  networking.hostName = "${builtins.currentSystem}";
+}