remote-builds: Refactor to support SSH config

Every time I touch this it's a ticking time bomb so let's see!
This commit is contained in:
Geoffrey Frogeye 2025-11-30 18:55:41 +01:00
parent a291391a7d
commit de6ddc085b

View file

@ -1,22 +1,23 @@
{
lib,
config,
options,
...
}:
let
vivariumBuilderDefault = {
systems = [
"x86_64-linux"
];
protocol = "ssh-ng";
sshUser = "nixremote";
};
# MANU ssh-keygen -y -f /etc/ssh/ssh_host_ed25519_key
# TODO Proper configuration option instead of pile of defs and hacks
vivariumBuilders = [
cfg = config.vivarium.remoteBuilders;
in
{
config = {
vivarium.remoteBuilders.morton = {
enable = true;
hostName = "morton.frogeye.fr";
publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEetvIp4ZrP+ofXNDypnrLxdU034SBYg7fx9FxClDJA3";
sshHostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEetvIp4ZrP+ofXNDypnrLxdU034SBYg7fx9FxClDJA3";
sshPort = 2277; # Could use 22 too since morton exposes it for git, but just making sure this option works for now.
nixPublicKey = "rSjbCZ4mgXkb+ENKI7sk/KIbftlQzCTQA7pWkdfS2r4=";
buildMachineConfig = {
systems = [ "x86_64-linux" ];
supportedFeatures = [
"nixos-test"
"benchmark"
@ -24,43 +25,89 @@ let
"kvm"
];
maxJobs = 12; # 8 cores, 16 with hyperthreading, trying not to overload the thing
}
];
# MANU pass vivarium/lemmy/remote-builds/cache | nix key convert-secret-to-public | cat
publicKeys = [
"morton.frogeye.fr:rSjbCZ4mgXkb+ENKI7sk/KIbftlQzCTQA7pWkdfS2r4="
];
in
{
config = {
};
};
boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
programs.ssh.knownHosts = lib.trivial.pipe vivariumBuilders [
(builtins.map (builder: {
name = builder.hostName;
value.publicKey = builder.publicKey;
}))
builtins.listToAttrs
programs.ssh = {
knownHosts = lib.attrsets.concatMapAttrs (name: builder: {
${builder.hostName}.publicKey = builder.sshHostKey;
}) cfg;
extraConfig = lib.trivial.pipe cfg [
(lib.attrsets.mapAttrsToList (
name: builder: ''
Host nix_builder_${name}
HostName ${builder.hostName}
User nixremote
Port ${builtins.toString builder.sshPort}
ControlMaster auto
ControlPath /tmp/ssh-%r@%h:%p
ControlPersist 120
''
))
lib.strings.concatLines
];
# Currently using port 22 only because:
# - Morton has to use it for git
# - Hopefully allowed on some firewalls
# - Thought you couldn't set SSH config
# You might be able to set SSH config with porgrams.ssh, although I only tried creating a /root/.ssh/config file
# (which does not work, unless logged in as root. host keys from root are used regardless of the user, though)
# FIXME 25.11 (assuming Lix 2.93+ is stable there): might want to add Control* options as no longer built in Lix
};
nix = {
buildMachines = builtins.map (
vivariumBuilder:
lib.attrsets.filterAttrs (k: v: k != "publicKey") (vivariumBuilderDefault // vivariumBuilder)
) vivariumBuilders;
buildMachines = lib.trivial.pipe cfg [
(lib.attrsets.filterAttrs (name: builder: builder.enable))
(lib.attrsets.mapAttrsToList (
name: builder:
builder.buildMachineConfig
// {
hostName = "nix_builder_${name}";
protocol = "ssh-ng";
sshUser = "nixremote"; # DEBUG To see if it use SSH config
}
))
];
distributedBuilds = true;
settings = {
builders-use-substitutes = true;
trusted-public-keys = publicKeys;
trusted-public-keys = lib.mapAttrsToList (
name: builder: "nix_builder_${name}:${builder.nixPublicKey}"
) cfg;
substituters = builtins.map (
builder: "${builder.protocol}://${builder.sshUser}@${builder.hostName}"
) config.nix.buildMachines;
};
};
};
options = {
vivarium.remoteBuilders = lib.mkOption {
default = { };
type = lib.types.attrsOf (
lib.types.submodule (
{ config, name, ... }:
{
options = {
enable = lib.mkOption {
type = lib.types.bool;
default = true;
description = "Whether to use for building. If disabled, keys will still be trusted.";
};
hostName = lib.mkOption {
type = lib.types.str;
};
sshPort = lib.mkOption {
type = lib.types.ints.positive;
default = 22;
};
sshHostKey = lib.mkOption {
type = lib.types.str;
};
nixPublicKey = lib.mkOption {
type = lib.types.str;
# MANU pass vivarium/lemmy/remote-builds/cache | nix key convert-secret-to-public | cat
};
buildMachineConfig = lib.mkOption {
type = options.nix.buildMachines.type.nestedTypes.elemType;
default = { };
};
};
}
)
);
};
};
}