Simplify flake and expose host pkgs

The configuration of the machines is now moved to m/
2023-06-14 17:28:00 +02:00
parent f787343f29
commit f4ac9f3186
31 changed files with 39 additions and 112 deletions
--- a/m/common/boot.nix
+++ b/m/common/boot.nix
@@ -0,0 +1,36 @@
+{ lib, pkgs, ... }:
+
+{
+  # Use the GRUB 2 boot loader.
+  boot.loader.grub.enable = lib.mkForce true;
+  boot.loader.grub.version = 2;
+
+  # Enable GRUB2 serial console
+  boot.loader.grub.extraConfig = ''
+    serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
+    terminal_input --append serial
+    terminal_output --append serial
+  '';
+
+  # Enable serial console
+  boot.kernelParams = [
+    "console=tty1"
+    "console=ttyS0,115200"
+  ];
+
+  boot.kernelPackages = pkgs.linuxPackages_latest;
+
+  #boot.kernelPatches = lib.singleton {
+  #  name = "osnoise-tracer";
+  #  patch = null;
+  #  extraStructuredConfig = with lib.kernel; {
+  #    OSNOISE_TRACER = yes;
+  #    HWLAT_TRACER = yes;
+  #  };
+  #};
+
+  boot.initrd.availableKernelModules = [ "ahci" "xhci_pci" "ehci_pci" "nvme" "usbhid" "sd_mod" ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+}
--- a/m/common/fs.nix
+++ b/m/common/fs.nix
@@ -0,0 +1,25 @@
+{ ... }:
+
+{
+  fileSystems."/" =
+    { device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+    };
+
+  swapDevices =
+    [ { device = "/dev/disk/by-label/swap"; }
+    ];
+
+  # Mount the home via NFS
+  fileSystems."/home" = {
+    device = "10.0.40.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+
+  # Tracing
+  fileSystems."/sys/kernel/tracing" = {
+    device = "none";
+    fsType = "tracefs";
+  };
+}
--- a/m/common/hw.nix
+++ b/m/common/hw.nix
@@ -0,0 +1,14 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{ config, lib, pkgs, modulesPath, ... }:
+
+{
+  imports =
+    [ (modulesPath + "/installer/scan/not-detected.nix")
+    ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}
--- a/m/common/main.nix
+++ b/m/common/main.nix
@@ -0,0 +1,87 @@
+{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }:
+
+{
+  imports = [
+    ./boot.nix
+    ./fs.nix
+    ./hw.nix
+    ./net.nix
+    ./slurm.nix
+    ./ssh.nix
+    ./users.nix
+  ];
+
+  nixpkgs.overlays = [ bscpkgs.bscOverlay ];
+
+  nix.nixPath = [
+    "nixpkgs=${nixpkgs}"
+    "bscpkgs=${bscpkgs}"
+    "jungle=${theFlake.outPath}"
+  ];
+
+  nix.registry.nixpkgs.flake = nixpkgs;
+  nix.registry.bscpkgs.flake = bscpkgs;
+  nix.registry.jungle.flake = theFlake;
+
+  system.configurationRevision =
+    if theFlake ? rev
+    then theFlake.rev
+    else throw ("Refusing to build from a dirty Git tree!");
+
+  environment.systemPackages = with pkgs; [
+    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
+    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
+    ncdu
+  ];
+
+  systemd.services."serial-getty@ttyS0" = {
+    enable = true;
+    wantedBy = [ "getty.target" ];
+    serviceConfig.Restart = "always";
+  };
+
+  # Increase limits
+  security.pam.loginLimits = [
+    {
+      domain = "*";
+      type = "-";
+      item = "memlock";
+      value = "1048576"; # 1 GiB of mem locked
+    }
+  ];
+
+  time.timeZone = "Europe/Madrid";
+  i18n.defaultLocale = "en_DK.UTF-8";
+
+  environment.variables = {
+    EDITOR = "vim";
+    VISUAL = "vim";
+  };
+
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.settings.sandbox = "relaxed";
+  nix.settings.trusted-users = [ "@wheel" ];
+  nix.gc.automatic = true;
+  nix.gc.dates = "weekly";
+  nix.gc.options = "--delete-older-than 30d";
+
+  programs.zsh.enable = true;
+  programs.zsh.histSize = 100000;
+
+  programs.bash.promptInit = ''
+    PS1="\h\\$ "
+  '';
+
+  # Copy the NixOS configuration file and link it from the resulting system
+  # (/run/current-system/configuration.nix). This is useful in case you
+  # accidentally delete configuration.nix.
+  #system.copySystemConfiguration = true;
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "22.11"; # Did you read the comment?
+}
--- a/m/common/net.nix
+++ b/m/common/net.nix
@@ -0,0 +1,86 @@
+{ pkgs, ... }:
+
+{
+  # Infiniband (IPoIB)
+  environment.systemPackages = [ pkgs.rdma-core ];
+  boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
+
+  networking = {
+    enableIPv6 = false;
+    useDHCP = false;
+    #defaultGateway = "10.0.40.30";
+    nameservers = ["8.8.8.8"];
+    proxy = {
+      default = "http://localhost:23080/";
+      noProxy = "127.0.0.1,localhost,internal.domain";
+    };
+
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [ 22 ];
+
+      # FIXME: For slurmd as it requests the compute nodes to connect to us
+      allowedTCPPortRanges = [ { from=1024; to=65535; } ];
+    };
+
+    extraHosts = ''
+      10.0.40.30      ssfhead
+      84.88.53.236    ssfhead.bsc.es ssfhead
+      
+      # Node Entry for node: mds01 (ID=72)
+      10.0.40.40              mds01 mds01-eth0
+      10.0.42.40              mds01-ib0
+      10.0.40.141             mds01-ipmi0
+      
+      # Node Entry for node: oss01 (ID=73)
+      10.0.40.41              oss01 oss01-eth0
+      10.0.42.41              oss01-ib0
+      10.0.40.142             oss01-ipmi0
+      
+      # Node Entry for node: oss02 (ID=74)
+      10.0.40.42              oss02 oss02-eth0
+      10.0.42.42              oss02-ib0
+      10.0.40.143             oss02-ipmi0
+      
+      # Node Entry for node: xeon01 (ID=15)
+      10.0.40.1               xeon01 xeon01-eth0 owl1
+      10.0.42.1               xeon01-ib0
+      10.0.40.101             xeon01-ipmi0
+      
+      # Node Entry for node: xeon02 (ID=16)
+      10.0.40.2               xeon02 xeon02-eth0 owl2
+      10.0.42.2               xeon02-ib0
+      10.0.40.102             xeon02-ipmi0
+      
+      # Node Entry for node: xeon03 (ID=17)
+      10.0.40.3               xeon03 xeon03-eth0
+      10.0.42.3               xeon03-ib0
+      10.0.40.103             xeon03-ipmi0
+      
+      # Node Entry for node: xeon04 (ID=18)
+      10.0.40.4               xeon04 xeon04-eth0
+      10.0.42.4               xeon04-ib0
+      10.0.40.104             xeon04-ipmi0
+      
+      # Node Entry for node: xeon05 (ID=19)
+      10.0.40.5               xeon05 xeon05-eth0
+      10.0.42.5               xeon05-ib0
+      10.0.40.105             xeon05-ipmi0
+      
+      # Node Entry for node: xeon06 (ID=20)
+      10.0.40.6               xeon06 xeon06-eth0
+      10.0.42.6               xeon06-ib0
+      10.0.40.106             xeon06-ipmi0
+      
+      # Node Entry for node: xeon07 (ID=21)
+      10.0.40.7               xeon07 xeon07-eth0 hut
+      10.0.42.7               xeon07-ib0
+      10.0.40.107             xeon07-ipmi0
+      
+      # Node Entry for node: xeon08 (ID=22)
+      10.0.40.8               xeon08 xeon08-eth0
+      10.0.42.8               xeon08-ib0
+      10.0.40.108             xeon08-ipmi0
+    '';
+  };
+}
--- a/m/common/slurm.nix
+++ b/m/common/slurm.nix
@@ -0,0 +1,17 @@
+{ ... }:
+
+{
+  services.slurm = {
+    client.enable = true;
+    controlMachine = "hut";
+    clusterName = "owl";
+    nodeName = [
+      "owl[1,2]  Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=owl"
+      "hut       Sockets=2 CoresPerSocket=14 ThreadsPerCore=2"
+    ];
+    extraConfig = ''
+      MpiDefault=pmix
+      ReturnToService=2
+    '';
+  };
+}
--- a/m/common/ssh.nix
+++ b/m/common/ssh.nix
@@ -0,0 +1,36 @@
+{ ... }:
+
+{
+  # Enable the OpenSSH daemon.
+  services.openssh.enable = true;
+
+  # Connect to intranet git hosts via proxy
+  programs.ssh.extraConfig = ''
+    Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
+      User git
+      ProxyCommand nc -X connect -x localhost:23080 %h %p
+  '';
+
+  # Authorize keys
+  users.users = {
+    root.openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
+    ];
+    rarias.openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKBOf4r4lzQfyO0bx5BaREePREw8Zw5+xYgZhXwOZoBO ram@hop"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINa0tvnNgwkc5xOwd6xTtaIdFi5jv0j2FrE7jl5MTLoE ram@mio"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGYcXIxe0poOEGLpk8NjiRozls7fMRX0N3j3Ar94U+Gl rarias@hal"
+    ];
+    arocanon.openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3zeB5KSimMBAjvzsp1GCkepVaquVZGPYwRIzyzaCba aleix@bsc"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
+    ];
+  };
+
+  programs.ssh.knownHosts = {
+    "gitlab-internal.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3";
+    "bscpm03.bsc.es".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIM2NuSUPsEhqz1j5b4Gqd+MWFnRqyqY57+xMvBUqHYUS";
+  };
+}
--- a/m/common/users.nix
+++ b/m/common/users.nix
@@ -0,0 +1,32 @@
+{ ... }:
+
+{
+  users = {
+    mutableUsers = false;
+    users = {
+      rarias = {
+        uid = 1880;
+        isNormalUser = true;
+        home = "/home/Computational/rarias";
+        description = "Rodrigo Arias";
+        group = "Computational";
+        extraGroups = [ "wheel" ];
+        hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/";
+      };
+
+      arocanon = {
+        uid = 1042;
+        isNormalUser = true;
+        home = "/home/Computational/arocanon";
+        description = "Aleix Roca";
+        group = "Computational";
+        extraGroups = [ "wheel" ];
+        hashedPassword = "$6$hliZiW4tULC/tH7p$pqZarwJkNZ7vS0G5llWQKx08UFG9DxDYgad7jplMD8WkZh5k58i4dfPoWtnEShfjTO6JHiIin05ny5lmSXzGM/";
+      };
+    };
+
+    groups = {
+      Computational = { gid = 564; };
+    };
+  };
+}
--- a/m/hut/configuration.nix
+++ b/m/hut/configuration.nix
@@ -0,0 +1,28 @@
+{ config, pkgs, agenix, ... }:
+
+{
+  imports = [
+    ../common/main.nix
+
+    ./gitlab-runner.nix
+    ./monitoring.nix
+    ./nfs.nix
+    ./slurm-daemon.nix
+    agenix.nixosModules.default
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
+
+  networking = {
+    hostName = "hut";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.7";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.7";
+      prefixLength = 24;
+    } ];
+  };
+}
--- a/m/hut/gitlab-runner.nix
+++ b/m/hut/gitlab-runner.nix
@@ -0,0 +1,62 @@
+{ pkgs, lib, config, ... }:
+
+{
+  age.secrets."secrets/ovni-token".file = ./secrets/ovni-token.age;
+  age.secrets."secrets/nosv-token".file = ./secrets/nosv-token.age;
+
+  services.gitlab-runner = {
+    enable = true;
+    settings.concurrent = 5;
+    services = {
+      ovni-shell = {
+        registrationConfigFile = config.age.secrets."secrets/ovni-token".path;
+        executor = "shell";
+        tagList = [ "nix" "xeon" ];
+        environmentVariables = {
+          SHELL = "${pkgs.bash}/bin/bash";
+        };
+      };
+      ovni-docker = {
+        registrationConfigFile = config.age.secrets."secrets/ovni-token".path;
+        dockerImage = "debian:stable";
+        tagList = [ "docker" "xeon" ];
+        registrationFlags = [ "--docker-network-mode host" ];
+        environmentVariables = {
+          https_proxy = "http://localhost:23080";
+          http_proxy = "http://localhost:23080";
+        };
+      };
+      nosv-docker = {
+        registrationConfigFile = config.age.secrets."secrets/nosv-token".path;
+        dockerImage = "debian:stable";
+        tagList = [ "docker" "xeon" ];
+        registrationFlags = [
+          "--docker-network-mode host"
+          "--docker-cpus 56"
+        ];
+        environmentVariables = {
+          https_proxy = "http://localhost:23080";
+          http_proxy = "http://localhost:23080";
+        };
+      };
+    };
+  };
+
+  #systemd.services.gitlab-runner.serviceConfig.Shell = "${pkgs.bash}/bin/bash";
+  systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
+  systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";
+  systemd.services.gitlab-runner.serviceConfig.Group = "gitlab-runner";
+  systemd.services.gitlab-runner.serviceConfig.ExecStart = lib.mkForce
+    ''${pkgs.gitlab-runner}/bin/gitlab-runner run --config ''${HOME}/.gitlab-runner/config.toml --listen-address "127.0.0.1:9252" --working-directory ''${HOME}'';
+
+  users.users.gitlab-runner = {
+    uid = config.ids.uids.gitlab-runner;
+    #isNormalUser = true;
+    home = "/var/lib/gitlab-runner";
+    description = "Gitlab Runner";
+    group = "gitlab-runner";
+    extraGroups = [ "docker" ];
+    createHome = true;
+  };
+  users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner;
+}
--- a/m/hut/monitoring.nix
+++ b/m/hut/monitoring.nix
@@ -0,0 +1,66 @@
+{ config, lib, ... }:
+
+{
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        domain = "jungle.bsc.es";
+        root_url = "%(protocol)s://%(domain)s/grafana";
+        serve_from_sub_path = true;
+        http_port = 2342;
+        http_addr = "127.0.0.1";
+      };
+      feature_toggles.publicDashboards = true;
+    };
+  };
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+  };
+
+  systemd.services.prometheus-ipmi-exporter.serviceConfig.DynamicUser = lib.mkForce false;
+  systemd.services.prometheus-ipmi-exporter.serviceConfig.PrivateDevices = lib.mkForce false;
+
+  virtualisation.docker.daemon.settings = {
+    metrics-addr = "127.0.0.1:9323";
+  };
+
+  # Required to allow the smartctl exporter to read the nvme0 character device,
+  # see the commit message on:
+  # https://github.com/NixOS/nixpkgs/commit/12c26aca1fd55ab99f831bedc865a626eee39f80
+  services.udev.extraRules = ''
+    SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", GROUP="disk"
+  '';
+
+  services.prometheus = {
+
+    exporters = {
+      ipmi.enable = true;
+      ipmi.group = "root";
+      ipmi.user = "root";
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 9002;
+      };
+      smartctl.enable = true;
+    };
+
+    scrapeConfigs = [
+      {
+        job_name = "xeon07";
+        static_configs = [{
+          targets = [
+            "127.0.0.1:${toString config.services.prometheus.exporters.node.port}"
+            "127.0.0.1:${toString config.services.prometheus.exporters.ipmi.port}"
+            "127.0.0.1:9323"
+            "127.0.0.1:9252"
+            "127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
+          ];
+        }];
+      }
+    ];
+  };
+}
--- a/m/hut/nfs.nix
+++ b/m/hut/nfs.nix
@@ -0,0 +1,9 @@
+{ ... }:
+
+{
+  services.nfs.server.enable = true;
+  services.nfs.server.exports = ''
+    /nix 10.0.40.0/24(ro,sync,no_subtree_check,root_squash)
+  '';
+  networking.firewall.allowedTCPPorts = [ 2049 ];
+}
--- a/m/hut/secrets.nix
+++ b/m/hut/secrets.nix
@@ -0,0 +1,9 @@
+let
+  root = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIII/1TNArcwA6D47mgW4TArwlxQRpwmIGiZDysah40Gb";
+  system = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1";
+  systems = [ root system ];
+in
+{
+  "secrets/ovni-token.age".publicKeys = systems;
+  "secrets/nosv-token.age".publicKeys = systems;
+}
--- a/m/hut/secrets/nosv-token.age
+++ b/m/hut/secrets/nosv-token.age
--- a/m/hut/secrets/ovni-token.age
+++ b/m/hut/secrets/ovni-token.age
@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 MSF3dg Ivlduky3TjzCthY9RB/Jb0+MouX2FYW06hoNdQ+f818
+NKnuQrTQBXjTArXG6/5KV5cdg/9JUk/l3vVdYq0fXOE
+-> ssh-ed25519 HY2yRg 1ZCKpZ7sXNPgllHoozCgyW8NqK2TCoyCYZdug6YeJkM
+BEeThDkjfaK9S5a81HcyaZv9zobKANVMEimduc/IO54
+-> &eB%}y-grease o;.XY Yirz }Xh\DG
+CkLRClqWRkCr7n8o5UV9+kdCik2iTG/dI1s666CKcgxbAPohmryJzOKdgRLyzCf0
+CSPMUfrixmuQtuShigtmY6Pm2A
+--- GEuNMnWZ3+B6QNXv7s7bfJdJ2bJAAW+jbfHQZ0UQB+k
+<EFBFBD><EFBFBD>3<EFBFBD><EFBFBD>.<2E>-<2D>Ӯ<EFBFBD>ƿD<C6BF><44><EFBFBD>{\<5C><><EFBFBD>%<25><>R0<>߷<EFBFBD><DFB7><EFBFBD><EFBFBD><EFBFBD><EFBFBD>|<7C>P<EFBFBD>F<EFBFBD><46>x<EFBFBD>s<1E>_P<5F><50>x`4<>,<2C>z35<33>L<EFBFBD><4C><01>drj<72>2<EFBFBD><02>^<5E>
+<EFBFBD><EFBFBD>]<5D>h<>4~AP<41><EFBFBD>e3fT<66>E<EFBFBD>l*<2A>8z.<2E><><EFBFBD>x2<78>0<>7
--- a/m/hut/slurm-daemon.nix
+++ b/m/hut/slurm-daemon.nix
@@ -0,0 +1,10 @@
+{ ... }:
+
+{
+  services.slurm = {
+    server.enable = true;
+    partitionName = [
+      "xeon Nodes=xeon[01-02,07] Default=YES MaxTime=INFINITE State=UP"
+    ];
+  };
+}
--- a/m/owl1/configuration.nix
+++ b/m/owl1/configuration.nix
@@ -0,0 +1,20 @@
+{ config, pkgs, ... }:
+
+{
+  imports = [ ../common/main.nix ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53566c";
+
+  networking = {
+    hostName = "xeon01";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.1";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.1";
+      prefixLength = 24;
+    } ];
+  };
+}
--- a/m/owl2/configuration.nix
+++ b/m/owl2/configuration.nix
@@ -0,0 +1,25 @@
+{ config, pkgs, modulesPath, lib, ... }:
+
+{
+  imports = [
+    #(modulesPath + "/installer/netboot/netboot-minimal.nix")
+    ../common/main.nix
+  ];
+
+  # Select the this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d535629";
+  #programs.ssh.forwardX11 = false;
+  #programs.ssh.setXAuthLocation = lib.mkForce true;
+
+  networking = {
+    hostName = "xeon02";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.2";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp129s0.ipv4.addresses = [ {
+      address = "10.0.42.2";
+      prefixLength = 24;
+    } ];
+  };
+}
--- a/m/xeon08/configuration.nix
+++ b/m/xeon08/configuration.nix
@@ -0,0 +1,37 @@
+{ config, pkgs, lib, modulesPath, ... }:
+
+{
+  imports = [
+    ../common/main.nix
+    #(modulesPath + "/installer/netboot/netboot-minimal.nix")
+
+    ./kernel/kernel.nix
+    ./cpufreq.nix
+    ./fs.nix
+    ./users.nix
+    ./slurm.nix
+  ];
+
+  # Select this using the ID to avoid mismatches
+  boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53564b";
+
+  # disable automatic garbage collector
+  nix.gc.automatic = lib.mkForce false;
+
+  # members of the tracing group can use the lttng-provided kernel events
+  # without root permissions
+  users.groups.tracing.members = [ "arocanon" ];
+
+  # set up both ethernet and infiniband ips
+  networking = {
+    hostName = "xeon08";
+    interfaces.eno1.ipv4.addresses = [ {
+      address = "10.0.40.8";
+      prefixLength = 24;
+    } ];
+    interfaces.ibp5s0.ipv4.addresses = [ {
+      address = "10.0.42.8";
+      prefixLength = 24;
+    } ];
+  };
+}
--- a/m/xeon08/cpufreq.nix
+++ b/m/xeon08/cpufreq.nix
@@ -0,0 +1,40 @@
+{ lib, ... }:
+
+{
+  # Disable frequency boost by default. Use the intel_pstate driver instead of
+  # acpi_cpufreq driver because the acpi_cpufreq driver does not read the
+  # complete range of P-States [1]. Use the intel_pstate passive mode [2] to
+  # disable HWP, which allows a core to "select P-states by itself". Also, this
+  # disables intel governors, which confusingly, have the same names as the
+  # generic ones but behave differently [3].
+
+  # Essentially, we use the generic governors, but use the intel driver to read
+  # the P-state list.
+
+  # [1] - https://www.kernel.org/doc/html/latest/admin-guide/pm/intel_pstate.html#intel-pstate-vs-acpi-cpufreq
+  # [2] - https://www.kernel.org/doc/html/latest/admin-guide/pm/intel_pstate.html#passive-mode
+  # [3] - https://www.kernel.org/doc/html/latest/admin-guide/pm/intel_pstate.html#active-mode
+  # https://www.kernel.org/doc/html/latest/admin-guide/pm/cpufreq.html
+
+  # set intel_pstate to passive mode
+  boot.kernelParams = [
+    "intel_pstate=passive"
+  ];
+  # Disable frequency boost
+  system.activationScripts = {
+    disableFrequencyBoost.text = ''
+      echo 1 > /sys/devices/system/cpu/intel_pstate/no_turbo
+    '';
+  };
+
+  ## disable intel_pstate
+  #boot.kernelParams = [
+  #  "intel_pstate=disable"
+  #];
+  ## Disable frequency boost
+  #system.activationScripts = {
+  #  disableFrequencyBoost.text = ''
+  #    echo 0 > /sys/devices/system/cpu/cpufreq/boost
+  #  '';
+  #};
+}
--- a/m/xeon08/fs.nix
+++ b/m/xeon08/fs.nix
@@ -0,0 +1,13 @@
+{ ... }:
+
+{
+  fileSystems."/nix" = {
+    device = "/dev/disk/by-label/optane";
+    fsType = "ext4";
+    neededForBoot = true;
+  };
+  fileSystems."/mnt/data" = {
+    device = "/dev/disk/by-label/data";
+    fsType = "ext4";
+  };
+}
--- a/m/xeon08/kernel/configs/defconfig
+++ b/m/xeon08/kernel/configs/defconfig
--- a/m/xeon08/kernel/configs/lockdep
+++ b/m/xeon08/kernel/configs/lockdep
--- a/m/xeon08/kernel/kernel.nix
+++ b/m/xeon08/kernel/kernel.nix
@@ -0,0 +1,68 @@
+{ pkgs, lib, ... }:
+
+let
+  #fcs-devel = pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = /mnt/data/kernel/fcs/kernel/src;
+  #   configfile = /mnt/data/kernel/fcs/kernel/configs/defconfig;
+  #};
+
+  #fcsv1 = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" false;
+  #fcsv2 = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" false;
+  #fcsv1-lockdep = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" true;
+  #fcsv2-lockdep = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" true;
+  #fcs-kernel = gitCommit: lockdep: pkgs.linuxPackages_custom {
+  #   version = "6.2.8";
+  #   src = builtins.fetchGit {
+  #     url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
+  #     rev = gitCommit;
+  #     ref = "fcs";
+  #   };
+  #   configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
+  #};
+
+  kernel = nixos-fcsv2;
+
+  nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
+    version = "6.2.8";
+    src = builtins.fetchGit {
+      url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
+      rev = gitCommit;
+      ref = "fcs";
+    };
+    structuredExtraConfig = with lib.kernel; {
+      # add general custom kernel options here
+    } // lib.optionalAttrs lockStat {
+      LOCK_STAT = yes;
+    } // lib.optionalAttrs preempt {
+      PREEMPT = lib.mkForce yes;
+      PREEMPT_VOLUNTARY = lib.mkForce no;
+    };
+    kernelPatches = [];
+    extraMeta.branch = lib.versions.majorMinor version;
+  });
+
+  nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
+  nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
+  nixos-fcsv1-lockstat = nixos-fcs-kernel {
+    gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
+    lockStat = true;
+  };
+  nixos-fcsv2-lockstat = nixos-fcs-kernel {
+    gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
+    lockStat = true;
+  };
+  nixos-fcsv2-lockstat-preempt = nixos-fcs-kernel {
+    gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
+    lockStat = true;
+    preempt = true;
+  };
+  latest = pkgs.linuxPackages_latest;
+
+in {
+  imports = [
+    ./lttng.nix
+    ./perf.nix
+  ];
+  boot.kernelPackages = lib.mkForce kernel;
+}
--- a/m/xeon08/kernel/lttng.nix
+++ b/m/xeon08/kernel/lttng.nix
@@ -0,0 +1,43 @@
+{ config, pkgs, lib, ... }:
+
+let
+
+  # The lttng btrfs probe crashes at compile time because of an undefined
+  # function. This disables the btrfs tracepoints to avoid the issue.
+
+  # Also enable lockdep tracepoints, this is disabled by default because it
+  # does not work well on architectures other than x86_64 (i think that arm) as
+  # I was told on the mailing list.
+  lttng-modules-fixed = config.boot.kernelPackages.lttng-modules.overrideAttrs (finalAttrs: previousAttrs: {
+    patchPhase = (lib.optionalString (previousAttrs ? patchPhase) previousAttrs.patchPhase) + ''
+      # disable btrfs
+      substituteInPlace src/probes/Kbuild \
+        --replace "  obj-\$(CONFIG_LTTNG) += lttng-probe-btrfs.o" "  #obj-\$(CONFIG_LTTNG) += lttng-probe-btrfs.o"
+
+      # enable lockdep tracepoints
+      substituteInPlace src/probes/Kbuild \
+        --replace "#ifneq (\$(CONFIG_LOCKDEP),)"                  "ifneq (\$(CONFIG_LOCKDEP),)" \
+        --replace "#  obj-\$(CONFIG_LTTNG) += lttng-probe-lock.o" "  obj-\$(CONFIG_LTTNG) += lttng-probe-lock.o" \
+        --replace "#endif # CONFIG_LOCKDEP"                       "endif # CONFIG_LOCKDEP"
+    '';
+  });
+in {
+
+  # add the lttng tools and modules to the system environment
+  boot.extraModulePackages = [ lttng-modules-fixed ];
+  environment.systemPackages = with pkgs; [
+    lttng-tools lttng-ust babeltrace
+  ];
+
+  # start the lttng root daemon to manage kernel events
+  systemd.services.lttng-sessiond = {
+    wantedBy = [ "multi-user.target" ];
+    description = "LTTng session daemon for the root user";
+    serviceConfig = {
+      User = "root";
+      ExecStart = ''
+        ${pkgs.lttng-tools}/bin/lttng-sessiond
+      '';
+    };
+  };
+}
--- a/m/xeon08/kernel/perf.nix
+++ b/m/xeon08/kernel/perf.nix
@@ -0,0 +1,22 @@
+{ config, pkgs, lib, ... }:
+
+{
+  # add the perf tool
+  environment.systemPackages = with pkgs; [
+    config.boot.kernelPackages.perf
+  ];
+
+  # allow non-root users to read tracing data from the kernel
+  boot.kernel.sysctl."kernel.perf_event_paranoid" = -2;
+  boot.kernel.sysctl."kernel.kptr_restrict" = 0;
+
+  # specify additionl options to the tracefs directory to allow members of the
+  # tracing group to access tracefs.
+  fileSystems."/sys/kernel/tracing" = {
+    options = [
+      "mode=755"
+      "gid=tracing"
+    ];
+  };
+}
+
--- a/m/xeon08/slurm.nix
+++ b/m/xeon08/slurm.nix
@@ -0,0 +1,7 @@
+{ lib, ... }:
+
+{
+  services.slurm = {
+    client.enable = lib.mkForce false;
+  };
+}
--- a/m/xeon08/users.nix
+++ b/m/xeon08/users.nix
@@ -0,0 +1,11 @@
+{ ... }:
+
+{
+  security.sudo.extraRules= [{
+    users = [ "arocanon" ];
+    commands = [{
+      command = "ALL" ;
+      options= [ "NOPASSWD" ]; # "SETENV" # Adding the following could be a good idea
+    }];
+  }];
+}