Split xeon specific configuration from base

To accomodate the raccoon knights workstation, some of the configuration
pulled by m/common/main.nix has to be removed. To solve it, the xeon
specific parts are placed into m/common/xeon.nix and only the common
configuration is at m/common/base.nix.

Reviewed-by: Aleix Roca Nonell <aleix.rocanonell@bsc.es>

doc/install.md

@@ -150,3 +150,13 @@ And update grub.
 ```
 # nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
 ```
+
+## Chain NixOS in same disk
+
+```
+menuentry 'NixOS' {
+        insmod chain
+        set root=(hd3,1)
+        configfile /boot/grub/grub.cfg
+}
+```

flake.nix

@@ -17,13 +17,14 @@ let
 in
   {
     nixosConfigurations = {
-      hut   = mkConf "hut";
+      hut     = mkConf "hut";
-      owl1  = mkConf "owl1";
+      owl1    = mkConf "owl1";
-      owl2  = mkConf "owl2";
+      owl2    = mkConf "owl2";
-      eudy  = mkConf "eudy";
+      eudy    = mkConf "eudy";
-      koro  = mkConf "koro";
+      koro    = mkConf "koro";
-      bay   = mkConf "bay";
+      bay     = mkConf "bay";
-      lake2 = mkConf "lake2";
+      lake2   = mkConf "lake2";
+      raccoon = mkConf "raccoon";
     };
 
     packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {

m/bay/configuration.nix

@@ -2,8 +2,8 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
-    ../common/monitoring.nix
+    ../module/monitoring.nix
   ];
 
   # Select the this using the ID to avoid mismatches

m/common/base.nix

@@ -0,0 +1,19 @@
+{
+  # All machines should include this profile.
+  # Includes the basic configuration for an Intel server.
+  imports = [
+    ./base/agenix.nix
+    ./base/boot.nix
+    ./base/env.nix
+    ./base/fs.nix
+    ./base/hw.nix
+    ./base/net.nix
+    ./base/nix.nix
+    ./base/ntp.nix
+    ./base/rev.nix
+    ./base/ssh.nix
+    ./base/users.nix
+    ./base/watchdog.nix
+    ./base/zsh.nix
+  ];
+}

m/common/base/boot.nix

@@ -2,7 +2,7 @@
 
 {
   # Use the GRUB 2 boot loader.
-  boot.loader.grub.enable = lib.mkForce true;
+  boot.loader.grub.enable = true;
 
   # Enable GRUB2 serial console
   boot.loader.grub.extraConfig = ''

m/common/base/env.nix

@@ -0,0 +1,35 @@
+{ pkgs, config, ... }:
+
+{
+  environment.systemPackages = with pkgs; [
+    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
+    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
+    ncdu config.boot.kernelPackages.perf ldns
+    # From bsckgs overlay
+    osumb
+  ];
+
+  programs.direnv.enable = true;
+
+  # Increase limits
+  security.pam.loginLimits = [
+    {
+      domain = "*";
+      type = "-";
+      item = "memlock";
+      value = "1048576"; # 1 GiB of mem locked
+    }
+  ];
+
+  environment.variables = {
+    EDITOR = "vim";
+    VISUAL = "vim";
+  };
+
+  programs.bash.promptInit = ''
+    PS1="\h\\$ "
+  '';
+
+  time.timeZone = "Europe/Madrid";
+  i18n.defaultLocale = "en_DK.UTF-8";
+}

m/common/base/fs.nix

@@ -13,13 +13,6 @@
     [ { device = "/dev/disk/by-label/swap"; }
     ];
 
-  # Mount the home via NFS
-  fileSystems."/home" = {
-    device = "10.0.40.30:/home";
-    fsType = "nfs";
-    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
-  };
-
   # Tracing
   fileSystems."/sys/kernel/tracing" = {
     device = "none";

m/common/base/net.nix

@@ -0,0 +1,19 @@
+{ pkgs, ... }:
+
+{
+  networking = {
+    enableIPv6 = false;
+    useDHCP = false;
+
+    firewall = {
+      enable = true;
+      allowedTCPPorts = [ 22 ];
+    };
+
+    hosts = {
+      "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
+      "84.88.51.152" = [ "raccoon" ];
+      "84.88.51.142" = [ "raccoon-ipmi" ];
+    };
+  };
+}

m/common/base/nix.nix

@@ -0,0 +1,42 @@
+{ pkgs, nixpkgs, bscpkgs, theFlake,  ... }:
+
+{
+  nixpkgs.overlays = [
+    bscpkgs.bscOverlay
+    (import ../../../pkgs/overlay.nix)
+  ];
+
+  nix = {
+    nixPath = [
+      "nixpkgs=${nixpkgs}"
+      "jungle=${theFlake.outPath}"
+    ];
+
+    registry = {
+      nixpkgs.flake = nixpkgs;
+      jungle.flake = theFlake;
+    };
+
+    settings = {
+      experimental-features = [ "nix-command" "flakes" ];
+      sandbox = "relaxed";
+      trusted-users = [ "@wheel" ];
+      flake-registry = pkgs.writeText "global-registry.json"
+        ''{"flakes":[],"version":2}'';
+    };
+
+    gc = {
+      automatic = true;
+      dates = "weekly";
+      options = "--delete-older-than 30d";
+    };
+  };
+
+  # This value determines the NixOS release from which the default
+  # settings for stateful data, like file locations and database versions
+  # on your system were taken. It‘s perfectly fine and recommended to leave
+  # this value at the release version of the first install of this system.
+  # Before changing this value read the documentation for this option
+  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+  system.stateVersion = "22.11"; # Did you read the comment?
+}

m/common/base/rev.nix

@@ -1,6 +1,7 @@
 { theFlake, ... }:
 
 let
+  # Prevent building a configuration without revision
   rev = if theFlake ? rev then theFlake.rev
     else throw ("Refusing to build from a dirty Git tree!");
 in {
@@ -15,4 +16,6 @@ in {
     DATENOW=$(date --iso-8601=seconds)
     echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log
   '';
+
+  system.configurationRevision = rev;
 }

m/common/base/ssh.nix

@@ -1,7 +1,7 @@
 { lib, ... }:
 
 let
-  keys = import ../../keys.nix;
+  keys = import ../../../keys.nix;
   hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts;
 in
 {

m/common/base/users.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../module/jungle-users.nix
+    ../../module/jungle-users.nix
   ];
 
   users = {

m/common/main.nix

@@ -1,96 +0,0 @@
-{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }:
-
-{
-  imports = [
-    ./agenix.nix
-    ./boot.nix
-    ./fs.nix
-    ./hw.nix
-    ./net.nix
-    ./ntp.nix
-    ./ssh.nix
-    ./users.nix
-    ./watchdog.nix
-    ./rev.nix
-    ./zsh.nix
-  ];
-
-  nixpkgs.overlays = [
-    bscpkgs.bscOverlay
-    (import ../../pkgs/overlay.nix)
-  ];
-
-  system.configurationRevision =
-    if theFlake ? rev
-    then theFlake.rev
-    else throw ("Refusing to build from a dirty Git tree!");
-
-  nix.nixPath = [
-    "nixpkgs=${nixpkgs}"
-    "jungle=${theFlake.outPath}"
-  ];
-
-  nix.settings.flake-registry =
-    pkgs.writeText "global-registry.json" ''{"flakes":[],"version":2}'';
-
-  nix.registry.nixpkgs.flake = nixpkgs;
-  nix.registry.jungle.flake = theFlake;
-
-  environment.systemPackages = with pkgs; [
-    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
-    nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
-    ncdu config.boot.kernelPackages.perf ldns
-    # From bsckgs overlay
-    osumb
-  ];
-
-  programs.direnv.enable = true;
-
-  systemd.services."serial-getty@ttyS0" = {
-    enable = true;
-    wantedBy = [ "getty.target" ];
-    serviceConfig.Restart = "always";
-  };
-
-  # Increase limits
-  security.pam.loginLimits = [
-    {
-      domain = "*";
-      type = "-";
-      item = "memlock";
-      value = "1048576"; # 1 GiB of mem locked
-    }
-  ];
-
-  time.timeZone = "Europe/Madrid";
-  i18n.defaultLocale = "en_DK.UTF-8";
-
-  environment.variables = {
-    EDITOR = "vim";
-    VISUAL = "vim";
-  };
-
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-  nix.settings.sandbox = "relaxed";
-  nix.settings.trusted-users = [ "@wheel" ];
-  nix.gc.automatic = true;
-  nix.gc.dates = "weekly";
-  nix.gc.options = "--delete-older-than 30d";
-
-  programs.bash.promptInit = ''
-    PS1="\h\\$ "
-  '';
-
-  # Copy the NixOS configuration file and link it from the resulting system
-  # (/run/current-system/configuration.nix). This is useful in case you
-  # accidentally delete configuration.nix.
-  #system.copySystemConfiguration = true;
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "22.11"; # Did you read the comment?
-}

m/common/xeon.nix

@@ -0,0 +1,9 @@
+{
+  # Provides the base system for a xeon node.
+  imports = [
+    ./base.nix
+    ./xeon/fs.nix
+    ./xeon/getty.nix
+    ./xeon/net.nix
+  ];
+}

m/common/xeon/fs.nix

@@ -0,0 +1,8 @@
+{
+  # Mount the home via NFS
+  fileSystems."/home" = {
+    device = "10.0.40.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+}

m/common/xeon/getty.nix

@@ -0,0 +1,8 @@
+{
+  # Restart the serial console
+  systemd.services."serial-getty@ttyS0" = {
+    enable = true;
+    wantedBy = [ "getty.target" ];
+    serviceConfig.Restart = "always";
+  };
+}

m/common/xeon/net.nix

@@ -6,10 +6,9 @@
   boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
 
   networking = {
-    enableIPv6 = false;
-    useDHCP = false;
     defaultGateway = "10.0.40.30";
     nameservers = ["8.8.8.8"];
+
     proxy = {
       default = "http://localhost:23080/";
       noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
@@ -19,8 +18,6 @@
     };
 
     firewall = {
-      enable = true;
-      allowedTCPPorts = [ 22 ];
       extraCommands = ''
         # Prevent ssfhead from contacting our slurmd daemon
         iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
@@ -32,8 +29,7 @@
     };
 
     extraHosts = ''
-      10.0.40.30      ssfhead
+      10.0.40.30              ssfhead
-      84.88.53.236    ssfhead.bsc.es ssfhead
       
       # Node Entry for node: mds01 (ID=72)
       10.0.40.40              bay mds01 mds01-eth0

m/eudy/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ./kernel/kernel.nix

m/hut/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
 
     ../module/ceph.nix
     ../module/debuginfod.nix

m/koro/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
     #(modulesPath + "/installer/netboot/netboot-minimal.nix")
 
     ../eudy/cpufreq.nix

m/lake2/configuration.nix

@@ -2,8 +2,8 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
-    ../common/monitoring.nix
+    ../module/monitoring.nix
   ];
 
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";

m/owl1/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
     ../module/ceph.nix
     ../module/slurm-client.nix
     ../module/slurm-firewall.nix

m/owl2/configuration.nix

@@ -2,7 +2,7 @@
 
 {
   imports = [
-    ../common/main.nix
+    ../common/xeon.nix
     ../module/ceph.nix
     ../module/slurm-client.nix
     ../module/slurm-firewall.nix

m/raccoon/configuration.nix

@@ -0,0 +1,32 @@
+{ config, pkgs, lib, modulesPath, ... }:
+
+{
+  imports = [
+    ../common/base.nix
+  ];
+
+  # Don't install Grub on the disk yet
+  boot.loader.grub.device = "nodev";
+
+  networking = {
+    hostName = "raccoon";
+    # Only BSC DNSs seem to be reachable from the office VLAN
+    nameservers = [ "84.88.52.35" "84.88.52.36" ];
+    defaultGateway = "84.88.51.129";
+    interfaces.eno0.ipv4.addresses = [ {
+      address = "84.88.51.152";
+      prefixLength = 25;
+    } ];
+  };
+
+  # Configure Nvidia driver to use with CUDA
+  hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
+  hardware.opengl = {
+    enable = true;
+    driSupport = true;
+    setLdLibraryPath = true;
+  };
+  nixpkgs.config.allowUnfree = true;
+  nixpkgs.config.nvidia.acceptLicense = true;
+  services.xserver.videoDrivers = [ "nvidia" ];
+}