Setup slurm and gitlab-runner

boot.nix

@@ -1,4 +1,4 @@
-{ ... }:
+{ lib, ... }:
 
 {
   # Use the GRUB 2 boot loader.
@@ -8,9 +8,25 @@
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
 
+  # Enable GRUB2 serial console
+  boot.loader.grub.extraConfig = ''
+    serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
+    terminal_input --append serial
+    terminal_output --append serial
+  '';
+
   # Enable serial console
   boot.kernelParams = [
     "console=tty1"
     "console=ttyS0,115200"
   ];
+
+  boot.kernelPatches = lib.singleton {
+    name = "osnoise-tracer";
+    patch = null;
+    extraStructuredConfig = with lib.kernel; {
+      OSNOISE_TRACER = yes;
+      HWLAT_TRACER = yes;
+    };
+  };
 }

configuration.nix

@@ -8,8 +8,10 @@
     ./fs.nix
     ./gitlab-runner.nix
     ./net.nix
+    ./slurm.nix
     ./ssh.nix
     ./users.nix
+    ./overlays.nix
   ];
 
   systemd.services."serial-getty@ttyS0" = {
@@ -19,13 +21,17 @@
   };
 
   time.timeZone = "Europe/Madrid";
-  i18n.defaultLocale = "en_US.UTF-8";
+  i18n.defaultLocale = "en_DK.UTF-8";
 
   environment.systemPackages = with pkgs; [
-    vim wget git htop
+    vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
   ];
 
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
+  nix.gc.automatic = true;
+
+  programs.zsh.enable = true;
+  programs.zsh.histSize = 100000;
 
   # Copy the NixOS configuration file and link it from the resulting system
   # (/run/current-system/configuration.nix). This is useful in case you

fs.nix

@@ -5,6 +5,12 @@
   fileSystems."/home" = {
     device = "10.0.40.30:/home";
     fsType = "nfs";
-    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" ];
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
+  };
+
+  # Tracing
+  fileSystems."/sys/kernel/tracing" = {
+    device = "none";
+    fsType = "tracefs";
   };
 }

gitlab-runner.nix

@@ -18,14 +18,21 @@
           SHELL = "${pkgs.bash}/bin/bash";
         };
       };
-    #  # runner for everything else
+
-    #  default = {
+      # runner for everything else
-    #    # File should contain at least these two variables:
+      default = {
-    #    # `CI_SERVER_URL`
+        # File should contain at least these two variables:
-    #    # `REGISTRATION_TOKEN`
+        # `CI_SERVER_URL`
-    #    registrationConfigFile = "/run/secrets/gitlab-runner-registration";
+        # `REGISTRATION_TOKEN`
-    #    dockerImage = "debian:stable";
+        registrationConfigFile = "/run/secrets/gitlab-runner-registration";
-    #  };
+        dockerImage = "debian:stable";
+        tagList = [ "docker" "xeon" ];
+        registrationFlags = [ "--docker-network-mode host" ];
+        environmentVariables = {
+          https_proxy = "http://localhost:23080";
+          http_proxy = "http://localhost:23080";
+        };
+      };
     };
   };
 
@@ -33,6 +40,22 @@
   systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
   systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";
   systemd.services.gitlab-runner.serviceConfig.Group = "gitlab-runner";
+  #systemd.services.gitlab-runner.serviceConfig.ExecStart = lib.mkForce
+  #  ''${pkgs.gitlab-runner}/bin/gitlab-runner --debug run --config ''${HOME}/.gitlab-runner/config.toml --working-directory ''${HOME}'';
+
+  # TODO https://docs.gitlab.com/runner/configuration/proxy.html
+  #systemd.services.docker.environment = {
+  #  HTTP_PROXY="http://localhost:23080/";
+  #  HTTPS_PROXY="http://localhost:23080/";
+  #};
+
+  #virtualisation.docker.daemon.settings = {
+  #  proxies.default = {
+  #    httpProxy = "http://localhost:23080/";
+  #    httpsProxy = "http://localhost:23080/";
+  #    noProxy = "localhost,127.0.0.0/8";
+  #  };
+  #};
 
   users.users.gitlab-runner = {
     uid = config.ids.uids.gitlab-runner;
@@ -40,6 +63,7 @@
     home = "/var/lib/gitlab-runner";
     description = "Gitlab Runner";
     group = "gitlab-runner";
+    extraGroups = [ "docker" ];
     createHome = true;
   };
   users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner;

net.nix

@@ -4,8 +4,9 @@
   networking = {
     hostName = "xeon07";
 
+    enableIPv6 = false;
     useDHCP = false;
-    defaultGateway = "10.0.40.30";
+    #defaultGateway = "10.0.40.30";
     nameservers = ["8.8.8.8"];
     interfaces.eno1.useDHCP = false;
     interfaces.eno1.ipv4.addresses = [ {
@@ -20,7 +21,70 @@
 
     firewall = {
       enable = true;
-      allowedTCPPorts = [ 22 80 443 ];
+      allowedTCPPorts = [ 22 ];
+
+      # FIXME: For slurmd as it requests the compute nodes to connect to us
+      allowedTCPPortRanges = [ { from=1024; to=65535; } ];
     };
+
+    extraHosts = ''
+      10.0.40.30      ssfhead
+      84.88.53.236    ssfhead.bsc.es ssfhead
+      
+      # Node Entry for node: mds01 (ID=72)
+      10.0.40.40              mds01 mds01-eth0
+      10.0.42.40              mds01-ib0
+      10.0.40.141             mds01-ipmi0
+      
+      # Node Entry for node: oss01 (ID=73)
+      10.0.40.41              oss01 oss01-eth0
+      10.0.42.41              oss01-ib0
+      10.0.40.142             oss01-ipmi0
+      
+      # Node Entry for node: oss02 (ID=74)
+      10.0.40.42              oss02 oss02-eth0
+      10.0.42.42              oss02-ib0
+      10.0.40.143             oss02-ipmi0
+      
+      # Node Entry for node: xeon01 (ID=15)
+      10.0.40.1               xeon01 xeon01-eth0
+      10.0.42.1               xeon01-ib0
+      10.0.40.101             xeon01-ipmi0
+      
+      # Node Entry for node: xeon02 (ID=16)
+      10.0.40.2               xeon02 xeon02-eth0
+      10.0.42.2               xeon02-ib0
+      10.0.40.102             xeon02-ipmi0
+      
+      # Node Entry for node: xeon03 (ID=17)
+      10.0.40.3               xeon03 xeon03-eth0
+      10.0.42.3               xeon03-ib0
+      10.0.40.103             xeon03-ipmi0
+      
+      # Node Entry for node: xeon04 (ID=18)
+      10.0.40.4               xeon04 xeon04-eth0
+      10.0.42.4               xeon04-ib0
+      10.0.40.104             xeon04-ipmi0
+      
+      # Node Entry for node: xeon05 (ID=19)
+      10.0.40.5               xeon05 xeon05-eth0
+      10.0.42.5               xeon05-ib0
+      10.0.40.105             xeon05-ipmi0
+      
+      # Node Entry for node: xeon06 (ID=20)
+      10.0.40.6               xeon06 xeon06-eth0
+      10.0.42.6               xeon06-ib0
+      10.0.40.106             xeon06-ipmi0
+      
+      # Node Entry for node: xeon07 (ID=21)
+      10.0.40.7               xeon07 xeon07-eth0
+      10.0.42.7               xeon07-ib0
+      10.0.40.107             xeon07-ipmi0
+      
+      # Node Entry for node: xeon08 (ID=22)
+      10.0.40.8               xeon08 xeon08-eth0
+      10.0.42.8               xeon08-ib0
+      10.0.40.108             xeon08-ipmi0
+    '';
   };
 }

overlays-compat/overlays.nix

@@ -0,0 +1,8 @@
+self: super:
+with super.lib;
+let
+  # Load the system config and get the `nixpkgs.overlays` option
+  overlays = (import <nixpkgs/nixos> { }).config.nixpkgs.overlays;
+in
+  # Apply all overlays to the input of the current "main" overlay
+  foldl' (flip extends) (_: super) overlays self

overlays.nix

@@ -0,0 +1,25 @@
+{ options, ... }:
+
+let
+
+  bscpkgsSrc = builtins.fetchTarball "https://pm.bsc.es/gitlab/rarias/bscpkgs/-/archive/slurm-xeon07/bscpkgs-master.tar.gz";
+  bscpkgs = import "${bscpkgsSrc}/overlay.nix";
+
+  xeon07Overlay = (self: super: {
+    slurm = super.bsc.slurm-16-05-8-1;
+  });
+
+in
+
+{
+  nix.nixPath =
+    # Prepend default nixPath values.
+    options.nix.nixPath.default ++
+    # Append our nixpkgs-overlays.
+    [ "nixpkgs-overlays=/etc/nixos/overlays-compat/" ]
+  ;
+
+  nixpkgs.overlays = [
+    bscpkgs xeon07Overlay
+  ];
+}

slurm.nix

@@ -0,0 +1,12 @@
+{ ... }:
+
+{
+  services.slurm = {
+    client.enable = true;
+    controlMachine = "ssfhead";
+    clusterName = "owl";
+    nodeName = [
+      "xeon[01-08] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
+    ];
+  };
+}

users.nix

@@ -8,8 +8,13 @@
       isNormalUser = true;
       home = "/home/Computational/rarias";
       description = "Rodrigo Arias";
+      group = "Computational";
       extraGroups = [ "wheel" ];
       hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/";
     };
+
+    groups = {
+      Computational = { gid = 564; };
+    };
   };
 }