From df371c950fc164dc2a75d2a43cb5390d9fd6bfbe Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Date: Mon, 3 Apr 2023 12:51:44 +0200 Subject: [PATCH] Setup slurm and gitlab-runner --- boot.nix | 18 +++++++++- configuration.nix | 10 ++++-- fs.nix | 8 ++++- gitlab-runner.nix | 40 ++++++++++++++++----- net.nix | 68 ++++++++++++++++++++++++++++++++++-- overlays-compat/overlays.nix | 8 +++++ overlays.nix | 25 +++++++++++++ slurm.nix | 12 +++++++ users.nix | 5 +++ 9 files changed, 180 insertions(+), 14 deletions(-) create mode 100644 overlays-compat/overlays.nix create mode 100644 overlays.nix create mode 100644 slurm.nix diff --git a/boot.nix b/boot.nix index 033e9b3..56a2a31 100644 --- a/boot.nix +++ b/boot.nix @@ -1,4 +1,4 @@ -{ ... }: +{ lib, ... }: { # Use the GRUB 2 boot loader. @@ -8,9 +8,25 @@ # Select the this using the ID to avoid mismatches boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN"; + # Enable GRUB2 serial console + boot.loader.grub.extraConfig = '' + serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1 + terminal_input --append serial + terminal_output --append serial + ''; + # Enable serial console boot.kernelParams = [ "console=tty1" "console=ttyS0,115200" ]; + + boot.kernelPatches = lib.singleton { + name = "osnoise-tracer"; + patch = null; + extraStructuredConfig = with lib.kernel; { + OSNOISE_TRACER = yes; + HWLAT_TRACER = yes; + }; + }; } diff --git a/configuration.nix b/configuration.nix index 4a20d1d..4e1e341 100644 --- a/configuration.nix +++ b/configuration.nix @@ -8,8 +8,10 @@ ./fs.nix ./gitlab-runner.nix ./net.nix + ./slurm.nix ./ssh.nix ./users.nix + ./overlays.nix ]; systemd.services."serial-getty@ttyS0" = { @@ -19,13 +21,17 @@ }; time.timeZone = "Europe/Madrid"; - i18n.defaultLocale = "en_US.UTF-8"; + i18n.defaultLocale = "en_DK.UTF-8"; environment.systemPackages = with pkgs; [ - vim wget git htop + vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option ]; nix.settings.experimental-features = [ "nix-command" "flakes" ]; + nix.gc.automatic = true; + + programs.zsh.enable = true; + programs.zsh.histSize = 100000; # Copy the NixOS configuration file and link it from the resulting system # (/run/current-system/configuration.nix). This is useful in case you diff --git a/fs.nix b/fs.nix index 7a459dd..7d8e36d 100644 --- a/fs.nix +++ b/fs.nix @@ -5,6 +5,12 @@ fileSystems."/home" = { device = "10.0.40.30:/home"; fsType = "nfs"; - options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" ]; + options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ]; + }; + + # Tracing + fileSystems."/sys/kernel/tracing" = { + device = "none"; + fsType = "tracefs"; }; } diff --git a/gitlab-runner.nix b/gitlab-runner.nix index 67c400d..ae1f1a3 100644 --- a/gitlab-runner.nix +++ b/gitlab-runner.nix @@ -18,14 +18,21 @@ SHELL = "${pkgs.bash}/bin/bash"; }; }; - # # runner for everything else - # default = { - # # File should contain at least these two variables: - # # `CI_SERVER_URL` - # # `REGISTRATION_TOKEN` - # registrationConfigFile = "/run/secrets/gitlab-runner-registration"; - # dockerImage = "debian:stable"; - # }; + + # runner for everything else + default = { + # File should contain at least these two variables: + # `CI_SERVER_URL` + # `REGISTRATION_TOKEN` + registrationConfigFile = "/run/secrets/gitlab-runner-registration"; + dockerImage = "debian:stable"; + tagList = [ "docker" "xeon" ]; + registrationFlags = [ "--docker-network-mode host" ]; + environmentVariables = { + https_proxy = "http://localhost:23080"; + http_proxy = "http://localhost:23080"; + }; + }; }; }; @@ -33,6 +40,22 @@ systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false; systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner"; systemd.services.gitlab-runner.serviceConfig.Group = "gitlab-runner"; + #systemd.services.gitlab-runner.serviceConfig.ExecStart = lib.mkForce + # ''${pkgs.gitlab-runner}/bin/gitlab-runner --debug run --config ''${HOME}/.gitlab-runner/config.toml --working-directory ''${HOME}''; + + # TODO https://docs.gitlab.com/runner/configuration/proxy.html + #systemd.services.docker.environment = { + # HTTP_PROXY="http://localhost:23080/"; + # HTTPS_PROXY="http://localhost:23080/"; + #}; + + #virtualisation.docker.daemon.settings = { + # proxies.default = { + # httpProxy = "http://localhost:23080/"; + # httpsProxy = "http://localhost:23080/"; + # noProxy = "localhost,127.0.0.0/8"; + # }; + #}; users.users.gitlab-runner = { uid = config.ids.uids.gitlab-runner; @@ -40,6 +63,7 @@ home = "/var/lib/gitlab-runner"; description = "Gitlab Runner"; group = "gitlab-runner"; + extraGroups = [ "docker" ]; createHome = true; }; users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner; diff --git a/net.nix b/net.nix index a9ae546..23547b8 100644 --- a/net.nix +++ b/net.nix @@ -4,8 +4,9 @@ networking = { hostName = "xeon07"; + enableIPv6 = false; useDHCP = false; - defaultGateway = "10.0.40.30"; + #defaultGateway = "10.0.40.30"; nameservers = ["8.8.8.8"]; interfaces.eno1.useDHCP = false; interfaces.eno1.ipv4.addresses = [ { @@ -20,7 +21,70 @@ firewall = { enable = true; - allowedTCPPorts = [ 22 80 443 ]; + allowedTCPPorts = [ 22 ]; + + # FIXME: For slurmd as it requests the compute nodes to connect to us + allowedTCPPortRanges = [ { from=1024; to=65535; } ]; }; + + extraHosts = '' + 10.0.40.30 ssfhead + 84.88.53.236 ssfhead.bsc.es ssfhead + + # Node Entry for node: mds01 (ID=72) + 10.0.40.40 mds01 mds01-eth0 + 10.0.42.40 mds01-ib0 + 10.0.40.141 mds01-ipmi0 + + # Node Entry for node: oss01 (ID=73) + 10.0.40.41 oss01 oss01-eth0 + 10.0.42.41 oss01-ib0 + 10.0.40.142 oss01-ipmi0 + + # Node Entry for node: oss02 (ID=74) + 10.0.40.42 oss02 oss02-eth0 + 10.0.42.42 oss02-ib0 + 10.0.40.143 oss02-ipmi0 + + # Node Entry for node: xeon01 (ID=15) + 10.0.40.1 xeon01 xeon01-eth0 + 10.0.42.1 xeon01-ib0 + 10.0.40.101 xeon01-ipmi0 + + # Node Entry for node: xeon02 (ID=16) + 10.0.40.2 xeon02 xeon02-eth0 + 10.0.42.2 xeon02-ib0 + 10.0.40.102 xeon02-ipmi0 + + # Node Entry for node: xeon03 (ID=17) + 10.0.40.3 xeon03 xeon03-eth0 + 10.0.42.3 xeon03-ib0 + 10.0.40.103 xeon03-ipmi0 + + # Node Entry for node: xeon04 (ID=18) + 10.0.40.4 xeon04 xeon04-eth0 + 10.0.42.4 xeon04-ib0 + 10.0.40.104 xeon04-ipmi0 + + # Node Entry for node: xeon05 (ID=19) + 10.0.40.5 xeon05 xeon05-eth0 + 10.0.42.5 xeon05-ib0 + 10.0.40.105 xeon05-ipmi0 + + # Node Entry for node: xeon06 (ID=20) + 10.0.40.6 xeon06 xeon06-eth0 + 10.0.42.6 xeon06-ib0 + 10.0.40.106 xeon06-ipmi0 + + # Node Entry for node: xeon07 (ID=21) + 10.0.40.7 xeon07 xeon07-eth0 + 10.0.42.7 xeon07-ib0 + 10.0.40.107 xeon07-ipmi0 + + # Node Entry for node: xeon08 (ID=22) + 10.0.40.8 xeon08 xeon08-eth0 + 10.0.42.8 xeon08-ib0 + 10.0.40.108 xeon08-ipmi0 + ''; }; } diff --git a/overlays-compat/overlays.nix b/overlays-compat/overlays.nix new file mode 100644 index 0000000..8a606c5 --- /dev/null +++ b/overlays-compat/overlays.nix @@ -0,0 +1,8 @@ +self: super: +with super.lib; +let + # Load the system config and get the `nixpkgs.overlays` option + overlays = (import { }).config.nixpkgs.overlays; +in + # Apply all overlays to the input of the current "main" overlay + foldl' (flip extends) (_: super) overlays self diff --git a/overlays.nix b/overlays.nix new file mode 100644 index 0000000..38fe262 --- /dev/null +++ b/overlays.nix @@ -0,0 +1,25 @@ +{ options, ... }: + +let + + bscpkgsSrc = builtins.fetchTarball "https://pm.bsc.es/gitlab/rarias/bscpkgs/-/archive/slurm-xeon07/bscpkgs-master.tar.gz"; + bscpkgs = import "${bscpkgsSrc}/overlay.nix"; + + xeon07Overlay = (self: super: { + slurm = super.bsc.slurm-16-05-8-1; + }); + +in + +{ + nix.nixPath = + # Prepend default nixPath values. + options.nix.nixPath.default ++ + # Append our nixpkgs-overlays. + [ "nixpkgs-overlays=/etc/nixos/overlays-compat/" ] + ; + + nixpkgs.overlays = [ + bscpkgs xeon07Overlay + ]; +} diff --git a/slurm.nix b/slurm.nix new file mode 100644 index 0000000..379681d --- /dev/null +++ b/slurm.nix @@ -0,0 +1,12 @@ +{ ... }: + +{ + services.slurm = { + client.enable = true; + controlMachine = "ssfhead"; + clusterName = "owl"; + nodeName = [ + "xeon[01-08] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon" + ]; + }; +} diff --git a/users.nix b/users.nix index 617ff5e..03a5eb1 100644 --- a/users.nix +++ b/users.nix @@ -8,8 +8,13 @@ isNormalUser = true; home = "/home/Computational/rarias"; description = "Rodrigo Arias"; + group = "Computational"; extraGroups = [ "wheel" ]; hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/"; }; + + groups = { + Computational = { gid = 564; }; + }; }; }