From 72faf8365b16cbbd9df6caf282fd15f949490492 Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Mon, 3 Jun 2024 09:20:11 +0200 Subject: [PATCH] Split xeon specific configuration from base To accomodate the raccoon knights workstation, some of the configuration pulled by m/common/main.nix has to be removed. To solve it, the xeon specific parts are placed into m/common/xeon.nix and only the common configuration is at m/common/base.nix. Reviewed-by: Aleix Roca Nonell --- doc/install.md | 10 +++ flake.nix | 15 ++--- m/bay/configuration.nix | 4 +- m/common/base.nix | 19 ++++++ m/common/{ => base}/agenix.nix | 0 m/common/{ => base}/boot.nix | 2 +- m/common/base/env.nix | 35 +++++++++++ m/common/{ => base}/fs.nix | 7 --- m/common/{ => base}/hw.nix | 0 m/common/base/net.nix | 19 ++++++ m/common/base/nix.nix | 42 +++++++++++++ m/common/{ => base}/ntp.nix | 0 m/common/{ => base}/rev.nix | 3 + m/common/{ => base}/ssh.nix | 2 +- m/common/{ => base}/users.nix | 2 +- m/common/{ => base}/watchdog.nix | 0 m/common/{ => base}/zsh.nix | 0 m/common/main.nix | 96 ----------------------------- m/common/xeon.nix | 9 +++ m/common/xeon/fs.nix | 8 +++ m/common/xeon/getty.nix | 8 +++ m/common/{ => xeon}/net.nix | 8 +-- m/eudy/configuration.nix | 2 +- m/hut/configuration.nix | 2 +- m/koro/configuration.nix | 2 +- m/lake2/configuration.nix | 4 +- m/{common => module}/monitoring.nix | 0 m/owl1/configuration.nix | 2 +- m/owl2/configuration.nix | 2 +- m/raccoon/configuration.nix | 32 ++++++++++ 30 files changed, 207 insertions(+), 128 deletions(-) create mode 100644 m/common/base.nix rename m/common/{ => base}/agenix.nix (100%) rename m/common/{ => base}/boot.nix (95%) create mode 100644 m/common/base/env.nix rename m/common/{ => base}/fs.nix (68%) rename m/common/{ => base}/hw.nix (100%) create mode 100644 m/common/base/net.nix create mode 100644 m/common/base/nix.nix rename m/common/{ => base}/ntp.nix (100%) rename m/common/{ => base}/rev.nix (87%) rename m/common/{ => base}/ssh.nix (95%) rename m/common/{ => base}/users.nix (98%) rename m/common/{ => base}/watchdog.nix (100%) rename m/common/{ => base}/zsh.nix (100%) delete mode 100644 m/common/main.nix create mode 100644 m/common/xeon.nix create mode 100644 m/common/xeon/fs.nix create mode 100644 m/common/xeon/getty.nix rename m/common/{ => xeon}/net.nix (94%) rename m/{common => module}/monitoring.nix (100%) create mode 100644 m/raccoon/configuration.nix diff --git a/doc/install.md b/doc/install.md index d5e279d..66a66c9 100644 --- a/doc/install.md +++ b/doc/install.md @@ -150,3 +150,13 @@ And update grub. ``` # nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v ``` + +## Chain NixOS in same disk + +``` +menuentry 'NixOS' { + insmod chain + set root=(hd3,1) + configfile /boot/grub/grub.cfg +} +``` diff --git a/flake.nix b/flake.nix index 807721a..9ff6538 100644 --- a/flake.nix +++ b/flake.nix @@ -17,13 +17,14 @@ let in { nixosConfigurations = { - hut = mkConf "hut"; - owl1 = mkConf "owl1"; - owl2 = mkConf "owl2"; - eudy = mkConf "eudy"; - koro = mkConf "koro"; - bay = mkConf "bay"; - lake2 = mkConf "lake2"; + hut = mkConf "hut"; + owl1 = mkConf "owl1"; + owl2 = mkConf "owl2"; + eudy = mkConf "eudy"; + koro = mkConf "koro"; + bay = mkConf "bay"; + lake2 = mkConf "lake2"; + raccoon = mkConf "raccoon"; }; packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // { diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix index ac34f69..6aabe6b 100644 --- a/m/bay/configuration.nix +++ b/m/bay/configuration.nix @@ -2,8 +2,8 @@ { imports = [ - ../common/main.nix - ../common/monitoring.nix + ../common/xeon.nix + ../module/monitoring.nix ]; # Select the this using the ID to avoid mismatches diff --git a/m/common/base.nix b/m/common/base.nix new file mode 100644 index 0000000..f6b74ea --- /dev/null +++ b/m/common/base.nix @@ -0,0 +1,19 @@ +{ + # All machines should include this profile. + # Includes the basic configuration for an Intel server. + imports = [ + ./base/agenix.nix + ./base/boot.nix + ./base/env.nix + ./base/fs.nix + ./base/hw.nix + ./base/net.nix + ./base/nix.nix + ./base/ntp.nix + ./base/rev.nix + ./base/ssh.nix + ./base/users.nix + ./base/watchdog.nix + ./base/zsh.nix + ]; +} diff --git a/m/common/agenix.nix b/m/common/base/agenix.nix similarity index 100% rename from m/common/agenix.nix rename to m/common/base/agenix.nix diff --git a/m/common/boot.nix b/m/common/base/boot.nix similarity index 95% rename from m/common/boot.nix rename to m/common/base/boot.nix index 8b71901..a3408ab 100644 --- a/m/common/boot.nix +++ b/m/common/base/boot.nix @@ -2,7 +2,7 @@ { # Use the GRUB 2 boot loader. - boot.loader.grub.enable = lib.mkForce true; + boot.loader.grub.enable = true; # Enable GRUB2 serial console boot.loader.grub.extraConfig = '' diff --git a/m/common/base/env.nix b/m/common/base/env.nix new file mode 100644 index 0000000..98943ed --- /dev/null +++ b/m/common/base/env.nix @@ -0,0 +1,35 @@ +{ pkgs, config, ... }: + +{ + environment.systemPackages = with pkgs; [ + vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option + nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree + ncdu config.boot.kernelPackages.perf ldns + # From bsckgs overlay + osumb + ]; + + programs.direnv.enable = true; + + # Increase limits + security.pam.loginLimits = [ + { + domain = "*"; + type = "-"; + item = "memlock"; + value = "1048576"; # 1 GiB of mem locked + } + ]; + + environment.variables = { + EDITOR = "vim"; + VISUAL = "vim"; + }; + + programs.bash.promptInit = '' + PS1="\h\\$ " + ''; + + time.timeZone = "Europe/Madrid"; + i18n.defaultLocale = "en_DK.UTF-8"; +} diff --git a/m/common/fs.nix b/m/common/base/fs.nix similarity index 68% rename from m/common/fs.nix rename to m/common/base/fs.nix index c6fea28..0c785b9 100644 --- a/m/common/fs.nix +++ b/m/common/base/fs.nix @@ -13,13 +13,6 @@ [ { device = "/dev/disk/by-label/swap"; } ]; - # Mount the home via NFS - fileSystems."/home" = { - device = "10.0.40.30:/home"; - fsType = "nfs"; - options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ]; - }; - # Tracing fileSystems."/sys/kernel/tracing" = { device = "none"; diff --git a/m/common/hw.nix b/m/common/base/hw.nix similarity index 100% rename from m/common/hw.nix rename to m/common/base/hw.nix diff --git a/m/common/base/net.nix b/m/common/base/net.nix new file mode 100644 index 0000000..e49d204 --- /dev/null +++ b/m/common/base/net.nix @@ -0,0 +1,19 @@ +{ pkgs, ... }: + +{ + networking = { + enableIPv6 = false; + useDHCP = false; + + firewall = { + enable = true; + allowedTCPPorts = [ 22 ]; + }; + + hosts = { + "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ]; + "84.88.51.152" = [ "raccoon" ]; + "84.88.51.142" = [ "raccoon-ipmi" ]; + }; + }; +} diff --git a/m/common/base/nix.nix b/m/common/base/nix.nix new file mode 100644 index 0000000..aef1397 --- /dev/null +++ b/m/common/base/nix.nix @@ -0,0 +1,42 @@ +{ pkgs, nixpkgs, bscpkgs, theFlake, ... }: + +{ + nixpkgs.overlays = [ + bscpkgs.bscOverlay + (import ../../../pkgs/overlay.nix) + ]; + + nix = { + nixPath = [ + "nixpkgs=${nixpkgs}" + "jungle=${theFlake.outPath}" + ]; + + registry = { + nixpkgs.flake = nixpkgs; + jungle.flake = theFlake; + }; + + settings = { + experimental-features = [ "nix-command" "flakes" ]; + sandbox = "relaxed"; + trusted-users = [ "@wheel" ]; + flake-registry = pkgs.writeText "global-registry.json" + ''{"flakes":[],"version":2}''; + }; + + gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 30d"; + }; + }; + + # This value determines the NixOS release from which the default + # settings for stateful data, like file locations and database versions + # on your system were taken. It‘s perfectly fine and recommended to leave + # this value at the release version of the first install of this system. + # Before changing this value read the documentation for this option + # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). + system.stateVersion = "22.11"; # Did you read the comment? +} diff --git a/m/common/ntp.nix b/m/common/base/ntp.nix similarity index 100% rename from m/common/ntp.nix rename to m/common/base/ntp.nix diff --git a/m/common/rev.nix b/m/common/base/rev.nix similarity index 87% rename from m/common/rev.nix rename to m/common/base/rev.nix index 80d019b..f2be747 100644 --- a/m/common/rev.nix +++ b/m/common/base/rev.nix @@ -1,6 +1,7 @@ { theFlake, ... }: let + # Prevent building a configuration without revision rev = if theFlake ? rev then theFlake.rev else throw ("Refusing to build from a dirty Git tree!"); in { @@ -15,4 +16,6 @@ in { DATENOW=$(date --iso-8601=seconds) echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log ''; + + system.configurationRevision = rev; } diff --git a/m/common/ssh.nix b/m/common/base/ssh.nix similarity index 95% rename from m/common/ssh.nix rename to m/common/base/ssh.nix index b8cb5c1..13f2d4d 100644 --- a/m/common/ssh.nix +++ b/m/common/base/ssh.nix @@ -1,7 +1,7 @@ { lib, ... }: let - keys = import ../../keys.nix; + keys = import ../../../keys.nix; hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts; in { diff --git a/m/common/users.nix b/m/common/base/users.nix similarity index 98% rename from m/common/users.nix rename to m/common/base/users.nix index 02680fc..71b9749 100644 --- a/m/common/users.nix +++ b/m/common/base/users.nix @@ -2,7 +2,7 @@ { imports = [ - ../module/jungle-users.nix + ../../module/jungle-users.nix ]; users = { diff --git a/m/common/watchdog.nix b/m/common/base/watchdog.nix similarity index 100% rename from m/common/watchdog.nix rename to m/common/base/watchdog.nix diff --git a/m/common/zsh.nix b/m/common/base/zsh.nix similarity index 100% rename from m/common/zsh.nix rename to m/common/base/zsh.nix diff --git a/m/common/main.nix b/m/common/main.nix deleted file mode 100644 index 4a4671d..0000000 --- a/m/common/main.nix +++ /dev/null @@ -1,96 +0,0 @@ -{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }: - -{ - imports = [ - ./agenix.nix - ./boot.nix - ./fs.nix - ./hw.nix - ./net.nix - ./ntp.nix - ./ssh.nix - ./users.nix - ./watchdog.nix - ./rev.nix - ./zsh.nix - ]; - - nixpkgs.overlays = [ - bscpkgs.bscOverlay - (import ../../pkgs/overlay.nix) - ]; - - system.configurationRevision = - if theFlake ? rev - then theFlake.rev - else throw ("Refusing to build from a dirty Git tree!"); - - nix.nixPath = [ - "nixpkgs=${nixpkgs}" - "jungle=${theFlake.outPath}" - ]; - - nix.settings.flake-registry = - pkgs.writeText "global-registry.json" ''{"flakes":[],"version":2}''; - - nix.registry.nixpkgs.flake = nixpkgs; - nix.registry.jungle.flake = theFlake; - - environment.systemPackages = with pkgs; [ - vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option - nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree - ncdu config.boot.kernelPackages.perf ldns - # From bsckgs overlay - osumb - ]; - - programs.direnv.enable = true; - - systemd.services."serial-getty@ttyS0" = { - enable = true; - wantedBy = [ "getty.target" ]; - serviceConfig.Restart = "always"; - }; - - # Increase limits - security.pam.loginLimits = [ - { - domain = "*"; - type = "-"; - item = "memlock"; - value = "1048576"; # 1 GiB of mem locked - } - ]; - - time.timeZone = "Europe/Madrid"; - i18n.defaultLocale = "en_DK.UTF-8"; - - environment.variables = { - EDITOR = "vim"; - VISUAL = "vim"; - }; - - nix.settings.experimental-features = [ "nix-command" "flakes" ]; - nix.settings.sandbox = "relaxed"; - nix.settings.trusted-users = [ "@wheel" ]; - nix.gc.automatic = true; - nix.gc.dates = "weekly"; - nix.gc.options = "--delete-older-than 30d"; - - programs.bash.promptInit = '' - PS1="\h\\$ " - ''; - - # Copy the NixOS configuration file and link it from the resulting system - # (/run/current-system/configuration.nix). This is useful in case you - # accidentally delete configuration.nix. - #system.copySystemConfiguration = true; - - # This value determines the NixOS release from which the default - # settings for stateful data, like file locations and database versions - # on your system were taken. It‘s perfectly fine and recommended to leave - # this value at the release version of the first install of this system. - # Before changing this value read the documentation for this option - # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html). - system.stateVersion = "22.11"; # Did you read the comment? -} diff --git a/m/common/xeon.nix b/m/common/xeon.nix new file mode 100644 index 0000000..25d4121 --- /dev/null +++ b/m/common/xeon.nix @@ -0,0 +1,9 @@ +{ + # Provides the base system for a xeon node. + imports = [ + ./base.nix + ./xeon/fs.nix + ./xeon/getty.nix + ./xeon/net.nix + ]; +} diff --git a/m/common/xeon/fs.nix b/m/common/xeon/fs.nix new file mode 100644 index 0000000..c50b3ff --- /dev/null +++ b/m/common/xeon/fs.nix @@ -0,0 +1,8 @@ +{ + # Mount the home via NFS + fileSystems."/home" = { + device = "10.0.40.30:/home"; + fsType = "nfs"; + options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ]; + }; +} diff --git a/m/common/xeon/getty.nix b/m/common/xeon/getty.nix new file mode 100644 index 0000000..40c5701 --- /dev/null +++ b/m/common/xeon/getty.nix @@ -0,0 +1,8 @@ +{ + # Restart the serial console + systemd.services."serial-getty@ttyS0" = { + enable = true; + wantedBy = [ "getty.target" ]; + serviceConfig.Restart = "always"; + }; +} diff --git a/m/common/net.nix b/m/common/xeon/net.nix similarity index 94% rename from m/common/net.nix rename to m/common/xeon/net.nix index 1c9c569..dbfb5ea 100644 --- a/m/common/net.nix +++ b/m/common/xeon/net.nix @@ -6,10 +6,9 @@ boot.kernelModules = [ "ib_umad" "ib_ipoib" ]; networking = { - enableIPv6 = false; - useDHCP = false; defaultGateway = "10.0.40.30"; nameservers = ["8.8.8.8"]; + proxy = { default = "http://localhost:23080/"; noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40"; @@ -19,8 +18,6 @@ }; firewall = { - enable = true; - allowedTCPPorts = [ 22 ]; extraCommands = '' # Prevent ssfhead from contacting our slurmd daemon iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse @@ -32,8 +29,7 @@ }; extraHosts = '' - 10.0.40.30 ssfhead - 84.88.53.236 ssfhead.bsc.es ssfhead + 10.0.40.30 ssfhead # Node Entry for node: mds01 (ID=72) 10.0.40.40 bay mds01 mds01-eth0 diff --git a/m/eudy/configuration.nix b/m/eudy/configuration.nix index b0889d8..29d495a 100644 --- a/m/eudy/configuration.nix +++ b/m/eudy/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/main.nix + ../common/xeon.nix #(modulesPath + "/installer/netboot/netboot-minimal.nix") ./kernel/kernel.nix diff --git a/m/hut/configuration.nix b/m/hut/configuration.nix index aa438c0..fe0f5d9 100644 --- a/m/hut/configuration.nix +++ b/m/hut/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/main.nix + ../common/xeon.nix ../module/ceph.nix ../module/debuginfod.nix diff --git a/m/koro/configuration.nix b/m/koro/configuration.nix index 9c92ef3..864efe5 100644 --- a/m/koro/configuration.nix +++ b/m/koro/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/main.nix + ../common/xeon.nix #(modulesPath + "/installer/netboot/netboot-minimal.nix") ../eudy/cpufreq.nix diff --git a/m/lake2/configuration.nix b/m/lake2/configuration.nix index 8e19a12..04627a8 100644 --- a/m/lake2/configuration.nix +++ b/m/lake2/configuration.nix @@ -2,8 +2,8 @@ { imports = [ - ../common/main.nix - ../common/monitoring.nix + ../common/xeon.nix + ../module/monitoring.nix ]; boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a"; diff --git a/m/common/monitoring.nix b/m/module/monitoring.nix similarity index 100% rename from m/common/monitoring.nix rename to m/module/monitoring.nix diff --git a/m/owl1/configuration.nix b/m/owl1/configuration.nix index b208139..1b9c4f3 100644 --- a/m/owl1/configuration.nix +++ b/m/owl1/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/main.nix + ../common/xeon.nix ../module/ceph.nix ../module/slurm-client.nix ../module/slurm-firewall.nix diff --git a/m/owl2/configuration.nix b/m/owl2/configuration.nix index fac678b..1b7d4be 100644 --- a/m/owl2/configuration.nix +++ b/m/owl2/configuration.nix @@ -2,7 +2,7 @@ { imports = [ - ../common/main.nix + ../common/xeon.nix ../module/ceph.nix ../module/slurm-client.nix ../module/slurm-firewall.nix diff --git a/m/raccoon/configuration.nix b/m/raccoon/configuration.nix new file mode 100644 index 0000000..556e9ab --- /dev/null +++ b/m/raccoon/configuration.nix @@ -0,0 +1,32 @@ +{ config, pkgs, lib, modulesPath, ... }: + +{ + imports = [ + ../common/base.nix + ]; + + # Don't install Grub on the disk yet + boot.loader.grub.device = "nodev"; + + networking = { + hostName = "raccoon"; + # Only BSC DNSs seem to be reachable from the office VLAN + nameservers = [ "84.88.52.35" "84.88.52.36" ]; + defaultGateway = "84.88.51.129"; + interfaces.eno0.ipv4.addresses = [ { + address = "84.88.51.152"; + prefixLength = 25; + } ]; + }; + + # Configure Nvidia driver to use with CUDA + hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production; + hardware.opengl = { + enable = true; + driSupport = true; + setLdLibraryPath = true; + }; + nixpkgs.config.allowUnfree = true; + nixpkgs.config.nvidia.acceptLicense = true; + services.xserver.videoDrivers = [ "nvidia" ]; +}