weasel: use tent cache

/home is a nfs mount, which does not support extra filesystem arguments
needed to run podman. We need to have a local home.

keys.nix

@@ -2,21 +2,22 @@
 # here all the public keys
 rec {
   hosts = {
-    hut    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
-    owl1   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
-    owl2   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
-    eudy   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
-    koro   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
-    bay    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
-    lake2  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
-    fox    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
-    tent   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
-    apex   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex";
-    weasel = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel";
+    hut     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICO7jIp6JRnRWTMDsTB/aiaICJCl4x8qmKMPSs4lCqP1 hut";
+    owl1    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMqMEXO0ApVsBA6yjmb0xP2kWyoPDIWxBB0Q3+QbHVhv owl1";
+    owl2    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHurEYpQzNHqWYF6B9Pd7W8UPgF3BxEg0BvSbsA7BAdK owl2";
+    eudy    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL+WYPRRvZupqLAG0USKmd/juEPmisyyJaP8hAgYwXsG eudy";
+    koro    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIImiTFDbxyUYPumvm8C4mEnHfuvtBY1H8undtd6oDd67 koro";
+    bay     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICvGBzpRQKuQYHdlUQeAk6jmdbkrhmdLwTBqf3el7IgU bay";
+    lake2   = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINo66//S1yatpQHE/BuYD/Gfq64TY7ZN5XOGXmNchiO0 lake2";
+    fox     = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDwItIk5uOJcQEVPoy/CVGRzfmE1ojrdDcI06FrU4NFT fox";
+    tent    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFAtTpHtdYoelbknD/IcfBlThwLKJv/dSmylOgpg3FRM tent";
+    apex    = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBvUFjSfoxXnKwXhEFXx5ckRKJ0oewJ82mRitSMNMKjh apex";
+    weasel  = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFLJrQ8BF6KcweQV8pLkSbFT+tbDxSG9qxrdQE65zJZp weasel";
+    raccoon = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGNQttFvL0dNEyy7klIhLoK4xXOeM2/K9R7lPMTG3qvK raccoon";
   };
 
   hostGroup = with hosts; rec {
-    compute    = [ owl1 owl2 fox ];
+    compute    = [ owl1 owl2 fox raccoon ];
     playground = [ eudy koro weasel ];
     storage    = [ bay lake2 ];
     monitor    = [ hut ];

m/apex/configuration.nix

@@ -5,6 +5,7 @@
     ../common/xeon.nix
     ../common/ssf/hosts.nix
     ../module/ceph.nix
+    ../module/hut-substituter.nix
     ../module/slurm-server.nix
     ./nfs.nix
     ./wireguard.nix
@@ -56,17 +57,6 @@
     };
   };
 
-  # Use SSH tunnel to reach internal hosts
-  programs.ssh.extraConfig = ''
-    Host bscpm04.bsc.es gitlab-internal.bsc.es knights3.bsc.es
-      ProxyCommand nc -X connect -x localhost:23080 %h %p
-    Host raccoon
-      HostName knights3.bsc.es
-      ProxyCommand nc -X connect -x localhost:23080 %h %p
-    Host tent
-      ProxyJump raccoon
-  '';
-
   networking.firewall = {
     extraCommands = ''
       # Blackhole BSC vulnerability scanner (OpenVAS) as it is spamming our
@@ -76,10 +66,4 @@
       iptables -I nixos-fw 2 -p tcp -s 84.88.52.176 -j nixos-fw-refuse
     '';
   };
-
-  # Use tent for cache
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
-  };
 }

m/apex/wireguard.nix

@@ -18,18 +18,25 @@
       # Public key: VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=
       peers = [
         {
-          name = "Fox";
+          name = "fox";
           publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=";
-          allowedIPs = [ "10.106.0.0/24" ];
+          allowedIPs = [ "10.106.0.1/32" ];
           endpoint = "fox.ac.upc.edu:666";
           # Send keepalives every 25 seconds. Important to keep NAT tables alive.
           persistentKeepalive = 25;
         }
+        {
+          name = "raccoon";
+          publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=";
+          allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ];
+        }
       ];
     };
   };
 
   networking.hosts = {
     "10.106.0.1" = [ "fox" ];
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
   };
 }

m/bay/configuration.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ../common/ssf.nix
+    ../module/hut-substituter.nix
     ../module/monitoring.nix
   ];
 

m/common/base.nix

@@ -11,11 +11,13 @@
     ./base/hw.nix
     ./base/net.nix
     ./base/nix.nix
+    ./base/nosv.nix
     ./base/ntp.nix
     ./base/rev.nix
     ./base/ssh.nix
     ./base/users.nix
     ./base/watchdog.nix
     ./base/zsh.nix
+    ./base/fish.nix
   ];
 }

m/common/base/env.nix

@@ -5,6 +5,8 @@
     vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
     nix-diff ipmitool freeipmi ethtool lm_sensors cmake gnumake file tree
     ncdu config.boot.kernelPackages.perf ldns pv
+    nix-output-monitor
+    nixfmt-rfc-style
     # From bsckgs overlay
     osumb
   ];

m/common/base/fish.nix

@@ -0,0 +1,4 @@
+{ ... }:
+{
+  programs.fish.enable = true;
+}

m/common/base/net.nix

@@ -15,8 +15,9 @@
 
     hosts = {
       "84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
-      "84.88.51.152" = [ "raccoon" ];
       "84.88.51.142" = [ "raccoon-ipmi" ];
+      "192.168.11.12" = [ "bscpm04.bsc.es" ];
+      "192.168.11.15" = [ "gitlab-internal.bsc.es" ];
     };
   };
 }

m/common/base/nosv.nix

@@ -0,0 +1,9 @@
+{ ... }:
+{
+  nix.settings.system-features = [ "nosv" ];
+  programs.nix-required-mounts.enable = true;
+  programs.nix-required-mounts.allowedPatterns.nosv.paths = [
+    "/sys/devices/system/cpu"
+    "/sys/devices/system/node"
+  ];
+}

m/common/base/users.nix

@@ -87,6 +87,12 @@
         openssh.authorizedKeys.keys = [
           "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
         ];
+        shell = pkgs.fish;
+        packages = with pkgs; [
+          starship
+          jujutsu
+          neovim
+        ];
       };
 
       vlopez = {

m/common/ssf.nix

@@ -4,7 +4,7 @@
     ./xeon.nix
     ./ssf/fs.nix
     ./ssf/hosts.nix
+    ./ssf/hosts-remote.nix
     ./ssf/net.nix
-    ./ssf/ssh.nix
   ];
 }

m/common/ssf/hosts-remote.nix

@@ -0,0 +1,9 @@
+{ pkgs, ... }:
+
+{
+  networking.hosts = {
+    # Remote hosts visible from compute nodes
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
+  };
+}

m/common/ssf/ssh.nix

@@ -1,16 +0,0 @@
-{
-  # Use SSH tunnel to apex to reach internal hosts
-  programs.ssh.extraConfig = ''
-    Host tent
-      ProxyJump raccoon
-
-    # Access raccoon via the HTTP proxy
-    Host raccoon knights3.bsc.es
-      HostName knights3.bsc.es
-      ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p'
-
-    # Make sure we can reach gitlab even if we don't have SSH access to raccoon
-    Host bscpm04.bsc.es gitlab-internal.bsc.es
-      ProxyCommand=ssh apex 'nc -X connect -x localhost:23080 %h %p'
-  '';
-}

m/eudy/configuration.nix

@@ -9,6 +9,7 @@
     ./cpufreq.nix
     ./fs.nix
     ./users.nix
+    ../module/hut-substituter.nix
     ../module/debuginfod.nix
   ];
 

m/fox/configuration.nix

@@ -8,6 +8,7 @@
     ../module/emulation.nix
     ../module/nvidia.nix
     ../module/slurm-client.nix
+    ../module/hut-substituter.nix
     ./wireguard.nix
   ];
 
@@ -45,16 +46,6 @@
 
   services.fail2ban.enable = true;
 
-  # Use SSH tunnel to reach internal hosts
-  programs.ssh.extraConfig = ''
-    Host bscpm04.bsc.es gitlab-internal.bsc.es tent
-      ProxyJump raccoon
-    Host raccoon
-      ProxyJump apex
-      HostName 127.0.0.1
-      Port 22022
-  '';
-
   networking = {
     timeServers = [ "ntp1.upc.edu" "ntp2.upc.edu" ];
     hostName = "fox";
@@ -72,12 +63,6 @@
     interfaces.enp1s0f0np0.useDHCP = true;
   };
 
-  # Use hut for cache
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
-  };
-
   # Recommended for new graphics cards
   hardware.nvidia.open = true;
 

m/fox/wireguard.nix

@@ -24,17 +24,24 @@
       peers = [
         # List of allowed peers.
         { 
-          name = "Apex";
+          name = "apex";
           publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=";
           # List of IPs assigned to this peer within the tunnel subnet. Used to configure routing.
           allowedIPs = [ "10.106.0.30/32" ];
         }
+        {
+          name = "raccoon";
+          publicKey = "QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=";
+          allowedIPs = [ "10.106.0.236/32" "192.168.0.0/16" "10.0.44.0/24" ];
+        }
       ];
     };
   };
 
   networking.hosts = {
     "10.106.0.30" = [ "apex" ];
+    "10.106.0.236" = [ "raccoon" ];
+    "10.0.44.4" = [ "tent" ];
   };
 
   networking.firewall = {

m/module/ssh-hut-extern.nix

@@ -1,8 +0,0 @@
-{
-  programs.ssh.extraConfig = ''
-    Host apex ssfhead
-      HostName ssflogin.bsc.es
-    Host hut
-      ProxyJump apex
-  '';
-}

m/raccoon/configuration.nix

@@ -3,11 +3,13 @@
 {
   imports = [
     ../common/base.nix
+    ../common/ssf/hosts.nix
     ../module/emulation.nix
     ../module/debuginfod.nix
-    ../module/ssh-hut-extern.nix
     ../module/nvidia.nix
     ../eudy/kernel/perf.nix
+    ./wireguard.nix
+    ../module/hut-substituter.nix
   ];
 
   # Don't install Grub on the disk yet
@@ -43,9 +45,11 @@
     };
   };
 
-  nix.settings = {
-    extra-substituters = [ "https://jungle.bsc.es/cache" ];
-    extra-trusted-public-keys = [ "jungle.bsc.es:pEc7MlAT0HEwLQYPtpkPLwRsGf80ZI26aj29zMw/HH0=" ];
+  # Mount the NFS home
+  fileSystems."/nfs/home" = {
+    device = "10.106.0.30:/home";
+    fsType = "nfs";
+    options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
   };
 
   # Enable performance governor

m/raccoon/wireguard.nix

@@ -0,0 +1,48 @@
+{ config, pkgs, ... }:
+
+{
+  networking.nat = {
+    enable = true;
+    enableIPv6 = false;
+    externalInterface = "eno0";
+    internalInterfaces = [ "wg0" ];
+  };
+
+  networking.firewall = {
+    allowedUDPPorts = [ 666 ];
+  };
+
+  age.secrets.wgRaccoon.file = ../../secrets/wg-raccoon.age;
+
+  # Enable WireGuard
+  networking.wireguard.enable = true;
+  networking.wireguard.interfaces = {
+    wg0 = {
+      ips = [ "10.106.0.236/24" ];
+      listenPort = 666;
+      privateKeyFile = config.age.secrets.wgRaccoon.path;
+      # Public key: QUfnGXSMEgu2bviglsaSdCjidB51oEDBFpnSFcKGfDI=
+      peers = [
+        {
+          name = "fox";
+          publicKey = "VfMPBQLQTKeyXJSwv8wBhc6OV0j2qAxUpX3kLHunK2Y=";
+          allowedIPs = [ "10.106.0.1/32" ];
+          endpoint = "fox.ac.upc.edu:666";
+          persistentKeepalive = 25;
+        }
+        {
+          name = "apex";
+          publicKey = "VwhcN8vSOzdJEotQTpmPHBC52x3Hbv1lkFIyKubrnUA=";
+          allowedIPs = [ "10.106.0.30/32" "10.0.40.0/24" ];
+          endpoint = "ssfhead.bsc.es:666";
+          persistentKeepalive = 25;
+        }
+      ];
+    };
+  };
+
+  networking.hosts = {
+    "10.106.0.1"  = [ "fox.wg" ];
+    "10.106.0.30" = [ "apex.wg" ];
+  };
+}

m/tent/configuration.nix

@@ -3,9 +3,9 @@
 {
   imports = [
     ../common/xeon.nix
+    ../common/ssf/hosts.nix
     ../module/emulation.nix
     ../module/debuginfod.nix
-    ../module/ssh-hut-extern.nix
     ./monitoring.nix
     ./nginx.nix
     ./nix-serve.nix
@@ -15,6 +15,7 @@
     ../hut/msmtp.nix
     ../module/p.nix
     ../module/vpn-dac.nix
+    ../module/hut-substituter.nix
   ];
 
   # Select the this using the ID to avoid mismatches
@@ -35,6 +36,7 @@
     defaultGateway = "10.0.44.1";
     hosts = {
       "84.88.53.236" = [ "apex" ];
+      "10.0.44.1" = [ "raccoon" ];
     };
   };
 

m/weasel/configuration.nix

@@ -3,6 +3,8 @@
 {
   imports = [
     ../common/ssf.nix
+    ../module/hut-substituter.nix
+    ./virtualization.nix
   ];
 
   # Select this using the ID to avoid mismatches
@@ -29,4 +31,5 @@
       prefixLength = 24;
     } ];
   };
+
 }

m/weasel/virtualization.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+
+{
+  # Enable common container config files in /etc/containers
+  virtualisation.containers.enable = true;
+  virtualisation = {
+    podman = {
+      enable = true;
+
+      # Required for containers under podman-compose to be able to talk to each other.
+      defaultNetwork.settings.dns_enabled = true;
+    };
+  };
+
+  # We cannot use /home since nfs does not support fileattrs needed by podman
+  systemd.tmpfiles.settings = {
+    "podman-users" = lib.mapAttrs' (
+      name: value:
+      lib.nameValuePair ("/var/lib/podman-users/" + name) {
+        d = {
+          group = value.group;
+          mode = value.homeMode;
+          user = name;
+        };
+      }
+    ) (lib.filterAttrs (_: x: x.isNormalUser) config.users.users);
+  };
+
+  # Useful other development tools
+  environment.systemPackages = with pkgs; [
+    dive # look into docker image layers
+    podman-tui # status of containers in the terminal
+    podman-compose # start group of containers for dev
+  ];
+}

secrets/secrets.nix

@@ -4,6 +4,7 @@ let
   hut = [ keys.hosts.hut ] ++ adminsKeys;
   fox = [ keys.hosts.fox ] ++ adminsKeys;
   apex = [ keys.hosts.apex ] ++ adminsKeys;
+  raccoon = [ keys.hosts.raccoon ] ++ adminsKeys;
   mon = [ keys.hosts.hut keys.hosts.tent ] ++ adminsKeys;
   tent = [ keys.hosts.tent ] ++ adminsKeys;
   # Only expose ceph keys to safe nodes and admins
@@ -29,4 +30,5 @@ in
 
   "wg-fox.age".publicKeys = fox;
   "wg-apex.age".publicKeys = apex;
+  "wg-raccoon.age".publicKeys = raccoon;
 }