7 Commits

Author SHA1 Message Date
44e518873c Move vlopez user to jungleUsers for koro host
Access to other machines can be easily added into the "hosts" attribute
without the need to replicate the configuration.
2024-06-07 10:47:13 +02:00
9f065fd7fb Add raccoon motd file 2024-06-07 10:47:13 +02:00
ccf9b1cf4d Add documentation section about GRUB chain loading 2024-06-07 10:47:13 +02:00
37073f017d Add raccoon workstation with Nvidia GPU
The raccoon workstation has a Nvidia GTX 960 GPU which will be used for
CUDA experiments. The configuration uses the production Nvidia driver
at version 550 which still supports the GPU. The current CUDA 12.2
version is also supported by the driver.

The workstation has Internet access directly from the gateway, but name
resolution via Google DNS servers seems to be blocked, so we use BSC
servers for now.

The NixOS system is installed in a partition alongside the old Debian
system, until we decide that is no longer neccesary to keep both. The
old /home partition is not used as we are using the same UIDs and groups
from the xeon machines, which don't match the ones here.
2024-06-07 10:47:10 +02:00
4077a87021 Split xeon specific configuration from base
To accomodate the raccoon knights workstation, some of the configuration
pulled by m/common/main.nix has to be removed. To solve it, the xeon
specific parts are placed into m/common/xeon.nix and only the common
configuration is at m/common/base.nix.
2024-06-07 10:45:46 +02:00
cc31cb30c3 Control user access to each machine
The users.jungleUsers configuration option behaves like the users.users
option, but defines the list attribute `hosts` for each user, which
filters users so that only the user can only access those hosts.
2024-06-07 10:45:46 +02:00
1dd7550459 Add PostgreSQL DB for performance tests results
The database will hold the performance results of the execution of the
benchmarks. We follow the same setup on knights3 for now.
2024-06-07 10:44:22 +02:00
33 changed files with 315 additions and 145 deletions

View File

@@ -150,3 +150,27 @@ And update grub.
```
# nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
```
## Chain NixOS in same disk with other systems
To install NixOS on a partition along another system which controls the GRUB,
first disable the grub device, so the GRUB is not installed in the disk by
NixOS (only the /boot files will be generated):
```
boot.loader.grub.device = "nodev";
```
Then add the following entry to the old GRUB configuration:
```
menuentry 'NixOS' {
insmod chain
search --no-floppy --label nixos --set root
configfile /boot/grub/grub.cfg
}
```
The partition with NixOS must have the label "nixos" for it to be found. New
system configuration entries will be stored in the GRUB configuration managed
by NixOS, so there is no need to change the old GRUB settings.

View File

@@ -17,13 +17,14 @@ let
in
{
nixosConfigurations = {
hut = mkConf "hut";
owl1 = mkConf "owl1";
owl2 = mkConf "owl2";
eudy = mkConf "eudy";
koro = mkConf "koro";
bay = mkConf "bay";
lake2 = mkConf "lake2";
hut = mkConf "hut";
owl1 = mkConf "owl1";
owl2 = mkConf "owl2";
eudy = mkConf "eudy";
koro = mkConf "koro";
bay = mkConf "bay";
lake2 = mkConf "lake2";
raccoon = mkConf "raccoon";
};
packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {

View File

@@ -2,8 +2,8 @@
{
imports = [
../common/main.nix
../common/monitoring.nix
../common/xeon.nix
../module/monitoring.nix
];
# Select the this using the ID to avoid mismatches

19
m/common/base.nix Normal file
View File

@@ -0,0 +1,19 @@
{
# All machines should include this profile.
# Includes the basic configuration for an Intel server.
imports = [
./base/agenix.nix
./base/boot.nix
./base/env.nix
./base/fs.nix
./base/hw.nix
./base/net.nix
./base/nix.nix
./base/ntp.nix
./base/rev.nix
./base/ssh.nix
./base/users.nix
./base/watchdog.nix
./base/zsh.nix
];
}

View File

@@ -2,7 +2,7 @@
{
# Use the GRUB 2 boot loader.
boot.loader.grub.enable = lib.mkForce true;
boot.loader.grub.enable = true;
# Enable GRUB2 serial console
boot.loader.grub.extraConfig = ''

35
m/common/base/env.nix Normal file
View File

@@ -0,0 +1,35 @@
{ pkgs, config, ... }:
{
environment.systemPackages = with pkgs; [
vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
ncdu config.boot.kernelPackages.perf ldns
# From bsckgs overlay
osumb
];
programs.direnv.enable = true;
# Increase limits
security.pam.loginLimits = [
{
domain = "*";
type = "-";
item = "memlock";
value = "1048576"; # 1 GiB of mem locked
}
];
environment.variables = {
EDITOR = "vim";
VISUAL = "vim";
};
programs.bash.promptInit = ''
PS1="\h\\$ "
'';
time.timeZone = "Europe/Madrid";
i18n.defaultLocale = "en_DK.UTF-8";
}

View File

@@ -13,13 +13,6 @@
[ { device = "/dev/disk/by-label/swap"; }
];
# Mount the home via NFS
fileSystems."/home" = {
device = "10.0.40.30:/home";
fsType = "nfs";
options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
};
# Tracing
fileSystems."/sys/kernel/tracing" = {
device = "none";

19
m/common/base/net.nix Normal file
View File

@@ -0,0 +1,19 @@
{ pkgs, ... }:
{
networking = {
enableIPv6 = false;
useDHCP = false;
firewall = {
enable = true;
allowedTCPPorts = [ 22 ];
};
hosts = {
"84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
"84.88.51.152" = [ "knights3.bsc.es" "raccoon" ];
"84.88.51.142" = [ "raccoon-ipmi" ];
};
};
}

42
m/common/base/nix.nix Normal file
View File

@@ -0,0 +1,42 @@
{ pkgs, nixpkgs, bscpkgs, theFlake, ... }:
{
nixpkgs.overlays = [
bscpkgs.bscOverlay
(import ../../../pkgs/overlay.nix)
];
nix = {
nixPath = [
"nixpkgs=${nixpkgs}"
"jungle=${theFlake.outPath}"
];
registry = {
nixpkgs.flake = nixpkgs;
jungle.flake = theFlake;
};
settings = {
experimental-features = [ "nix-command" "flakes" ];
sandbox = "relaxed";
trusted-users = [ "@wheel" ];
flake-registry = pkgs.writeText "global-registry.json"
''{"flakes":[],"version":2}'';
};
gc = {
automatic = true;
dates = "weekly";
options = "--delete-older-than 30d";
};
};
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions
# on your system were taken. Its perfectly fine and recommended to leave
# this value at the release version of the first install of this system.
# Before changing this value read the documentation for this option
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
system.stateVersion = "22.11"; # Did you read the comment?
}

View File

@@ -1,6 +1,7 @@
{ theFlake, ... }:
let
# Prevent building a configuration without revision
rev = if theFlake ? rev then theFlake.rev
else throw ("Refusing to build from a dirty Git tree!");
in {
@@ -15,4 +16,6 @@ in {
DATENOW=$(date --iso-8601=seconds)
echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log
'';
system.configurationRevision = rev;
}

View File

@@ -1,7 +1,7 @@
{ lib, ... }:
let
keys = import ../../keys.nix;
keys = import ../../../keys.nix;
hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts;
in
{

View File

@@ -1,6 +1,10 @@
{ pkgs, ... }:
{
imports = [
../../module/jungle-users.nix
];
users = {
mutableUsers = false;
users = {
@@ -42,13 +46,16 @@
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
];
};
};
jungleUsers = {
rpenacob = {
uid = 2761;
isNormalUser = true;
home = "/home/Computational/rpenacob";
description = "Raúl Peñacoba";
group = "Computational";
hosts = [ "hut" ];
hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
@@ -61,11 +68,25 @@
home = "/home/Computational/anavarro";
description = "Antoni Navarro";
group = "Computational";
hosts = [ "hut" "raccoon" ];
hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
];
};
vlopez = {
uid = 4334;
isNormalUser = true;
home = "/home/Computational/vlopez";
description = "Victor López";
group = "Computational";
hosts = [ "koro" ];
hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
];
};
};
groups = {

View File

@@ -1,96 +0,0 @@
{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }:
{
imports = [
./agenix.nix
./boot.nix
./fs.nix
./hw.nix
./net.nix
./ntp.nix
./ssh.nix
./users.nix
./watchdog.nix
./rev.nix
./zsh.nix
];
nixpkgs.overlays = [
bscpkgs.bscOverlay
(import ../../pkgs/overlay.nix)
];
system.configurationRevision =
if theFlake ? rev
then theFlake.rev
else throw ("Refusing to build from a dirty Git tree!");
nix.nixPath = [
"nixpkgs=${nixpkgs}"
"jungle=${theFlake.outPath}"
];
nix.settings.flake-registry =
pkgs.writeText "global-registry.json" ''{"flakes":[],"version":2}'';
nix.registry.nixpkgs.flake = nixpkgs;
nix.registry.jungle.flake = theFlake;
environment.systemPackages = with pkgs; [
vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
ncdu config.boot.kernelPackages.perf ldns
# From bsckgs overlay
osumb
];
programs.direnv.enable = true;
systemd.services."serial-getty@ttyS0" = {
enable = true;
wantedBy = [ "getty.target" ];
serviceConfig.Restart = "always";
};
# Increase limits
security.pam.loginLimits = [
{
domain = "*";
type = "-";
item = "memlock";
value = "1048576"; # 1 GiB of mem locked
}
];
time.timeZone = "Europe/Madrid";
i18n.defaultLocale = "en_DK.UTF-8";
environment.variables = {
EDITOR = "vim";
VISUAL = "vim";
};
nix.settings.experimental-features = [ "nix-command" "flakes" ];
nix.settings.sandbox = "relaxed";
nix.settings.trusted-users = [ "@wheel" ];
nix.gc.automatic = true;
nix.gc.dates = "weekly";
nix.gc.options = "--delete-older-than 30d";
programs.bash.promptInit = ''
PS1="\h\\$ "
'';
# Copy the NixOS configuration file and link it from the resulting system
# (/run/current-system/configuration.nix). This is useful in case you
# accidentally delete configuration.nix.
#system.copySystemConfiguration = true;
# This value determines the NixOS release from which the default
# settings for stateful data, like file locations and database versions
# on your system were taken. Its perfectly fine and recommended to leave
# this value at the release version of the first install of this system.
# Before changing this value read the documentation for this option
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
system.stateVersion = "22.11"; # Did you read the comment?
}

9
m/common/xeon.nix Normal file
View File

@@ -0,0 +1,9 @@
{
# Provides the base system for a xeon node.
imports = [
./base.nix
./xeon/fs.nix
./xeon/getty.nix
./xeon/net.nix
];
}

8
m/common/xeon/fs.nix Normal file
View File

@@ -0,0 +1,8 @@
{
# Mount the home via NFS
fileSystems."/home" = {
device = "10.0.40.30:/home";
fsType = "nfs";
options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
};
}

8
m/common/xeon/getty.nix Normal file
View File

@@ -0,0 +1,8 @@
{
# Restart the serial console
systemd.services."serial-getty@ttyS0" = {
enable = true;
wantedBy = [ "getty.target" ];
serviceConfig.Restart = "always";
};
}

View File

@@ -6,10 +6,9 @@
boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
networking = {
enableIPv6 = false;
useDHCP = false;
defaultGateway = "10.0.40.30";
nameservers = ["8.8.8.8"];
proxy = {
default = "http://localhost:23080/";
noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
@@ -19,8 +18,6 @@
};
firewall = {
enable = true;
allowedTCPPorts = [ 22 ];
extraCommands = ''
# Prevent ssfhead from contacting our slurmd daemon
iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
@@ -32,8 +29,7 @@
};
extraHosts = ''
10.0.40.30 ssfhead
84.88.53.236 ssfhead.bsc.es ssfhead
10.0.40.30 ssfhead
# Node Entry for node: mds01 (ID=72)
10.0.40.40 bay mds01 mds01-eth0

View File

@@ -2,7 +2,7 @@
{
imports = [
../common/main.nix
../common/xeon.nix
#(modulesPath + "/installer/netboot/netboot-minimal.nix")
./kernel/kernel.nix

View File

@@ -2,7 +2,7 @@
{
imports = [
../common/main.nix
../common/xeon.nix
../module/ceph.nix
../module/debuginfod.nix
@@ -15,6 +15,7 @@
./public-inbox.nix
./gitea.nix
./msmtp.nix
./postgresql.nix
#./pxe.nix
];

19
m/hut/postgresql.nix Normal file
View File

@@ -0,0 +1,19 @@
{ lib, ... }:
{
services.postgresql = {
enable = true;
ensureDatabases = [ "perftestsdb" ];
ensureUsers = [
{ name = "anavarro"; ensureClauses.superuser = true; }
{ name = "rarias"; ensureClauses.superuser = true; }
{ name = "grafana"; }
];
authentication = ''
#type database DBuser auth-method
local perftestsdb rarias trust
local perftestsdb anavarro trust
local perftestsdb grafana trust
'';
};
}

View File

@@ -2,12 +2,11 @@
{
imports = [
../common/main.nix
../common/xeon.nix
#(modulesPath + "/installer/netboot/netboot-minimal.nix")
../eudy/cpufreq.nix
../eudy/users.nix
./users.nix
./kernel.nix
];

View File

@@ -1,17 +0,0 @@
{ ... }:
{
users.users = {
vlopez = {
uid = 4334;
isNormalUser = true;
home = "/home/Computational/vlopez";
description = "Victor López";
group = "Computational";
hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
openssh.authorizedKeys.keys = [
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
];
};
};
}

View File

@@ -2,8 +2,8 @@
{
imports = [
../common/main.nix
../common/monitoring.nix
../common/xeon.nix
../module/monitoring.nix
];
boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";

24
m/module/jungle-users.nix Normal file
View File

@@ -0,0 +1,24 @@
{ config, lib, ... }:
with lib;
{
options = {
users.jungleUsers = mkOption {
type = types.attrsOf (types.anything // { check = (x: x ? "hosts"); });
description = ''
Same as users.users but with the extra `hosts` attribute, which controls
access to the nodes by `networking.hostName`.
'';
};
};
config = let
allowedUser = host: userConf: builtins.elem host userConf.hosts;
filterUsers = host: users: filterAttrs (n: v: allowedUser host v) users;
removeHosts = users: mapAttrs (n: v: builtins.removeAttrs v [ "hosts" ]) users;
currentHost = config.networking.hostName;
in {
users.users = removeHosts (filterUsers currentHost config.users.jungleUsers);
};
}

View File

@@ -2,7 +2,7 @@
{
imports = [
../common/main.nix
../common/xeon.nix
../module/ceph.nix
../module/slurm-client.nix
../module/slurm-firewall.nix

View File

@@ -2,7 +2,7 @@
{
imports = [
../common/main.nix
../common/xeon.nix
../module/ceph.nix
../module/slurm-client.nix
../module/slurm-firewall.nix

View File

@@ -0,0 +1,62 @@
{ config, pkgs, lib, modulesPath, ... }:
{
imports = [
../common/base.nix
];
# Don't install Grub on the disk yet
boot.loader.grub.device = "nodev";
networking = {
hostName = "raccoon";
# Only BSC DNSs seem to be reachable from the office VLAN
nameservers = [ "84.88.52.35" "84.88.52.36" ];
defaultGateway = "84.88.51.129";
interfaces.eno0.ipv4.addresses = [ {
address = "84.88.51.152";
prefixLength = 25;
} ];
};
# Configure Nvidia driver to use with CUDA
hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
hardware.opengl = {
enable = true;
driSupport = true;
setLdLibraryPath = true;
};
nixpkgs.config.allowUnfree = true;
nixpkgs.config.nvidia.acceptLicense = true;
services.xserver.videoDrivers = [ "nvidia" ];
users.motd = ''
DO YOU BRING FEEDS?
'';
}