rarias/jungle
rarias/jungle
1
0
Fork 0
Code Issues 27 Pull Requests Packages Projects Releases Wiki Activity

Setup slurm and gitlab-runner

Browse Source
This commit is contained in:
Rodrigo Arias 2023-04-03 12:51:44 +02:00
parent 52eed708f0
commit df371c950f
9 changed files with 180 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
boot.nix
View File

@ -1,4 +1,4 @@
{ ... }:
{ lib, ... }:
{
# Use the GRUB 2 boot loader.
@ -8,9 +8,25 @@
# Select the this using the ID to avoid mismatches
boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
# Enable GRUB2 serial console
boot.loader.grub.extraConfig = ''
serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
terminal_input --append serial
terminal_output --append serial
'';
# Enable serial console
boot.kernelParams = [
"console=tty1"
"console=ttyS0,115200"
];
boot.kernelPatches = lib.singleton {
name = "osnoise-tracer";
patch = null;
extraStructuredConfig = with lib.kernel; {
OSNOISE_TRACER = yes;
HWLAT_TRACER = yes;
};
};
}

10
configuration.nix
View File

@ -8,8 +8,10 @@
./fs.nix
./gitlab-runner.nix
./net.nix
./slurm.nix
./ssh.nix
./users.nix
./overlays.nix
];
systemd.services."serial-getty@ttyS0" = {
@ -19,13 +21,17 @@
};
time.timeZone = "Europe/Madrid";
i18n.defaultLocale = "en_US.UTF-8";
i18n.defaultLocale = "en_DK.UTF-8";
environment.systemPackages = with pkgs; [
vim wget git htop
vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
];
nix.settings.experimental-features = [ "nix-command" "flakes" ];
nix.gc.automatic = true;
programs.zsh.enable = true;
programs.zsh.histSize = 100000;
# Copy the NixOS configuration file and link it from the resulting system
# (/run/current-system/configuration.nix). This is useful in case you

8
fs.nix
View File

@ -5,6 +5,12 @@
fileSystems."/home" = {
device = "10.0.40.30:/home";
fsType = "nfs";
options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" ];
options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
};
# Tracing
fileSystems."/sys/kernel/tracing" = {
device = "none";
fsType = "tracefs";
};
}

40
gitlab-runner.nix
View File

@ -18,14 +18,21 @@
SHELL = "${pkgs.bash}/bin/bash";
};
};
# # runner for everything else
# default = {
# # File should contain at least these two variables:
# # `CI_SERVER_URL`
# # `REGISTRATION_TOKEN`
# registrationConfigFile = "/run/secrets/gitlab-runner-registration";
# dockerImage = "debian:stable";
# };
# runner for everything else
default = {
# File should contain at least these two variables:
# `CI_SERVER_URL`
# `REGISTRATION_TOKEN`
registrationConfigFile = "/run/secrets/gitlab-runner-registration";
dockerImage = "debian:stable";
tagList = [ "docker" "xeon" ];
registrationFlags = [ "--docker-network-mode host" ];
environmentVariables = {
https_proxy = "http://localhost:23080";
http_proxy = "http://localhost:23080";
};
};
};
};
@ -33,6 +40,22 @@
systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false;
systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner";
systemd.services.gitlab-runner.serviceConfig.Group = "gitlab-runner";
#systemd.services.gitlab-runner.serviceConfig.ExecStart = lib.mkForce
# ''${pkgs.gitlab-runner}/bin/gitlab-runner --debug run --config ''${HOME}/.gitlab-runner/config.toml --working-directory ''${HOME}'';
# TODO https://docs.gitlab.com/runner/configuration/proxy.html
#systemd.services.docker.environment = {
# HTTP_PROXY="http://localhost:23080/";
# HTTPS_PROXY="http://localhost:23080/";
#};
#virtualisation.docker.daemon.settings = {
# proxies.default = {
# httpProxy = "http://localhost:23080/";
# httpsProxy = "http://localhost:23080/";
# noProxy = "localhost,127.0.0.0/8";
# };
#};
users.users.gitlab-runner = {
uid = config.ids.uids.gitlab-runner;
@ -40,6 +63,7 @@
home = "/var/lib/gitlab-runner";
description = "Gitlab Runner";
group = "gitlab-runner";
extraGroups = [ "docker" ];
createHome = true;
};
users.groups.gitlab-runner.gid = config.ids.gids.gitlab-runner;

68
net.nix
View File

@ -4,8 +4,9 @@
networking = {
hostName = "xeon07";
enableIPv6 = false;
useDHCP = false;
defaultGateway = "10.0.40.30";
#defaultGateway = "10.0.40.30";
nameservers = ["8.8.8.8"];
interfaces.eno1.useDHCP = false;
interfaces.eno1.ipv4.addresses = [ {
@ -20,7 +21,70 @@
firewall = {
enable = true;
allowedTCPPorts = [ 22 80 443 ];
allowedTCPPorts = [ 22 ];
# FIXME: For slurmd as it requests the compute nodes to connect to us
allowedTCPPortRanges = [ { from=1024; to=65535; } ];
};
extraHosts = ''
10.0.40.30 ssfhead
84.88.53.236 ssfhead.bsc.es ssfhead
# Node Entry for node: mds01 (ID=72)
10.0.40.40 mds01 mds01-eth0
10.0.42.40 mds01-ib0
10.0.40.141 mds01-ipmi0
# Node Entry for node: oss01 (ID=73)
10.0.40.41 oss01 oss01-eth0
10.0.42.41 oss01-ib0
10.0.40.142 oss01-ipmi0
# Node Entry for node: oss02 (ID=74)
10.0.40.42 oss02 oss02-eth0
10.0.42.42 oss02-ib0
10.0.40.143 oss02-ipmi0
# Node Entry for node: xeon01 (ID=15)
10.0.40.1 xeon01 xeon01-eth0
10.0.42.1 xeon01-ib0
10.0.40.101 xeon01-ipmi0
# Node Entry for node: xeon02 (ID=16)
10.0.40.2 xeon02 xeon02-eth0
10.0.42.2 xeon02-ib0
10.0.40.102 xeon02-ipmi0
# Node Entry for node: xeon03 (ID=17)
10.0.40.3 xeon03 xeon03-eth0
10.0.42.3 xeon03-ib0
10.0.40.103 xeon03-ipmi0
# Node Entry for node: xeon04 (ID=18)
10.0.40.4 xeon04 xeon04-eth0
10.0.42.4 xeon04-ib0
10.0.40.104 xeon04-ipmi0
# Node Entry for node: xeon05 (ID=19)
10.0.40.5 xeon05 xeon05-eth0
10.0.42.5 xeon05-ib0
10.0.40.105 xeon05-ipmi0
# Node Entry for node: xeon06 (ID=20)
10.0.40.6 xeon06 xeon06-eth0
10.0.42.6 xeon06-ib0
10.0.40.106 xeon06-ipmi0
# Node Entry for node: xeon07 (ID=21)
10.0.40.7 xeon07 xeon07-eth0
10.0.42.7 xeon07-ib0
10.0.40.107 xeon07-ipmi0
# Node Entry for node: xeon08 (ID=22)
10.0.40.8 xeon08 xeon08-eth0
10.0.42.8 xeon08-ib0
10.0.40.108 xeon08-ipmi0
'';
};
}

8
overlays-compat/overlays.nix Normal file
View File

@ -0,0 +1,8 @@
self: super:
with super.lib;
let
# Load the system config and get the `nixpkgs.overlays` option
overlays = (import <nixpkgs/nixos> { }).config.nixpkgs.overlays;
in
# Apply all overlays to the input of the current "main" overlay
foldl' (flip extends) (_: super) overlays self

25
overlays.nix Normal file
View File

@ -0,0 +1,25 @@
{ options, ... }:
let
bscpkgsSrc = builtins.fetchTarball "https://pm.bsc.es/gitlab/rarias/bscpkgs/-/archive/slurm-xeon07/bscpkgs-master.tar.gz";
bscpkgs = import "${bscpkgsSrc}/overlay.nix";
xeon07Overlay = (self: super: {
slurm = super.bsc.slurm-16-05-8-1;
});
in
{
nix.nixPath =
# Prepend default nixPath values.
options.nix.nixPath.default ++
# Append our nixpkgs-overlays.
[ "nixpkgs-overlays=/etc/nixos/overlays-compat/" ]
;
nixpkgs.overlays = [
bscpkgs xeon07Overlay
];
}

12
slurm.nix Normal file
View File

@ -0,0 +1,12 @@
{ ... }:
{
services.slurm = {
client.enable = true;
controlMachine = "ssfhead";
clusterName = "owl";
nodeName = [
"xeon[01-08] Sockets=2 CoresPerSocket=14 ThreadsPerCore=2 Feature=xeon"
];
};
}

5
users.nix
View File

@ -8,8 +8,13 @@
isNormalUser = true;
home = "/home/Computational/rarias";
description = "Rodrigo Arias";
group = "Computational";
extraGroups = [ "wheel" ];
hashedPassword = "$6$u06tkCy13enReBsb$xiI.twRvvTfH4jdS3s68NZ7U9PSbGKs5.LXU/UgoawSwNWhZo2hRAjNL5qG0/lAckzcho2LjD0r3NfVPvthY6/";
};
groups = {
Computational = { gid = 564; };
};
};
}