Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4da36899f5 | |||
| 3906876a69 |
Binary file not shown.
@ -150,27 +150,3 @@ And update grub.
|
|||||||
```
|
```
|
||||||
# nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
|
# nix build .#nixosConfigurations.xeon02.config.system.build.kexecTree -v
|
||||||
```
|
```
|
||||||
|
|
||||||
## Chain NixOS in same disk with other systems
|
|
||||||
|
|
||||||
To install NixOS on a partition along another system which controls the GRUB,
|
|
||||||
first disable the grub device, so the GRUB is not installed in the disk by
|
|
||||||
NixOS (only the /boot files will be generated):
|
|
||||||
|
|
||||||
```
|
|
||||||
boot.loader.grub.device = "nodev";
|
|
||||||
```
|
|
||||||
|
|
||||||
Then add the following entry to the old GRUB configuration:
|
|
||||||
|
|
||||||
```
|
|
||||||
menuentry 'NixOS' {
|
|
||||||
insmod chain
|
|
||||||
search --no-floppy --label nixos --set root
|
|
||||||
configfile /boot/grub/grub.cfg
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The partition with NixOS must have the label "nixos" for it to be found. New
|
|
||||||
system configuration entries will be stored in the GRUB configuration managed
|
|
||||||
by NixOS, so there is no need to change the old GRUB settings.
|
|
||||||
|
|||||||
54
flake.lock
54
flake.lock
@ -6,15 +6,14 @@
|
|||||||
"home-manager": "home-manager",
|
"home-manager": "home-manager",
|
||||||
"nixpkgs": [
|
"nixpkgs": [
|
||||||
"nixpkgs"
|
"nixpkgs"
|
||||||
],
|
]
|
||||||
"systems": "systems"
|
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1720546205,
|
"lastModified": 1690228878,
|
||||||
"narHash": "sha256-boCXsjYVxDviyzoEyAk624600f3ZBo/DKtUdvMTpbGY=",
|
"narHash": "sha256-9Xe7JV0krp4RJC9W9W9WutZVlw6BlHTFMiUP/k48LQY=",
|
||||||
"owner": "ryantm",
|
"owner": "ryantm",
|
||||||
"repo": "agenix",
|
"repo": "agenix",
|
||||||
"rev": "de96bd907d5fbc3b14fc33ad37d1b9a3cb15edc6",
|
"rev": "d8c973fd228949736dedf61b7f8cc1ece3236792",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@ -30,17 +29,17 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1713974364,
|
"lastModified": 1694077645,
|
||||||
"narHash": "sha256-ilZTVWSaNP1ibhQIIRXE+q9Lj2XOH+F9W3Co4QyY1eU=",
|
"narHash": "sha256-72bvRBhq8Q8V6ibsR9lyBE92V2EC6C6Ek3J5cOM79So=",
|
||||||
"ref": "refs/heads/master",
|
"ref": "refs/heads/master",
|
||||||
"rev": "de89197a4a7b162db7df9d41c9d07759d87c5709",
|
"rev": "6122fef92701701e1a0622550ac0fc5c2beb5906",
|
||||||
"revCount": 937,
|
"revCount": 860,
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://git.sr.ht/~rodarima/bscpkgs"
|
"url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://git.sr.ht/~rodarima/bscpkgs"
|
"url": "https://pm.bsc.es/gitlab/rarias/bscpkgs.git"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"darwin": {
|
"darwin": {
|
||||||
@ -51,11 +50,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1700795494,
|
"lastModified": 1673295039,
|
||||||
"narHash": "sha256-gzGLZSiOhf155FW7262kdHo2YDeugp3VuIFb4/GGng0=",
|
"narHash": "sha256-AsdYgE8/GPwcelGgrntlijMg4t3hLFJFCRF3tL5WVjA=",
|
||||||
"owner": "lnl7",
|
"owner": "lnl7",
|
||||||
"repo": "nix-darwin",
|
"repo": "nix-darwin",
|
||||||
"rev": "4b9b83d5a92e8c1fbfd8eb27eda375908c11ec4d",
|
"rev": "87b9d090ad39b25b2400029c64825fc2a8868943",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@ -73,11 +72,11 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1703113217,
|
"lastModified": 1682203081,
|
||||||
"narHash": "sha256-7ulcXOk63TIT2lVDSExj7XzFx09LpdSAPtvgtM7yQPE=",
|
"narHash": "sha256-kRL4ejWDhi0zph/FpebFYhzqlOBrk0Pl3dzGEKSAlEw=",
|
||||||
"owner": "nix-community",
|
"owner": "nix-community",
|
||||||
"repo": "home-manager",
|
"repo": "home-manager",
|
||||||
"rev": "3bfaacf46133c037bb356193bd2f1765d9dc82c1",
|
"rev": "32d3e39c491e2f91152c84f8ad8b003420eab0a1",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@ -88,11 +87,11 @@
|
|||||||
},
|
},
|
||||||
"nixpkgs": {
|
"nixpkgs": {
|
||||||
"locked": {
|
"locked": {
|
||||||
"lastModified": 1720957393,
|
"lastModified": 1693663421,
|
||||||
"narHash": "sha256-oedh2RwpjEa+TNxhg5Je9Ch6d3W1NKi7DbRO1ziHemA=",
|
"narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=",
|
||||||
"owner": "NixOS",
|
"owner": "NixOS",
|
||||||
"repo": "nixpkgs",
|
"repo": "nixpkgs",
|
||||||
"rev": "693bc46d169f5af9c992095736e82c3488bf7dbb",
|
"rev": "e56990880811a451abd32515698c712788be5720",
|
||||||
"type": "github"
|
"type": "github"
|
||||||
},
|
},
|
||||||
"original": {
|
"original": {
|
||||||
@ -108,21 +107,6 @@
|
|||||||
"bscpkgs": "bscpkgs",
|
"bscpkgs": "bscpkgs",
|
||||||
"nixpkgs": "nixpkgs"
|
"nixpkgs": "nixpkgs"
|
||||||
}
|
}
|
||||||
},
|
|
||||||
"systems": {
|
|
||||||
"locked": {
|
|
||||||
"lastModified": 1681028828,
|
|
||||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
|
||||||
"owner": "nix-systems",
|
|
||||||
"repo": "default",
|
|
||||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
|
||||||
"type": "github"
|
|
||||||
},
|
|
||||||
"original": {
|
|
||||||
"owner": "nix-systems",
|
|
||||||
"repo": "default",
|
|
||||||
"type": "github"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"root": "root",
|
"root": "root",
|
||||||
|
|||||||
22
flake.nix
22
flake.nix
@ -3,7 +3,7 @@
|
|||||||
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||||
agenix.url = "github:ryantm/agenix";
|
agenix.url = "github:ryantm/agenix";
|
||||||
agenix.inputs.nixpkgs.follows = "nixpkgs";
|
agenix.inputs.nixpkgs.follows = "nixpkgs";
|
||||||
bscpkgs.url = "git+https://git.sr.ht/~rodarima/bscpkgs";
|
bscpkgs.url = "git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git";
|
||||||
bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
|
bscpkgs.inputs.nixpkgs.follows = "nixpkgs";
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -17,19 +17,15 @@ let
|
|||||||
in
|
in
|
||||||
{
|
{
|
||||||
nixosConfigurations = {
|
nixosConfigurations = {
|
||||||
hut = mkConf "hut";
|
hut = mkConf "hut";
|
||||||
owl1 = mkConf "owl1";
|
owl1 = mkConf "owl1";
|
||||||
owl2 = mkConf "owl2";
|
owl2 = mkConf "owl2";
|
||||||
eudy = mkConf "eudy";
|
eudy = mkConf "eudy";
|
||||||
koro = mkConf "koro";
|
koro = mkConf "koro";
|
||||||
bay = mkConf "bay";
|
bay = mkConf "bay";
|
||||||
lake2 = mkConf "lake2";
|
lake2 = mkConf "lake2";
|
||||||
raccoon = mkConf "raccoon";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
packages.x86_64-linux = self.nixosConfigurations.hut.pkgs // {
|
packages.x86_64-linux.hut = self.nixosConfigurations.hut.pkgs;
|
||||||
bscpkgs = bscpkgs.packages.x86_64-linux;
|
|
||||||
nixpkgs = nixpkgs.legacyPackages.x86_64-linux;
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@ -2,21 +2,21 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../common/xeon.nix
|
../common/main.nix
|
||||||
../module/monitoring.nix
|
../common/monitoring.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
# Select the this using the ID to avoid mismatches
|
# Select the this using the ID to avoid mismatches
|
||||||
boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d";
|
boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d";
|
||||||
|
|
||||||
boot.kernel.sysctl = {
|
|
||||||
"kernel.yama.ptrace_scope" = lib.mkForce "1";
|
|
||||||
};
|
|
||||||
|
|
||||||
environment.systemPackages = with pkgs; [
|
environment.systemPackages = with pkgs; [
|
||||||
ceph
|
ceph
|
||||||
];
|
];
|
||||||
|
|
||||||
|
services.slurm = {
|
||||||
|
client.enable = lib.mkForce false;
|
||||||
|
};
|
||||||
|
|
||||||
networking = {
|
networking = {
|
||||||
hostName = "bay";
|
hostName = "bay";
|
||||||
interfaces.eno1.ipv4.addresses = [ {
|
interfaces.eno1.ipv4.addresses = [ {
|
||||||
@ -27,16 +27,6 @@
|
|||||||
address = "10.0.42.40";
|
address = "10.0.42.40";
|
||||||
prefixLength = 24;
|
prefixLength = 24;
|
||||||
} ];
|
} ];
|
||||||
firewall = {
|
|
||||||
extraCommands = ''
|
|
||||||
# Accept all incoming TCP traffic from lake2
|
|
||||||
iptables -A nixos-fw -p tcp -s lake2 -j nixos-fw-accept
|
|
||||||
# Accept monitoring requests from hut
|
|
||||||
iptables -A nixos-fw -p tcp -s hut -m multiport --dport 9283,9002 -j nixos-fw-accept
|
|
||||||
# Accept all Ceph traffic from the local network
|
|
||||||
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
services.ceph = {
|
services.ceph = {
|
||||||
|
|||||||
@ -1,20 +0,0 @@
|
|||||||
{
|
|
||||||
# All machines should include this profile.
|
|
||||||
# Includes the basic configuration for an Intel server.
|
|
||||||
imports = [
|
|
||||||
./base/agenix.nix
|
|
||||||
./base/august-shutdown.nix
|
|
||||||
./base/boot.nix
|
|
||||||
./base/env.nix
|
|
||||||
./base/fs.nix
|
|
||||||
./base/hw.nix
|
|
||||||
./base/net.nix
|
|
||||||
./base/nix.nix
|
|
||||||
./base/ntp.nix
|
|
||||||
./base/rev.nix
|
|
||||||
./base/ssh.nix
|
|
||||||
./base/users.nix
|
|
||||||
./base/watchdog.nix
|
|
||||||
./base/zsh.nix
|
|
||||||
];
|
|
||||||
}
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
{
|
|
||||||
# Shutdown all machines on August 2nd at 11:00 AM, so we can protect the
|
|
||||||
# hardware from spurious electrical peaks on the yearly electrical cut for
|
|
||||||
# manteinance that starts on August 4th.
|
|
||||||
systemd.timers.august-shutdown = {
|
|
||||||
description = "Shutdown on August 2nd for maintenance";
|
|
||||||
wantedBy = [ "timers.target" ];
|
|
||||||
timerConfig = {
|
|
||||||
OnCalendar = "*-08-02 11:00:00";
|
|
||||||
RandomizedDelaySec = "10min";
|
|
||||||
Unit = "systemd-poweroff.service";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -1,35 +0,0 @@
|
|||||||
{ pkgs, config, ... }:
|
|
||||||
|
|
||||||
{
|
|
||||||
environment.systemPackages = with pkgs; [
|
|
||||||
vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
|
|
||||||
nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
|
|
||||||
ncdu config.boot.kernelPackages.perf ldns
|
|
||||||
# From bsckgs overlay
|
|
||||||
osumb
|
|
||||||
];
|
|
||||||
|
|
||||||
programs.direnv.enable = true;
|
|
||||||
|
|
||||||
# Increase limits
|
|
||||||
security.pam.loginLimits = [
|
|
||||||
{
|
|
||||||
domain = "*";
|
|
||||||
type = "-";
|
|
||||||
item = "memlock";
|
|
||||||
value = "1048576"; # 1 GiB of mem locked
|
|
||||||
}
|
|
||||||
];
|
|
||||||
|
|
||||||
environment.variables = {
|
|
||||||
EDITOR = "vim";
|
|
||||||
VISUAL = "vim";
|
|
||||||
};
|
|
||||||
|
|
||||||
programs.bash.promptInit = ''
|
|
||||||
PS1="\h\\$ "
|
|
||||||
'';
|
|
||||||
|
|
||||||
time.timeZone = "Europe/Madrid";
|
|
||||||
i18n.defaultLocale = "en_DK.UTF-8";
|
|
||||||
}
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
{ pkgs, ... }:
|
|
||||||
|
|
||||||
{
|
|
||||||
networking = {
|
|
||||||
enableIPv6 = false;
|
|
||||||
useDHCP = false;
|
|
||||||
|
|
||||||
firewall = {
|
|
||||||
enable = true;
|
|
||||||
allowedTCPPorts = [ 22 ];
|
|
||||||
};
|
|
||||||
|
|
||||||
hosts = {
|
|
||||||
"84.88.53.236" = [ "ssfhead.bsc.es" "ssfhead" ];
|
|
||||||
"84.88.51.152" = [ "raccoon" ];
|
|
||||||
"84.88.51.142" = [ "raccoon-ipmi" ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -1,57 +0,0 @@
|
|||||||
{ pkgs, nixpkgs, bscpkgs, theFlake, ... }:
|
|
||||||
|
|
||||||
{
|
|
||||||
nixpkgs.overlays = [
|
|
||||||
bscpkgs.bscOverlay
|
|
||||||
(import ../../../pkgs/overlay.nix)
|
|
||||||
];
|
|
||||||
|
|
||||||
nix = {
|
|
||||||
nixPath = [
|
|
||||||
"nixpkgs=${nixpkgs}"
|
|
||||||
"jungle=${theFlake.outPath}"
|
|
||||||
];
|
|
||||||
|
|
||||||
registry = {
|
|
||||||
nixpkgs.flake = nixpkgs;
|
|
||||||
jungle.flake = theFlake;
|
|
||||||
};
|
|
||||||
|
|
||||||
settings = {
|
|
||||||
experimental-features = [ "nix-command" "flakes" ];
|
|
||||||
sandbox = "relaxed";
|
|
||||||
trusted-users = [ "@wheel" ];
|
|
||||||
flake-registry = pkgs.writeText "global-registry.json"
|
|
||||||
''{"flakes":[],"version":2}'';
|
|
||||||
};
|
|
||||||
|
|
||||||
gc = {
|
|
||||||
automatic = true;
|
|
||||||
dates = "weekly";
|
|
||||||
options = "--delete-older-than 30d";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
# The nix-gc.service can begin its execution *before* /home is mounted,
|
|
||||||
# causing it to remove all gcroots considering them as stale, as it cannot
|
|
||||||
# access the symlink. To prevent this problem, we force the service to wait
|
|
||||||
# until /home is mounted as well as other remote FS like /ceph.
|
|
||||||
systemd.services.nix-gc = {
|
|
||||||
# Start remote-fs.target if not already being started and fail if it fails
|
|
||||||
# to start. It will also be stopped if the remote-fs.target fails after
|
|
||||||
# starting successfully.
|
|
||||||
bindsTo = [ "remote-fs.target" ];
|
|
||||||
# Wait until remote-fs.target fully starts before starting this one.
|
|
||||||
after = [ "remote-fs.target"];
|
|
||||||
# Ensure we can access a remote path inside /home
|
|
||||||
unitConfig.ConditionPathExists = "/home/Computational";
|
|
||||||
};
|
|
||||||
|
|
||||||
# This value determines the NixOS release from which the default
|
|
||||||
# settings for stateful data, like file locations and database versions
|
|
||||||
# on your system were taken. It‘s perfectly fine and recommended to leave
|
|
||||||
# this value at the release version of the first install of this system.
|
|
||||||
# Before changing this value read the documentation for this option
|
|
||||||
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
|
|
||||||
system.stateVersion = "22.11"; # Did you read the comment?
|
|
||||||
}
|
|
||||||
@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
# Use the GRUB 2 boot loader.
|
# Use the GRUB 2 boot loader.
|
||||||
boot.loader.grub.enable = true;
|
boot.loader.grub.enable = lib.mkForce true;
|
||||||
|
|
||||||
# Enable GRUB2 serial console
|
# Enable GRUB2 serial console
|
||||||
boot.loader.grub.extraConfig = ''
|
boot.loader.grub.extraConfig = ''
|
||||||
@ -11,12 +11,14 @@
|
|||||||
terminal_output --append serial
|
terminal_output --append serial
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
# Enable serial console
|
||||||
|
boot.kernelParams = [
|
||||||
|
"console=tty1"
|
||||||
|
"console=ttyS0,115200"
|
||||||
|
];
|
||||||
|
|
||||||
boot.kernel.sysctl = {
|
boot.kernel.sysctl = {
|
||||||
"kernel.perf_event_paranoid" = lib.mkDefault "-1";
|
"kernel.perf_event_paranoid" = lib.mkDefault "-1";
|
||||||
|
|
||||||
# Allow ptracing (i.e. attach with GDB) any process of the same user, see:
|
|
||||||
# https://www.kernel.org/doc/Documentation/security/Yama.txt
|
|
||||||
"kernel.yama.ptrace_scope" = "0";
|
|
||||||
};
|
};
|
||||||
|
|
||||||
boot.kernelPackages = pkgs.linuxPackages_latest;
|
boot.kernelPackages = pkgs.linuxPackages_latest;
|
||||||
@ -13,12 +13,16 @@
|
|||||||
[ { device = "/dev/disk/by-label/swap"; }
|
[ { device = "/dev/disk/by-label/swap"; }
|
||||||
];
|
];
|
||||||
|
|
||||||
|
# Mount the home via NFS
|
||||||
|
fileSystems."/home" = {
|
||||||
|
device = "10.0.40.30:/home";
|
||||||
|
fsType = "nfs";
|
||||||
|
options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
|
||||||
|
};
|
||||||
|
|
||||||
# Tracing
|
# Tracing
|
||||||
fileSystems."/sys/kernel/tracing" = {
|
fileSystems."/sys/kernel/tracing" = {
|
||||||
device = "none";
|
device = "none";
|
||||||
fsType = "tracefs";
|
fsType = "tracefs";
|
||||||
};
|
};
|
||||||
|
|
||||||
# Mount a tmpfs into /tmp
|
|
||||||
boot.tmp.useTmpfs = true;
|
|
||||||
}
|
}
|
||||||
94
m/common/main.nix
Normal file
94
m/common/main.nix
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
{ config, pkgs, nixpkgs, bscpkgs, agenix, theFlake, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
imports = [
|
||||||
|
./agenix.nix
|
||||||
|
./boot.nix
|
||||||
|
./fs.nix
|
||||||
|
./hw.nix
|
||||||
|
./net.nix
|
||||||
|
./ntp.nix
|
||||||
|
./slurm.nix
|
||||||
|
./ssh.nix
|
||||||
|
./users.nix
|
||||||
|
./watchdog.nix
|
||||||
|
./rev.nix
|
||||||
|
./zsh.nix
|
||||||
|
];
|
||||||
|
|
||||||
|
nixpkgs.overlays = [
|
||||||
|
bscpkgs.bscOverlay
|
||||||
|
(import ../../pkgs/overlay.nix)
|
||||||
|
];
|
||||||
|
|
||||||
|
system.configurationRevision =
|
||||||
|
if theFlake ? rev
|
||||||
|
then theFlake.rev
|
||||||
|
else throw ("Refusing to build from a dirty Git tree!");
|
||||||
|
|
||||||
|
nix.nixPath = [
|
||||||
|
"nixpkgs=${nixpkgs}"
|
||||||
|
"bscpkgs=${bscpkgs}"
|
||||||
|
"jungle=${theFlake.outPath}"
|
||||||
|
];
|
||||||
|
|
||||||
|
nix.registry.nixpkgs.flake = nixpkgs;
|
||||||
|
nix.registry.bscpkgs.flake = bscpkgs;
|
||||||
|
nix.registry.jungle.flake = theFlake;
|
||||||
|
|
||||||
|
environment.systemPackages = with pkgs; [
|
||||||
|
vim wget git htop tmux pciutils tcpdump ripgrep nix-index nixos-option
|
||||||
|
nix-diff ipmitool freeipmi ethtool lm_sensors ix cmake gnumake file tree
|
||||||
|
ncdu config.boot.kernelPackages.perf ldns
|
||||||
|
# From bsckgs overlay
|
||||||
|
bsc.osumb
|
||||||
|
];
|
||||||
|
|
||||||
|
systemd.services."serial-getty@ttyS0" = {
|
||||||
|
enable = true;
|
||||||
|
wantedBy = [ "getty.target" ];
|
||||||
|
serviceConfig.Restart = "always";
|
||||||
|
};
|
||||||
|
|
||||||
|
# Increase limits
|
||||||
|
security.pam.loginLimits = [
|
||||||
|
{
|
||||||
|
domain = "*";
|
||||||
|
type = "-";
|
||||||
|
item = "memlock";
|
||||||
|
value = "1048576"; # 1 GiB of mem locked
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
time.timeZone = "Europe/Madrid";
|
||||||
|
i18n.defaultLocale = "en_DK.UTF-8";
|
||||||
|
|
||||||
|
environment.variables = {
|
||||||
|
EDITOR = "vim";
|
||||||
|
VISUAL = "vim";
|
||||||
|
};
|
||||||
|
|
||||||
|
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
||||||
|
nix.settings.sandbox = "relaxed";
|
||||||
|
nix.settings.trusted-users = [ "@wheel" ];
|
||||||
|
nix.gc.automatic = true;
|
||||||
|
nix.gc.dates = "weekly";
|
||||||
|
nix.gc.options = "--delete-older-than 30d";
|
||||||
|
|
||||||
|
programs.bash.promptInit = ''
|
||||||
|
PS1="\h\\$ "
|
||||||
|
'';
|
||||||
|
|
||||||
|
# Copy the NixOS configuration file and link it from the resulting system
|
||||||
|
# (/run/current-system/configuration.nix). This is useful in case you
|
||||||
|
# accidentally delete configuration.nix.
|
||||||
|
#system.copySystemConfiguration = true;
|
||||||
|
|
||||||
|
# This value determines the NixOS release from which the default
|
||||||
|
# settings for stateful data, like file locations and database versions
|
||||||
|
# on your system were taken. It‘s perfectly fine and recommended to leave
|
||||||
|
# this value at the release version of the first install of this system.
|
||||||
|
# Before changing this value read the documentation for this option
|
||||||
|
# (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
|
||||||
|
system.stateVersion = "22.11"; # Did you read the comment?
|
||||||
|
}
|
||||||
@ -6,11 +6,12 @@
|
|||||||
boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
|
boot.kernelModules = [ "ib_umad" "ib_ipoib" ];
|
||||||
|
|
||||||
networking = {
|
networking = {
|
||||||
|
enableIPv6 = false;
|
||||||
|
useDHCP = false;
|
||||||
defaultGateway = "10.0.40.30";
|
defaultGateway = "10.0.40.30";
|
||||||
nameservers = ["8.8.8.8"];
|
nameservers = ["8.8.8.8"];
|
||||||
|
|
||||||
proxy = {
|
proxy = {
|
||||||
default = "http://hut:23080/";
|
default = "http://localhost:23080/";
|
||||||
noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
|
noProxy = "127.0.0.1,localhost,internal.domain,10.0.40.40";
|
||||||
# Don't set all_proxy as go complains and breaks the gitlab runner, see:
|
# Don't set all_proxy as go complains and breaks the gitlab runner, see:
|
||||||
# https://github.com/golang/go/issues/16715
|
# https://github.com/golang/go/issues/16715
|
||||||
@ -18,9 +19,11 @@
|
|||||||
};
|
};
|
||||||
|
|
||||||
firewall = {
|
firewall = {
|
||||||
|
enable = true;
|
||||||
|
allowedTCPPorts = [ 22 ];
|
||||||
extraCommands = ''
|
extraCommands = ''
|
||||||
# Prevent ssfhead from contacting our slurmd daemon
|
# Prevent ssfhead from contacting our slurmd daemon
|
||||||
iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-refuse
|
iptables -A nixos-fw -p tcp -s ssfhead --dport 6817:6819 -j nixos-fw-log-refuse
|
||||||
# But accept traffic to slurm ports from any other node in the subnet
|
# But accept traffic to slurm ports from any other node in the subnet
|
||||||
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
|
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 6817:6819 -j nixos-fw-accept
|
||||||
# We also need to open the srun port range
|
# We also need to open the srun port range
|
||||||
@ -29,7 +32,8 @@
|
|||||||
};
|
};
|
||||||
|
|
||||||
extraHosts = ''
|
extraHosts = ''
|
||||||
10.0.40.30 ssfhead
|
10.0.40.30 ssfhead
|
||||||
|
84.88.53.236 ssfhead.bsc.es ssfhead
|
||||||
|
|
||||||
# Node Entry for node: mds01 (ID=72)
|
# Node Entry for node: mds01 (ID=72)
|
||||||
10.0.40.40 bay mds01 mds01-eth0
|
10.0.40.40 bay mds01 mds01-eth0
|
||||||
@ -1,7 +1,6 @@
|
|||||||
{ theFlake, ... }:
|
{ theFlake, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
# Prevent building a configuration without revision
|
|
||||||
rev = if theFlake ? rev then theFlake.rev
|
rev = if theFlake ? rev then theFlake.rev
|
||||||
else throw ("Refusing to build from a dirty Git tree!");
|
else throw ("Refusing to build from a dirty Git tree!");
|
||||||
in {
|
in {
|
||||||
@ -16,6 +15,4 @@ in {
|
|||||||
DATENOW=$(date --iso-8601=seconds)
|
DATENOW=$(date --iso-8601=seconds)
|
||||||
echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log
|
echo "$DATENOW booted=$BOOTED current=$CURRENT next=$NEXT" >> /var/configrev.log
|
||||||
'';
|
'';
|
||||||
|
|
||||||
system.configurationRevision = rev;
|
|
||||||
}
|
}
|
||||||
@ -47,8 +47,8 @@ in {
|
|||||||
];
|
];
|
||||||
|
|
||||||
partitionName = [
|
partitionName = [
|
||||||
"owl Nodes=owl[1-2] Default=YES DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
|
"owl Nodes=owl[1-2] Default=YES MaxTime=INFINITE State=UP"
|
||||||
"all Nodes=owl[1-2],hut Default=NO DefaultTime=01:00:00 MaxTime=INFINITE State=UP"
|
"all Nodes=owl[1-2],hut Default=NO MaxTime=INFINITE State=UP"
|
||||||
];
|
];
|
||||||
|
|
||||||
# See slurm.conf(5) for more details about these options.
|
# See slurm.conf(5) for more details about these options.
|
||||||
@ -83,14 +83,6 @@ in {
|
|||||||
|
|
||||||
# Reduce port range so we can allow only this range in the firewall
|
# Reduce port range so we can allow only this range in the firewall
|
||||||
SrunPortRange=60000-61000
|
SrunPortRange=60000-61000
|
||||||
|
|
||||||
# Use cores as consumable resources. In SLURM terms, a core may have
|
|
||||||
# multiple hardware threads (or CPUs).
|
|
||||||
SelectType=select/cons_tres
|
|
||||||
|
|
||||||
# Ignore memory constraints and only use unused cores to share a node with
|
|
||||||
# other jobs.
|
|
||||||
SelectTypeParameters=CR_Core
|
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -1,7 +1,7 @@
|
|||||||
{ lib, ... }:
|
{ lib, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
keys = import ../../../keys.nix;
|
keys = import ../../keys.nix;
|
||||||
hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts;
|
hostsKeys = lib.mapAttrs (name: value: { publicKey = value; }) keys.hosts;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
@ -12,7 +12,7 @@ in
|
|||||||
programs.ssh.extraConfig = ''
|
programs.ssh.extraConfig = ''
|
||||||
Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
|
Host bscpm02.bsc.es bscpm03.bsc.es gitlab-internal.bsc.es alya.gitlab.bsc.es
|
||||||
User git
|
User git
|
||||||
ProxyCommand nc -X connect -x hut:23080 %h %p
|
ProxyCommand nc -X connect -x localhost:23080 %h %p
|
||||||
'';
|
'';
|
||||||
|
|
||||||
programs.ssh.knownHosts = hostsKeys // {
|
programs.ssh.knownHosts = hostsKeys // {
|
||||||
@ -1,10 +1,6 @@
|
|||||||
{ pkgs, ... }:
|
{ pkgs, ... }:
|
||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
|
||||||
../../module/jungle-users.nix
|
|
||||||
];
|
|
||||||
|
|
||||||
users = {
|
users = {
|
||||||
mutableUsers = false;
|
mutableUsers = false;
|
||||||
users = {
|
users = {
|
||||||
@ -46,16 +42,13 @@
|
|||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGdphWxLAEekicZ/WBrvP7phMyxKSSuLAZBovNX+hZXQ aleix@kerneland"
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
};
|
|
||||||
|
|
||||||
jungleUsers = {
|
|
||||||
rpenacob = {
|
rpenacob = {
|
||||||
uid = 2761;
|
uid = 2761;
|
||||||
isNormalUser = true;
|
isNormalUser = true;
|
||||||
home = "/home/Computational/rpenacob";
|
home = "/home/Computational/rpenacob";
|
||||||
description = "Raúl Peñacoba";
|
description = "Raúl Peñacoba";
|
||||||
group = "Computational";
|
group = "Computational";
|
||||||
hosts = [ "owl1" "owl2" "hut" ];
|
|
||||||
hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
|
hashedPassword = "$6$TZm3bDIFyPrMhj1E$uEDXoYYd1z2Wd5mMPfh3DZAjP7ztVjJ4ezIcn82C0ImqafPA.AnTmcVftHEzLB3tbe2O4SxDyPSDEQgJ4GOtj/";
|
||||||
openssh.authorizedKeys.keys = [
|
openssh.authorizedKeys.keys = [
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFYfXg37mauGeurqsLpedgA2XQ9d4Nm0ZGo/hI1f7wwH rpenacob@bsc"
|
||||||
@ -68,51 +61,11 @@
|
|||||||
home = "/home/Computational/anavarro";
|
home = "/home/Computational/anavarro";
|
||||||
description = "Antoni Navarro";
|
description = "Antoni Navarro";
|
||||||
group = "Computational";
|
group = "Computational";
|
||||||
hosts = [ "hut" "raccoon" ];
|
|
||||||
hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
|
hashedPassword = "$6$QdNDsuLehoZTYZlb$CDhCouYDPrhoiB7/seu7RF.Gqg4zMQz0n5sA4U1KDgHaZOxy2as9pbIGeF8tOHJKRoZajk5GiaZv0rZMn7Oq31";
|
||||||
openssh.authorizedKeys.keys = [
|
openssh.authorizedKeys.keys = [
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILWjRSlKgzBPZQhIeEtk6Lvws2XNcYwHcwPv4osSgst5 anavarro@ssfhead"
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
||||||
abonerib = {
|
|
||||||
uid = 4541;
|
|
||||||
isNormalUser = true;
|
|
||||||
home = "/home/Computational/abonerib";
|
|
||||||
description = "Aleix Boné";
|
|
||||||
group = "Computational";
|
|
||||||
hosts = [ "owl1" "owl2" "hut" "raccoon" ];
|
|
||||||
hashedPassword = "$6$V1EQWJr474whv7XJ$OfJ0wueM2l.dgiJiiah0Tip9ITcJ7S7qDvtSycsiQ43QBFyP4lU0e0HaXWps85nqB4TypttYR4hNLoz3bz662/";
|
|
||||||
openssh.authorizedKeys.keys = [
|
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIIFiqXqt88VuUfyANkZyLJNiuroIITaGlOOTMhVDKjf abonerib@bsc"
|
|
||||||
];
|
|
||||||
};
|
|
||||||
|
|
||||||
vlopez = {
|
|
||||||
uid = 4334;
|
|
||||||
isNormalUser = true;
|
|
||||||
home = "/home/Computational/vlopez";
|
|
||||||
description = "Victor López";
|
|
||||||
group = "Computational";
|
|
||||||
hosts = [ "koro" ];
|
|
||||||
hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
|
|
||||||
openssh.authorizedKeys.keys = [
|
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
|
|
||||||
];
|
|
||||||
};
|
|
||||||
|
|
||||||
dbautist = {
|
|
||||||
uid = 5649;
|
|
||||||
isNormalUser = true;
|
|
||||||
home = "/home/Computational/dbautist";
|
|
||||||
description = "Dylan Bautista Cases";
|
|
||||||
group = "Computational";
|
|
||||||
hosts = [ "hut" ];
|
|
||||||
hashedPassword = "$6$a2lpzMRVkG9nSgIm$12G6.ka0sFX1YimqJkBAjbvhRKZ.Hl090B27pdbnQOW0wzyxVWySWhyDDCILjQELky.HKYl9gqOeVXW49nW7q/";
|
|
||||||
openssh.authorizedKeys.keys = [
|
|
||||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAb+EQBoS98zrCwnGKkHKwMLdYABMTqv7q9E0+T0QmkS dbautist@bsc-848818791"
|
|
||||||
];
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
groups = {
|
groups = {
|
||||||
@ -1,9 +0,0 @@
|
|||||||
{
|
|
||||||
# Provides the base system for a xeon node.
|
|
||||||
imports = [
|
|
||||||
./base.nix
|
|
||||||
./xeon/fs.nix
|
|
||||||
./xeon/console.nix
|
|
||||||
./xeon/net.nix
|
|
||||||
];
|
|
||||||
}
|
|
||||||
@ -1,14 +0,0 @@
|
|||||||
{
|
|
||||||
# Restart the serial console
|
|
||||||
systemd.services."serial-getty@ttyS0" = {
|
|
||||||
enable = true;
|
|
||||||
wantedBy = [ "getty.target" ];
|
|
||||||
serviceConfig.Restart = "always";
|
|
||||||
};
|
|
||||||
|
|
||||||
# Enable serial console
|
|
||||||
boot.kernelParams = [
|
|
||||||
"console=tty1"
|
|
||||||
"console=ttyS0,115200"
|
|
||||||
];
|
|
||||||
}
|
|
||||||
@ -1,8 +0,0 @@
|
|||||||
{
|
|
||||||
# Mount the home via NFS
|
|
||||||
fileSystems."/home" = {
|
|
||||||
device = "10.0.40.30:/home";
|
|
||||||
fsType = "nfs";
|
|
||||||
options = [ "nfsvers=3" "rsize=1024" "wsize=1024" "cto" "nofail" ];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
environment.systemPackages = with pkgs; [
|
environment.systemPackages = with pkgs; [
|
||||||
|
direnv
|
||||||
zsh-completions
|
zsh-completions
|
||||||
nix-zsh-completions
|
nix-zsh-completions
|
||||||
];
|
];
|
||||||
@ -2,14 +2,14 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../common/xeon.nix
|
../common/main.nix
|
||||||
#(modulesPath + "/installer/netboot/netboot-minimal.nix")
|
#(modulesPath + "/installer/netboot/netboot-minimal.nix")
|
||||||
|
|
||||||
./kernel/kernel.nix
|
./kernel/kernel.nix
|
||||||
./cpufreq.nix
|
./cpufreq.nix
|
||||||
./fs.nix
|
./fs.nix
|
||||||
./users.nix
|
./users.nix
|
||||||
../module/debuginfod.nix
|
./slurm.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
# Select this using the ID to avoid mismatches
|
# Select this using the ID to avoid mismatches
|
||||||
|
|||||||
@ -21,9 +21,9 @@ let
|
|||||||
# configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
|
# configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
|
||||||
#};
|
#};
|
||||||
|
|
||||||
kernel = nixos-fcs;
|
kernel = nixos-fcsv3;
|
||||||
|
|
||||||
nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
|
nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
|
||||||
version = "6.2.8";
|
version = "6.2.8";
|
||||||
src = builtins.fetchGit {
|
src = builtins.fetchGit {
|
||||||
url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
|
url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
|
||||||
@ -40,13 +40,35 @@ let
|
|||||||
};
|
};
|
||||||
kernelPatches = [];
|
kernelPatches = [];
|
||||||
extraMeta.branch = lib.versions.majorMinor version;
|
extraMeta.branch = lib.versions.majorMinor version;
|
||||||
}));
|
});
|
||||||
|
|
||||||
nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
|
nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
|
||||||
nixos-fcs-lockstat = nixos-fcs.override {
|
nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
|
||||||
|
nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";};
|
||||||
|
|
||||||
|
# always use fcs_sched_setaffinity
|
||||||
|
#nixos-debug = nixos-fcs-kernel {gitCommit = "7d0bf285fca92badc8df3c9907a9ab30db4418aa";};
|
||||||
|
# remove need_check_cgroup
|
||||||
|
#nixos-debug = nixos-fcs-kernel {gitCommit = "4cc4efaab5e4a0bfa3089e935215b981c1922919";};
|
||||||
|
# merge again fcs_wake and fcs_wait
|
||||||
|
#nixos-debug = nixos-fcs-kernel {gitCommit = "40c6f72f4ae54b0b636b193ac0648fb5730c810d";};
|
||||||
|
# start from scratch, this is the working version with split fcs_wake and fcs_wait
|
||||||
|
nixos-debug = nixos-fcs-kernel {gitCommit = "c9a39d6a4ca83845b4e71fcc268fb0a76aff1bdf"; branch = "fcs-test"; };
|
||||||
|
|
||||||
|
nixos-fcsv1-lockstat = nixos-fcs-kernel {
|
||||||
|
gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
|
||||||
lockStat = true;
|
lockStat = true;
|
||||||
};
|
};
|
||||||
nixos-fcs-lockstat-preempt = nixos-fcs.override {
|
nixos-fcsv2-lockstat = nixos-fcs-kernel {
|
||||||
|
gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
|
||||||
|
lockStat = true;
|
||||||
|
};
|
||||||
|
nixos-fcsv3-lockstat = nixos-fcs-kernel {
|
||||||
|
gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
|
||||||
|
lockStat = true;
|
||||||
|
};
|
||||||
|
nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel {
|
||||||
|
gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
|
||||||
lockStat = true;
|
lockStat = true;
|
||||||
preempt = true;
|
preempt = true;
|
||||||
};
|
};
|
||||||
|
|||||||
7
m/eudy/slurm.nix
Normal file
7
m/eudy/slurm.nix
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
{ lib, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
services.slurm = {
|
||||||
|
client.enable = lib.mkForce false;
|
||||||
|
};
|
||||||
|
}
|
||||||
@ -1,162 +0,0 @@
|
|||||||
modules:
|
|
||||||
http_2xx:
|
|
||||||
prober: http
|
|
||||||
timeout: 5s
|
|
||||||
http:
|
|
||||||
proxy_url: "http://127.0.0.1:23080"
|
|
||||||
skip_resolve_phase_with_proxy: true
|
|
||||||
follow_redirects: true
|
|
||||||
valid_status_codes: [] # Defaults to 2xx
|
|
||||||
method: GET
|
|
||||||
http_with_proxy:
|
|
||||||
prober: http
|
|
||||||
http:
|
|
||||||
proxy_url: "http://127.0.0.1:3128"
|
|
||||||
skip_resolve_phase_with_proxy: true
|
|
||||||
http_with_proxy_and_headers:
|
|
||||||
prober: http
|
|
||||||
http:
|
|
||||||
proxy_url: "http://127.0.0.1:3128"
|
|
||||||
proxy_connect_header:
|
|
||||||
Proxy-Authorization:
|
|
||||||
- Bearer token
|
|
||||||
http_post_2xx:
|
|
||||||
prober: http
|
|
||||||
timeout: 5s
|
|
||||||
http:
|
|
||||||
method: POST
|
|
||||||
headers:
|
|
||||||
Content-Type: application/json
|
|
||||||
body: '{}'
|
|
||||||
http_post_body_file:
|
|
||||||
prober: http
|
|
||||||
timeout: 5s
|
|
||||||
http:
|
|
||||||
method: POST
|
|
||||||
body_file: "/files/body.txt"
|
|
||||||
http_basic_auth_example:
|
|
||||||
prober: http
|
|
||||||
timeout: 5s
|
|
||||||
http:
|
|
||||||
method: POST
|
|
||||||
headers:
|
|
||||||
Host: "login.example.com"
|
|
||||||
basic_auth:
|
|
||||||
username: "username"
|
|
||||||
password: "mysecret"
|
|
||||||
http_2xx_oauth_client_credentials:
|
|
||||||
prober: http
|
|
||||||
timeout: 5s
|
|
||||||
http:
|
|
||||||
valid_http_versions: ["HTTP/1.1", "HTTP/2"]
|
|
||||||
follow_redirects: true
|
|
||||||
preferred_ip_protocol: "ip4"
|
|
||||||
valid_status_codes:
|
|
||||||
- 200
|
|
||||||
- 201
|
|
||||||
oauth2:
|
|
||||||
client_id: "client_id"
|
|
||||||
client_secret: "client_secret"
|
|
||||||
token_url: "https://api.example.com/token"
|
|
||||||
endpoint_params:
|
|
||||||
grant_type: "client_credentials"
|
|
||||||
http_custom_ca_example:
|
|
||||||
prober: http
|
|
||||||
http:
|
|
||||||
method: GET
|
|
||||||
tls_config:
|
|
||||||
ca_file: "/certs/my_cert.crt"
|
|
||||||
http_gzip:
|
|
||||||
prober: http
|
|
||||||
http:
|
|
||||||
method: GET
|
|
||||||
compression: gzip
|
|
||||||
http_gzip_with_accept_encoding:
|
|
||||||
prober: http
|
|
||||||
http:
|
|
||||||
method: GET
|
|
||||||
compression: gzip
|
|
||||||
headers:
|
|
||||||
Accept-Encoding: gzip
|
|
||||||
tls_connect:
|
|
||||||
prober: tcp
|
|
||||||
timeout: 5s
|
|
||||||
tcp:
|
|
||||||
tls: true
|
|
||||||
tcp_connect_example:
|
|
||||||
prober: tcp
|
|
||||||
timeout: 5s
|
|
||||||
imap_starttls:
|
|
||||||
prober: tcp
|
|
||||||
timeout: 5s
|
|
||||||
tcp:
|
|
||||||
query_response:
|
|
||||||
- expect: "OK.*STARTTLS"
|
|
||||||
- send: ". STARTTLS"
|
|
||||||
- expect: "OK"
|
|
||||||
- starttls: true
|
|
||||||
- send: ". capability"
|
|
||||||
- expect: "CAPABILITY IMAP4rev1"
|
|
||||||
smtp_starttls:
|
|
||||||
prober: tcp
|
|
||||||
timeout: 5s
|
|
||||||
tcp:
|
|
||||||
query_response:
|
|
||||||
- expect: "^220 ([^ ]+) ESMTP (.+)$"
|
|
||||||
- send: "EHLO prober\r"
|
|
||||||
- expect: "^250-STARTTLS"
|
|
||||||
- send: "STARTTLS\r"
|
|
||||||
- expect: "^220"
|
|
||||||
- starttls: true
|
|
||||||
- send: "EHLO prober\r"
|
|
||||||
- expect: "^250-AUTH"
|
|
||||||
- send: "QUIT\r"
|
|
||||||
irc_banner_example:
|
|
||||||
prober: tcp
|
|
||||||
timeout: 5s
|
|
||||||
tcp:
|
|
||||||
query_response:
|
|
||||||
- send: "NICK prober"
|
|
||||||
- send: "USER prober prober prober :prober"
|
|
||||||
- expect: "PING :([^ ]+)"
|
|
||||||
send: "PONG ${1}"
|
|
||||||
- expect: "^:[^ ]+ 001"
|
|
||||||
icmp:
|
|
||||||
prober: icmp
|
|
||||||
timeout: 5s
|
|
||||||
icmp:
|
|
||||||
preferred_ip_protocol: "ip4"
|
|
||||||
dns_udp_example:
|
|
||||||
prober: dns
|
|
||||||
timeout: 5s
|
|
||||||
dns:
|
|
||||||
query_name: "www.prometheus.io"
|
|
||||||
query_type: "A"
|
|
||||||
valid_rcodes:
|
|
||||||
- NOERROR
|
|
||||||
validate_answer_rrs:
|
|
||||||
fail_if_matches_regexp:
|
|
||||||
- ".*127.0.0.1"
|
|
||||||
fail_if_all_match_regexp:
|
|
||||||
- ".*127.0.0.1"
|
|
||||||
fail_if_not_matches_regexp:
|
|
||||||
- "www.prometheus.io.\t300\tIN\tA\t127.0.0.1"
|
|
||||||
fail_if_none_matches_regexp:
|
|
||||||
- "127.0.0.1"
|
|
||||||
validate_authority_rrs:
|
|
||||||
fail_if_matches_regexp:
|
|
||||||
- ".*127.0.0.1"
|
|
||||||
validate_additional_rrs:
|
|
||||||
fail_if_matches_regexp:
|
|
||||||
- ".*127.0.0.1"
|
|
||||||
dns_soa:
|
|
||||||
prober: dns
|
|
||||||
dns:
|
|
||||||
query_name: "prometheus.io"
|
|
||||||
query_type: "SOA"
|
|
||||||
dns_tcp_example:
|
|
||||||
prober: dns
|
|
||||||
dns:
|
|
||||||
transport_protocol: "tcp" # defaults to "udp"
|
|
||||||
preferred_ip_protocol: "ip4" # defaults to "ip6"
|
|
||||||
query_name: "www.prometheus.io"
|
|
||||||
@ -2,24 +2,19 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../common/xeon.nix
|
../common/main.nix
|
||||||
|
|
||||||
../module/ceph.nix
|
../module/ceph.nix
|
||||||
../module/debuginfod.nix
|
|
||||||
../module/emulation.nix
|
|
||||||
../module/slurm-client.nix
|
|
||||||
./gitlab-runner.nix
|
./gitlab-runner.nix
|
||||||
./monitoring.nix
|
./monitoring.nix
|
||||||
./nfs.nix
|
./nfs.nix
|
||||||
./slurm-server.nix
|
./slurm-daemon.nix
|
||||||
./nix-serve.nix
|
./nix-serve.nix
|
||||||
./public-inbox.nix
|
|
||||||
./gitea.nix
|
|
||||||
./msmtp.nix
|
|
||||||
./postgresql.nix
|
|
||||||
#./pxe.nix
|
#./pxe.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
|
boot.binfmt.emulatedSystems = [ "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
|
||||||
|
|
||||||
# Select the this using the ID to avoid mismatches
|
# Select the this using the ID to avoid mismatches
|
||||||
boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
|
boot.loader.grub.device = "/dev/disk/by-id/ata-INTEL_SSDSC2BB240G7_PHDV6462004Y240AGN";
|
||||||
|
|
||||||
@ -33,15 +28,5 @@
|
|||||||
address = "10.0.42.7";
|
address = "10.0.42.7";
|
||||||
prefixLength = 24;
|
prefixLength = 24;
|
||||||
} ];
|
} ];
|
||||||
firewall = {
|
|
||||||
extraCommands = ''
|
|
||||||
# Accept all proxy traffic from compute nodes but not the login
|
|
||||||
iptables -A nixos-fw -p tcp -s 10.0.40.30 --dport 23080 -j nixos-fw-log-refuse
|
|
||||||
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 --dport 23080 -j nixos-fw-accept
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# Allow proxy to bind to the ethernet interface
|
|
||||||
services.openssh.settings.GatewayPorts = "clientspecified";
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,63 +0,0 @@
|
|||||||
{ config, lib, ... }:
|
|
||||||
{
|
|
||||||
age.secrets.giteaRunnerToken.file = ../../secrets/gitea-runner-token.age;
|
|
||||||
|
|
||||||
services.gitea = {
|
|
||||||
enable = true;
|
|
||||||
appName = "Gitea in the jungle";
|
|
||||||
|
|
||||||
settings = {
|
|
||||||
server = {
|
|
||||||
ROOT_URL = "https://jungle.bsc.es/git/";
|
|
||||||
LOCAL_ROOT_URL = "https://jungle.bsc.es/git/";
|
|
||||||
LANDING_PAGE = "explore";
|
|
||||||
};
|
|
||||||
metrics.ENABLED = true;
|
|
||||||
service = {
|
|
||||||
REGISTER_MANUAL_CONFIRM = true;
|
|
||||||
ENABLE_NOTIFY_MAIL = true;
|
|
||||||
};
|
|
||||||
log.LEVEL = "Warn";
|
|
||||||
|
|
||||||
mailer = {
|
|
||||||
ENABLED = true;
|
|
||||||
FROM = "jungle-robot@bsc.es";
|
|
||||||
PROTOCOL = "sendmail";
|
|
||||||
SENDMAIL_PATH = "/run/wrappers/bin/sendmail";
|
|
||||||
SENDMAIL_ARGS = "--";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
services.gitea-actions-runner.instances = {
|
|
||||||
runrun = {
|
|
||||||
enable = true;
|
|
||||||
name = "runrun";
|
|
||||||
url = "https://jungle.bsc.es/git/";
|
|
||||||
tokenFile = config.age.secrets.giteaRunnerToken.path;
|
|
||||||
labels = [ "native:host" ];
|
|
||||||
settings.runner.capacity = 8;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
systemd.services.gitea-runner-runrun = {
|
|
||||||
path = [ "/run/current-system/sw" ];
|
|
||||||
serviceConfig = {
|
|
||||||
# DynamicUser doesn't work well with SSH
|
|
||||||
DynamicUser = lib.mkForce false;
|
|
||||||
User = "gitea-runner";
|
|
||||||
Group = "gitea-runner";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
users.users.gitea-runner = {
|
|
||||||
isSystemUser = true;
|
|
||||||
home = "/var/lib/gitea-runner";
|
|
||||||
description = "Gitea Runner";
|
|
||||||
group = "gitea-runner";
|
|
||||||
extraGroups = [ "docker" ];
|
|
||||||
createHome = true;
|
|
||||||
};
|
|
||||||
users.groups.gitea-runner = {};
|
|
||||||
}
|
|
||||||
|
|
||||||
@ -1,23 +1,31 @@
|
|||||||
{ pkgs, lib, config, ... }:
|
{ pkgs, lib, config, ... }:
|
||||||
|
|
||||||
{
|
{
|
||||||
age.secrets.gitlabRunnerShellToken.file = ../../secrets/gitlab-runner-shell-token.age;
|
age.secrets.ovniToken.file = ../../secrets/ovni-token.age;
|
||||||
age.secrets.gitlabRunnerDockerToken.file = ../../secrets/gitlab-runner-docker-token.age;
|
age.secrets.nosvToken.file = ../../secrets/nosv-token.age;
|
||||||
|
|
||||||
services.gitlab-runner = {
|
services.gitlab-runner = {
|
||||||
enable = true;
|
enable = true;
|
||||||
settings.concurrent = 5;
|
settings.concurrent = 5;
|
||||||
services = let
|
services = {
|
||||||
common-shell = {
|
ovni-shell = {
|
||||||
|
registrationConfigFile = config.age.secrets.ovniToken.path;
|
||||||
executor = "shell";
|
executor = "shell";
|
||||||
|
tagList = [ "nix" "xeon" ];
|
||||||
|
registrationFlags = [
|
||||||
|
# Using space doesn't work, and causes it to misread the next flag
|
||||||
|
"--locked='false'"
|
||||||
|
];
|
||||||
environmentVariables = {
|
environmentVariables = {
|
||||||
SHELL = "${pkgs.bash}/bin/bash";
|
SHELL = "${pkgs.bash}/bin/bash";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
common-docker = {
|
ovni-docker = {
|
||||||
executor = "docker";
|
registrationConfigFile = config.age.secrets.ovniToken.path;
|
||||||
dockerImage = "debian:stable";
|
dockerImage = "debian:stable";
|
||||||
|
tagList = [ "docker" "xeon" ];
|
||||||
registrationFlags = [
|
registrationFlags = [
|
||||||
|
"--locked='false'"
|
||||||
"--docker-network-mode host"
|
"--docker-network-mode host"
|
||||||
];
|
];
|
||||||
environmentVariables = {
|
environmentVariables = {
|
||||||
@ -25,13 +33,18 @@
|
|||||||
http_proxy = "http://localhost:23080";
|
http_proxy = "http://localhost:23080";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
in {
|
nosv-docker = {
|
||||||
# For pm.bsc.es/gitlab
|
registrationConfigFile = config.age.secrets.nosvToken.path;
|
||||||
gitlab-pm-shell = common-shell // {
|
dockerImage = "debian:stable";
|
||||||
authenticationTokenConfigFile = config.age.secrets.gitlabRunnerShellToken.path;
|
tagList = [ "docker" "xeon" ];
|
||||||
};
|
registrationFlags = [
|
||||||
gitlab-pm-docker = common-docker // {
|
"--docker-network-mode host"
|
||||||
authenticationTokenConfigFile = config.age.secrets.gitlabRunnerDockerToken.path;
|
"--docker-cpus 56"
|
||||||
|
];
|
||||||
|
environmentVariables = {
|
||||||
|
https_proxy = "http://localhost:23080";
|
||||||
|
http_proxy = "http://localhost:23080";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@ -1,14 +1,6 @@
|
|||||||
{ config, lib, ... }:
|
{ config, lib, ... }:
|
||||||
|
|
||||||
{
|
{
|
||||||
imports = [ ../module/slurm-exporter.nix ];
|
|
||||||
|
|
||||||
age.secrets.grafanaJungleRobotPassword = {
|
|
||||||
file = ../../secrets/jungle-robot-password.age;
|
|
||||||
owner = "grafana";
|
|
||||||
mode = "400";
|
|
||||||
};
|
|
||||||
|
|
||||||
services.grafana = {
|
services.grafana = {
|
||||||
enable = true;
|
enable = true;
|
||||||
settings = {
|
settings = {
|
||||||
@ -19,25 +11,10 @@
|
|||||||
http_port = 2342;
|
http_port = 2342;
|
||||||
http_addr = "127.0.0.1";
|
http_addr = "127.0.0.1";
|
||||||
};
|
};
|
||||||
smtp = {
|
|
||||||
enabled = true;
|
|
||||||
from_address = "jungle-robot@bsc.es";
|
|
||||||
user = "jungle-robot";
|
|
||||||
# Read the password from a file, which is only readable by grafana user
|
|
||||||
# https://grafana.com/docs/grafana/latest/setup-grafana/configure-grafana/#file-provider
|
|
||||||
password = "$__file{${config.age.secrets.grafanaJungleRobotPassword.path}}";
|
|
||||||
host = "mail.bsc.es:465";
|
|
||||||
startTLS_policy = "NoStartTLS";
|
|
||||||
};
|
|
||||||
feature_toggles.publicDashboards = true;
|
feature_toggles.publicDashboards = true;
|
||||||
"auth.anonymous".enabled = true;
|
|
||||||
log.level = "warn";
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
# Make grafana alerts also use the proxy
|
|
||||||
systemd.services.grafana.environment = config.networking.proxy.envVars;
|
|
||||||
|
|
||||||
services.prometheus = {
|
services.prometheus = {
|
||||||
enable = true;
|
enable = true;
|
||||||
port = 9001;
|
port = 9001;
|
||||||
@ -84,11 +61,6 @@
|
|||||||
enable = true;
|
enable = true;
|
||||||
listenAddress = "127.0.0.1";
|
listenAddress = "127.0.0.1";
|
||||||
};
|
};
|
||||||
blackbox = {
|
|
||||||
enable = true;
|
|
||||||
listenAddress = "127.0.0.1";
|
|
||||||
configFile = ./blackbox.yml;
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
scrapeConfigs = [
|
scrapeConfigs = [
|
||||||
@ -101,8 +73,6 @@
|
|||||||
"127.0.0.1:9323"
|
"127.0.0.1:9323"
|
||||||
"127.0.0.1:9252"
|
"127.0.0.1:9252"
|
||||||
"127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
|
"127.0.0.1:${toString config.services.prometheus.exporters.smartctl.port}"
|
||||||
"127.0.0.1:9341" # Slurm exporter
|
|
||||||
"127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}"
|
|
||||||
];
|
];
|
||||||
}];
|
}];
|
||||||
}
|
}
|
||||||
@ -116,71 +86,6 @@
|
|||||||
];
|
];
|
||||||
}];
|
}];
|
||||||
}
|
}
|
||||||
{
|
|
||||||
job_name = "blackbox-http";
|
|
||||||
metrics_path = "/probe";
|
|
||||||
params = { module = [ "http_2xx" ]; };
|
|
||||||
static_configs = [{
|
|
||||||
targets = [
|
|
||||||
"https://www.google.com/robots.txt"
|
|
||||||
"https://pm.bsc.es/"
|
|
||||||
"https://pm.bsc.es/gitlab/"
|
|
||||||
"https://jungle.bsc.es/"
|
|
||||||
"https://gitlab.bsc.es/"
|
|
||||||
];
|
|
||||||
}];
|
|
||||||
relabel_configs = [
|
|
||||||
{
|
|
||||||
# Takes the address and sets it in the "target=<xyz>" URL parameter
|
|
||||||
source_labels = [ "__address__" ];
|
|
||||||
target_label = "__param_target";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
# Sets the "instance" label with the remote host we are querying
|
|
||||||
source_labels = [ "__param_target" ];
|
|
||||||
target_label = "instance";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
# Shows the host target address instead of the blackbox address
|
|
||||||
target_label = "__address__";
|
|
||||||
replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
job_name = "blackbox-icmp";
|
|
||||||
metrics_path = "/probe";
|
|
||||||
params = { module = [ "icmp" ]; };
|
|
||||||
static_configs = [{
|
|
||||||
targets = [
|
|
||||||
"1.1.1.1"
|
|
||||||
"8.8.8.8"
|
|
||||||
"ssfhead"
|
|
||||||
"anella-bsc.cesca.cat"
|
|
||||||
];
|
|
||||||
}];
|
|
||||||
relabel_configs = [
|
|
||||||
{
|
|
||||||
# Takes the address and sets it in the "target=<xyz>" URL parameter
|
|
||||||
source_labels = [ "__address__" ];
|
|
||||||
target_label = "__param_target";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
# Sets the "instance" label with the remote host we are querying
|
|
||||||
source_labels = [ "__param_target" ];
|
|
||||||
target_label = "instance";
|
|
||||||
}
|
|
||||||
{
|
|
||||||
# Shows the host target address instead of the blackbox address
|
|
||||||
target_label = "__address__";
|
|
||||||
replacement = "127.0.0.1:${toString config.services.prometheus.exporters.blackbox.port}";
|
|
||||||
}
|
|
||||||
];
|
|
||||||
}
|
|
||||||
{
|
|
||||||
job_name = "gitea";
|
|
||||||
static_configs = [{ targets = [ "127.0.0.1:3000" ]; }];
|
|
||||||
}
|
|
||||||
{
|
{
|
||||||
# Scrape the IPMI info of the hosts remotely via LAN
|
# Scrape the IPMI info of the hosts remotely via LAN
|
||||||
job_name = "ipmi-lan";
|
job_name = "ipmi-lan";
|
||||||
@ -233,17 +138,6 @@
|
|||||||
}
|
}
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
{
|
|
||||||
job_name = "ipmi-raccoon";
|
|
||||||
metrics_path = "/ipmi";
|
|
||||||
static_configs = [
|
|
||||||
{ targets = [ "127.0.0.1:9291" ]; }
|
|
||||||
];
|
|
||||||
params = {
|
|
||||||
target = [ "84.88.51.142" ];
|
|
||||||
module = [ "raccoon" ];
|
|
||||||
};
|
|
||||||
}
|
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,24 +0,0 @@
|
|||||||
{ config, lib, ... }:
|
|
||||||
{
|
|
||||||
age.secrets.jungleRobotPassword = {
|
|
||||||
file = ../../secrets/jungle-robot-password.age;
|
|
||||||
group = "gitea";
|
|
||||||
mode = "440";
|
|
||||||
};
|
|
||||||
|
|
||||||
programs.msmtp = {
|
|
||||||
enable = true;
|
|
||||||
accounts = {
|
|
||||||
default = {
|
|
||||||
auth = true;
|
|
||||||
tls = true;
|
|
||||||
tls_starttls = false;
|
|
||||||
port = 465;
|
|
||||||
host = "mail.bsc.es";
|
|
||||||
user = "jungle-robot";
|
|
||||||
passwordeval = "cat ${config.age.secrets.jungleRobotPassword.path}";
|
|
||||||
from = "jungle-robot@bsc.es";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
{ lib, ... }:
|
|
||||||
|
|
||||||
{
|
|
||||||
services.postgresql = {
|
|
||||||
enable = true;
|
|
||||||
ensureDatabases = [ "perftestsdb" ];
|
|
||||||
ensureUsers = [
|
|
||||||
{ name = "anavarro"; ensureClauses.superuser = true; }
|
|
||||||
{ name = "rarias"; ensureClauses.superuser = true; }
|
|
||||||
{ name = "grafana"; }
|
|
||||||
];
|
|
||||||
authentication = ''
|
|
||||||
#type database DBuser auth-method
|
|
||||||
local perftestsdb rarias trust
|
|
||||||
local perftestsdb anavarro trust
|
|
||||||
local perftestsdb grafana trust
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -1,79 +0,0 @@
|
|||||||
/*
|
|
||||||
* CC0-1.0 <https://creativecommons.org/publicdomain/zero/1.0/legalcode>
|
|
||||||
* Dark color scheme using 216 web-safe colors, inspired
|
|
||||||
* somewhat by the default color scheme in mutt.
|
|
||||||
* It reduces eyestrain for me, and energy usage for all:
|
|
||||||
* https://en.wikipedia.org/wiki/Light-on-dark_color_scheme
|
|
||||||
*/
|
|
||||||
|
|
||||||
* {
|
|
||||||
font-size: 14px;
|
|
||||||
font-family: monospace;
|
|
||||||
}
|
|
||||||
|
|
||||||
pre {
|
|
||||||
white-space: pre-wrap;
|
|
||||||
padding: 10px;
|
|
||||||
background: #f5f5f5;
|
|
||||||
}
|
|
||||||
|
|
||||||
hr {
|
|
||||||
margin: 30px 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
body {
|
|
||||||
max-width: 120ex; /* 120 columns wide */
|
|
||||||
margin: 50px auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
* Underlined links add visual noise which make them hard-to-read.
|
|
||||||
* Use colors to make them stand out, instead.
|
|
||||||
*/
|
|
||||||
a:link {
|
|
||||||
color: #007;
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
a:visited {
|
|
||||||
color:#504;
|
|
||||||
}
|
|
||||||
a:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* quoted text in emails gets a different color */
|
|
||||||
*.q { color:gray }
|
|
||||||
|
|
||||||
/*
|
|
||||||
* these may be used with cgit <https://git.zx2c4.com/cgit/>, too.
|
|
||||||
* (cgit uses <div>, public-inbox uses <span>)
|
|
||||||
*/
|
|
||||||
*.add { color:darkgreen } /* diff post-image lines */
|
|
||||||
*.del { color:darkred } /* diff pre-image lines */
|
|
||||||
*.head { color:black } /* diff header (metainformation) */
|
|
||||||
*.hunk { color:gray } /* diff hunk-header */
|
|
||||||
|
|
||||||
/*
|
|
||||||
* highlight 3.x colors (tested 3.18) for displaying blobs.
|
|
||||||
* This doesn't use most of the colors available, as I find too
|
|
||||||
* many colors overwhelming, so the default is commented out.
|
|
||||||
*/
|
|
||||||
.hl.num { color:#f30 } /* number */
|
|
||||||
.hl.esc { color:#f0f } /* escape character */
|
|
||||||
.hl.str { color:#f30 } /* string */
|
|
||||||
.hl.ppc { color:#f0f } /* preprocessor */
|
|
||||||
.hl.pps { color:#f30 } /* preprocessor string */
|
|
||||||
.hl.slc { color:#09f } /* single-line comment */
|
|
||||||
.hl.com { color:#09f } /* multi-line comment */
|
|
||||||
/* .hl.opt { color:#ccc } */ /* operator */
|
|
||||||
/* .hl.ipl { color:#ccc } */ /* interpolation */
|
|
||||||
|
|
||||||
/* keyword groups kw[a-z] */
|
|
||||||
.hl.kwa { color:#ff0 }
|
|
||||||
.hl.kwb { color:#0f0 }
|
|
||||||
.hl.kwc { color:#ff0 }
|
|
||||||
/* .hl.kwd { color:#ccc } */
|
|
||||||
|
|
||||||
/* line-number (unused by public-inbox) */
|
|
||||||
/* .hl.lin { color:#ccc } */
|
|
||||||
|
|
||||||
@ -1,47 +0,0 @@
|
|||||||
{ lib, ... }:
|
|
||||||
|
|
||||||
{
|
|
||||||
services.public-inbox = {
|
|
||||||
enable = true;
|
|
||||||
http = {
|
|
||||||
enable = true;
|
|
||||||
port = 8081;
|
|
||||||
mounts = [ "/lists" ];
|
|
||||||
};
|
|
||||||
settings.publicinbox = {
|
|
||||||
css = [ "${./public-inbox.css}" ];
|
|
||||||
wwwlisting = "all";
|
|
||||||
};
|
|
||||||
inboxes = {
|
|
||||||
bscpkgs = {
|
|
||||||
url = "https://jungle.bsc.es/lists/bscpkgs";
|
|
||||||
address = [ "~rodarima/bscpkgs@lists.sr.ht" ];
|
|
||||||
watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
|
|
||||||
description = "Patches for bscpkgs";
|
|
||||||
listid = "~rodarima/bscpkgs.lists.sr.ht";
|
|
||||||
};
|
|
||||||
jungle = {
|
|
||||||
url = "https://jungle.bsc.es/lists/jungle";
|
|
||||||
address = [ "~rodarima/jungle@lists.sr.ht" ];
|
|
||||||
watch = [ "imaps://jungle-robot%40gmx.com@imap.gmx.com/INBOX" ];
|
|
||||||
description = "Patches for jungle";
|
|
||||||
listid = "~rodarima/jungle.lists.sr.ht";
|
|
||||||
};
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
# We need access to the network for the watch service, as we will fetch the
|
|
||||||
# emails directly from the IMAP server.
|
|
||||||
systemd.services.public-inbox-watch.serviceConfig = {
|
|
||||||
PrivateNetwork = lib.mkForce false;
|
|
||||||
RestrictAddressFamilies = lib.mkForce [ "AF_UNIX" "AF_INET" "AF_INET6" ];
|
|
||||||
KillSignal = "SIGKILL"; # Avoid slow shutdown
|
|
||||||
|
|
||||||
# Required for chmod(..., 02750) on directories by git, from
|
|
||||||
# systemd.exec(8):
|
|
||||||
# > Note that this restricts marking of any type of file system object with
|
|
||||||
# > these bits, including both regular files and directories (where the SGID
|
|
||||||
# > is a different meaning than for files, see documentation).
|
|
||||||
RestrictSUIDSGID = lib.mkForce false;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -2,11 +2,13 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../common/xeon.nix
|
../common/main.nix
|
||||||
#(modulesPath + "/installer/netboot/netboot-minimal.nix")
|
#(modulesPath + "/installer/netboot/netboot-minimal.nix")
|
||||||
|
|
||||||
../eudy/cpufreq.nix
|
../eudy/cpufreq.nix
|
||||||
../eudy/users.nix
|
../eudy/users.nix
|
||||||
|
../eudy/slurm.nix
|
||||||
|
./users.nix
|
||||||
./kernel.nix
|
./kernel.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
@ -1,29 +1,9 @@
|
|||||||
{ pkgs, lib, ... }:
|
{ pkgs, lib, ... }:
|
||||||
|
|
||||||
let
|
let
|
||||||
#fcs-devel = pkgs.linuxPackages_custom {
|
kernel = nixos-fcsv4;
|
||||||
# version = "6.2.8";
|
|
||||||
# src = /mnt/data/kernel/fcs/kernel/src;
|
|
||||||
# configfile = /mnt/data/kernel/fcs/kernel/configs/defconfig;
|
|
||||||
#};
|
|
||||||
|
|
||||||
#fcsv1 = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" false;
|
nixos-fcs-kernel = {gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
|
||||||
#fcsv2 = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" false;
|
|
||||||
#fcsv1-lockdep = fcs-kernel "bc11660676d3d68ce2459b9fb5d5e654e3f413be" true;
|
|
||||||
#fcsv2-lockdep = fcs-kernel "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1" true;
|
|
||||||
#fcs-kernel = gitCommit: lockdep: pkgs.linuxPackages_custom {
|
|
||||||
# version = "6.2.8";
|
|
||||||
# src = builtins.fetchGit {
|
|
||||||
# url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
|
|
||||||
# rev = gitCommit;
|
|
||||||
# ref = "fcs";
|
|
||||||
# };
|
|
||||||
# configfile = if lockdep then ./configs/lockdep else ./configs/defconfig;
|
|
||||||
#};
|
|
||||||
|
|
||||||
kernel = nixos-fcs;
|
|
||||||
|
|
||||||
nixos-fcs-kernel = lib.makeOverridable ({gitCommit, lockStat ? false, preempt ? false, branch ? "fcs"}: pkgs.linuxPackagesFor (pkgs.buildLinux rec {
|
|
||||||
version = "6.2.8";
|
version = "6.2.8";
|
||||||
src = builtins.fetchGit {
|
src = builtins.fetchGit {
|
||||||
url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
|
url = "git@bscpm03.bsc.es:ompss-kernel/linux.git";
|
||||||
@ -40,13 +20,27 @@ let
|
|||||||
};
|
};
|
||||||
kernelPatches = [];
|
kernelPatches = [];
|
||||||
extraMeta.branch = lib.versions.majorMinor version;
|
extraMeta.branch = lib.versions.majorMinor version;
|
||||||
}));
|
});
|
||||||
|
|
||||||
nixos-fcs = nixos-fcs-kernel {gitCommit = "8a09822dfcc8f0626b209d6d2aec8b5da459dfee";};
|
nixos-fcsv1 = nixos-fcs-kernel {gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";};
|
||||||
nixos-fcs-lockstat = nixos-fcs.override {
|
nixos-fcsv2 = nixos-fcs-kernel {gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";};
|
||||||
|
nixos-fcsv3 = nixos-fcs-kernel {gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";};
|
||||||
|
nixos-fcsv4 = nixos-fcs-kernel {gitCommit = "c94c3d946f33ac3e5782a02ee002cc1164c0cb4f";};
|
||||||
|
|
||||||
|
nixos-fcsv1-lockstat = nixos-fcs-kernel {
|
||||||
|
gitCommit = "bc11660676d3d68ce2459b9fb5d5e654e3f413be";
|
||||||
lockStat = true;
|
lockStat = true;
|
||||||
};
|
};
|
||||||
nixos-fcs-lockstat-preempt = nixos-fcs.override {
|
nixos-fcsv2-lockstat = nixos-fcs-kernel {
|
||||||
|
gitCommit = "db0f2eca0cd57a58bf456d7d2c7d5d8fdb25dfb1";
|
||||||
|
lockStat = true;
|
||||||
|
};
|
||||||
|
nixos-fcsv3-lockstat = nixos-fcs-kernel {
|
||||||
|
gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
|
||||||
|
lockStat = true;
|
||||||
|
};
|
||||||
|
nixos-fcsv3-lockstat-preempt = nixos-fcs-kernel {
|
||||||
|
gitCommit = "6c17394890704c3345ac1a521bb547164b36b154";
|
||||||
lockStat = true;
|
lockStat = true;
|
||||||
preempt = true;
|
preempt = true;
|
||||||
};
|
};
|
||||||
@ -66,5 +60,5 @@ in {
|
|||||||
|
|
||||||
# enable memory overcommit, needed to build a taglibc system using nix after
|
# enable memory overcommit, needed to build a taglibc system using nix after
|
||||||
# increasing the openblas memory footprint
|
# increasing the openblas memory footprint
|
||||||
boot.kernel.sysctl."vm.overcommit_memory" = 1;
|
boot.kernel.sysctl."vm.overcommit_memory" = lib.mkForce 1;
|
||||||
}
|
}
|
||||||
|
|||||||
17
m/koro/users.nix
Normal file
17
m/koro/users.nix
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
{ ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
users.users = {
|
||||||
|
vlopez = {
|
||||||
|
uid = 4334;
|
||||||
|
isNormalUser = true;
|
||||||
|
home = "/home/Computational/vlopez";
|
||||||
|
description = "Victor López";
|
||||||
|
group = "Computational";
|
||||||
|
hashedPassword = "$6$0ZBkgIYE/renVqtt$1uWlJsb0FEezRVNoETTzZMx4X2SvWiOsKvi0ppWCRqI66S6TqMBXBdP4fcQyvRRBt0e4Z7opZIvvITBsEtO0f0";
|
||||||
|
openssh.authorizedKeys.keys = [
|
||||||
|
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGMwlUZRf9jfG666Qa5Sb+KtEhXqkiMlBV2su3x/dXHq victor@arch"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
@ -2,20 +2,20 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../common/xeon.nix
|
../common/main.nix
|
||||||
../module/monitoring.nix
|
../common/monitoring.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";
|
boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53563a";
|
||||||
|
|
||||||
boot.kernel.sysctl = {
|
|
||||||
"kernel.yama.ptrace_scope" = lib.mkForce "1";
|
|
||||||
};
|
|
||||||
|
|
||||||
environment.systemPackages = with pkgs; [
|
environment.systemPackages = with pkgs; [
|
||||||
ceph
|
ceph
|
||||||
];
|
];
|
||||||
|
|
||||||
|
services.slurm = {
|
||||||
|
client.enable = lib.mkForce false;
|
||||||
|
};
|
||||||
|
|
||||||
services.ceph = {
|
services.ceph = {
|
||||||
enable = true;
|
enable = true;
|
||||||
global = {
|
global = {
|
||||||
@ -49,16 +49,6 @@
|
|||||||
address = "10.0.42.42";
|
address = "10.0.42.42";
|
||||||
prefixLength = 24;
|
prefixLength = 24;
|
||||||
} ];
|
} ];
|
||||||
firewall = {
|
|
||||||
extraCommands = ''
|
|
||||||
# Accept all incoming TCP traffic from bay
|
|
||||||
iptables -A nixos-fw -p tcp -s bay -j nixos-fw-accept
|
|
||||||
# Accept monitoring requests from hut
|
|
||||||
iptables -A nixos-fw -p tcp -s hut --dport 9002 -j nixos-fw-accept
|
|
||||||
# Accept all Ceph traffic from the local network
|
|
||||||
iptables -A nixos-fw -p tcp -s 10.0.40.0/24 -m multiport --dport 3300,6789,6800:7568 -j nixos-fw-accept
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# Missing service for volumes, see:
|
# Missing service for volumes, see:
|
||||||
|
|||||||
@ -3,6 +3,7 @@
|
|||||||
# Mounts the /ceph filesystem at boot
|
# Mounts the /ceph filesystem at boot
|
||||||
{
|
{
|
||||||
environment.systemPackages = with pkgs; [
|
environment.systemPackages = with pkgs; [
|
||||||
|
ceph
|
||||||
ceph-client
|
ceph-client
|
||||||
fio # For benchmarks
|
fio # For benchmarks
|
||||||
];
|
];
|
||||||
|
|||||||
@ -1,3 +0,0 @@
|
|||||||
{
|
|
||||||
services.nixseparatedebuginfod.enable = true;
|
|
||||||
}
|
|
||||||
@ -1,3 +0,0 @@
|
|||||||
{
|
|
||||||
boot.binfmt.emulatedSystems = [ "armv7l-linux" "aarch64-linux" "powerpc64le-linux" "riscv64-linux" ];
|
|
||||||
}
|
|
||||||
@ -1,24 +0,0 @@
|
|||||||
{ config, lib, ... }:
|
|
||||||
|
|
||||||
with lib;
|
|
||||||
|
|
||||||
{
|
|
||||||
options = {
|
|
||||||
users.jungleUsers = mkOption {
|
|
||||||
type = types.attrsOf (types.anything // { check = (x: x ? "hosts"); });
|
|
||||||
description = ''
|
|
||||||
Same as users.users but with the extra `hosts` attribute, which controls
|
|
||||||
access to the nodes by `networking.hostName`.
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
config = let
|
|
||||||
allowedUser = host: userConf: builtins.elem host userConf.hosts;
|
|
||||||
filterUsers = host: users: filterAttrs (n: v: allowedUser host v) users;
|
|
||||||
removeHosts = users: mapAttrs (n: v: builtins.removeAttrs v [ "hosts" ]) users;
|
|
||||||
currentHost = config.networking.hostName;
|
|
||||||
in {
|
|
||||||
users.users = removeHosts (filterUsers currentHost config.users.jungleUsers);
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -1,28 +0,0 @@
|
|||||||
{ config, lib, pkgs, ... }:
|
|
||||||
|
|
||||||
# See also: https://github.com/NixOS/nixpkgs/pull/112010
|
|
||||||
# And: https://github.com/NixOS/nixpkgs/pull/115839
|
|
||||||
|
|
||||||
with lib;
|
|
||||||
|
|
||||||
{
|
|
||||||
systemd.services."prometheus-slurm-exporter" = {
|
|
||||||
wantedBy = [ "multi-user.target" ];
|
|
||||||
after = [ "network.target" ];
|
|
||||||
serviceConfig = {
|
|
||||||
Restart = mkDefault "always";
|
|
||||||
PrivateTmp = mkDefault true;
|
|
||||||
WorkingDirectory = mkDefault "/tmp";
|
|
||||||
DynamicUser = mkDefault true;
|
|
||||||
ExecStart = ''
|
|
||||||
${pkgs.prometheus-slurm-exporter}/bin/prometheus-slurm-exporter --listen-address "127.0.0.1:9341"
|
|
||||||
'';
|
|
||||||
Environment = [
|
|
||||||
"PATH=${pkgs.slurm}/bin"
|
|
||||||
# We need to specify the slurm config to be able to talk to the slurmd
|
|
||||||
# daemon.
|
|
||||||
"SLURM_CONF=${config.services.slurm.etcSlurm}/slurm.conf"
|
|
||||||
];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
{ ... }:
|
|
||||||
|
|
||||||
{
|
|
||||||
# Mount the hut nix store via NFS
|
|
||||||
fileSystems."/mnt/hut-nix-store" = {
|
|
||||||
device = "hut:/nix/store";
|
|
||||||
fsType = "nfs";
|
|
||||||
options = [ "ro" ];
|
|
||||||
};
|
|
||||||
|
|
||||||
systemd.services.slurmd.serviceConfig = {
|
|
||||||
# When running a job, bind the hut store in /nix/store so the paths are
|
|
||||||
# available too.
|
|
||||||
# FIXME: This doesn't keep the programs in /run/current-system/sw/bin
|
|
||||||
# available in the store. Ideally they should be merged but the overlay FS
|
|
||||||
# doesn't work when the underlying directories change.
|
|
||||||
BindReadOnlyPaths = "/mnt/hut-nix-store:/nix/store";
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -2,12 +2,9 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../common/xeon.nix
|
../common/main.nix
|
||||||
../module/ceph.nix
|
../module/ceph.nix
|
||||||
../module/emulation.nix
|
|
||||||
../module/slurm-client.nix
|
|
||||||
../module/slurm-firewall.nix
|
../module/slurm-firewall.nix
|
||||||
../module/debuginfod.nix
|
|
||||||
];
|
];
|
||||||
|
|
||||||
# Select the this using the ID to avoid mismatches
|
# Select the this using the ID to avoid mismatches
|
||||||
|
|||||||
@ -2,12 +2,9 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
../common/xeon.nix
|
../common/main.nix
|
||||||
../module/ceph.nix
|
../module/ceph.nix
|
||||||
../module/emulation.nix
|
|
||||||
../module/slurm-client.nix
|
|
||||||
../module/slurm-firewall.nix
|
../module/slurm-firewall.nix
|
||||||
../module/debuginfod.nix
|
|
||||||
];
|
];
|
||||||
|
|
||||||
# Select the this using the ID to avoid mismatches
|
# Select the this using the ID to avoid mismatches
|
||||||
|
|||||||
@ -1,64 +0,0 @@
|
|||||||
{ config, pkgs, lib, modulesPath, ... }:
|
|
||||||
|
|
||||||
{
|
|
||||||
imports = [
|
|
||||||
../common/base.nix
|
|
||||||
];
|
|
||||||
|
|
||||||
# Don't install Grub on the disk yet
|
|
||||||
boot.loader.grub.device = "nodev";
|
|
||||||
|
|
||||||
# Enable serial console
|
|
||||||
boot.kernelParams = [
|
|
||||||
"console=tty1"
|
|
||||||
"console=ttyS1,115200"
|
|
||||||
];
|
|
||||||
|
|
||||||
networking = {
|
|
||||||
hostName = "raccoon";
|
|
||||||
# Only BSC DNSs seem to be reachable from the office VLAN
|
|
||||||
nameservers = [ "84.88.52.35" "84.88.52.36" ];
|
|
||||||
defaultGateway = "84.88.51.129";
|
|
||||||
interfaces.eno0.ipv4.addresses = [ {
|
|
||||||
address = "84.88.51.152";
|
|
||||||
prefixLength = 25;
|
|
||||||
} ];
|
|
||||||
};
|
|
||||||
|
|
||||||
# Configure Nvidia driver to use with CUDA
|
|
||||||
hardware.nvidia.package = config.boot.kernelPackages.nvidiaPackages.production;
|
|
||||||
hardware.graphics.enable = true;
|
|
||||||
nixpkgs.config.allowUnfree = true;
|
|
||||||
nixpkgs.config.nvidia.acceptLicense = true;
|
|
||||||
services.xserver.videoDrivers = [ "nvidia" ];
|
|
||||||
|
|
||||||
users.motd = ''
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⣀⣀⣄⣠⣀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⢰⠇⡀⠀⠙⠻⡿⣦⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⡀⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⡎⢰⣧⠀⠀⠀⠁⠈⠛⢿⣦⡀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣠⣴⡦⠶⠟⠓⠚⠻⡄⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⣧⠀⣱⣀⣰⣧⠀⢀⠀⣘⣿⣿⣦⣶⣄⣠⡀⠀⠀⣀⣀⣤⣴⣄⣀⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⢀⣴⣿⠿⠏⠁⠀⣀⣠⣶⣿⡶⣿⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⣹⣆⠘⣿⣿⣿⣇⢸⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣾⣿⣿⣿⣿⣿⣿⣿⣿⣶⣶⣦⡀⣀⣤⣠⣤⡾⠋⠀⢀⣤⣶⣿⣿⣿⣿⣿⣿⣿⡀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠘⢿⡄⢼⣿⣿⣿⣿⣿⡟⠻⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣵⣾⡾⠙⣋⣩⣽⣿⣿⣿⣿⢋⡼⠁
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠈⢻⣄⠸⢿⣿⣿⠿⠷⠀⠈⠀⣭⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣾⣿⣿⣿⣿⣿⣿⠇⡼⠁⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⢾⣯⡀⠀⢼⡿⠀⠀⠀⢼⠿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⣿⡿⣿⣿⣿⠿⣿⣯⣼⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⢋⡼⠁⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⢻⡏⠠⣦⠁⠀⠀⠀⠀⠀⠟⠛⠛⣿⣿⣿⣿⣿⠿⠁⠀⠁⢿⠙⠁⠀⠛⠹⣿⣏⣾⣿⣿⣿⣿⣿⣿⣿⣿⠿⠃⣹⠁⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⣘⣧⠀⠙⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣿⣿⡿⡿⠀⠀⠀⠀⠈⠀⠀⠀⠀⠀⠀⢹⣿⠿⢿⣿⣿⣿⣿⣿⠋⢀⡤⠛⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⢹⡯⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⣿⣿⣿⠇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠁⠀⢸⣿⣿⣿⠛⠉⠀⣰⠷⠀⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⢸⠇⠀⠀⠀⠀⠀⢀⣿⡇⠀⠀⢻⣿⣿⠁⠀⠀⢠⣾⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠸⠟⢿⣿⣄⡀⢸⣿⡀⠀⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⢀⣿⠀⠀⠀⢰⣿⣿⡛⣿⣿⡄⢠⡺⠿⡍⠁⢀⣤⣿⣿⣿⠿⣷⣮⣉⣀⡀⠀⠀⠀⠀⠀⠀⠀⠀⠈⣿⠀⠀⠈⣧⠀⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⢾⠉⠃⠀⣴⣿⣟⠻⣿⣿⣿⡇⢸⣿⣶⠀⢀⣾⣿⣿⣟⠿⣷⣾⣿⣿⣿⣿⣦⣤⣤⡤⠀⠀⠀⠀⠀⠁⠀⠀⠀⣼⠗⠀⠀⠀⠀
|
|
||||||
⠀⠀⠐⢄⡀⠀⠀⠀⢘⡀⠀⢶⣾⣿⣿⣿⣿⡿⠋⠁⠈⠻⠉⠀⠚⠻⣿⣿⣿⣶⣾⣿⣿⣿⣿⣿⣿⣷⣬⣤⣶⣦⡀⣾⣶⣇⠀⠀⠈⢉⣷⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠈⠓⠶⢦⡽⠄⣈⣿⣿⣿⣿⣿⠏⠀⠀⠀⠀⠀⠀⠀⠀⠀⠹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡓⠙⣿⡟⠀⠀⠀⠈⠛⣷⣶⡄⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⢀⣬⠆⢠⣍⣛⠻⣿⡇⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⢿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣉⣀⡀⠀⠀⠈⠛⢿⣦⡀
|
|
||||||
⠐⠒⠒⠶⠶⠶⢦⣬⣟⣥⣀⡉⠛⠻⠶⢁⣤⣾⣿⣿⣿⣷⡄⠀⠀⠀⠀⠀⢸⣿⣿⣿⣿⣿⣟⡛⠿⠭⠭⠭⠭⠭⠿⠿⠿⢿⣿⣟⠃⠀⠀⠀⠹⣟⠓
|
|
||||||
⠀⣀⣠⠤⠤⢤⣤⣾⣤⡄⣉⣉⣙⣓⡂⣿⣿⣭⣹⣿⣿⣿⣿⡰⣂⣀⢀⠀⠻⣿⠛⠻⠟⠡⣶⣾⣿⣿⣿⣿⣿⣿⣿⡖⠒⠒⠒⠛⠷⢤⡀⢰⣴⣿⡆
|
|
||||||
⠀⠀⠀⢀⣠⡴⠾⠟⠻⣟⡉⠉⠉⠉⢁⢿⣿⣿⣿⣿⣿⣿⡿⣱⣿⣭⡌⠤⠀⠀⠐⣶⣌⡻⣶⣭⡻⢿⣿⣿⣿⣿⣿⣯⣥⣤⣦⠀⠠⣴⣶⣶⣿⡟⢿
|
|
||||||
⢀⠔⠊⠉⠀⠀⠀⠀⢸⣯⣤⠀⠀⠠⣼⣮⣟⣿⣿⣿⣻⣭⣾⣿⣿⣷⣶⣦⠶⣚⣾⣿⣿⣷⣜⣿⣿⣶⣝⢿⣿⣿⣿⣿⣷⣦⣄⣰⡄⠈⢿⣿⡿⣇⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠈⢡⢇⠀⠀⣠⣿⣿⣿⣯⣟⣛⣛⣛⣛⣛⣩⣭⣴⣶⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣷⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣻⣿⣧⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⣾⠏⠀⢹⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣦⣍⣿⣿⣿⣿⡄⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⣿⣾⡁⢈⣾⣿⡿⠛⣛⣿⣿⣿⣿ DO YOU BRING FEEDS? ⣿⣿⣿⣿⣿⣿⡏⠈⠙⠈⠁⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠛⡿⠛⠉⣽⣿⣷⣾⡿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⡿⠷⠌⠛⠉⠀⠁⠀⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠈⠀⠀⠹⠋⠀⢻⣿⣿⣿⣿⠿⢿⣿⣿⣿⣿⣿⣿⠿⣿⣿⣿⣿⠿⠛⠋⠉⠁⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
|
|
||||||
⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠉⠉⠁⠀⠀⠀⠀⠀⠈⠉⠉⠀⠀⠈⠋⠉⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀
|
|
||||||
'';
|
|
||||||
}
|
|
||||||
405
pkgs/ceph.nix
Normal file
405
pkgs/ceph.nix
Normal file
@ -0,0 +1,405 @@
|
|||||||
|
{ lib
|
||||||
|
, stdenv
|
||||||
|
, runCommand
|
||||||
|
, fetchurl
|
||||||
|
, fetchFromGitHub
|
||||||
|
, fetchPypi
|
||||||
|
|
||||||
|
# Build time
|
||||||
|
, cmake
|
||||||
|
, ensureNewerSourcesHook
|
||||||
|
, fmt
|
||||||
|
, git
|
||||||
|
, makeWrapper
|
||||||
|
, nasm
|
||||||
|
, pkg-config
|
||||||
|
, which
|
||||||
|
|
||||||
|
# Tests
|
||||||
|
, nixosTests
|
||||||
|
|
||||||
|
# Runtime dependencies
|
||||||
|
, arrow-cpp
|
||||||
|
, babeltrace
|
||||||
|
, boost179
|
||||||
|
, bzip2
|
||||||
|
, cryptsetup
|
||||||
|
, cunit
|
||||||
|
, doxygen
|
||||||
|
, gperf
|
||||||
|
, graphviz
|
||||||
|
, gtest
|
||||||
|
, icu
|
||||||
|
, libcap
|
||||||
|
, libcap_ng
|
||||||
|
, libnl
|
||||||
|
, libxml2
|
||||||
|
, lttng-ust
|
||||||
|
, lua
|
||||||
|
, lz4
|
||||||
|
, oath-toolkit
|
||||||
|
, openldap
|
||||||
|
, python310
|
||||||
|
, rdkafka
|
||||||
|
, rocksdb
|
||||||
|
, snappy
|
||||||
|
, sqlite
|
||||||
|
, utf8proc
|
||||||
|
, zlib
|
||||||
|
, zstd
|
||||||
|
|
||||||
|
# Optional Dependencies
|
||||||
|
, curl ? null
|
||||||
|
, expat ? null
|
||||||
|
, fuse ? null
|
||||||
|
, libatomic_ops ? null
|
||||||
|
, libedit ? null
|
||||||
|
, libs3 ? null
|
||||||
|
, yasm ? null
|
||||||
|
|
||||||
|
# Mallocs
|
||||||
|
, gperftools ? null
|
||||||
|
, jemalloc ? null
|
||||||
|
|
||||||
|
# Crypto Dependencies
|
||||||
|
, cryptopp ? null
|
||||||
|
, nspr ? null
|
||||||
|
, nss ? null
|
||||||
|
|
||||||
|
# Linux Only Dependencies
|
||||||
|
, linuxHeaders
|
||||||
|
, util-linux
|
||||||
|
, libuuid
|
||||||
|
, udev
|
||||||
|
, keyutils
|
||||||
|
, rdma-core
|
||||||
|
, rabbitmq-c
|
||||||
|
, libaio ? null
|
||||||
|
, libxfs ? null
|
||||||
|
, liburing ? null
|
||||||
|
, zfs ? null
|
||||||
|
, ...
|
||||||
|
}:
|
||||||
|
|
||||||
|
# We must have one crypto library
|
||||||
|
assert cryptopp != null || (nss != null && nspr != null);
|
||||||
|
|
||||||
|
let
|
||||||
|
shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null;
|
||||||
|
|
||||||
|
optYasm = shouldUsePkg yasm;
|
||||||
|
optExpat = shouldUsePkg expat;
|
||||||
|
optCurl = shouldUsePkg curl;
|
||||||
|
optFuse = shouldUsePkg fuse;
|
||||||
|
optLibedit = shouldUsePkg libedit;
|
||||||
|
optLibatomic_ops = shouldUsePkg libatomic_ops;
|
||||||
|
optLibs3 = shouldUsePkg libs3;
|
||||||
|
|
||||||
|
optJemalloc = shouldUsePkg jemalloc;
|
||||||
|
optGperftools = shouldUsePkg gperftools;
|
||||||
|
|
||||||
|
optCryptopp = shouldUsePkg cryptopp;
|
||||||
|
optNss = shouldUsePkg nss;
|
||||||
|
optNspr = shouldUsePkg nspr;
|
||||||
|
|
||||||
|
optLibaio = shouldUsePkg libaio;
|
||||||
|
optLibxfs = shouldUsePkg libxfs;
|
||||||
|
optZfs = shouldUsePkg zfs;
|
||||||
|
|
||||||
|
# Downgrade rocksdb, 7.10 breaks ceph
|
||||||
|
rocksdb' = rocksdb.overrideAttrs {
|
||||||
|
version = "7.9.2";
|
||||||
|
src = fetchFromGitHub {
|
||||||
|
owner = "facebook";
|
||||||
|
repo = "rocksdb";
|
||||||
|
rev = "refs/tags/v7.9.2";
|
||||||
|
hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
|
||||||
|
|
||||||
|
# Malloc implementation (can be jemalloc, tcmalloc or null)
|
||||||
|
malloc = if optJemalloc != null then optJemalloc else optGperftools;
|
||||||
|
|
||||||
|
# We prefer nss over cryptopp
|
||||||
|
cryptoStr = if optNss != null && optNspr != null then "nss" else
|
||||||
|
if optCryptopp != null then "cryptopp" else "none";
|
||||||
|
|
||||||
|
cryptoLibsMap = {
|
||||||
|
nss = [ optNss optNspr ];
|
||||||
|
cryptopp = [ optCryptopp ];
|
||||||
|
none = [ ];
|
||||||
|
};
|
||||||
|
|
||||||
|
getMeta = description: with lib; {
|
||||||
|
homepage = "https://ceph.io/en/";
|
||||||
|
inherit description;
|
||||||
|
license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
|
||||||
|
maintainers = with maintainers; [ adev ak johanot krav ];
|
||||||
|
platforms = [ "x86_64-linux" "aarch64-linux" ];
|
||||||
|
};
|
||||||
|
|
||||||
|
ceph-common = with python.pkgs; buildPythonPackage {
|
||||||
|
pname = "ceph-common";
|
||||||
|
inherit src version;
|
||||||
|
|
||||||
|
sourceRoot = "ceph-${version}/src/python-common";
|
||||||
|
|
||||||
|
propagatedBuildInputs = [
|
||||||
|
pyyaml
|
||||||
|
];
|
||||||
|
|
||||||
|
nativeCheckInputs = [
|
||||||
|
pytestCheckHook
|
||||||
|
];
|
||||||
|
|
||||||
|
disabledTests = [
|
||||||
|
# requires network access
|
||||||
|
"test_valid_addr"
|
||||||
|
];
|
||||||
|
|
||||||
|
meta = getMeta "Ceph common module for code shared by manager modules";
|
||||||
|
};
|
||||||
|
|
||||||
|
# Watch out for python <> boost compatibility
|
||||||
|
python = python310.override {
|
||||||
|
packageOverrides = self: super: {
|
||||||
|
sqlalchemy = super.sqlalchemy.overridePythonAttrs rec {
|
||||||
|
version = "1.4.46";
|
||||||
|
src = fetchPypi {
|
||||||
|
pname = "SQLAlchemy";
|
||||||
|
inherit version;
|
||||||
|
hash = "sha256-aRO4JH2KKS74MVFipRkx4rQM6RaB8bbxj2lwRSAMSjA=";
|
||||||
|
};
|
||||||
|
disabledTestPaths = [
|
||||||
|
"test/aaa_profiling"
|
||||||
|
"test/ext/mypy"
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
boost = boost179.override {
|
||||||
|
enablePython = true;
|
||||||
|
inherit python;
|
||||||
|
};
|
||||||
|
|
||||||
|
# TODO: split this off in build and runtime environment
|
||||||
|
ceph-python-env = python.withPackages (ps: with ps; [
|
||||||
|
ceph-common
|
||||||
|
|
||||||
|
# build time
|
||||||
|
cython
|
||||||
|
|
||||||
|
# debian/control
|
||||||
|
bcrypt
|
||||||
|
cherrypy
|
||||||
|
influxdb
|
||||||
|
jinja2
|
||||||
|
kubernetes
|
||||||
|
natsort
|
||||||
|
numpy
|
||||||
|
pecan
|
||||||
|
prettytable
|
||||||
|
pyjwt
|
||||||
|
pyopenssl
|
||||||
|
python-dateutil
|
||||||
|
pyyaml
|
||||||
|
requests
|
||||||
|
routes
|
||||||
|
scikit-learn
|
||||||
|
scipy
|
||||||
|
setuptools
|
||||||
|
sphinx
|
||||||
|
virtualenv
|
||||||
|
werkzeug
|
||||||
|
|
||||||
|
# src/pybind/mgr/requirements-required.txt
|
||||||
|
cryptography
|
||||||
|
jsonpatch
|
||||||
|
|
||||||
|
# src/tools/cephfs/shell/setup.py
|
||||||
|
cmd2
|
||||||
|
colorama
|
||||||
|
]);
|
||||||
|
inherit (ceph-python-env.python) sitePackages;
|
||||||
|
|
||||||
|
version = "18.2.0";
|
||||||
|
src = fetchurl {
|
||||||
|
url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz";
|
||||||
|
hash = "sha256:0k9nl6xi5brva51rr14m7ig27mmmd7vrpchcmqc40q3c2khn6ns9";
|
||||||
|
};
|
||||||
|
in rec {
|
||||||
|
ceph = stdenv.mkDerivation {
|
||||||
|
pname = "ceph";
|
||||||
|
inherit src version;
|
||||||
|
|
||||||
|
nativeBuildInputs = [
|
||||||
|
cmake
|
||||||
|
fmt
|
||||||
|
git
|
||||||
|
makeWrapper
|
||||||
|
nasm
|
||||||
|
pkg-config
|
||||||
|
python
|
||||||
|
python.pkgs.python # for the toPythonPath function
|
||||||
|
python.pkgs.wrapPython
|
||||||
|
which
|
||||||
|
(ensureNewerSourcesHook { year = "1980"; })
|
||||||
|
# for building docs/man-pages presumably
|
||||||
|
doxygen
|
||||||
|
graphviz
|
||||||
|
];
|
||||||
|
|
||||||
|
enableParallelBuilding = true;
|
||||||
|
|
||||||
|
buildInputs = cryptoLibsMap.${cryptoStr} ++ [
|
||||||
|
arrow-cpp
|
||||||
|
babeltrace
|
||||||
|
boost
|
||||||
|
bzip2
|
||||||
|
ceph-python-env
|
||||||
|
cryptsetup
|
||||||
|
cunit
|
||||||
|
gperf
|
||||||
|
gtest
|
||||||
|
icu
|
||||||
|
libcap
|
||||||
|
libnl
|
||||||
|
libxml2
|
||||||
|
lttng-ust
|
||||||
|
lua
|
||||||
|
lz4
|
||||||
|
malloc
|
||||||
|
oath-toolkit
|
||||||
|
openldap
|
||||||
|
optLibatomic_ops
|
||||||
|
optLibs3
|
||||||
|
optYasm
|
||||||
|
rdkafka
|
||||||
|
rocksdb'
|
||||||
|
snappy
|
||||||
|
sqlite
|
||||||
|
utf8proc
|
||||||
|
zlib
|
||||||
|
zstd
|
||||||
|
] ++ lib.optionals stdenv.isLinux [
|
||||||
|
keyutils
|
||||||
|
libcap_ng
|
||||||
|
liburing
|
||||||
|
libuuid
|
||||||
|
linuxHeaders
|
||||||
|
optLibaio
|
||||||
|
optLibxfs
|
||||||
|
optZfs
|
||||||
|
rabbitmq-c
|
||||||
|
rdma-core
|
||||||
|
udev
|
||||||
|
util-linux
|
||||||
|
] ++ lib.optionals hasRadosgw [
|
||||||
|
optCurl
|
||||||
|
optExpat
|
||||||
|
optFuse
|
||||||
|
optLibedit
|
||||||
|
];
|
||||||
|
|
||||||
|
pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ];
|
||||||
|
|
||||||
|
preConfigure =''
|
||||||
|
substituteInPlace src/common/module.c --replace "/sbin/modinfo" "modinfo"
|
||||||
|
substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe"
|
||||||
|
substituteInPlace src/common/module.c --replace "/bin/grep" "grep"
|
||||||
|
|
||||||
|
# install target needs to be in PYTHONPATH for "*.pth support" check to succeed
|
||||||
|
# set PYTHONPATH, so the build system doesn't silently skip installing ceph-volume and others
|
||||||
|
export PYTHONPATH=${ceph-python-env}/${sitePackages}:$lib/${sitePackages}:$out/${sitePackages}
|
||||||
|
patchShebangs src/
|
||||||
|
'';
|
||||||
|
|
||||||
|
cmakeFlags = [
|
||||||
|
"-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
|
||||||
|
|
||||||
|
"-DWITH_CEPHFS_SHELL:BOOL=ON"
|
||||||
|
"-DWITH_SYSTEMD:BOOL=OFF"
|
||||||
|
# `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with:
|
||||||
|
# CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message):
|
||||||
|
# Command failed: 2
|
||||||
|
#
|
||||||
|
# 'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace'
|
||||||
|
#
|
||||||
|
# See also
|
||||||
|
#
|
||||||
|
# /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log
|
||||||
|
# and that file contains:
|
||||||
|
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()':
|
||||||
|
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'?
|
||||||
|
# 71 | ::THRIFT_CLOSESOCKET(socket_);
|
||||||
|
# | ^~~~~~~~~~~~~~~~~~
|
||||||
|
# Looks like `close()` is somehow not included.
|
||||||
|
# But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031
|
||||||
|
# So it's proably not worth trying to fix that for this Ceph version,
|
||||||
|
# and instead just disable Ceph's Jaeger support.
|
||||||
|
"-DWITH_JAEGER:BOOL=OFF"
|
||||||
|
"-DWITH_TESTS:BOOL=OFF"
|
||||||
|
|
||||||
|
# Use our own libraries, where possible
|
||||||
|
"-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support.
|
||||||
|
"-DWITH_SYSTEM_BOOST:BOOL=ON"
|
||||||
|
"-DWITH_SYSTEM_GTEST:BOOL=ON"
|
||||||
|
"-DWITH_SYSTEM_ROCKSDB:BOOL=ON"
|
||||||
|
"-DWITH_SYSTEM_UTF8PROC:BOOL=ON"
|
||||||
|
"-DWITH_SYSTEM_ZSTD:BOOL=ON"
|
||||||
|
|
||||||
|
# TODO breaks with sandbox, tries to download stuff with npm
|
||||||
|
"-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF"
|
||||||
|
# WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
|
||||||
|
''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}''
|
||||||
|
] ++ lib.optional stdenv.isLinux "-DWITH_SYSTEM_LIBURING=ON";
|
||||||
|
|
||||||
|
postFixup = ''
|
||||||
|
wrapPythonPrograms
|
||||||
|
wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
|
||||||
|
|
||||||
|
# Test that ceph-volume exists since the build system has a tendency to
|
||||||
|
# silently drop it with misconfigurations.
|
||||||
|
test -f $out/bin/ceph-volume
|
||||||
|
'';
|
||||||
|
|
||||||
|
outputs = [ "out" "lib" "dev" "doc" "man" ];
|
||||||
|
|
||||||
|
doCheck = false; # uses pip to install things from the internet
|
||||||
|
|
||||||
|
# Takes 7+h to build with 2 cores.
|
||||||
|
requiredSystemFeatures = [ "big-parallel" ];
|
||||||
|
|
||||||
|
meta = getMeta "Distributed storage system";
|
||||||
|
|
||||||
|
passthru = {
|
||||||
|
inherit version;
|
||||||
|
tests = {
|
||||||
|
inherit (nixosTests)
|
||||||
|
ceph-multi-node
|
||||||
|
ceph-single-node
|
||||||
|
ceph-single-node-bluestore;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
ceph-client = runCommand "ceph-client-${version}" {
|
||||||
|
meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs";
|
||||||
|
} ''
|
||||||
|
mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions}
|
||||||
|
cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
|
||||||
|
cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
|
||||||
|
cp -r ${ceph}/bin/rbd-replay* $out/bin
|
||||||
|
cp -r ${ceph}/sbin/mount.ceph $out/bin
|
||||||
|
cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin
|
||||||
|
ln -s bin $out/sbin
|
||||||
|
cp -r ${ceph}/${sitePackages}/* $out/${sitePackages}
|
||||||
|
cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions
|
||||||
|
# wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
|
||||||
|
substituteInPlace $out/bin/ceph --replace ${ceph} $out
|
||||||
|
substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out
|
||||||
|
'';
|
||||||
|
}
|
||||||
@ -1,36 +0,0 @@
|
|||||||
diff --git a/src/util/mpir_hwtopo.c b/src/util/mpir_hwtopo.c
|
|
||||||
index 33e88bc..ee3641c 100644
|
|
||||||
--- a/src/util/mpir_hwtopo.c
|
|
||||||
+++ b/src/util/mpir_hwtopo.c
|
|
||||||
@@ -200,18 +200,6 @@ int MPII_hwtopo_init(void)
|
|
||||||
#ifdef HAVE_HWLOC
|
|
||||||
bindset = hwloc_bitmap_alloc();
|
|
||||||
hwloc_topology_init(&hwloc_topology);
|
|
||||||
- char *xmlfile = MPIR_pmi_get_jobattr("PMI_hwloc_xmlfile");
|
|
||||||
- if (xmlfile != NULL) {
|
|
||||||
- int rc;
|
|
||||||
- rc = hwloc_topology_set_xml(hwloc_topology, xmlfile);
|
|
||||||
- if (rc == 0) {
|
|
||||||
- /* To have hwloc still actually call OS-specific hooks, the
|
|
||||||
- * HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM has to be set to assert that the loaded
|
|
||||||
- * file is really the underlying system. */
|
|
||||||
- hwloc_topology_set_flags(hwloc_topology, HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM);
|
|
||||||
- }
|
|
||||||
- MPL_free(xmlfile);
|
|
||||||
- }
|
|
||||||
|
|
||||||
hwloc_topology_set_io_types_filter(hwloc_topology, HWLOC_TYPE_FILTER_KEEP_ALL);
|
|
||||||
if (!hwloc_topology_load(hwloc_topology))
|
|
||||||
|
|
||||||
--- a/src/mpi/init/local_proc_attrs.c
|
|
||||||
+++ b/src/mpi/init/local_proc_attrs.c
|
|
||||||
@@ -79,10 +79,6 @@ int MPII_init_local_proc_attrs(int *p_thread_required)
|
|
||||||
/* Set the number of tag bits. The device may override this value. */
|
|
||||||
MPIR_Process.tag_bits = MPIR_TAG_BITS_DEFAULT;
|
|
||||||
|
|
||||||
- char *requested_kinds = MPIR_pmi_get_jobattr("PMI_mpi_memory_alloc_kinds");
|
|
||||||
- MPIR_get_supported_memory_kinds(requested_kinds, &MPIR_Process.memory_alloc_kinds);
|
|
||||||
- MPL_free(requested_kinds);
|
|
||||||
-
|
|
||||||
return mpi_errno;
|
|
||||||
}
|
|
||||||
@ -1,45 +1,35 @@
|
|||||||
final: prev:
|
final: prev:
|
||||||
{
|
{
|
||||||
# Set MPICH as default
|
bsc = prev.bsc.extend (bscFinal: bscPrev: {
|
||||||
mpi = final.mpich;
|
# Set MPICH as default
|
||||||
|
mpi = bscFinal.mpich;
|
||||||
|
|
||||||
# Configure the network for MPICH
|
# Configure the network for MPICH
|
||||||
mpich = with final; let
|
mpich = with final; prev.mpich.overrideAttrs (old: {
|
||||||
# pmix comes with the libraries in .out and headers in .dev
|
buildInput = old.buildInputs ++ [
|
||||||
pmixAll = symlinkJoin {
|
libfabric
|
||||||
name = "pmix-all";
|
pmix
|
||||||
paths = [ pmix.dev pmix.out ];
|
];
|
||||||
};
|
configureFlags = [
|
||||||
in prev.mpich.overrideAttrs (old: {
|
"--enable-shared"
|
||||||
patches = [
|
"--enable-sharedlib"
|
||||||
# See https://github.com/pmodels/mpich/issues/6946
|
"--with-pm=no"
|
||||||
./mpich-fix-hwtopo.patch
|
"--with-device=ch4:ofi"
|
||||||
];
|
"--with-pmi=pmix"
|
||||||
buildInput = old.buildInputs ++ [
|
"--with-pmix=${final.pmix}"
|
||||||
libfabric
|
"--with-libfabric=${final.libfabric}"
|
||||||
pmixAll
|
"--enable-g=log"
|
||||||
];
|
] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
|
||||||
configureFlags = [
|
"FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
|
||||||
"--enable-shared"
|
"FCFLAGS=-fallow-argument-mismatch"
|
||||||
"--enable-sharedlib"
|
];
|
||||||
"--with-pm=no"
|
});
|
||||||
"--with-device=ch4:ofi"
|
|
||||||
"--with-pmi=pmix"
|
|
||||||
"--with-pmix=${pmixAll}"
|
|
||||||
"--with-libfabric=${libfabric}"
|
|
||||||
"--enable-g=log"
|
|
||||||
] ++ lib.optionals (lib.versionAtLeast gfortran.version "10") [
|
|
||||||
"FFLAGS=-fallow-argument-mismatch" # https://github.com/pmodels/mpich/issues/4300
|
|
||||||
"FCFLAGS=-fallow-argument-mismatch"
|
|
||||||
];
|
|
||||||
});
|
});
|
||||||
|
|
||||||
slurm = prev.slurm.overrideAttrs (old: {
|
# Update ceph to 18.2.0 until it lands in nixpkgs, see:
|
||||||
patches = (old.patches or []) ++ [
|
# https://github.com/NixOS/nixpkgs/pull/247849
|
||||||
# See https://bugs.schedmd.com/show_bug.cgi?id=19324
|
inherit (prev.callPackage ./ceph.nix {
|
||||||
./slurm-rank-expansion.patch
|
lua = prev.lua5_4;
|
||||||
];
|
fmt = prev.fmt_8;
|
||||||
});
|
}) ceph ceph-client;
|
||||||
|
|
||||||
prometheus-slurm-exporter = prev.callPackage ./slurm-exporter.nix { };
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,22 +0,0 @@
|
|||||||
{ buildGoModule, fetchFromGitHub, lib }:
|
|
||||||
|
|
||||||
buildGoModule rec {
|
|
||||||
pname = "prometheus-slurm-exporter";
|
|
||||||
version = "0.20";
|
|
||||||
|
|
||||||
src = fetchFromGitHub {
|
|
||||||
rev = version;
|
|
||||||
owner = "vpenso";
|
|
||||||
repo = pname;
|
|
||||||
sha256 = "sha256-KS9LoDuLQFq3KoKpHd8vg1jw20YCNRJNJrnBnu5vxvs=";
|
|
||||||
};
|
|
||||||
|
|
||||||
vendorHash = "sha256-A1dd9T9SIEHDCiVT2UwV6T02BSLh9ej6LC/2l54hgwI=";
|
|
||||||
doCheck = false;
|
|
||||||
|
|
||||||
meta = with lib; {
|
|
||||||
description = "Prometheus SLURM Exporter";
|
|
||||||
homepage = "https://github.com/vpenso/prometheus-slurm-exporter";
|
|
||||||
platforms = platforms.linux;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
--- a/src/plugins/mpi/pmix/pmixp_dmdx.c 2024-03-15 13:05:24.815313882 +0100
|
|
||||||
+++ b/src/plugins/mpi/pmix/pmixp_dmdx.c 2024-03-15 13:09:53.936900823 +0100
|
|
||||||
@@ -314,7 +314,7 @@ static void _dmdx_req(buf_t *buf, int no
|
|
||||||
}
|
|
||||||
|
|
||||||
nsptr = pmixp_nspaces_local();
|
|
||||||
- if (nsptr->ntasks <= rank) {
|
|
||||||
+ if ((long) nsptr->ntasks <= (long) rank) {
|
|
||||||
char *nodename = pmixp_info_job_host(nodeid);
|
|
||||||
PMIXP_ERROR("Bad request from %s: nspace \"%s\" has only %d ranks, asked for %d",
|
|
||||||
nodename, ns, nsptr->ntasks, rank);
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
age-encryption.org/v1
|
|
||||||
-> ssh-ed25519 HY2yRg DQdgCk16Yu524BsrWVf0krnwWzDM6SeaJCgQipOfwCA
|
|
||||||
Ab9ocqra/UWJZI+QGMlxUhBu5AzqfjPgXl+ENIiHYGs
|
|
||||||
-> ssh-ed25519 CAWG4Q KF9rGCenb3nf+wyz2hyVs/EUEbsmUs5R+1fBxlCibC8
|
|
||||||
7++Kxbr3FHVdVfnFdHYdAuR0Tgfd+sRcO6WRss6LhEw
|
|
||||||
-> ssh-ed25519 MSF3dg aUe4DhRsu4X8CFOEAnD/XM/o/0qHYSB522woCaAVh0I
|
|
||||||
GRcs5cm2YqA/lGhUtbpboBaz7mfgiLaCr+agaB7vACU
|
|
||||||
--- 9Q7Ou+Pxq+3RZilCb2dKC/pCFjZEt4rp5KnTUUU7WJ8
|
|
||||||
1¬Mw4‘Í ì:Hµ@Á/ägLtMÇ,ßÆ¥ô*¡žzñNV5ˆm‚ÍNŽoÞáj1 $÷TøG_³E{Œ%“‰1ǯ‘<>H£îAÛp™
|
|
||||||
@ -1,9 +0,0 @@
|
|||||||
age-encryption.org/v1
|
|
||||||
-> ssh-ed25519 HY2yRg WvKK6U1wQtx2pbUDfuaUIXTQiCulDkz7hgUCSwMfMzQ
|
|
||||||
jLktUMqKuVxukqzz++pHOKvmucUQqeKYy5IwBma7KxY
|
|
||||||
-> ssh-ed25519 CAWG4Q XKGuNNoYFl9bdZzsqYYTY7GsEt5sypLW4R+1uk78NmU
|
|
||||||
8dIA2GzRAwTGM5CDHSM2BUBsbXzEAUssWUz2PY2PaTg
|
|
||||||
-> ssh-ed25519 MSF3dg T630RsKuZIF/bp+KITnIIWWHsg6M/VQGqbWQZxqT+AA
|
|
||||||
SraZcgZJVtmUzHF/XR9J7aK5t5EDNpkC/av/WJUT/G8
|
|
||||||
--- /12G8pj9sbs591OM/ryhoLnSWWmzYcoqprk9uN/3g18
|
|
||||||
ä·ù¼Â‡%å]yi"ô<>»LÓ âùH`ªa$Æþ)¦9ve<76>.0úmÉK<EFBFBD>vƒÀ ïu"|1cÞ-%ÔÕ"åWFï¡ÞA«<41>hº$•ºj<eñ¶xÅLx«ç.?œÈâ:L…¬–ƒ,ëu»|³‹F|Õi²äÔ
|
|
||||||
Binary file not shown.
@ -1,10 +0,0 @@
|
|||||||
age-encryption.org/v1
|
|
||||||
-> ssh-ed25519 HY2yRg 3L1Y5upc5qN6fgiFAox5rD/W8n0eQUv5mT39QAdO5Ac
|
|
||||||
XkWsmPmzRgHjsvJgsDKJRgHZ7/sBZFmd1Doppj/y390
|
|
||||||
-> ssh-ed25519 CAWG4Q v03Qr+fckdIpsxvQG/viKxlF8WNpO4XUe//QcPzH4k0
|
|
||||||
afUwi3ccDCRfUxPDdF7ZkoL+0UX1XwqVtiyabDWjVQk
|
|
||||||
-> ssh-ed25519 MSF3dg c2hEUk4LslJpiL7v/4UpT8fK7ZiBJ8+uRhZ/vBoRUDE
|
|
||||||
YX9EpnJpHo1eDsZtapTVY6jD+81kb588Oik4NoY9jro
|
|
||||||
--- LhUkopNtCsyHCLzEYzBFs+vekOkAR4B3VBaiMF/ZF8w
|
|
||||||
oË<EFBFBD>…×à»ÂC‚ßHãáàùýy—LØ”ItMèÕåµI×±sMÆ\Í1-±K”ˆ¤‰G:õ™<02>¦
|
|
||||||
ÝgáºÙbpF¼Ó¶Í%Y·
|
|
||||||
11
secrets/nosv-token.age
Normal file
11
secrets/nosv-token.age
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
age-encryption.org/v1
|
||||||
|
-> ssh-ed25519 HY2yRg hrdS7Dl/j+u3XVfM79ZJpZSlre9TcD7DTQ+EEAT6kEE
|
||||||
|
avUO96P1h7w2BYWgrQ7GpUgdaCV9AZL7eOTTcF9gfro
|
||||||
|
-> ssh-ed25519 CAWG4Q A5raRY1CAgFYZgoQ92GMyNejYNdHx/7Y6uTS+EjLPWA
|
||||||
|
FRFqT2Jz7qRcybaxkQTKHGl797LVXoHpYG4RZSrX/70
|
||||||
|
-> ssh-ed25519 MSF3dg D+R80Bg7W9AuiOMAqtGFZQl994dRBIegYRLmmTaeZ3o
|
||||||
|
BHvZsugRiuZ91b4jk91h30o3eF3hadSnVCwxXge95T8
|
||||||
|
-> BT/El`a-grease W{nq|Vm )bld 2Nl}4 N$#JGB4t
|
||||||
|
oLG+0S1aGfO/ohCfgGmhDhwwLi4H
|
||||||
|
--- 2I5C+FvBG/K1ZHh7C5QD39feTSLoFGwcTeZAmeILNsI
|
||||||
|
¹õW©o÷ ÙÄd;ËÐC¾.¹¡_(“u
G¡€‰#ìvâœgÉ<67>†õõy¹Y‰žl9ŒÈ¡Ïµ.Œé0x<30>Þ½úN. /ü<>tB×b‡ü¼K¼ì:Q×—È\¹ÀÍT_´»Átx ïm’——_JñÞž-š
|
||||||
BIN
secrets/ovni-token.age
Normal file
BIN
secrets/ovni-token.age
Normal file
Binary file not shown.
@ -6,11 +6,9 @@ let
|
|||||||
safe = keys.hostGroup.safe ++ adminsKeys;
|
safe = keys.hostGroup.safe ++ adminsKeys;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
"gitea-runner-token.age".publicKeys = hut;
|
"ovni-token.age".publicKeys = hut;
|
||||||
"gitlab-runner-docker-token.age".publicKeys = hut;
|
"nosv-token.age".publicKeys = hut;
|
||||||
"gitlab-runner-shell-token.age".publicKeys = hut;
|
|
||||||
"nix-serve.age".publicKeys = hut;
|
"nix-serve.age".publicKeys = hut;
|
||||||
"jungle-robot-password.age".publicKeys = hut;
|
|
||||||
|
|
||||||
"ceph-user.age".publicKeys = safe;
|
"ceph-user.age".publicKeys = safe;
|
||||||
"munge-key.age".publicKeys = safe;
|
"munge-key.age".publicKeys = safe;
|
||||||
|
|||||||
@ -1,6 +0,0 @@
|
|||||||
---
|
|
||||||
title: "Git"
|
|
||||||
description: "Gitea instance"
|
|
||||||
---
|
|
||||||
|
|
||||||
If you are reading this page, the proxy to the Gitea service is not working.
|
|
||||||
7
web/content/intro-nix/_index.md
Normal file
7
web/content/intro-nix/_index.md
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
---
|
||||||
|
title: "Intro to nix"
|
||||||
|
date: 2023-09-15
|
||||||
|
---
|
||||||
|
|
||||||
|
Basic introduction to Nix for users of the jungle machines. You should be able
|
||||||
|
to access the jungle machines, otherwise [request access](/access).
|
||||||
100
web/content/intro-nix/ch1.md
Normal file
100
web/content/intro-nix/ch1.md
Normal file
@ -0,0 +1,100 @@
|
|||||||
|
---
|
||||||
|
title: "Chapter 1: Packages"
|
||||||
|
description: "Here we show where packages come from"
|
||||||
|
date: 2023-06-13T19:36:57+02:00
|
||||||
|
weight: 1
|
||||||
|
---
|
||||||
|
|
||||||
|
In this chapter we describe where the packages available in the cluster come
|
||||||
|
from and how to load them.
|
||||||
|
|
||||||
|
## Where packages come from
|
||||||
|
|
||||||
|
The packages in the jungle cluster are constructed by *layers*. Each layer
|
||||||
|
applies some changes over the previous one:
|
||||||
|
|
||||||
|
1. The first layer is [nixpkgs][1], a large repository of packages maintained by
|
||||||
|
the NixOS community. It provides packages like gcc, bash, gcc or the linux
|
||||||
|
kernel.
|
||||||
|
|
||||||
|
[1]: https://github.com/NixOS/nixpkgs/
|
||||||
|
|
||||||
|
2. The second layer is [bscpkgs][2], it takes the nixpkgs set of packages and
|
||||||
|
expands it by adding custom packages from the BSC such as Nanos6, nOS-V,
|
||||||
|
NODES, ovni or wxparaver.
|
||||||
|
|
||||||
|
[2]: https://pm.bsc.es/gitlab/rarias/bscpkgs
|
||||||
|
|
||||||
|
3. The third layer is [jungle][3], it takes the extended packages from bscpkgs
|
||||||
|
and configures them for the jungle cluster. For example, we configure MPICH
|
||||||
|
to use the OmniPath network and set it as the default implementation.
|
||||||
|
|
||||||
|
[3]: https://pm.bsc.es/gitlab/rarias/jungle
|
||||||
|
|
||||||
|
These layers are called *overlays* in Nix and they are the default mechanism
|
||||||
|
used to modify the packages. Generally you will use the packages defined in the
|
||||||
|
last layer (jungle) but you can define your own additional layer to specify
|
||||||
|
custom changes. For example, instead of choosing MPICH, you may want to use
|
||||||
|
Intel MPI instead by default.
|
||||||
|
|
||||||
|
## Loading packages in an ephemeral shell
|
||||||
|
|
||||||
|
You can manually load packages in a *new* shell with `nix shell jungle#<pkg>`,
|
||||||
|
for example:
|
||||||
|
|
||||||
|
```
|
||||||
|
hut% which ovniemu
|
||||||
|
ovniemu not found
|
||||||
|
hut% nix shell jungle#bsc.ovni
|
||||||
|
hut% which ovniemu
|
||||||
|
/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/bin/ovniemu
|
||||||
|
hut% exit
|
||||||
|
hut%
|
||||||
|
```
|
||||||
|
|
||||||
|
You can also specify multiple packages by listing them as parameters of `nix
|
||||||
|
shell`:
|
||||||
|
|
||||||
|
```
|
||||||
|
hut% nix shell jungle#bsc.ovni jungle#bsc.osumb
|
||||||
|
hut% which osu_bw
|
||||||
|
/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1/bin/osu_bw
|
||||||
|
hut% exit
|
||||||
|
```
|
||||||
|
|
||||||
|
Or make the bash (zsh in this case) shell expand them:
|
||||||
|
|
||||||
|
```
|
||||||
|
hut% echo nix shell jungle#bsc.{ovni,osumb}
|
||||||
|
nix shell jungle#bsc.ovni jungle#bsc.osumb
|
||||||
|
hut% nix shell jungle#bsc.{ovni,osumb}
|
||||||
|
hut% which osu_bw
|
||||||
|
/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1/bin/osu_bw
|
||||||
|
hut% exit
|
||||||
|
```
|
||||||
|
|
||||||
|
You can use TAB to see which packages are available:
|
||||||
|
|
||||||
|
```
|
||||||
|
hut% nix shell jungle#bsc.n<TAB>
|
||||||
|
jungle\#bsc.nanos6 jungle\#bsc.nixtools
|
||||||
|
jungle\#bsc.nanos6Debug jungle\#bsc.nix-wrap
|
||||||
|
jungle\#bsc.nanos6Git jungle\#bsc.nodes
|
||||||
|
jungle\#bsc.nanos6GlibcxxDebug jungle\#bsc.nodesGit
|
||||||
|
jungle\#bsc.nanos6-icc jungle\#bsc.nodesRelease
|
||||||
|
jungle\#bsc.nanos6-icx jungle\#bsc.nodesWithOvni
|
||||||
|
jungle\#bsc.nanos6Release jungle\#bsc.nosv
|
||||||
|
jungle\#bsc.nix-mn4
|
||||||
|
```
|
||||||
|
|
||||||
|
Notice that these packages are evaluated at the moment the command is invoked.
|
||||||
|
So if you come back a month later and run the same command, you may find that
|
||||||
|
the packages have been updated and that could be problematic.
|
||||||
|
|
||||||
|
In the next section we will create a new flake that defines the packages of the
|
||||||
|
shell and also records the exact version of the packages that we used at the
|
||||||
|
evaluation time for future use.
|
||||||
|
|
||||||
|
In the [next chapter](../ch2) we will see how to create a permanent shell that
|
||||||
|
will retain the same packages even if they are upgraded in the cluster, until we
|
||||||
|
decide to upgrade them.
|
||||||
155
web/content/intro-nix/ch2.md
Normal file
155
web/content/intro-nix/ch2.md
Normal file
@ -0,0 +1,155 @@
|
|||||||
|
---
|
||||||
|
title: "Chapter 2: Your first shell"
|
||||||
|
date: 2023-09-15
|
||||||
|
weight: 2
|
||||||
|
---
|
||||||
|
|
||||||
|
## Creating a shell with flake.nix
|
||||||
|
|
||||||
|
First, create an empty git repository where your shells will live:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% mkdir jungle-examples
|
||||||
|
hut% cd jungle-examples
|
||||||
|
hut% git init
|
||||||
|
Initialized empty Git repository in /home/Computational/rarias/jungle-examples/.git/
|
||||||
|
```
|
||||||
|
|
||||||
|
And then, place a file named `flake.nix` on the repo with this content:
|
||||||
|
|
||||||
|
```nix
|
||||||
|
{
|
||||||
|
inputs.jungle.url = "jungle";
|
||||||
|
nixConfig.bash-prompt = "\[nix-develop\]$ ";
|
||||||
|
|
||||||
|
outputs = { self, jungle }:
|
||||||
|
let
|
||||||
|
pkgs = jungle.outputs.packages.x86_64-linux;
|
||||||
|
in {
|
||||||
|
devShells.x86_64-linux.default = pkgs.mkShell rec {
|
||||||
|
pname = "my-shell";
|
||||||
|
buildInputs = with pkgs.bsc; [
|
||||||
|
ovni osumb # other packages here...
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This file defines a how to create a shell in the Nix language with the
|
||||||
|
`pkgs.mkShell` function using the packages listed in `buildInputs`. It also
|
||||||
|
requests the packages to be taken from the *jungle* input, which corresponds to
|
||||||
|
the set of packages that [we defined earlier](../ch1#where-packages-come-from),
|
||||||
|
tuned for the cluster. We will describe it in more detail later.
|
||||||
|
|
||||||
|
The tool `nix develop` tries to find a flake.nix in the current directory and
|
||||||
|
enter the shell described by `devShells.x86_64-linux.default` (or the
|
||||||
|
corresponding architecture).
|
||||||
|
|
||||||
|
Now, **it is important that all the files of the repository are committed in
|
||||||
|
git**, as nix will only read what is in the index of git. If we try to enter the
|
||||||
|
shell with the `nix develop` command, it will complain and fail:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% nix develop
|
||||||
|
warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
|
||||||
|
error: getting status of '/nix/store/0ccnxa25whszw7mgbgyzdm4nqc0zwnm8-source/flake.nix': No such file or directory
|
||||||
|
```
|
||||||
|
|
||||||
|
The first warning states that the git directory has modified files not added to
|
||||||
|
the index. Then the error occurs because the flake.nix is not in the index of
|
||||||
|
git, so `nix develop` doesn't see it. So let's add it to a commit and try again:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% git add flake.nix
|
||||||
|
hut% git commit flake.nix -m 'First shell'
|
||||||
|
[master (root-commit) eb8a4ac] First shell
|
||||||
|
1 file changed, 13 insertions(+)
|
||||||
|
create mode 100644 flake.nix
|
||||||
|
hut% nix develop
|
||||||
|
warning: creating lock file '/home/Computational/rarias/jungle-examples/flake.lock'
|
||||||
|
warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
|
||||||
|
[nix-develop]$
|
||||||
|
```
|
||||||
|
|
||||||
|
In the `flake.nix` we have set the shell prompt to `[nix-develop]` so we can
|
||||||
|
easily spot that we are inside a `nix develop` shell. To exit:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
[nix-develop]$ exit
|
||||||
|
hut%
|
||||||
|
```
|
||||||
|
|
||||||
|
## Using the flake.lock file
|
||||||
|
|
||||||
|
Now we see the `creating lock file` message and the git tree becomes dirty
|
||||||
|
again (however, we enter the shell successfully).
|
||||||
|
|
||||||
|
This `flake.lock` file that has been created collects the current state of the
|
||||||
|
jungle packages in a file, so future invocations will use the same versions. We
|
||||||
|
can see more details with `nix flake metadata`:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% nix flake metadata
|
||||||
|
warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
|
||||||
|
Resolved URL: git+file:///home/Computational/rarias/jungle-examples
|
||||||
|
Locked URL: git+file:///home/Computational/rarias/jungle-examples
|
||||||
|
Path: /nix/store/bckxqjkkv52hy4pzgb96r7fchhmvmql8-source
|
||||||
|
Revision: eb8a4ac544a74e3995d859c751e9ff4339de6509-dirty
|
||||||
|
Last modified: 2023-09-15 13:06:12
|
||||||
|
Inputs:
|
||||||
|
└───jungle: path:/nix/store/3wv6q0f3pkgw840nnkn4jsp9xi650dyj-source?lastModified=1694772033&narHash=sha256-7a09O0Jb8WncxeB32ywmQEMqJdEFLrOG/XVT9bdII6I%3D&rev=653d411b9e46076a7878be9574ed6b3bd627cff1&revCount=195
|
||||||
|
├───agenix: github:ryantm/agenix/d8c973fd228949736dedf61b7f8cc1ece3236792
|
||||||
|
│ ├───darwin: github:lnl7/nix-darwin/87b9d090ad39b25b2400029c64825fc2a8868943
|
||||||
|
│ │ └───nixpkgs follows input 'jungle/agenix/nixpkgs'
|
||||||
|
│ ├───home-manager: github:nix-community/home-manager/32d3e39c491e2f91152c84f8ad8b003420eab0a1
|
||||||
|
│ │ └───nixpkgs follows input 'jungle/agenix/nixpkgs'
|
||||||
|
│ └───nixpkgs follows input 'jungle/nixpkgs'
|
||||||
|
├───bscpkgs: git+https://pm.bsc.es/gitlab/rarias/bscpkgs.git?ref=refs/heads/master&rev=3a4062ac04be6263c64a481420d8e768c2521b80
|
||||||
|
│ └───nixpkgs follows input 'jungle/nixpkgs'
|
||||||
|
└───nixpkgs: github:NixOS/nixpkgs/e56990880811a451abd32515698c712788be5720
|
||||||
|
```
|
||||||
|
|
||||||
|
Now, as long as we keep these two files `flake.nix` and `flake.lock`, we can
|
||||||
|
reproduce the same shell in the future, so let's add the lock file into git too.
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% git commit -m 'Add flake.lock file'
|
||||||
|
[master d3725ec] Add flake.lock file
|
||||||
|
1 file changed, 135 insertions(+)
|
||||||
|
create mode 100644 flake.lock
|
||||||
|
hut% git status
|
||||||
|
On branch master
|
||||||
|
nothing to commit, working tree clean
|
||||||
|
```
|
||||||
|
|
||||||
|
## Using the shell with nix develop
|
||||||
|
|
||||||
|
Now, the invocations of `nix develop` won't complain that the git tree is dirty
|
||||||
|
anymore and will enter the shell:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% nix develop
|
||||||
|
[nix-develop]$
|
||||||
|
```
|
||||||
|
|
||||||
|
And the requested packages are now available:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
[nix-develop]$ which ovniemu
|
||||||
|
/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/bin/ovniemu
|
||||||
|
```
|
||||||
|
|
||||||
|
The packages of the shell are listed in the `$buildInputs` variable, in case you
|
||||||
|
need to examine them:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
[nix-develop]$ printf '%s\n' $buildInputs
|
||||||
|
/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0
|
||||||
|
/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1
|
||||||
|
[nix-develop]$ exit
|
||||||
|
hut%
|
||||||
|
```
|
||||||
|
|
||||||
|
In the [next chapter](../ch3) we will see how to add more packages and also how to modify
|
||||||
|
them.
|
||||||
160
web/content/intro-nix/ch3.md
Normal file
160
web/content/intro-nix/ch3.md
Normal file
@ -0,0 +1,160 @@
|
|||||||
|
---
|
||||||
|
title: "Chapter 3: Custom packages"
|
||||||
|
date: 2023-09-15
|
||||||
|
weight: 3
|
||||||
|
---
|
||||||
|
|
||||||
|
## Adding more packages
|
||||||
|
|
||||||
|
So far we have define all the packages using:
|
||||||
|
|
||||||
|
```nix
|
||||||
|
pkgs.mkShell rec {
|
||||||
|
pname = "my-shell";
|
||||||
|
buildInputs = with pkgs.bsc; [
|
||||||
|
ovni osumb # other packages here...
|
||||||
|
];
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
This line specifies that all packages come from the `pkgs.bsc` set. We can add
|
||||||
|
additional packages adding them to the list:
|
||||||
|
|
||||||
|
```nix
|
||||||
|
pkgs.mkShell rec {
|
||||||
|
pname = "my-shell";
|
||||||
|
buildInputs = with pkgs.bsc; [
|
||||||
|
ovni osumb sonar
|
||||||
|
];
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
And running `nix develop` again:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% nix develop
|
||||||
|
warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
|
||||||
|
[nix-develop]$ printf '%s\n' $buildInputs
|
||||||
|
/nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0
|
||||||
|
/nix/store/lnjirzllhjn2fadlqzrz7a547iawl8jc-osu-micro-benchmarks-7.1-1
|
||||||
|
/nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0
|
||||||
|
```
|
||||||
|
|
||||||
|
In the jungle cluster, the default MPI implementation is currently set to MPICH,
|
||||||
|
as it can be shown with ldd:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
[nix-develop]$ ldd $(which ovnisync) | grep mpi
|
||||||
|
libmpi.so.12 => /nix/store/nnnaly6hgylravdrmqkhpx1ndg5p79nc-mpich-4.1.2/lib/libmpi.so.12 (0x00007ffff5200000)
|
||||||
|
```
|
||||||
|
|
||||||
|
Now, what if we want to replace the MPI implementation by another one?
|
||||||
|
|
||||||
|
## Modifying a package
|
||||||
|
|
||||||
|
You notice that the packages we are using are coming directly from the ones
|
||||||
|
specified in jungle. However, what if we need to modify some option at build
|
||||||
|
time or change a dependency?
|
||||||
|
|
||||||
|
The Nix language is used to describe how to build each package, and can be
|
||||||
|
extended to create derived versions very easily.
|
||||||
|
|
||||||
|
Let's focus on the `ovni` package. First, to load the definition we can use the
|
||||||
|
`nix edit` command, which opens the definition file using the editor defined in
|
||||||
|
`$EDITOR`:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% nix edit jungle#bsc.ovni
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
This particular package has several inputs that can be modified directly:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
{
|
||||||
|
stdenv
|
||||||
|
, lib
|
||||||
|
, cmake
|
||||||
|
, mpi
|
||||||
|
, fetchFromGitHub
|
||||||
|
, useGit ? false
|
||||||
|
, gitBranch ? "master"
|
||||||
|
, gitUrl ? "ssh://git@bscpm03.bsc.es/rarias/ovni.git"
|
||||||
|
, gitCommit ? "d0a47783f20f8b177a48418966dae45454193a6a"
|
||||||
|
, enableDebug ? false
|
||||||
|
}:
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
For example, the `enableDebug` flag, currently set to false, affects how the
|
||||||
|
build is configured:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
cmakeBuildType = if (enableDebug) then "Debug" else "Release";
|
||||||
|
```
|
||||||
|
|
||||||
|
Now, to change this option we could replace `ovni` for our version:
|
||||||
|
|
||||||
|
```nix
|
||||||
|
{
|
||||||
|
inputs.jungle.url = "jungle";
|
||||||
|
nixConfig.bash-prompt = "\[nix-develop\]$ ";
|
||||||
|
|
||||||
|
outputs = { self, jungle }:
|
||||||
|
let
|
||||||
|
pkgs = jungle.outputs.packages.x86_64-linux;
|
||||||
|
ovniDebug = pkgs.bsc.ovni.override { enableDebug = true; };
|
||||||
|
in {
|
||||||
|
devShells.x86_64-linux.default = pkgs.mkShell rec {
|
||||||
|
pname = "my-shell";
|
||||||
|
buildInputs = with pkgs.bsc; [
|
||||||
|
ovniDebug osumb sonar
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
And then, when we now enter the develop shell we can see that ovni gets build
|
||||||
|
with the Debug option:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
hut% nix develop -L
|
||||||
|
warning: Git tree '/home/Computational/rarias/jungle-examples' is dirty
|
||||||
|
ovni> unpacking sources
|
||||||
|
ovni> unpacking source archive /nix/store/cz4si0vsw85r9s6dyiqr5ybngh9aympi-source
|
||||||
|
ovni> source root is source
|
||||||
|
ovni> patching sources
|
||||||
|
ovni> updateAutotoolsGnuConfigScriptsPhase
|
||||||
|
ovni> configuring
|
||||||
|
ovni> fixing cmake files...
|
||||||
|
ovni> cmake flags: ... -DCMAKE_BUILD_TYPE=Debug ...
|
||||||
|
...
|
||||||
|
[nix-develop]$ which ovniver
|
||||||
|
/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/bin/ovniver
|
||||||
|
|
||||||
|
[nix-develop]$ file $(which ovniver) | fold
|
||||||
|
/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/bin/ovniver: ELF 64-bit L
|
||||||
|
SB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /nix/st
|
||||||
|
ore/9la894yvmmksqlapd4v16wvxpaw3rg70-glibc-2.37-8/lib/ld-linux-x86-64.so.2, for
|
||||||
|
GNU/Linux 3.10.0, with debug_info, not stripped
|
||||||
|
```
|
||||||
|
|
||||||
|
And we see that the ovniver program is now compiled with debug symbols.
|
||||||
|
|
||||||
|
However, this *only* replaces the ovni package that we specify in the shell. The
|
||||||
|
sonar library also depends on ovni, but that package is still using the old one:
|
||||||
|
|
||||||
|
```txt
|
||||||
|
[nix-develop]$ find $buildInputs -name 'libovni.so.1'
|
||||||
|
/nix/store/hg0xs7fpibwjhsp9ajqfcbffsh69mrsm-ovni-1.3.0/lib/libovni.so.1
|
||||||
|
|
||||||
|
[nix-develop]$ find $buildInputs -name 'libsonar-mpi.so'
|
||||||
|
/nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0/lib/libsonar-mpi.so
|
||||||
|
|
||||||
|
[nix-develop]$ ldd /nix/store/fjxj4xs0wblw3jyhp4vsrsfnlfwawifa-sonar-0.1.0/lib/libsonar-mpi.so | grep ovni
|
||||||
|
libovni.so.1 => /nix/store/0yzas8007x9djlpbb0pckcr1vhd0mcfy-ovni-1.3.0/lib/libovni.so.1 (0x00007ffff7f8d000)
|
||||||
|
```
|
||||||
|
|
||||||
|
In the [next chapter](../ch4) we will see how to replace packages in such a way
|
||||||
|
that all the dependences are automatically updated too.
|
||||||
29
web/content/intro-nix/ch4.md
Normal file
29
web/content/intro-nix/ch4.md
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
---
|
||||||
|
title: "Chapter 4: Adding an overlay"
|
||||||
|
date: 2023-09-15
|
||||||
|
weight: 4
|
||||||
|
---
|
||||||
|
|
||||||
|
NOTE: We shouldn't be instructing users to use an overlay to replace packages in
|
||||||
|
`bsc.` until we have determined if we move them to the root attribute set
|
||||||
|
first!
|
||||||
|
|
||||||
|
```nix
|
||||||
|
{
|
||||||
|
inputs.jungle.url = "jungle";
|
||||||
|
nixConfig.bash-prompt = "\[nix-develop\]$ ";
|
||||||
|
|
||||||
|
outputs = { self, jungle }:
|
||||||
|
let
|
||||||
|
pkgs = jungle.outputs.packages.x86_64-linux;
|
||||||
|
ovniDebug = pkgs.bsc.ovni.override { enableDebug = true; };
|
||||||
|
in {
|
||||||
|
devShells.x86_64-linux.default = pkgs.mkShell rec {
|
||||||
|
pname = "my-shell";
|
||||||
|
buildInputs = with pkgs.bsc; [
|
||||||
|
ovniDebug osumb sonar
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
```
|
||||||
@ -1,6 +0,0 @@
|
|||||||
---
|
|
||||||
title: "Lists"
|
|
||||||
description: "Mailing lists"
|
|
||||||
---
|
|
||||||
|
|
||||||
If you are reading this page, the proxy to the public-inbox service is not working.
|
|
||||||
@ -8,11 +8,3 @@ description: "Low system noise"
|
|||||||
Much like the silent flight of an owl at night, these nodes are configured to
|
Much like the silent flight of an owl at night, these nodes are configured to
|
||||||
minimize the system noise and let programs run undisturbed. The list of nodes is
|
minimize the system noise and let programs run undisturbed. The list of nodes is
|
||||||
`owl[1-2]` and are available for jobs with SLURM.
|
`owl[1-2]` and are available for jobs with SLURM.
|
||||||
|
|
||||||
The contents of the nix store of the hut node is made available in the owl nodes
|
|
||||||
when a job is running. This allows jobs to access the same paths that are on hut
|
|
||||||
to load dependencies.
|
|
||||||
|
|
||||||
For now, only the hut node can be used to build new derivations so that they
|
|
||||||
appear in the compute nodes. This applies to the `nix build`, `nix develop` and
|
|
||||||
`nix shell` commands.
|
|
||||||
|
|||||||
@ -2,27 +2,7 @@ baseURL = 'https://jungle.bsc.es/'
|
|||||||
languageCode = 'en-us'
|
languageCode = 'en-us'
|
||||||
title = 'The jungle'
|
title = 'The jungle'
|
||||||
theme = 'PaperMod'
|
theme = 'PaperMod'
|
||||||
|
sectionPagesMenu = "main"
|
||||||
|
|
||||||
[[menu.main]]
|
[params]
|
||||||
identifier = "grafana"
|
ShowBreadCrumbs = true
|
||||||
name = "Grafana"
|
|
||||||
url = "/grafana/"
|
|
||||||
weight = 10
|
|
||||||
|
|
||||||
[[menu.main]]
|
|
||||||
identifier = "Git"
|
|
||||||
name = "Git"
|
|
||||||
url = "/git/"
|
|
||||||
weight = 20
|
|
||||||
|
|
||||||
[[menu.main]]
|
|
||||||
identifier = "Lists"
|
|
||||||
name = "Lists"
|
|
||||||
url = "/lists/"
|
|
||||||
weight = 30
|
|
||||||
|
|
||||||
[[menu.main]]
|
|
||||||
identifier = "Posts"
|
|
||||||
name = "Posts"
|
|
||||||
url = "/posts/"
|
|
||||||
weight = 40
|
|
||||||
|
|||||||
@ -100,9 +100,9 @@
|
|||||||
{{- $separator := or $label_text (not site.Params.disableThemeToggle)}}
|
{{- $separator := or $label_text (not site.Params.disableThemeToggle)}}
|
||||||
{{- with site.Home.AllTranslations }}
|
{{- with site.Home.AllTranslations }}
|
||||||
<ul class="lang-switch">
|
<ul class="lang-switch">
|
||||||
|
{{- if $separator }}<li>|</li>{{ end }}
|
||||||
{{- range . -}}
|
{{- range . -}}
|
||||||
{{- if ne $lang .Lang }}
|
{{- if ne $lang .Lang }}
|
||||||
{{- if $separator }}<li>|</li>{{ end }}
|
|
||||||
<li>
|
<li>
|
||||||
<a href="{{- .Permalink -}}" title="{{ .Language.Params.languageAltTitle | default (.Language.LanguageName | emojify) | default (.Lang | title) }}"
|
<a href="{{- .Permalink -}}" title="{{ .Language.Params.languageAltTitle | default (.Language.LanguageName | emojify) | default (.Lang | title) }}"
|
||||||
aria-label="{{ .Language.LanguageName | default (.Lang | title) }}">
|
aria-label="{{ .Language.LanguageName | default (.Lang | title) }}">
|
||||||
|
|||||||
@ -17,5 +17,5 @@
|
|||||||
{{- end }}
|
{{- end }}
|
||||||
|
|
||||||
{{- with ($scratch.Get "meta") }}
|
{{- with ($scratch.Get "meta") }}
|
||||||
{{- delimit . " · " | safeHTML -}}
|
{{- delimit . " · " -}}
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user