From cd5853cf53714bc6c1a99591d4f13ce59ad2d8bf Mon Sep 17 00:00:00 2001
From: Rodrigo Arias Mallo <rodrigo.arias@bsc.es>
Date: Tue, 22 Aug 2023 15:57:25 +0200
Subject: [PATCH] Add ceph config in bay

---
 m/bay/configuration.nix | 68 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 67 insertions(+), 1 deletion(-)

diff --git a/m/bay/configuration.nix b/m/bay/configuration.nix
index b42abf1..847e162 100644
--- a/m/bay/configuration.nix
+++ b/m/bay/configuration.nix
@@ -1,4 +1,4 @@
-{ config, pkgs, ... }:
+{ config, pkgs, lib, ... }:
 
 {
   imports = [
@@ -8,6 +8,14 @@
   # Select the this using the ID to avoid mismatches
   boot.loader.grub.device = "/dev/disk/by-id/wwn-0x55cd2e414d53562d";
 
+  environment.systemPackages = with pkgs; [
+    ceph
+  ];
+
+  services.slurm = {
+    client.enable = lib.mkForce false;
+  };
+
   networking = {
     hostName = "bay";
     interfaces.eno1.ipv4.addresses = [ {
@@ -19,4 +27,62 @@
       prefixLength = 24;
     } ];
   };
+
+  services.ceph = {
+    enable = true;
+    global = {
+      fsid = "9c8d06e0-485f-4aaf-b16b-06d6daf1232b";
+      monHost = "10.0.40.40";
+      monInitialMembers = "bay";
+      clusterNetwork = "10.0.40.40/24"; # Use Ethernet only
+    };
+    mds = {
+      enable = true;
+      daemons = [ "mds0" "mds1" ];
+      extraConfig = {
+        "host" = "bay";
+      };
+    };
+    mgr = {
+      enable = true;
+      daemons = [ "bay" ];
+    };
+    mon = {
+      enable = true;
+      daemons = [ "bay" ];
+    };
+    osd = {
+      enable = true;
+      # One daemon per NVME disk
+      daemons = [ "0" "1" "2" "3" ];
+      extraConfig = {
+        "osd crush chooseleaf type" = "0";
+        "osd journal size" = "10000";
+        "osd pool default min size" = "2";
+        "osd pool default pg num" = "200";
+        "osd pool default pgp num" = "200";
+        "osd pool default size" = "3";
+      };
+    };
+  };
+
+  # Missing service for volumes, see:
+  # https://www.reddit.com/r/ceph/comments/14otjyo/comment/jrd69vt/
+  systemd.services.ceph-volume = {
+    enable = true;
+    description = "Ceph Volume activation";
+    unitConfig = {
+      Type = "oneshot";
+      After = "local-fs.target";
+      Wants = "local-fs.target";
+    };
+    path = [ pkgs.ceph pkgs.util-linux pkgs.lvm2 pkgs.cryptsetup ];
+    serviceConfig = {
+      KillMode = "none";
+      Environment = "CEPH_VOLUME_TIMEOUT=10000";
+      ExecStart = "/bin/sh -c 'timeout $CEPH_VOLUME_TIMEOUT ${pkgs.ceph}/bin/ceph-volume lvm activate --all --no-systemd'";
+      TimeoutSec = "0";
+    };
+    wantedBy = [ "multi-user.target" ];
+  };
 }