From 3f4b4fb810440cefef68d60023b8d34af0e3bf7b Mon Sep 17 00:00:00 2001 From: Rodrigo Arias Mallo Date: Fri, 24 Jan 2025 13:00:54 +0100 Subject: [PATCH] Add new GitLab runner for gitlab.bsc.es MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It uses docker based on alpine and the host nix store, so we can perform builds but isolate them from the system. Reviewed-by: Aleix Boné --- m/hut/gitlab-runner.nix | 81 ++++++++++++++++++++++++++--- secrets/gitlab-bsc-docker-token.age | 11 ++++ secrets/secrets.nix | 1 + 3 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 secrets/gitlab-bsc-docker-token.age diff --git a/m/hut/gitlab-runner.nix b/m/hut/gitlab-runner.nix index 226099b..9241e5d 100644 --- a/m/hut/gitlab-runner.nix +++ b/m/hut/gitlab-runner.nix @@ -1,8 +1,9 @@ { pkgs, lib, config, ... }: { - age.secrets.gitlabRunnerShellToken.file = ../../secrets/gitlab-runner-shell-token.age; - age.secrets.gitlabRunnerDockerToken.file = ../../secrets/gitlab-runner-docker-token.age; + age.secrets.gitlab-pm-shell.file = ../../secrets/gitlab-runner-shell-token.age; + age.secrets.gitlab-pm-docker.file = ../../secrets/gitlab-runner-docker-token.age; + age.secrets.gitlab-bsc-docker.file = ../../secrets/gitlab-bsc-docker-token.age; services.gitlab-runner = { enable = true; @@ -21,21 +22,89 @@ "--docker-network-mode host" ]; environmentVariables = { - https_proxy = "http://localhost:23080"; - http_proxy = "http://localhost:23080"; + https_proxy = "http://hut:23080"; + http_proxy = "http://hut:23080"; }; }; in { # For pm.bsc.es/gitlab gitlab-pm-shell = common-shell // { - authenticationTokenConfigFile = config.age.secrets.gitlabRunnerShellToken.path; + authenticationTokenConfigFile = config.age.secrets.gitlab-pm-shell.path; }; gitlab-pm-docker = common-docker // { - authenticationTokenConfigFile = config.age.secrets.gitlabRunnerDockerToken.path; + authenticationTokenConfigFile = config.age.secrets.gitlab-pm-docker.path; + }; + + gitlab-bsc-docker = { + # gitlab.bsc.es still uses the old token mechanism + registrationConfigFile = config.age.secrets.gitlab-bsc-docker.path; + tagList = [ "docker" "hut" ]; + environmentVariables = { + # We cannot access the hut local interface from docker, so we connect + # to hut directly via the ethernet one. + https_proxy = "http://hut:23080"; + http_proxy = "http://hut:23080"; + }; + executor = "docker"; + dockerImage = "alpine"; + dockerVolumes = [ + "/nix/store:/nix/store:ro" + "/nix/var/nix/db:/nix/var/nix/db:ro" + "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro" + ]; + dockerExtraHosts = [ + # Required to pass the proxy via hut + "hut:10.0.40.7" + ]; + dockerDisableCache = true; + registrationFlags = [ + # Increase build log length to 64 MiB + "--output-limit 65536" + ]; + preBuildScript = pkgs.writeScript "setup-container" '' + mkdir -p -m 0755 /nix/var/log/nix/drvs + mkdir -p -m 0755 /nix/var/nix/gcroots + mkdir -p -m 0755 /nix/var/nix/profiles + mkdir -p -m 0755 /nix/var/nix/temproots + mkdir -p -m 0755 /nix/var/nix/userpool + mkdir -p -m 1777 /nix/var/nix/gcroots/per-user + mkdir -p -m 1777 /nix/var/nix/profiles/per-user + mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root + mkdir -p -m 0700 "$HOME/.nix-defexpr" + mkdir -p -m 0700 "$HOME/.ssh" + cat > "$HOME/.ssh/config" << EOF + Host bscpm04.bsc.es gitlab-internal.bsc.es + User git + ProxyCommand nc -X connect -x hut:23080 %h %p + Host amdlogin1.bsc.es armlogin1.bsc.es hualogin1.bsc.es glogin1.bsc.es glogin2.bsc.es fpgalogin1.bsc.es + ProxyCommand nc -X connect -x hut:23080 %h %p + EOF + cat >> "$HOME/.ssh/known_hosts" << EOF + bscpm04.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPx4mC0etyyjYUT2Ztc/bs4ZXSbVMrogs1ZTP924PDgT + gitlab-internal.bsc.es ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9arsAOSRB06hdy71oTvJHG2Mg8zfebADxpvc37lZo3 + EOF + . ${pkgs.nix}/etc/profile.d/nix-daemon.sh + # Required to load SSL certificate paths + . ${pkgs.cacert}/nix-support/setup-hook + ''; + environmentVariables = { + ENV = "/etc/profile"; + USER = "root"; + NIX_REMOTE = "daemon"; + PATH = "${config.system.path}/bin:/bin:/sbin:/usr/bin:/usr/sbin"; + }; }; }; }; + # DOCKER* chains are useless, override at FORWARD + networking.firewall.extraCommands = '' + # Allow docker to use our proxy + iptables -I FORWARD 1 -p tcp -i docker0 -d hut --dport 23080 -j nixos-fw-accept + # Block anything else coming from docker + iptables -I FORWARD 2 -p all -i docker0 -j nixos-fw-log-refuse + ''; + #systemd.services.gitlab-runner.serviceConfig.Shell = "${pkgs.bash}/bin/bash"; systemd.services.gitlab-runner.serviceConfig.DynamicUser = lib.mkForce false; systemd.services.gitlab-runner.serviceConfig.User = "gitlab-runner"; diff --git a/secrets/gitlab-bsc-docker-token.age b/secrets/gitlab-bsc-docker-token.age new file mode 100644 index 0000000..74b83e0 --- /dev/null +++ b/secrets/gitlab-bsc-docker-token.age @@ -0,0 +1,11 @@ +age-encryption.org/v1 +-> ssh-ed25519 HY2yRg WSdjyQPzBJ4JbzQpGeq1AAYpWKoXmLI1ZtmNmM5QOzs +qGDlDT31DQF1DdHen0+5+52DdsQlabJdA2pOB5O1I6g +-> ssh-ed25519 CAWG4Q wioWMDxQjN+d4JdIbCwZg0DLQu1OH2mV6gukRprjuAs +670fE61hidOEh20hHiQAhP0+CjDF0WMBNzgwkGT8Yqg +-> ssh-ed25519 MSF3dg DN19uvAEtqq4708P6HpuX9i/o/qAvHX6dj69dCF2H1o +4Lu9GnjiFLMeXJ2C7aVPJsCHCQVlhylNWJi896Av92s +--- 7cKBwOYNOUZ2h3/kAY09aSMASZSxX7hZIT4kvlIiT6w +6fQF5=bX+v e`7/A~PѦ7 +A)h=oZ$ ^V0/܅r +kubĶ:R>^gik_*% a7KG&PIn \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index ac3eac3..58ec2d6 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -9,6 +9,7 @@ in "gitea-runner-token.age".publicKeys = hut; "gitlab-runner-docker-token.age".publicKeys = hut; "gitlab-runner-shell-token.age".publicKeys = hut; + "gitlab-bsc-docker-token.age".publicKeys = hut; "nix-serve.age".publicKeys = hut; "jungle-robot-password.age".publicKeys = hut; "ipmi.yml.age".publicKeys = hut;