From eea3af80ba1d0de358ade3c443f8e531f33ce5c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Sat, 20 Dec 2025 18:25:27 +0100 Subject: [PATCH 1/3] nixosTests.gitlab.runner: add gitlab runner tests & docs (with IFD) --- nixos/doc/manual/redirects.json | 6 + nixos/modules/module-list.nix | 2 +- .../runner.nix} | 0 nixos/modules/services/misc/gitlab.md | 113 +++++ nixos/release-combined.nix | 3 +- nixos/tests/all-tests.nix | 4 +- nixos/tests/gitlab/default.nix | 5 + nixos/tests/{ => gitlab}/gitlab.nix | 6 +- nixos/tests/gitlab/runner.nix | 178 +++++++ .../gitlab/runner/podman-runner/default.nix | 442 ++++++++++++++++++ .../podman-runner/files/basicRoot/etc/group | 21 + .../files/basicRoot/etc/nsswitch.conf | 11 + .../podman-runner/files/basicRoot/etc/passwd | 34 ++ .../containers/etc/containers/containers.conf | 2 + .../containers/etc/containers/mounts.conf | 2 + .../containers/etc/containers/policy.json | 12 + .../containers/etc/containers/registries.conf | 2 + .../registries.conf.d/000-shortnames.conf | 5 + .../etc/containers/registries.d/default.yaml | 27 ++ .../registry.access.redhat.com.yaml | 3 + .../registries.d/registry.redhat.io.yaml | 3 + .../containers/etc/containers/storage.conf | 15 + .../runner/podman-runner/files/default.nix | 46 ++ .../files/fake-nixpkgs/default.nix | 10 + .../runner/podman-runner/scripts/prebuild.nix | 55 +++ .../runner/podman-runner/virtualization.nix | 43 ++ nixos/tests/gitlab/runner/shell-runner.nix | 12 + nixos/tests/gitlab/runner_test.py | 146 ++++++ 28 files changed, 1202 insertions(+), 6 deletions(-) rename nixos/modules/services/continuous-integration/{gitlab-runner.nix => gitlab-runner/runner.nix} (100%) create mode 100644 nixos/tests/gitlab/default.nix rename nixos/tests/{ => gitlab}/gitlab.nix (99%) create mode 100644 nixos/tests/gitlab/runner.nix create mode 100644 nixos/tests/gitlab/runner/podman-runner/default.nix create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/group create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/nsswitch.conf create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/passwd create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/containers.conf create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/mounts.conf create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/policy.json create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.conf create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.conf.d/000-shortnames.conf create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/default.yaml create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/registry.access.redhat.com.yaml create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/registry.redhat.io.yaml create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/storage.conf create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/default.nix create mode 100644 nixos/tests/gitlab/runner/podman-runner/files/fake-nixpkgs/default.nix create mode 100644 nixos/tests/gitlab/runner/podman-runner/scripts/prebuild.nix create mode 100644 nixos/tests/gitlab/runner/podman-runner/virtualization.nix create mode 100644 nixos/tests/gitlab/runner/shell-runner.nix create mode 100644 nixos/tests/gitlab/runner_test.py diff --git a/nixos/doc/manual/redirects.json b/nixos/doc/manual/redirects.json index 4385b88e7cea..654debde8da7 100644 --- a/nixos/doc/manual/redirects.json +++ b/nixos/doc/manual/redirects.json @@ -1185,6 +1185,12 @@ "module-services-gitlab-maintenance-rake": [ "index.html#module-services-gitlab-maintenance-rake" ], + "module-services-gitlab-runner": [ + "index.html#module-services-gitlab-runner" + ], + "ex-gitlab-runner-podman": [ + "index.html#ex-gitlab-runner-podman" + ], "module-forgejo": [ "index.html#module-forgejo" ], diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index f0beafd77364..2bf95dc1c926 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -502,7 +502,7 @@ ./services/continuous-integration/buildkite-agents.nix ./services/continuous-integration/gitea-actions-runner.nix ./services/continuous-integration/github-runners.nix - ./services/continuous-integration/gitlab-runner.nix + ./services/continuous-integration/gitlab-runner/runner.nix ./services/continuous-integration/gocd-agent/default.nix ./services/continuous-integration/gocd-server/default.nix ./services/continuous-integration/hercules-ci-agent/default.nix diff --git a/nixos/modules/services/continuous-integration/gitlab-runner.nix b/nixos/modules/services/continuous-integration/gitlab-runner/runner.nix similarity index 100% rename from nixos/modules/services/continuous-integration/gitlab-runner.nix rename to nixos/modules/services/continuous-integration/gitlab-runner/runner.nix diff --git a/nixos/modules/services/misc/gitlab.md b/nixos/modules/services/misc/gitlab.md index 60c0e9310acd..0e29071d222d 100644 --- a/nixos/modules/services/misc/gitlab.md +++ b/nixos/modules/services/misc/gitlab.md @@ -10,6 +10,7 @@ configure a webserver to proxy HTTP requests to the socket. For instance, the following configuration could be used to use nginx as frontend proxy: + ```nix { services.nginx = { @@ -40,6 +41,7 @@ The default state dir is `/var/gitlab/state`. This is where all data like the repositories and uploads will be stored. A basic configuration with some custom settings could look like this: + ```nix { services.gitlab = { @@ -104,6 +106,7 @@ the [services.gitlab.backup.startAt](#opt-services.gitlab.backup.startAt) option to configure regular backups. To run a manual backup, start the `gitlab-backup` service: + ```ShellSession $ systemctl start gitlab-backup.service ``` @@ -116,6 +119,116 @@ will have to run the command as the user that you configured to run GitLab with. A list of all available rake tasks can be obtained by running: + ```ShellSession $ sudo -u git -H gitlab-rake -T ``` + +## Runner {#module-services-gitlab-runner} + +GitLab Runner is a CI runner which is an executable which you can host yourself. +A Gitlab pipeline runs operations over a Gitlab Runner. These can include +building an executable, running a test suite, pushing a docker image, etc. The +Gitlab Runner receives jobs from Gitlab which it then dispatches to the +configured executors +([`docker` (`podman`), or `shell` or `kubernetes`](https://docs.gitlab.com/runner/executors)). + +The +[services.gitlab-runner.services](https://search.nixos.org/options?query=services.gitlab-runner.services) +documents a number of typical setups to configure multiple runners with +different executors. + +The [below example](#ex-gitlab-runner-podman) gives a **more elaborate** example how to +configure a Gitlab Runner with caching and reasonably good security practices. + +::: {#ex-gitlab-runner-podman .example} + +## Example: Gitlab Runner with `podman` and Nix Store Caching + +The [VM tested `podman-runner`](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/continuous-integration/gitlab-runner/runner.nix) +(a NixOS module for reuse) configures an advanced Gitlab runner with the following features: + +- The executor is `podman` which gives you better additional safety than + `docker`. That means every job is run in a `podman` container. + +- The following container **images** are built with Nix: + + **Container Images for Gitlab Jobs**: + - `local/alpine`: An image based on Alpine with a Nix installation + (attribute `jobImages.alpine`). + - `local/ubuntu`: An image based on Ubuntu with a Nix installation + (attribute `jobImages.ubuntu`). + - `local/nix`: An image based on Nix which only comes with `nix` + installed (attribute `jobImages.nix`). + + **Images for VM Setup**: + - `local/nix-daemon-image`: An image with a Nix daemon which is + used to share the `/nix/store` across jobs (variable `nixDaemonImage`) setup with some essentials derivations `bootstrapPkgs`. + - `local/podman-daemon-image`: An image with `podman` running as a daemon which is + used to run `podman` inside the above job containers images + (variable `podmanDaemonImage`). + +- Every job container runs in a `podman` container instance based by default on + `jobImage.ubuntu`. A pipeline job can override this with `image: local/alpine`. + - Each job container will have the `/nix/store` mounted from the container + `nix-daemon-container` (see registration flags + `--docker-volumes-from "nix-daemon-container:ro"`). + + The `nix-daemon-container` is a single container instance of a + `nixDaemonImage`. This enables caching of `/nix/store` paths across all jobs + in **all** runners. This makes **the host VM's `/nix/store` independent of the + Nix store used in the jobs**, which is good. + + ::: {.note} + **Security:** If you don't want this you need multiple `nixDaemonImage` + containers for each registered runner (`gitlab-runner.services.`). + ::: + + - Each job container will have the `/run/podman/podman.sock` socket mounted from the + `podman-daemon-container`. + + The `podman-daemon-container` is a single container of a `podmanDaemonImage` which runs + `podman` as a daemon. Job containers can use this daemon to spawn nested containers as well (podman-in-podman). + **Keep in mind that `bind` mounts are local to the `podman-daemon-container`** + and can be be worked around with a `podman volume create ` and manual copy-to/copy-from this volume ``. + + If you only need to build containers you don't need this feature (`podman-daemon-container`), see below point. + + Container configuration files (`auxRootFiles`) are copied to all containers to + ensure `podman` works consistently inside the job containers. + + - The job containers do **not** mount the `podman` socket from the host (NixOS + VM) mounted for security reasons. + + ::: {.note} + Building container images with `buildah` (stripped + `podman` for building images) inside a job which runs `jobImage.alpine` + is still possible. + ::: + + - **Cleanup Disk Space**: + + With this setup its really easy to clean the `nix-daemon-container` + (e.g. if you run out of disk space), then reboot and have the runner in a clean state. + You can do the following to effectively clean everything and start with fresh volumes safely: + + ```bash + # Stop the Gitlab runner. + systemctl stop gitlab-runner.service + # Stop `systemd`-managed containers, such that they get not recreated + # when deleting below. + systemctl stop podman-podman-daemon-container.service \ + podman-nix-daemon-container.service \ + podman-nix-container.service \ + podman-alpine-container.service \ + podman-ubuntu-container.service || true + + podman container rm -f --all + podman image rm -f --all + podman volumes rm -f --all + + reboot + # Systemd will restart all containers and create volumes etc. + ``` + +::: diff --git a/nixos/release-combined.nix b/nixos/release-combined.nix index da66d7c323f3..6730a9faa4d8 100644 --- a/nixos/release-combined.nix +++ b/nixos/release-combined.nix @@ -98,7 +98,8 @@ rec { (onFullSupported "nixos.tests.firewall") (onFullSupported "nixos.tests.fontconfig-default-fonts") - (onFullSupported "nixos.tests.gitlab") + (onFullSupported "nixos.tests.gitlab.gitlab") + (onFullSupported "nixos.tests.gitlab.runner") (onFullSupported "nixos.tests.gnome") (onSystems [ "x86_64-linux" ] "nixos.tests.hibernate") (onFullSupported "nixos.tests.i3wm") diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 58cddfabe0b7..7717e9ee9253 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -643,7 +643,9 @@ in gitdaemon = runTest ./gitdaemon.nix; gitea = handleTest ./gitea.nix { giteaPackage = pkgs.gitea; }; github-runner = runTest ./github-runner.nix; - gitlab = runTest ./gitlab.nix; + gitlab = import ./gitlab { + inherit runTest; + }; gitolite = runTest ./gitolite.nix; gitolite-fcgiwrap = runTest ./gitolite-fcgiwrap.nix; glance = runTest ./glance.nix; diff --git a/nixos/tests/gitlab/default.nix b/nixos/tests/gitlab/default.nix new file mode 100644 index 000000000000..0e8841e95187 --- /dev/null +++ b/nixos/tests/gitlab/default.nix @@ -0,0 +1,5 @@ +{ runTest }: +{ + gitlab = runTest ./gitlab.nix; + runner = runTest ./runner.nix; +} diff --git a/nixos/tests/gitlab.nix b/nixos/tests/gitlab/gitlab.nix similarity index 99% rename from nixos/tests/gitlab.nix rename to nixos/tests/gitlab/gitlab.nix index 1a9c1605ec43..762d0a572854 100644 --- a/nixos/tests/gitlab.nix +++ b/nixos/tests/gitlab/gitlab.nix @@ -7,12 +7,12 @@ # - Opening and closing issues. # - Downloading repository archives as tar.gz and tar.bz2 # Run with -# [nixpkgs]$ nix-build -A nixosTests.gitlab +# [nixpkgs]$ nix-build -A nixosTests.gitlab.gitlab { pkgs, lib, ... }: let - inherit (import ./ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey; + inherit (import ../ssh-keys.nix pkgs) snakeOilPrivateKey snakeOilPublicKey; initialRootPassword = "notproduction"; rootProjectId = "2"; @@ -37,7 +37,7 @@ in gitlab = { ... }: { - imports = [ common/user-account.nix ]; + imports = [ ../common/user-account.nix ]; environment.systemPackages = with pkgs; [ git ]; diff --git a/nixos/tests/gitlab/runner.nix b/nixos/tests/gitlab/runner.nix new file mode 100644 index 000000000000..8a97bea4b596 --- /dev/null +++ b/nixos/tests/gitlab/runner.nix @@ -0,0 +1,178 @@ +# This test runs a gitlab-runner and performs the following tests in +# two machines `gitlab` and `gitlab-runner`: +# - Create runners in the `gitlab` machine for all runners in `./runner`. +# - Inject the runner tokens into the `gitlab-runner.service` (machine `gitlab-runner`) +# which runs all runners: +# - Shell runner in `./runner/shell-runner`. +# - Start the `gitlab-runner.service`. +# - Check that all runners in `gitlab` are `active`. +# +# Run with +# [nixpkgs]$ nix-build -A nixosTests.gitlab.runner + +{ + pkgs, + lib, + ... +}: + +let + initialRootPassword = "notproduction"; + + runnerTokenDir = "/run/secrets/gitlab-runner"; + + runnerConfigs = { + # The Gitlab runner where each job runs + # on the host (not containerized and very insecure). + shell = { + desc = "Shell runner (host NixOS shell, host Nix store)"; + name = "shell"; + tokenFile = "${runnerTokenDir}/token-shell.env"; + }; + + # The Gitlab runner which uses the Docker runner (we use podman). + # Features: + # - Daemonizes the Nix store into a container. + # - All jobs run in an unprivileged container, e.g. with image + # (`local/nix`, `local/alpine`, `local/ubuntu`) + podman = { + desc = "Podman runner (containers, shared containerized Nix store)"; + name = "podman"; + tokenFile = "${runnerTokenDir}/token-podman.env"; + }; + }; +in +{ + name = "gitlab-runner"; + meta.maintainers = with lib.maintainers; [ + gabyx + ]; + + nodes = { + gitlab-runner = + { ... }: + { + imports = [ + ../common/user-account.nix + (import ./runner/shell-runner.nix { + runnerConfig = runnerConfigs.shell; + }) + ] + # Only enable the podman runner on x86_64 + # cause of built images. + ++ (lib.optional pkgs.stdenv.buildPlatform.isx86_64 ( + import ./runner/podman-runner { + runnerConfig = runnerConfigs.podman; + } + )); + + virtualisation = { + diskSize = 10000; + }; + + # Define the Gitlab Runner. + services.gitlab-runner = { + enable = true; + + settings = { + log_level = "info"; + }; + + gracefulTermination = false; + }; + }; + gitlab = + { config, ... }: + { + imports = [ ../common/user-account.nix ]; + + networking.firewall.allowedTCPPorts = [ + config.services.nginx.defaultHTTPListenPort + ]; + + environment.systemPackages = with pkgs; [ git ]; + + virtualisation.memorySize = 6144; + virtualisation.cores = 4; + + systemd.services.gitlab.serviceConfig.Restart = lib.mkForce "no"; + systemd.services.gitlab-workhorse.serviceConfig.Restart = lib.mkForce "no"; + systemd.services.gitaly.serviceConfig.Restart = lib.mkForce "no"; + systemd.services.gitlab-sidekiq.serviceConfig.Restart = lib.mkForce "no"; + + services.nginx = { + enable = true; + recommendedProxySettings = true; + virtualHosts = { + localhost = { + locations."/".proxyPass = "http://unix:/run/gitlab/gitlab-workhorse.socket"; + }; + }; + }; + + services.gitlab = { + enable = true; + databasePasswordFile = pkgs.writeText "dbPassword" "xo0daiF4"; + initialRootPasswordFile = pkgs.writeText "rootPassword" initialRootPassword; + secrets = { + secretFile = pkgs.writeText "secret" "Aig5zaic"; + otpFile = pkgs.writeText "otpsecret" "Riew9mue"; + dbFile = pkgs.writeText "dbsecret" "we2quaeZ"; + jwsFile = pkgs.runCommand "oidcKeyBase" { } "${pkgs.openssl}/bin/openssl genrsa 2048 > $out"; + activeRecordPrimaryKeyFile = pkgs.writeText "arprimary" "vsaYPZjTRxcbG7W6gNr95AwBmzFUd4Eu"; + activeRecordDeterministicKeyFile = pkgs.writeText "ardeterministic" "kQarv9wb2JVP7XzLTh5f6DFcMHms4nEC"; + activeRecordSaltFile = pkgs.writeText "arsalt" "QkgR9CfFU3MXEWGqa7LbP24AntK5ZeYw"; + }; + + # reduce memory usage + sidekiq.concurrency = 1; + puma.workers = 2; + }; + }; + }; + + testScript = + { nodes, ... }: + let + authPayload = pkgs.writeText "auth.json" ( + builtins.toJSON { + grant_type = "password"; + username = "root"; + password = initialRootPassword; + } + ); + + runnerTokenEnv = pkgs.writeText "runner-token.env" '' + CI_SERVER_URL=http://gitlab + CI_SERVER_TOKEN=$token + ''; + + createRunnerPayload = pkgs.writeText "create-runner.json" ( + builtins.toJSON { + runner_type = "instance_type"; + } + ); + in + # python + '' + # Define some globals for the python script below. + JQ_BINARY="${pkgs.jq}/bin/jq" + GITLAB_STATE_PATH="${nodes.gitlab.services.gitlab.statePath}" + RUNNER_TOKEN_ENV_FILE="${runnerTokenEnv}" + AUTH_PAYLOAD_FILE="${authPayload}" + CREATE_RUNNER_PAYLOAD_FILE="${createRunnerPayload}" + + ${lib.readFile ./runner_test.py} + + start_all() + wait_for_services() + + # Run all tests. + test_connection() + test_register_runner(name="shell", tokenFile="${runnerConfigs.shell.tokenFile}") + test_register_runner(name="podman", tokenFile="${runnerConfigs.podman.tokenFile}") + restart_gitlab_runner_service(runnerConfigs) + test_runner_registered(runnerConfigs["shell"]) + test_runner_registered(runnerConfigs["podman"]) + ''; +} diff --git a/nixos/tests/gitlab/runner/podman-runner/default.nix b/nixos/tests/gitlab/runner/podman-runner/default.nix new file mode 100644 index 000000000000..bbc9d7d75fef --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/default.nix @@ -0,0 +1,442 @@ +{ runnerConfig }: +# Gitlab Runner Module +# +# This module will add a Gitlab-Runner +# configured similar to https://wiki.nixos.org/wiki/Gitlab_runner +# with a nix-daemon running in a podman container `nix-daemon-container`. +# +# - The volumes from the `nix-daemon-container` will get mounted to +# each job container which Gitlab starts, which gives them access +# to a commonly shared Nix store. +# +# - The `/nix/store` inside the job container +# (either image `alpineImage` or `ubuntuImage` or `nixImage`) +# will be read-only and nix can only store stuff into this path by using the +# `NIX_DAEMON` env. variable which lets it communicate through the +# mounted daemon socket. + +# - The `bootstrapPkgs` derivation is copied into the job containers +# but without the Nix store paths cause they get provided by the +# `nix-daemon-store` volume. +# I cannot denote these volumes because they overmount the +# shit which is in the image. +# TODO: make a systemd service which starts before +# that and creates some volumes and inits these from the image. +# +# - The `podman-daemon-socket` volume gets mounted to the job container +# enabling it to use `podman`. +# Note: The job container instance is not using the system `podman` running in NixOS. +# Its a dedicated podman service `podmanDaemonContainer` +# running as `--privileged` +# [non-rootless container](https://rootlesscontaine.rs/#what-are-rootless-containers-and-what-are-not). +# (TODO: This podman daemon instance could be maybe run as rootless +# container under a user `ci` and a separated Gitlab Runner could +# run over this socket, effectively run only rootless containers.) +# +# - There is also a job runner prebuild script which is started on every job. +# See `scripts/prebuild.nix` to setup some missing stuff. +# +# Debugging on the VM: +# +# - You can use `journalclt -u gitlab-runner.service`. +# +# - To run a job container use: +# ```bash +# podman run --rm -it +# --volumes-from 'nix-daemon-container' +# -v "podman-daemon-socket:/run/podman" +# "local/alpine" \ +# bash -c "export CI_PIPkELINE_ID=123456 && gitlab-runner-prebuild-script; echo hello" +# ``` +{ + lib, + pkgs, + ... +}: +let + nixRepo = pkgs.fetchFromGitHub { + owner = "NixOS"; + repo = "nix"; + rev = "2.32.4"; + hash = "sha256-8QYnRyGOTm3h/Dp8I6HCmQzlO7C009Odqyp28pTWgcY="; + }; + + # Either we use a Nix as the base image or Alpine. + imageNames = { + default = imageNames.alpine; + + alpine = "local/alpine"; + nix = "local/nix"; + ubuntu = "local/ubuntu"; + + all = with imageNames; [ + alpine + nix + ubuntu + ]; + }; + + noPruneLabels = { + no-prune = "true"; + }; + + # This derivation will contain a folder `/etc` + files = pkgs.callPackage ./files { }; + preBuildScript = pkgs.callPackage ./scripts/prebuild.nix { }; + + # These derivations are Linked into the job images root dir. + bootstrapPkgs = [ + pkgs.nix + # Runtime dependencies of nix. + pkgs.gnutar + pkgs.gzip + pkgs.openssh + pkgs.xz + pkgs.cacert + + # Other stuff. + (lib.hiPrio pkgs.coreutils) + (lib.hiPrio pkgs.findutils) + pkgs.openssh + pkgs.bashInteractive + (lib.hiPrio pkgs.git) + pkgs.cachix + + pkgs.just + pkgs.podman # For nested containers. + + preBuildScript + + files.containers + files.nixConfig + ]; + + # All these packages are added to the Nix daemon. + nixStorePkgs = bootstrapPkgs ++ [ + # These files + files.basicRoot + files.fakeNixpkgs + ]; + + toEnvList = envs: lib.mapAttrsToList (k: v: "${k}=${v}") envs; + + # This is the Nix base image. + nixImageBase = pkgs.callPackage (import (nixRepo + "/docker.nix")) { + name = "local/nix-base"; + tag = "latest"; + + bundleNixpkgs = false; + maxLayers = 2; + + # You can add here a user with uid,gid,uname,gname etc. + # We are using root. + + extraPkgs = nixStorePkgs; + + nixConf = { + cores = "0"; + experimental-features = [ + "nix-command" + "flakes" + ]; + }; + }; + + # This is the daemon image which provides the store + # as volumes. + nixDaemonImage = pkgs.dockerTools.buildLayeredImage { + fromImage = nixImageBase; + name = "local/nix-daemon"; + tag = "latest"; + + config = { + Volumes = { + "/nix/store" = { }; + "/nix/var/nix/db" = { }; + "/nix/var/nix/daemon-socket" = { }; + }; + Labels = noPruneLabels; + }; + maxLayers = 4; + }; + + # This is the podman daemon image which enables + # a job image to use `podman` internally. + podmanDaemonImage = + let + # Update with: + # ```shell + # nix run "github:nixos/nixpkgs/nixos-unstable#nix-prefetch-docker" -- \ + # --image-name quay.io/podman/stable --image-tag v5.6.0 + # ``` + base = pkgs.dockerTools.pullImage { + imageName = "quay.io/podman/stable"; + imageDigest = "sha256:7c9381b9af167cf2218831c3af3135856c99f488b543b78435c8f18e19ad739a"; + hash = "sha256-pXXCu13fB/RN9qx8iLhE5Kko6glTrFrRhR7fo2OS7V0="; + finalImageName = "quay.io/podman/stable"; + finalImageTag = "v5.6.0"; + }; + in + pkgs.dockerTools.buildLayeredImage { + fromImage = base; + name = "local/podman-daemon"; + tag = "latest"; + + config = { + Labels = noPruneLabels; + }; + }; + + jobImages = + let + extraCommands = '' + set -eu + # Set missing Nix directories. + mkdir -p -m 0755 nix/var/log/nix/drvs + mkdir -p -m 0755 nix/var/nix/{gcroots,profiles,temproots,userpool} + mkdir -p -m 1777 nix/var/nix/{gcroots,profiles}/per-user + mkdir -p -m 0755 nix/var/nix/profiles/per-user/root + + # Need a HOME. + mkdir -vp root + mkdir -p -m 0700 root/.nix-defexpr + ''; + in + { + # The Nix image. + # Similar to https://github.com/nix-community/docker-nixpkgs/blob/main/images/nix/default.nix. + nix = pkgs.dockerTools.buildLayeredImage { + name = imageNames.nix; + tag = "latest"; + + extraCommands = extraCommands + '' + set -eu + # For `/usr/bin/env`. + mkdir -p usr && ln -s ../bin usr/bin + ''; + + contents = bootstrapPkgs ++ [ files.basicRoot ]; + # No store paths are copied into. We provide them by mounting the + # /nix/store. + includeStorePaths = false; + + config = { + Labels = noPruneLabels; + Env = toEnvList envs.nix; + }; + maxLayers = 2; + }; + + # This is the analog image to `local/nix` but Alpine based. + alpine = + let + # Update with: + # ```shell + # nix run "github:nixos/nixpkgs/nixos-unstable#nix-prefetch-docker" -- --image-name alpine --image-tag latest + # ``` + alpineBase = pkgs.dockerTools.pullImage { + imageName = "alpine"; + imageDigest = "sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d"; + sha256 = "0gf7wbjp37zbni3pz8vdgq1mss6mz69wynms0gqhq7lsxfmg9xj9"; + finalImageName = "alpine"; + finalImageTag = "latest"; + }; + in + (pkgs.dockerTools.buildLayeredImage { + fromImage = alpineBase; + name = imageNames.alpine; + tag = "latest"; + + inherit extraCommands; + + contents = bootstrapPkgs; + # No store paths are copied into. We provide them by mounting the + # /nix/store. + includeStorePaths = false; + + config = { + Labels = noPruneLabels; + Env = toEnvList envs.nix; + }; + + # Only if `build buildLayeredImage`. + maxLayers = 3; + }); + + # This is the analog image to `local/nix` but Ubuntu based. + ubuntu = + let + # Update with: + # ```shell + # nix run "github:nixos/nixpkgs/nixos-unstable#nix-prefetch-docker" -- \ + # --image-name ubuntu --image-tag latest + # ``` + ubuntuBase = pkgs.dockerTools.pullImage { + imageName = "ubuntu"; + imageDigest = "sha256:1e622c5f073b4f6bfad6632f2616c7f59ef256e96fe78bf6a595d1dc4376ac02"; + hash = "sha256-aC8SgxdcMSaaU89YMr/uwE022Yqey2frmeZqr+L1xEU="; + finalImageName = "ubuntu"; + finalImageTag = "latest"; + }; + in + (pkgs.dockerTools.buildLayeredImage { + fromImage = ubuntuBase; + name = imageNames.ubuntu; + tag = "latest"; + + inherit extraCommands; + + contents = bootstrapPkgs; + # No store paths are copied into. We provide them by mounting the + # /nix/store. + includeStorePaths = false; + + config = { + Labels = noPruneLabels; + Env = toEnvList envs.ubuntu; + }; + + # Only if `build buildLayeredImage`. + maxLayers = 3; + }); + }; + + nixDaemonContainer = { + imageFile = nixDaemonImage; + image = "local/nix-daemon:latest"; + + volumes = [ + "nix-daemon-store:/nix/store" + "nix-daemon-db:/nix/var/nix/db" + "nix-daemon-socket:/nix/var/nix/daemon-socket" + ]; + cmd = [ + "nix" + "daemon" + ]; + }; + + podmanDaemonContainer = { + imageFile = podmanDaemonImage; + image = "local/podman-daemon:latest"; + volumes = [ + "podman-daemon-socket:/run/podman" + "podman-cache:/var/lib/container" + # Shared images, currently not needed. + "podman-shared:/var/lib/shared:ro" + ]; + privileged = true; + cmd = [ + "podman" + "system" + "service" + "--time=0" + "unix:///run/podman/podman.sock" + "--log-level" + "info" + ]; + }; + + # Environment variables for all job containers. + envs = rec { + common = { + # Access to the nix daemon. + NIX_REMOTE = "daemon"; + # Access to podman. + CONTAINER_HOST = "unix:///run/podman/podman.sock"; + + USER = "root"; + PATH = "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin"; + + SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt"; + + # For shells, source this file. + ENV = "${pkgs.nix}/etc/profile.d/nix-daemon.sh"; + BASH_ENV = "${pkgs.nix}/etc/profile.d/nix-daemon.sh"; + + # Make a fake nixpkgs which throws when using + # `nix repl -f ` for example. + NIX_PATH = "nixpkgs=${files.fakeNixpkgs}"; + }; + + nix = common // { + IMAGE_OS_DIST = "nix"; + }; + + alpine = common // { + IMAGE_OS_DIST = "alpine"; + }; + + ubuntu = common // { + IMAGE_OS_DIST = "ubuntu"; + }; + }; + + registrationFlags = [ + "--docker-volumes" + "gitlab-runner-scratch:/scratch" + + "--docker-volumes" + "podman-daemon-socket:/run/podman" + + "--docker-volumes-from" + "nix-daemon-container:ro" + + "--docker-pull-policy" + "if-not-present" + + "--docker-allowed-pull-policies" + "if-not-present" + + "--docker-host" + "unix:///var/run/podman/podman.sock" + + "--docker-network-mode" + "host" + ]; + +in +{ + imports = [ ./virtualization.nix ]; + + virtualisation.oci-containers = { + backend = "podman"; + + containers = { + nix-daemon-container = nixDaemonContainer; + podman-daemon-container = podmanDaemonContainer; + } + // + # Workaround to add the job images to the registry. + (lib.concatMapAttrs (name: image: { + "${name}-container" = { + imageFile = jobImages.${name}; + image = "${imageNames.${name}}:latest"; + extraOptions = [ + "--volumes-from" + "nix-daemon-container:ro" + ]; + dependsOn = [ "nix-daemon-container" ]; + cmd = [ "true" ]; + }; + }) jobImages); + }; + + # Define the Gitlab Runner. + services.gitlab-runner.services.podman-runner = { + description = runnerConfig.desc; + + inherit registrationFlags; + + authenticationTokenConfigFile = runnerConfig.tokenFile; + + executor = "docker"; + dockerImage = imageNames.default; + dockerAllowedImages = [ ]; + dockerPrivileged = false; + requestConcurrency = 4; + + preBuildScript = "${preBuildScript}/bin/gitlab-runner-pre-build-script"; + }; +} diff --git a/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/group b/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/group new file mode 100644 index 000000000000..162f79fd7086 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/group @@ -0,0 +1,21 @@ +root:x:0: +wheel:x:1: +kmem:x:2: +tty:x:3: +messagebus:x:4: +disk:x:6: +audio:x:17: +floppy:x:18: +uucp:x:19: +lp:x:20: +cdrom:x:24: +tape:x:25: +video:x:26: +dialout:x:27: +utmp:x:29: +adm:x:55: +keys:x:96: +users:x:100: +input:x:174: +nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9 +nogroup:x:65534: diff --git a/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/nsswitch.conf b/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/nsswitch.conf new file mode 100644 index 000000000000..59a21416fd8f --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/nsswitch.conf @@ -0,0 +1,11 @@ +passwd: files mymachines systemd +group: files mymachines systemd +shadow: files + +hosts: files mymachines dns myhostname +networks: files + +ethers: files +services: files +protocols: files +rpc: files diff --git a/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/passwd b/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/passwd new file mode 100644 index 000000000000..006b53f7bf82 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/basicRoot/etc/passwd @@ -0,0 +1,34 @@ +root:x:0:0:System administrator:/root:/bin/bash +nixbld1:x:30001:30000:Nix build user 1:/var/empty:/run/current-system/sw/bin/nologin +nixbld2:x:30002:30000:Nix build user 2:/var/empty:/run/current-system/sw/bin/nologin +nixbld3:x:30003:30000:Nix build user 3:/var/empty:/run/current-system/sw/bin/nologin +nixbld4:x:30004:30000:Nix build user 4:/var/empty:/run/current-system/sw/bin/nologin +nixbld5:x:30005:30000:Nix build user 5:/var/empty:/run/current-system/sw/bin/nologin +nixbld6:x:30006:30000:Nix build user 6:/var/empty:/run/current-system/sw/bin/nologin +nixbld7:x:30007:30000:Nix build user 7:/var/empty:/run/current-system/sw/bin/nologin +nixbld8:x:30008:30000:Nix build user 8:/var/empty:/run/current-system/sw/bin/nologin +nixbld9:x:30009:30000:Nix build user 9:/var/empty:/run/current-system/sw/bin/nologin +nixbld10:x:30010:30000:Nix build user 10:/var/empty:/run/current-system/sw/bin/nologin +nixbld11:x:30011:30000:Nix build user 11:/var/empty:/run/current-system/sw/bin/nologin +nixbld12:x:30012:30000:Nix build user 12:/var/empty:/run/current-system/sw/bin/nologin +nixbld13:x:30013:30000:Nix build user 13:/var/empty:/run/current-system/sw/bin/nologin +nixbld14:x:30014:30000:Nix build user 14:/var/empty:/run/current-system/sw/bin/nologin +nixbld15:x:30015:30000:Nix build user 15:/var/empty:/run/current-system/sw/bin/nologin +nixbld16:x:30016:30000:Nix build user 16:/var/empty:/run/current-system/sw/bin/nologin +nixbld17:x:30017:30000:Nix build user 17:/var/empty:/run/current-system/sw/bin/nologin +nixbld18:x:30018:30000:Nix build user 18:/var/empty:/run/current-system/sw/bin/nologin +nixbld19:x:30019:30000:Nix build user 19:/var/empty:/run/current-system/sw/bin/nologin +nixbld20:x:30020:30000:Nix build user 20:/var/empty:/run/current-system/sw/bin/nologin +nixbld21:x:30021:30000:Nix build user 21:/var/empty:/run/current-system/sw/bin/nologin +nixbld22:x:30022:30000:Nix build user 22:/var/empty:/run/current-system/sw/bin/nologin +nixbld23:x:30023:30000:Nix build user 23:/var/empty:/run/current-system/sw/bin/nologin +nixbld24:x:30024:30000:Nix build user 24:/var/empty:/run/current-system/sw/bin/nologin +nixbld25:x:30025:30000:Nix build user 25:/var/empty:/run/current-system/sw/bin/nologin +nixbld26:x:30026:30000:Nix build user 26:/var/empty:/run/current-system/sw/bin/nologin +nixbld27:x:30027:30000:Nix build user 27:/var/empty:/run/current-system/sw/bin/nologin +nixbld28:x:30028:30000:Nix build user 28:/var/empty:/run/current-system/sw/bin/nologin +nixbld29:x:30029:30000:Nix build user 29:/var/empty:/run/current-system/sw/bin/nologin +nixbld30:x:30030:30000:Nix build user 30:/var/empty:/run/current-system/sw/bin/nologin +nixbld31:x:30031:30000:Nix build user 31:/var/empty:/run/current-system/sw/bin/nologin +nixbld32:x:30032:30000:Nix build user 32:/var/empty:/run/current-system/sw/bin/nologin +nobody:x:65534:65534:Unprivileged account (don't use!):/var/empty:/run/current-system/sw/bin/nologin diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/containers.conf b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/containers.conf new file mode 100644 index 000000000000..0bf45cd2a1a1 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/containers.conf @@ -0,0 +1,2 @@ +[engine] +cgroup_manager = "cgroupfs" diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/mounts.conf b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/mounts.conf new file mode 100644 index 000000000000..b54e7222210b --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/mounts.conf @@ -0,0 +1,2 @@ +/run/secrets/etc-pki-entitlement:/run/secrets/etc-pki-entitlement +/run/secrets/rhsm:/run/secrets/rhsm diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/policy.json b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/policy.json new file mode 100644 index 000000000000..4724dd816814 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/policy.json @@ -0,0 +1,12 @@ +{ + "default": [ + { + "type": "insecureAcceptAnything" + } + ], + "transports": { + "docker-daemon": { + "": [{ "type": "insecureAcceptAnything" }] + } + } +} diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.conf b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.conf new file mode 100644 index 000000000000..c3a575800d86 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.conf @@ -0,0 +1,2 @@ +unqualified-search-registries = ["registry.fedoraproject.org", "registry.access.redhat.com", "docker.io"] +short-name-mode = "enforcing" diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.conf.d/000-shortnames.conf b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.conf.d/000-shortnames.conf new file mode 100644 index 000000000000..142e6158235c --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.conf.d/000-shortnames.conf @@ -0,0 +1,5 @@ +[aliases] + "buildah" = "quay.io/buildah/stable" + "podman" = "quay.io/podman/stable" + "alpine" = "docker.io/library/alpine" + "ubuntu" = "docker.io/library/ubuntu" diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/default.yaml b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/default.yaml new file mode 100644 index 000000000000..9e892d760b21 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/default.yaml @@ -0,0 +1,27 @@ +# This is a default registries.d configuration file. You may +# add to this file or create additional files in registries.d/. +# +# lookaside: for reading/writing simple signing signatures +# lookaside-staging: for writing simple signing signatures, preferred over lookaside +# +# lookaside and lookaside-staging take a value of the following: +# lookaside: {schema}://location +# +# For reading signatures, schema may be http, https, or file. +# For writing signatures, schema may only be file. + +# The default locations are built-in, for both reading and writing: +# /var/lib/containers/sigstore for root, or +# ~/.local/share/containers/sigstore for non-root users. +default-docker: +# lookaside: https://… +# lookaside-staging: file:///… + +# The 'docker' indicator here is the start of the configuration +# for docker registries. +# +# docker: +# +# privateregistry.com: +# lookaside: https://privateregistry.com/sigstore/ +# lookaside-staging: /mnt/nfs/privateregistry/sigstore diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/registry.access.redhat.com.yaml b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/registry.access.redhat.com.yaml new file mode 100644 index 000000000000..45018d5830a7 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/registry.access.redhat.com.yaml @@ -0,0 +1,3 @@ +docker: + registry.access.redhat.com: + lookaside: https://access.redhat.com/webassets/docker/content/sigstore diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/registry.redhat.io.yaml b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/registry.redhat.io.yaml new file mode 100644 index 000000000000..ba1769320ce7 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/registries.d/registry.redhat.io.yaml @@ -0,0 +1,3 @@ +docker: + registry.redhat.io: + lookaside: https://registry.redhat.io/containers/sigstore diff --git a/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/storage.conf b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/storage.conf new file mode 100644 index 000000000000..9db50278d80b --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/containers/etc/containers/storage.conf @@ -0,0 +1,15 @@ +[storage] +driver = "overlay" +runroot = "/run/containers/storage" +graphroot = "/var/lib/containers/storage" + +[storage.options] +additionalimagestores = [ +"/var/lib/shared", +"/usr/lib/containers/storage", +] +pull_options = {enable_partial_images = "true", use_hard_links = "false", ostree_repos=""} + +[storage.options.overlay] +mount_program = "/usr/bin/fuse-overlayfs" +mountopt = "nodev,fsync=0" diff --git a/nixos/tests/gitlab/runner/podman-runner/files/default.nix b/nixos/tests/gitlab/runner/podman-runner/files/default.nix new file mode 100644 index 000000000000..e791ae42367f --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/default.nix @@ -0,0 +1,46 @@ +# Specific files for the job images. +# +# - `basicRoot`: Some basic root files for the `jobImages.nix`. +# - `fakeNixpkgs`: A fake Nixpkg directory which is set as `NIX_PATH=nixpkgs:` +# which throws on load. +# - `nixConfig`: The Nix config with some options. +# - `containers`: +# These are some files which are copied to the job images needed for +# `buildah` (`podman`): +# +# ```bash +# podman create --name temp-buildah quay.io/buildah/stable:latest +# podman cp temp-buildah:/etc/containers ./etc/ +# find ./etc -type d -empty -delete +# podman container rm temp-buildah +#``` +# +{ pkgs, ... }: +let + + # We need proper derivations to add it to the nixImageBase. + mkDrv = + name: src: + pkgs.stdenv.mkDerivation { + inherit name src; + installPhase = '' + mkdir -p $out + cp -r $src/* $out/ + ''; + }; +in +{ + basicRoot = mkDrv "basic-root-files" ./basicRoot; + containers = mkDrv "containers-files" ./containers; + fakeNixpkgs = mkDrv "fake-nixpkgs" ./fake-nixpkgs; + + nixConfig = pkgs.writeTextFile { + name = "nix.conf"; + destination = "/etc/nix/nix.conf"; + text = '' + accept-flake-config = true + experimental-features = nix-command flakes + max-jobs = auto + ''; + }; +} diff --git a/nixos/tests/gitlab/runner/podman-runner/files/fake-nixpkgs/default.nix b/nixos/tests/gitlab/runner/podman-runner/files/fake-nixpkgs/default.nix new file mode 100644 index 000000000000..eee7aacaf920 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/files/fake-nixpkgs/default.nix @@ -0,0 +1,10 @@ +_: +throw '' + This container doesn't include nixpkgs. + + The best way to work around that is to pin your dependencies. See + https://nix.dev/tutorials/first-steps/towards-reproducibility-pinning-nixpkgs.html + + Or if you must, override the NIX_PATH environment variable with eg: + "NIX_PATH=nixpkgs=channel:nixos-unstable" +'' diff --git a/nixos/tests/gitlab/runner/podman-runner/scripts/prebuild.nix b/nixos/tests/gitlab/runner/podman-runner/scripts/prebuild.nix new file mode 100644 index 000000000000..78b8b0f41b18 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/scripts/prebuild.nix @@ -0,0 +1,55 @@ +{ writeShellScriptBin, nix }: +writeShellScriptBin "gitlab-runner-pre-build-script" + # bash + '' + set -e + set -u + + function section_start() { + local name="$1" + shift + echo -e "\e[0Ksection_start:$(date +%s):$name[collapsed=true]\r\e[0K$*" + } + + function section_end() { + local name="$1" + echo -e "\e[0Ksection_end:$(date +%s):$name\r\e[0K" + } + + function setup() { + # We need to allow modification of nix config for cachix as + # otherwise it is link to the read only file in the store. + cp --remove-destination \ + "$(readlink -f /etc/nix/nix.conf)" /etc/nix/nix.conf + + # shellcheck disable=SC1091 + . "${nix}/etc/profile.d/nix-daemon.sh" + } + + function setup_pipeline_scratch_dir() { + scratch_dir="/scratch/$CI_PIPELINE_ID" + + echo "Create scrtach directory for pipeline: $scratch_dir" + mkdir -p "$scratch_dir" || { + echo "Could not create scratch dir '$scratch_dir'." >&2 + exit 1 + } + + export CI_CUSTOM_SCRATCH_DIR="$scratch_dir" + } + + function print_info() { + echo "Nix version:" + nix --version + } + + function main() { + print_info + setup + setup_pipeline_scratch_dir + } + + section_start gitlab-runner-prebuild "Gitlab-Runner PreBuild Script" + main "$@" + section_end gitlab-runner-prebuild + '' diff --git a/nixos/tests/gitlab/runner/podman-runner/virtualization.nix b/nixos/tests/gitlab/runner/podman-runner/virtualization.nix new file mode 100644 index 000000000000..89a06023400d --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/virtualization.nix @@ -0,0 +1,43 @@ +{ lib, ... }: +{ + virtualisation.docker = { + enable = lib.mkForce false; + }; + + virtualisation.podman = { + enable = true; + + # Create a `docker` alias for podman, to use it as a drop-in replacement + # dockerCompat = true; + dockerSocket = { + enable = true; + }; + + # Required for containers under podman-compose to be able to talk to each other. + defaultNetwork.settings.dns_enabled = true; + + autoPrune = { + dates = "weekly"; + flags = [ + "--filter" + "label!=no-prune" + "--volumes" + "--log-level" + "debug" + ]; + }; + }; + + virtualisation.containers.storage.settings = { + storage = { + driver = "overlay"; + graphroot = "/var/lib/containers/storage"; + runroot = "/run/containers/storage"; + + # Does not work currently. + options.overlay = { + mountopt = "nodev,metacopy=on"; + }; + }; + }; +} diff --git a/nixos/tests/gitlab/runner/shell-runner.nix b/nixos/tests/gitlab/runner/shell-runner.nix new file mode 100644 index 000000000000..da2eaa1b94ea --- /dev/null +++ b/nixos/tests/gitlab/runner/shell-runner.nix @@ -0,0 +1,12 @@ +{ + runnerConfig, +}: +# This is the runner config for the `nixosConfiguration.services.gitlab-runner.services.X` +{ + services.gitlab-runner.services.shell-runner = { + description = runnerConfig.desc; + authenticationTokenConfigFile = runnerConfig.tokenFile; + + executor = "shell"; + }; +} diff --git a/nixos/tests/gitlab/runner_test.py b/nixos/tests/gitlab/runner_test.py new file mode 100644 index 000000000000..adc98e363915 --- /dev/null +++ b/nixos/tests/gitlab/runner_test.py @@ -0,0 +1,146 @@ +import json +import os +from dataclasses import dataclass +from pathlib import Path +from string import Template +from typing import Any + + +@dataclass +class Runner: + name: str + tokenFile: str + + id: str + token: str + + +@dataclass +class Machines: + gitlab: Any + gitlab_runner: Any + + +@dataclass +class Nix: + jq: str + gitlab_state_path: str + create_runner_payload_file: str + auth_payload_file: str + runner_token_env_file: str + + +# Some global variables to work in the tests. +out_dir = os.environ.get("out", os.getcwd()) +nix = Nix( + jq=JQ_BINARY, + gitlab_state_path=GITLAB_STATE_PATH, + auth_payload_file=AUTH_PAYLOAD_FILE, + create_runner_payload_file=CREATE_RUNNER_PAYLOAD_FILE, + runner_token_env_file=RUNNER_TOKEN_ENV_FILE, +) +vms = Machines(gitlab, gitlab_runner) +runnerConfigs: dict[str, Runner] = {} + + +def wait_for_services(): + vms.gitlab.wait_for_unit("gitaly.service") + vms.gitlab.wait_for_unit("gitlab-workhorse.service") + vms.gitlab.wait_for_unit("gitlab.service") + vms.gitlab.wait_for_unit("gitlab-sidekiq.service") + vms.gitlab.wait_for_file(f"{nix.gitlab_state_path}/tmp/sockets/gitlab.socket") + vms.gitlab.wait_until_succeeds("curl -sSf http://gitlab/users/sign_in") + + +def test_connection(): + """ + Test the connection to Gitlab and check if it is reachable from the runner VM. + """ + + print("==> Getting secrets and headers.") + vms.gitlab.succeed( + "cp /var/gitlab/state/config/secrets.yml /root/gitlab-secrets.yml" + ) + + vms.gitlab.succeed( + f"echo \"Authorization: Bearer $(curl -X POST -H 'Content-Type: application/json' -d @{nix.auth_payload_file} http://gitlab/oauth/token | {nix.jq} -r '.access_token')\" >/tmp/headers" + ) + + vms.gitlab.copy_from_vm("/tmp/headers") + out_dir = os.environ.get("out", os.getcwd()) + vms.gitlab_runner.copy_from_host(str(Path(out_dir, "headers")), "/tmp/headers") + + print("==> Testing connection.") + vms.gitlab_runner.succeed("curl -v -H @/tmp/headers http://gitlab/api/v4/version") + + +def test_register_runner(name: str, tokenFile: str): + """ + Register the runner in Gitlab and write the token file to be picked up by + the gitlab-runner service on the other VM. + """ + + r = Runner( + name=name, + tokenFile=tokenFile, + token="", + id="", + ) + runnerConfigs[r.name] = r + + print(f"==> Create Runner '{r.name}'") + resp = vms.gitlab.execute( + f""" + curl -s -X POST \ + -H 'Content-Type: application/json' \ + -H @/tmp/headers \ + -d @{nix.create_runner_payload_file} \ + http://gitlab/api/v4/user/runners + """ + )[1] + obj = json.loads(resp) + r.id = obj["id"] + r.token = obj["token"] + print(f"==> Registered runner '{r.id}' with token '{r.token}'.") + + # Push the token to the runner machine. + print("==> Push runner token to machine.") + tokenF = Path(out_dir, f"token-{r.name}.env") + with open(nix.runner_token_env_file, "r") as f: + tokenData = Template(f.read()).substitute({"token": r.token}) + with open(tokenF, "w") as w: + w.write(tokenData) + vms.gitlab_runner.copy_from_host(str(tokenF), r.tokenFile) + + +def restart_gitlab_runner_service(runnerConfigs): + print("==> Restart Gitlab Runner") + + if any([n == "podman" for n in runnerConfigs.keys()]): + vms.gitlab_runner.wait_for_unit("podman-nix-daemon-container.service") + vms.gitlab_runner.wait_for_unit("podman-podman-daemon-container.service") + + vms.gitlab_runner.systemctl("restart gitlab-runner.service") + vms.gitlab_runner.wait_for_unit("gitlab-runner.service") + + +def test_runner_registered(r: Runner): + """ + Test that the runner `r` is registered in Gitlab and its status is active. + """ + + print(f"==> Check that runner '{r.name}' is registered.") + + resp = vms.gitlab.execute( + f""" + curl -s -X GET \ + -H 'Content-Type: application/json' \ + -H @/tmp/headers \ + http://gitlab/api/v4/runners/{r.id}""" + )[1] + runnerStatus = json.loads(resp) + + if not runnerStatus["active"]: + raise Exception( + f"Runner '{r.name}' [id: '{r.id}'] status is not active: {resp}" + ) From 8fa348c4ba665fb5e7715c6b7871b0c97364e3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Sat, 20 Dec 2025 18:48:28 +0100 Subject: [PATCH 2/3] fix: apply review changes Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- nixos/modules/services/misc/gitlab.md | 2 +- nixos/tests/gitlab/runner/podman-runner/default.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/misc/gitlab.md b/nixos/modules/services/misc/gitlab.md index 0e29071d222d..b583836e2ff7 100644 --- a/nixos/modules/services/misc/gitlab.md +++ b/nixos/modules/services/misc/gitlab.md @@ -145,7 +145,7 @@ configure a Gitlab Runner with caching and reasonably good security practices. ## Example: Gitlab Runner with `podman` and Nix Store Caching -The [VM tested `podman-runner`](https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/continuous-integration/gitlab-runner/runner.nix) +The [VM tested `podman-runner`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/gitlab/runner/podman-runner/default.nix) (a NixOS module for reuse) configures an advanced Gitlab runner with the following features: - The executor is `podman` which gives you better additional safety than diff --git a/nixos/tests/gitlab/runner/podman-runner/default.nix b/nixos/tests/gitlab/runner/podman-runner/default.nix index bbc9d7d75fef..e9a0818d5c28 100644 --- a/nixos/tests/gitlab/runner/podman-runner/default.nix +++ b/nixos/tests/gitlab/runner/podman-runner/default.nix @@ -38,7 +38,7 @@ # # Debugging on the VM: # -# - You can use `journalclt -u gitlab-runner.service`. +# - You can use `journalctl -u gitlab-runner.service`. # # - To run a job container use: # ```bash From 1cbd52c09b24b99a7f11addf6cb53a4f9e80a784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20N=C3=BCtzi?= Date: Sat, 20 Dec 2025 20:38:50 +0100 Subject: [PATCH 3/3] fix: remove IFD (import from derivation) by vendoring the `docker.nix` from `NixOS/nix` --- .../gitlab/runner/podman-runner/default.nix | 67 ++- .../gitlab/runner/podman-runner/nix-image.nix | 400 ++++++++++++++++++ .../runner/podman-runner/scripts/prebuild.nix | 2 +- 3 files changed, 426 insertions(+), 43 deletions(-) create mode 100644 nixos/tests/gitlab/runner/podman-runner/nix-image.nix diff --git a/nixos/tests/gitlab/runner/podman-runner/default.nix b/nixos/tests/gitlab/runner/podman-runner/default.nix index e9a0818d5c28..0bdccfaeed3a 100644 --- a/nixos/tests/gitlab/runner/podman-runner/default.nix +++ b/nixos/tests/gitlab/runner/podman-runner/default.nix @@ -2,51 +2,20 @@ # Gitlab Runner Module # # This module will add a Gitlab-Runner -# configured similar to https://wiki.nixos.org/wiki/Gitlab_runner # with a nix-daemon running in a podman container `nix-daemon-container`. -# -# - The volumes from the `nix-daemon-container` will get mounted to -# each job container which Gitlab starts, which gives them access -# to a commonly shared Nix store. -# -# - The `/nix/store` inside the job container -# (either image `alpineImage` or `ubuntuImage` or `nixImage`) -# will be read-only and nix can only store stuff into this path by using the -# `NIX_DAEMON` env. variable which lets it communicate through the -# mounted daemon socket. - -# - The `bootstrapPkgs` derivation is copied into the job containers -# but without the Nix store paths cause they get provided by the -# `nix-daemon-store` volume. -# I cannot denote these volumes because they overmount the -# shit which is in the image. -# TODO: make a systemd service which starts before -# that and creates some volumes and inits these from the image. -# -# - The `podman-daemon-socket` volume gets mounted to the job container -# enabling it to use `podman`. -# Note: The job container instance is not using the system `podman` running in NixOS. -# Its a dedicated podman service `podmanDaemonContainer` -# running as `--privileged` -# [non-rootless container](https://rootlesscontaine.rs/#what-are-rootless-containers-and-what-are-not). -# (TODO: This podman daemon instance could be maybe run as rootless -# container under a user `ci` and a separated Gitlab Runner could -# run over this socket, effectively run only rootless containers.) -# -# - There is also a job runner prebuild script which is started on every job. -# See `scripts/prebuild.nix` to setup some missing stuff. +# Check the documentation in the NixOS Manual. # # Debugging on the VM: # # - You can use `journalctl -u gitlab-runner.service`. # -# - To run a job container use: +# - To run a job container inside the VM use: # ```bash # podman run --rm -it # --volumes-from 'nix-daemon-container' # -v "podman-daemon-socket:/run/podman" # "local/alpine" \ -# bash -c "export CI_PIPkELINE_ID=123456 && gitlab-runner-prebuild-script; echo hello" +# bash -c "export CI_PIPELINE_ID=123456 && gitlab-runner-pre-build-script; echo hello" # ``` { lib, @@ -54,12 +23,9 @@ ... }: let - nixRepo = pkgs.fetchFromGitHub { - owner = "NixOS"; - repo = "nix"; - rev = "2.32.4"; - hash = "sha256-8QYnRyGOTm3h/Dp8I6HCmQzlO7C009Odqyp28pTWgcY="; - }; + # Switch to not use IFD in nixpkgs test. + # NOTE: When reusing this runner, you can set this to `true`. + useIFD = false; # Either we use a Nix as the base image or Alpine. imageNames = { @@ -120,8 +86,25 @@ let toEnvList = envs: lib.mapAttrsToList (k: v: "${k}=${v}") envs; - # This is the Nix base image. - nixImageBase = pkgs.callPackage (import (nixRepo + "/docker.nix")) { + # This is the Nix base image used for the Nix Daemon. + # The build script for the nixos/nix image is vendored due to Hydra limitations. + # cause it is IFD (Import from Derivation) which is not allowed. + # NOTE: When reusing this runner you can set `useIFD` to true: + nixImageBaseFn = + if !useIFD then + import ./nix-image.nix + else + import ( + (pkgs.fetchFromGitHub { + owner = "NixOS"; + repo = "nix"; + rev = "2.32.4"; + hash = "sha256-8QYnRyGOTm3h/Dp8I6HCmQzlO7C009Odqyp28pTWgcY="; + }) + + "/docker.nix" + ); + + nixImageBase = pkgs.callPackage nixImageBaseFn { name = "local/nix-base"; tag = "latest"; diff --git a/nixos/tests/gitlab/runner/podman-runner/nix-image.nix b/nixos/tests/gitlab/runner/podman-runner/nix-image.nix new file mode 100644 index 000000000000..43c11ae25175 --- /dev/null +++ b/nixos/tests/gitlab/runner/podman-runner/nix-image.nix @@ -0,0 +1,400 @@ +# This is the vendored build script from +# https://raw.githubusercontent.com/NixOS/nix/refs/heads/master/docker.nix +# which builds the Nix image. +# This is only here to please Hydra which is not beeing able to build IFDs. +# `import (nixRepo + "./docker.nix")`. +{ + # Core dependencies + pkgs ? import { }, + lib ? pkgs.lib, + dockerTools ? pkgs.dockerTools, + runCommand ? pkgs.runCommand, + buildPackages ? pkgs.buildPackages, + # Image configuration + name ? "nix", + tag ? "latest", + bundleNixpkgs ? true, + channelName ? "nixpkgs", + channelURL ? "https://channels.nixos.org/nixpkgs-unstable", + extraPkgs ? [ ], + maxLayers ? 70, + nixConf ? { }, + flake-registry ? null, + uid ? 0, + gid ? 0, + uname ? "root", + gname ? "root", + Labels ? { + "org.opencontainers.image.title" = "Nix"; + "org.opencontainers.image.source" = "https://github.com/NixOS/nix"; + "org.opencontainers.image.vendor" = "Nix project"; + "org.opencontainers.image.version" = nix.version; + "org.opencontainers.image.description" = "Nix container image"; + }, + Cmd ? [ (lib.getExe bashInteractive) ], + # Default Packages + nix ? pkgs.nix, + bashInteractive ? pkgs.bashInteractive, + coreutils-full ? pkgs.coreutils-full, + gnutar ? pkgs.gnutar, + gzip ? pkgs.gzip, + gnugrep ? pkgs.gnugrep, + which ? pkgs.which, + curl ? pkgs.curl, + less ? pkgs.less, + wget ? pkgs.wget, + man ? pkgs.man, + cacert ? pkgs.cacert, + findutils ? pkgs.findutils, + iana-etc ? pkgs.iana-etc, + gitMinimal ? pkgs.gitMinimal, + openssh ? pkgs.openssh, + # Other dependencies + shadow ? pkgs.shadow, +}: +let + defaultPkgs = [ + nix + bashInteractive + coreutils-full + gnutar + gzip + gnugrep + which + curl + less + wget + man + cacert.out + findutils + iana-etc + gitMinimal + openssh + ] + ++ extraPkgs; + + users = { + + root = { + uid = 0; + shell = lib.getExe bashInteractive; + home = "/root"; + gid = 0; + groups = [ "root" ]; + description = "System administrator"; + }; + + nobody = { + uid = 65534; + shell = lib.getExe' shadow "nologin"; + home = "/var/empty"; + gid = 65534; + groups = [ "nobody" ]; + description = "Unprivileged account (don't use!)"; + }; + + } + // lib.optionalAttrs (uid != 0) { + "${uname}" = { + uid = uid; + shell = lib.getExe bashInteractive; + home = "/home/${uname}"; + gid = gid; + groups = [ "${gname}" ]; + description = "Nix user"; + }; + } + // lib.listToAttrs ( + map (n: { + name = "nixbld${toString n}"; + value = { + uid = 30000 + n; + gid = 30000; + groups = [ "nixbld" ]; + description = "Nix build user ${toString n}"; + }; + }) (lib.lists.range 1 32) + ); + + groups = { + root.gid = 0; + nixbld.gid = 30000; + nobody.gid = 65534; + } + // lib.optionalAttrs (gid != 0) { + "${gname}".gid = gid; + }; + + userToPasswd = ( + k: + { + uid, + gid ? 65534, + home ? "/var/empty", + description ? "", + shell ? "/bin/false", + groups ? [ ], + }: + "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}" + ); + passwdContents = (lib.concatStringsSep "\n" (lib.attrValues (lib.mapAttrs userToPasswd users))); + + userToShadow = k: { ... }: "${k}:!:1::::::"; + shadowContents = (lib.concatStringsSep "\n" (lib.attrValues (lib.mapAttrs userToShadow users))); + + # Map groups to members + # { + # group = [ "user1" "user2" ]; + # } + groupMemberMap = ( + let + # Create a flat list of user/group mappings + mappings = ( + builtins.foldl' ( + acc: user: + let + groups = users.${user}.groups or [ ]; + in + acc + ++ map (group: { + inherit user group; + }) groups + ) [ ] (lib.attrNames users) + ); + in + (builtins.foldl' ( + acc: v: + acc + // { + ${v.group} = acc.${v.group} or [ ] ++ [ v.user ]; + } + ) { } mappings) + ); + + groupToGroup = + k: + { gid }: + let + members = groupMemberMap.${k} or [ ]; + in + "${k}:x:${toString gid}:${lib.concatStringsSep "," members}"; + groupContents = (lib.concatStringsSep "\n" (lib.attrValues (lib.mapAttrs groupToGroup groups))); + + toConf = + with pkgs.lib.generators; + toKeyValue { + mkKeyValue = mkKeyValueDefault { + mkValueString = v: if lib.isList v then lib.concatStringsSep " " v else mkValueStringDefault { } v; + } " = "; + }; + + nixConfContents = toConf ( + { + sandbox = false; + build-users-group = "nixbld"; + trusted-public-keys = [ "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ]; + } + // nixConf + ); + + userHome = if uid == 0 then "/root" else "/home/${uname}"; + + baseSystem = + let + nixpkgs = pkgs.path; + channel = runCommand "channel-nixos" { inherit bundleNixpkgs; } '' + mkdir $out + if [ "$bundleNixpkgs" ]; then + ln -s ${ + builtins.path { + path = nixpkgs; + name = "source"; + } + } $out/nixpkgs + echo "[]" > $out/manifest.nix + fi + ''; + # doc/manual/source/command-ref/files/manifest.nix.md + manifest = buildPackages.runCommand "manifest.nix" { } '' + cat > $out < $out/etc/passwd + echo "" >> $out/etc/passwd + + cat $groupContentsPath > $out/etc/group + echo "" >> $out/etc/group + + cat $shadowContentsPath > $out/etc/shadow + echo "" >> $out/etc/shadow + + mkdir -p $out/usr + ln -s /nix/var/nix/profiles/share $out/usr/ + + mkdir -p $out/nix/var/nix/gcroots + + mkdir $out/tmp + + mkdir -p $out/var/tmp + + mkdir -p $out/etc/nix + cat $nixConfContentsPath > $out/etc/nix/nix.conf + + mkdir -p $out${userHome} + mkdir -p $out/nix/var/nix/profiles/per-user/${uname} + + # see doc/manual/source/command-ref/files/profiles.md + ln -s ${profile} $out/nix/var/nix/profiles/default-1-link + ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile + + # see doc/manual/source/command-ref/files/channels.md + ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link + ln -s /nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels + + # see doc/manual/source/command-ref/files/default-nix-expression.md + mkdir -p $out${userHome}/.nix-defexpr + ln -s /nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels + echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels + + # may get replaced by pkgs.dockerTools.binSh & pkgs.dockerTools.usrBinEnv + mkdir -p $out/bin $out/usr/bin + ln -s ${lib.getExe' coreutils-full "env"} $out/usr/bin/env + ln -s ${lib.getExe bashInteractive} $out/bin/sh + + '' + + (lib.optionalString (flake-registry-path != null) '' + nixCacheDir="${userHome}/.cache/nix" + mkdir -p $out$nixCacheDir + globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" + ln -s ${flake-registry-path} $out$globalFlakeRegistryPath + mkdir -p $out/nix/var/nix/gcroots/auto + rootName=$(${lib.getExe' nix "nix"} --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) + ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName + '') + ); + +in +dockerTools.buildLayeredImageWithNixDb { + + inherit + name + tag + maxLayers + uid + gid + uname + gname + ; + + contents = [ baseSystem ]; + + extraCommands = '' + rm -rf nix-support + ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles + ''; + fakeRootCommands = '' + chmod 1777 tmp + chmod 1777 var/tmp + chown -R ${toString uid}:${toString gid} .${userHome} + chown -R ${toString uid}:${toString gid} nix + ''; + + config = { + inherit Cmd Labels; + User = "${toString uid}:${toString gid}"; + Env = [ + "USER=${uname}" + "PATH=${ + lib.concatStringsSep ":" [ + "${userHome}/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ] + }" + "MANPATH=${ + lib.concatStringsSep ":" [ + "${userHome}/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ] + }" + "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_PATH=/nix/var/nix/profiles/per-user/${uname}/channels:${userHome}/.nix-defexpr/channels" + ]; + }; + +} diff --git a/nixos/tests/gitlab/runner/podman-runner/scripts/prebuild.nix b/nixos/tests/gitlab/runner/podman-runner/scripts/prebuild.nix index 78b8b0f41b18..4e78638f09a5 100644 --- a/nixos/tests/gitlab/runner/podman-runner/scripts/prebuild.nix +++ b/nixos/tests/gitlab/runner/podman-runner/scripts/prebuild.nix @@ -29,7 +29,7 @@ writeShellScriptBin "gitlab-runner-pre-build-script" function setup_pipeline_scratch_dir() { scratch_dir="/scratch/$CI_PIPELINE_ID" - echo "Create scrtach directory for pipeline: $scratch_dir" + echo "Create scratch directory for pipeline: $scratch_dir" mkdir -p "$scratch_dir" || { echo "Could not create scratch dir '$scratch_dir'." >&2 exit 1