fix: improvements on the images

This commit is contained in:
Gabriel Nützi
2025-12-16 01:06:30 +01:00
parent 7b3047b586
commit 762916259a
19 changed files with 290 additions and 214 deletions
+5 -5
View File
@@ -155,11 +155,11 @@ The [VM tested `podman-runner`](https://github.com/NixOS/nixpkgs/blob/master/nix
**Container Images for Gitlab Jobs**:
- `local/alpine`: An image based on Alpine with a Nix installation
(variable `alpineImage`).
(attribute `jobImages.alpine`).
- `local/ubuntu`: An image based on Ubuntu with a Nix installation
(variable `ubuntuImage`).
(attribute `jobImages.ubuntu`).
- `local/nix`: An image based on Nix which only comes with `nix`
installed (variable `nixImage`).
installed (attribute `jobImages.nix`).
**Images for VM Setup**:
- `local/nix-daemon-image`: An image with a Nix daemon which is
@@ -169,7 +169,7 @@ The [VM tested `podman-runner`](https://github.com/NixOS/nixpkgs/blob/master/nix
(variable `podmanDaemonImage`).
- Every job container runs in a `podman` container instance based by default on
`ubuntuImage`. A pipeline job can override this with `image: local/alpine`.
`jobImage.ubuntu`. A pipeline job can override this with `image: local/alpine`.
- Each job container will have the `/nix/store` mounted from the container
`nix-daemon-container` (see registration flags
`--docker-volumes-from "nix-daemon-container:ro"`).
@@ -202,7 +202,7 @@ The [VM tested `podman-runner`](https://github.com/NixOS/nixpkgs/blob/master/nix
::: {.note}
Building container images with `buildah` (stripped
`podman` for building images) inside a job which runs `alpineImage`
`podman` for building images) inside a job which runs `jobImage.alpine`
is still possible.
:::
@@ -18,6 +18,10 @@
# - The `bootstrapPkgs` derivation is copied into the job containers
# but without the Nix store paths cause they get provided by the
# `nix-daemon-store` volume.
# I cannot denote these volumes because they overmount the
# shit which is in the image.
# TODO: make a systemd service which starts before
# that and creates some volumes and inits these from the image.
#
# - The `podman-daemon-socket` volume gets mounted to the job container
# enabling it to use `podman`.
@@ -77,16 +81,24 @@ let
};
# This derivation will contain a folder `/etc`
auxRootFiles = pkgs.callPackage ./root { };
files = pkgs.callPackage ./files { };
preBuildScript = pkgs.callPackage ./scripts/prebuild.nix { };
# These derivations are Linked into the job images root dir.
bootstrapPkgs = [
pkgs.nix
# Runtime dependencies of nix.
pkgs.gnutar
pkgs.gzip
pkgs.openssh
pkgs.xz
pkgs.cacert
# Other stuff.
(lib.hiPrio pkgs.coreutils)
(lib.hiPrio pkgs.findutils)
pkgs.openssh
pkgs.bash
pkgs.bashInteractive
(lib.hiPrio pkgs.git)
pkgs.cachix
@@ -94,7 +106,16 @@ let
pkgs.podman # For nested containers.
preBuildScript
auxRootFiles
files.containers
files.nixConfig
];
# All these packages are added to the Nix daemon.
nixStorePkgs = bootstrapPkgs ++ [
# These files
files.basicRoot
files.fakeNixpkgs
];
toEnvList = envs: lib.mapAttrsToList (k: v: "${k}=${v}") envs;
@@ -110,6 +131,8 @@ let
# You can add here a user with uid,gid,uname,gname etc.
# We are using root.
extraPkgs = nixStorePkgs;
nixConf = {
cores = "0";
experimental-features = [
@@ -126,8 +149,6 @@ let
name = "local/nix-daemon";
tag = "latest";
contents = bootstrapPkgs;
config = {
Volumes = {
"/nix/store" = { };
@@ -166,95 +187,119 @@ let
};
};
jobImages = {
# The base image
nix = pkgs.dockerTools.buildLayeredImage {
fromImage = nixImageBase;
name = imageNames.nix;
tag = "latest";
jobImages =
let
extraCommands = ''
set -eu
# Set missing Nix directories.
mkdir -p -m 0755 nix/var/log/nix/drvs
mkdir -p -m 0755 nix/var/nix/{gcroots,profiles,temproots,userpool}
mkdir -p -m 1777 nix/var/nix/{gcroots,profiles}/per-user
mkdir -p -m 0755 nix/var/nix/profiles/per-user/root
contents = bootstrapPkgs;
# No store paths are copied into. We provide them by mounting the
# /nix/store.
includeStorePaths = false;
# Need a HOME.
mkdir -vp root
mkdir -p -m 0700 root/.nix-defexpr
'';
in
{
# The Nix image.
nix = pkgs.dockerTools.buildLayeredImage {
name = imageNames.nix;
tag = "latest";
config = {
Labels = noPruneLabels;
Env = toEnvList envs.nix;
extraCommands = extraCommands + ''
set -eu
# For `/usr/bin/env`.
mkdir -p usr && ln -s ../bin usr/bin
'';
contents = bootstrapPkgs ++ [ files.basicRoot ];
# No store paths are copied into. We provide them by mounting the
# /nix/store.
includeStorePaths = false;
config = {
Labels = noPruneLabels;
Env = toEnvList envs.nix;
};
maxLayers = 2;
};
maxLayers = 4;
# This is the analog image to `local/nix` but alpine based.
alpine =
let
# Update with:
# ```shell
# nix run "github:nixos/nixpkgs/nixos-unstable#nix-prefetch-docker" -- --image-name alpine --image-tag latest
# ```
alpineBase = pkgs.dockerTools.pullImage {
imageName = "alpine";
imageDigest = "sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d";
sha256 = "0gf7wbjp37zbni3pz8vdgq1mss6mz69wynms0gqhq7lsxfmg9xj9";
finalImageName = "alpine";
finalImageTag = "latest";
};
in
(pkgs.dockerTools.buildLayeredImage {
fromImage = alpineBase;
name = imageNames.alpine;
tag = "latest";
inherit extraCommands;
contents = bootstrapPkgs;
# No store paths are copied into. We provide them by mounting the
# /nix/store.
includeStorePaths = false;
config = {
Labels = noPruneLabels;
Env = toEnvList envs.nix;
};
# Only if `build buildLayeredImage`.
maxLayers = 3;
});
# This is the analog image to `local/nix` but ubuntu based.
ubuntu =
let
# Update with:
# ```shell
# nix run "github:nixos/nixpkgs/nixos-unstable#nix-prefetch-docker" -- \
# --image-name ubuntu --image-tag latest
# ```
ubuntuBase = pkgs.dockerTools.pullImage {
imageName = "ubuntu";
imageDigest = "sha256:1e622c5f073b4f6bfad6632f2616c7f59ef256e96fe78bf6a595d1dc4376ac02";
hash = "sha256-aC8SgxdcMSaaU89YMr/uwE022Yqey2frmeZqr+L1xEU=";
finalImageName = "ubuntu";
finalImageTag = "latest";
};
in
(pkgs.dockerTools.buildLayeredImage {
fromImage = ubuntuBase;
name = imageNames.ubuntu;
tag = "latest";
inherit extraCommands;
contents = bootstrapPkgs;
# No store paths are copied into. We provide them by mounting the
# /nix/store.
includeStorePaths = false;
config = {
Labels = noPruneLabels;
Env = toEnvList envs.ubuntu;
};
# Only if `build buildLayeredImage`.
maxLayers = 3;
});
};
# This is the analog image to `local/nix` but alpine based.
alpine =
let
# Update with:
# ```shell
# nix run "github:nixos/nixpkgs/nixos-unstable#nix-prefetch-docker" -- --image-name alpine --image-tag latest
# ```
alpineBase = pkgs.dockerTools.pullImage {
imageName = "alpine";
imageDigest = "sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d";
sha256 = "0gf7wbjp37zbni3pz8vdgq1mss6mz69wynms0gqhq7lsxfmg9xj9";
finalImageName = "alpine";
finalImageTag = "latest";
};
in
(pkgs.dockerTools.buildLayeredImage {
fromImage = alpineBase;
name = imageNames.alpine;
tag = "latest";
contents = bootstrapPkgs;
# No store paths are copied into. We provide them by mounting the
# /nix/store.
includeStorePaths = false;
config = {
Labels = noPruneLabels;
Env = toEnvList envs.alpine;
};
# Only if `build buildLayeredImage`.
maxLayers = 15;
});
# This is the analog image to `local/nix` but ubuntu based.
ubuntu =
let
# Update with:
# ```shell
# nix run "github:nixos/nixpkgs/nixos-unstable#nix-prefetch-docker" -- \
# --image-name ubuntu --image-tag latest
# ```
ubuntuBase = pkgs.dockerTools.pullImage {
imageName = "ubuntu";
imageDigest = "sha256:1e622c5f073b4f6bfad6632f2616c7f59ef256e96fe78bf6a595d1dc4376ac02";
hash = "sha256-aC8SgxdcMSaaU89YMr/uwE022Yqey2frmeZqr+L1xEU=";
finalImageName = "ubuntu";
finalImageTag = "latest";
};
in
(pkgs.dockerTools.buildLayeredImage {
fromImage = ubuntuBase;
name = imageNames.ubuntu;
tag = "latest";
contents = bootstrapPkgs;
# No store paths are copied into. We provide them by mounting the
# /nix/store.
includeStorePaths = false;
config = {
Labels = noPruneLabels;
Env = toEnvList envs.ubuntu;
};
# Only if `build buildLayeredImage`.
maxLayers = 15;
});
};
nixDaemonContainer = {
imageFile = nixDaemonImage;
image = "local/nix-daemon:latest";
@@ -293,27 +338,36 @@ let
# Environment variables for all job containers.
envs = rec {
daemons = {
common = {
# Access to the nix daemon.
NIX_REMOTE = "daemon";
# Access to podman.
CONTAINER_HOST = "unix:///run/podman/podman.sock";
};
nix = daemons // {
IMAGE_OS_DIST = "nixos";
};
alpine = daemons // {
IMAGE_OS_DIST = "alpine";
USER = "root";
PATH = "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin";
SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
NIX_SSL_CERT_FILE = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
# For shells, source this file.
ENV = "${pkgs.nix}/etc/profile.d/nix-daemon.sh";
BASH_ENV = "${pkgs.nix}/etc/profile.d/nix-daemon.sh";
# Make a fake nixpkgs which throws when using
# `nix repl -f <nixpkgs>` for example.
NIX_PATH = "nixpkgs=${files.fakeNixpkgs}";
};
ubuntu = alpine // {
nix = common // {
IMAGE_OS_DIST = "nix";
};
alpine = common // {
IMAGE_OS_DIST = "alpine";
};
ubuntu = common // {
IMAGE_OS_DIST = "ubuntu";
};
};
@@ -0,0 +1,21 @@
root:x:0:
wheel:x:1:
kmem:x:2:
tty:x:3:
messagebus:x:4:
disk:x:6:
audio:x:17:
floppy:x:18:
uucp:x:19:
lp:x:20:
cdrom:x:24:
tape:x:25:
video:x:26:
dialout:x:27:
utmp:x:29:
adm:x:55:
keys:x:96:
users:x:100:
input:x:174:
nixbld:x:30000:nixbld1,nixbld10,nixbld11,nixbld12,nixbld13,nixbld14,nixbld15,nixbld16,nixbld17,nixbld18,nixbld19,nixbld2,nixbld20,nixbld21,nixbld22,nixbld23,nixbld24,nixbld25,nixbld26,nixbld27,nixbld28,nixbld29,nixbld3,nixbld30,nixbld31,nixbld32,nixbld4,nixbld5,nixbld6,nixbld7,nixbld8,nixbld9
nogroup:x:65534:
@@ -0,0 +1,11 @@
passwd: files mymachines systemd
group: files mymachines systemd
shadow: files
hosts: files mymachines dns myhostname
networks: files
ethers: files
services: files
protocols: files
rpc: files
@@ -0,0 +1,34 @@
root:x:0:0:System administrator:/root:/bin/bash
nixbld1:x:30001:30000:Nix build user 1:/var/empty:/run/current-system/sw/bin/nologin
nixbld2:x:30002:30000:Nix build user 2:/var/empty:/run/current-system/sw/bin/nologin
nixbld3:x:30003:30000:Nix build user 3:/var/empty:/run/current-system/sw/bin/nologin
nixbld4:x:30004:30000:Nix build user 4:/var/empty:/run/current-system/sw/bin/nologin
nixbld5:x:30005:30000:Nix build user 5:/var/empty:/run/current-system/sw/bin/nologin
nixbld6:x:30006:30000:Nix build user 6:/var/empty:/run/current-system/sw/bin/nologin
nixbld7:x:30007:30000:Nix build user 7:/var/empty:/run/current-system/sw/bin/nologin
nixbld8:x:30008:30000:Nix build user 8:/var/empty:/run/current-system/sw/bin/nologin
nixbld9:x:30009:30000:Nix build user 9:/var/empty:/run/current-system/sw/bin/nologin
nixbld10:x:30010:30000:Nix build user 10:/var/empty:/run/current-system/sw/bin/nologin
nixbld11:x:30011:30000:Nix build user 11:/var/empty:/run/current-system/sw/bin/nologin
nixbld12:x:30012:30000:Nix build user 12:/var/empty:/run/current-system/sw/bin/nologin
nixbld13:x:30013:30000:Nix build user 13:/var/empty:/run/current-system/sw/bin/nologin
nixbld14:x:30014:30000:Nix build user 14:/var/empty:/run/current-system/sw/bin/nologin
nixbld15:x:30015:30000:Nix build user 15:/var/empty:/run/current-system/sw/bin/nologin
nixbld16:x:30016:30000:Nix build user 16:/var/empty:/run/current-system/sw/bin/nologin
nixbld17:x:30017:30000:Nix build user 17:/var/empty:/run/current-system/sw/bin/nologin
nixbld18:x:30018:30000:Nix build user 18:/var/empty:/run/current-system/sw/bin/nologin
nixbld19:x:30019:30000:Nix build user 19:/var/empty:/run/current-system/sw/bin/nologin
nixbld20:x:30020:30000:Nix build user 20:/var/empty:/run/current-system/sw/bin/nologin
nixbld21:x:30021:30000:Nix build user 21:/var/empty:/run/current-system/sw/bin/nologin
nixbld22:x:30022:30000:Nix build user 22:/var/empty:/run/current-system/sw/bin/nologin
nixbld23:x:30023:30000:Nix build user 23:/var/empty:/run/current-system/sw/bin/nologin
nixbld24:x:30024:30000:Nix build user 24:/var/empty:/run/current-system/sw/bin/nologin
nixbld25:x:30025:30000:Nix build user 25:/var/empty:/run/current-system/sw/bin/nologin
nixbld26:x:30026:30000:Nix build user 26:/var/empty:/run/current-system/sw/bin/nologin
nixbld27:x:30027:30000:Nix build user 27:/var/empty:/run/current-system/sw/bin/nologin
nixbld28:x:30028:30000:Nix build user 28:/var/empty:/run/current-system/sw/bin/nologin
nixbld29:x:30029:30000:Nix build user 29:/var/empty:/run/current-system/sw/bin/nologin
nixbld30:x:30030:30000:Nix build user 30:/var/empty:/run/current-system/sw/bin/nologin
nixbld31:x:30031:30000:Nix build user 31:/var/empty:/run/current-system/sw/bin/nologin
nixbld32:x:30032:30000:Nix build user 32:/var/empty:/run/current-system/sw/bin/nologin
nobody:x:65534:65534:Unprivileged account (don't use!):/var/empty:/run/current-system/sw/bin/nologin
@@ -0,0 +1,46 @@
# Specific files for the job images.
#
# - `basicRoot`: Some basic root files for the `jobImages.nix`.
# - `fakeNixpkgs`: A fake Nixpkg directory which is set as `NIX_PATH=nixpkgs:<path>`
# which throws on load.
# - `nixConfig`: The Nix config with some options.
# - `containers`:
# These are some files which are copied to the job images needed for
# `buildah` (`podman`):
#
# ```bash
# podman create --name temp-buildah quay.io/buildah/stable:latest
# podman cp temp-buildah:/etc/containers ./etc/
# find ./etc -type d -empty -delete
# podman container rm temp-buildah
#```
#
{ pkgs, ... }:
let
# We need proper derivations to add it to the nixImageBase.
mkDrv =
name: src:
pkgs.stdenv.mkDerivation {
inherit name src;
installPhase = ''
mkdir -p $out
cp -r $src/* $out/
'';
};
in
{
basicRoot = mkDrv "basic-root-files" ./basicRoot;
containers = mkDrv "containers-files" ./containers;
fakeNixpkgs = mkDrv "fake-nixpkgs" ./fake-nixpkgs;
nixConfig = pkgs.writeTextFile {
name = "nix.conf";
destination = "/etc/nix/nix.conf";
text = ''
accept-flake-config = true
experimental-features = nix-command flakes
max-jobs = auto
'';
};
}
@@ -0,0 +1,10 @@
_:
throw ''
This container doesn't include nixpkgs.
The best way to work around that is to pin your dependencies. See
https://nix.dev/tutorials/first-steps/towards-reproducibility-pinning-nixpkgs.html
Or if you must, override the NIX_PATH environment variable with eg:
"NIX_PATH=nixpkgs=channel:nixos-unstable"
''
@@ -1,29 +0,0 @@
## Root files for podman executions in job containers.
#
# These are some files which are copied to the image in the runner needed for
# `buildah` (`podman`):
#
# ```bash
# podman create --name temp-buildah quay.io/buildah/stable:latest
# podman cp temp-buildah:/etc/containers ./etc/
# find ./etc -type d -empty -delete
# podman container rm temp-buildah
#```
#
{ lib, pkgs, ... }:
let
fs = lib.fileset;
auxRootFiles = fs.toSource {
root = ./.;
fileset = ./etc;
};
in
pkgs.stdenv.mkDerivation {
name = "aux-root-files";
src = auxRootFiles;
installPhase = ''
mkdir -p $out
cp -r $src/* $out/
'';
}
@@ -16,7 +16,7 @@ writeShellScriptBin "gitlab-runner-pre-build-script"
echo -e "\e[0Ksection_end:$(date +%s):$name\r\e[0K"
}
function setup_nixos() {
function setup() {
# We need to allow modification of nix config for cachix as
# otherwise it is link to the read only file in the store.
cp --remove-destination \
@@ -26,55 +26,26 @@ writeShellScriptBin "gitlab-runner-pre-build-script"
. "${nix}/etc/profile.d/nix-daemon.sh"
}
function setup_non_nixos() {
# shellcheck disable=SC2174
{
echo "Set missing '/nix/var' directories."
mkdir -p -m 0755 /nix/var/log/nix/drvs
mkdir -p -m 0755 /nix/var/nix/gcroots
mkdir -p -m 0755 /nix/var/nix/profiles
mkdir -p -m 0755 /nix/var/nix/temproots
mkdir -p -m 0755 /nix/var/nix/userpool
mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
mkdir -p -m 1777 /nix/var/nix/profiles/per-user
mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
echo "Set missing '~/.nix-defexpr'."
mkdir -p -m 0700 ~/.nix-defexpr
}
echo "Source 'nix-daemon.sh'."
# shellcheck disable=SC1091
. "${nix}/etc/profile.d/nix-daemon.sh"
echo "Set nix experimental features."
mkdir -p ~/.config/nix
touch ~/.config/nix/nix.conf
echo "experimental-features = nix-command flakes" >>~/.config/nix/nix.conf
}
function setup_pipeline_scratch_dir() {
scratch_dir="/scratch/$CI_PIPELINE_ID"
echo "Create scratch directory for pipeline: $scratch_dir"
echo "Create scrtach directory for pipeline: $scratch_dir"
mkdir -p "$scratch_dir" || {
echo "Could not create scratch dir '$scratch_dir'." >&2
exit 1
}
export CI_CUSTODIAN_SCRATCH_DIR="$scratch_dir"
export CI_CUSTOM_SCRATCH_DIR="$scratch_dir"
}
function print_info() {
echo "Nix version:"
nix --version
}
function main() {
if [ "$IMAGE_OS_DIST" = "ubuntu" ] || [ "$IMAGE_OS_DIST" = "alpine" ]; then
setup_non_nixos
elif [ "$IMAGE_OS_DIST" = "nixos" ]; then
setup_nixos
else
echo "OS not implemented."
exit 1
fi
print_info
setup
setup_pipeline_scratch_dir
}
@@ -1,42 +0,0 @@
#!/usr/bin/env bash
set -eu
[ "$(hostname)" = "vm-gitlab-runner" ] || {
echo "error: you do not seem to be logged in on the vm."
exit 1
}
NIX_STORE_VOLUME_ID=$(podman container inspect -f "{{ json .Mounts }}" nix-daemon-container |
jq -r '.[] | select(.Type == "volume" and .Destination == "/nix/store") | .Name')
NIX_DB_VOLUME_ID=$(podman container inspect -f "{{ json .Mounts }}" nix-daemon-container |
jq -r '.[] | select(.Type == "volume" and .Destination == "/nix/var/nix/db") | .Name')
[ -n "$NIX_STORE_VOLUME_ID" ] || {
echo "Could not determine podman volume with nix store in 'nix-daemon-container'." >&2
exit 1
}
echo "NIX_STORE_VOLUME_ID: $NIX_STORE_VOLUME_ID"
echo "NIX_DB_VOLUME_ID: $NIX_DB_VOLUME_ID"
if [ "${1:-}" = "--force" ]; then
just podman-remove-containers
fi
echo "Stopping gitlab runner ..."
systemctl stop gitlab-runner.service
echo "Remove podman-nix-daemon-container..."
systemctl stop podman-nix-daemon-container.service
podman container stop nix-daemon-container || true
echo "Delete nix-daemon-store and nix-daemon-db..."
podman volume rm --force "$NIX_STORE_VOLUME_ID" || true
podman volume rm --force "$NIX_DB_VOLUME_ID" || true
echo "Recreate empty volumes ..."
podman volume create "$NIX_STORE_VOLUME_ID" --label="no-prune=true"
podman volume create "$NIX_DB_VOLUME_ID" --label="no-prune=true"
systemctl start podman-nix-daemon-container.service
systemctl start gitlab-runner.service