From 608db26178c297c306d3739d21df101b63fd1800 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 23 Jan 2024 17:03:08 +0100 Subject: [PATCH 1/3] grafana: build frontend from source Up until now, the frontend was taken from `srcStatic`, i.e. prebuilt from upstream. I recall at least three cases[1][2][3] where we got a hash mismatch eventually. Rather than spending time finding out whether or not it's a supply-chain attack or just a build issue, I decided to implement a source-build now with the following benefits: * It's now actually possible to apply patches for Grafana's frontend. * We rely a little less on third-party build systems. Of course, patching potential vulnerabilities in transitive frontend dependencies is still hard (let alone discovering that this package is affected!), but that's a fundamental issue we have in nixpkgs and I won't invent a half-baked solution just for this package, I still consider this a step into the right direction. The build itself mainly orients on the `yarn` commands used in the upstream Makefile[4]. However, we can't use `fetchYarnDeps` here because yarn v2 (a.k.a. `berry`) is in use which is why the same was done as in `hedgedoc`, writing a custom FoD that downloads all dependencies and writes the offline cache into `$out`[5]. Additionally there are two more notable differences to upstream: * We patch out every dependency to `@grafana/e2e` and `cypress`. The first is a dependency on the latter in another version and the latter downloads random blobs from the Internet in postInstall. Since it's a testing framework (and the `e2e` package apparently a testing library), I decided it's not worth the effort and patched it out everywhere. * There was a `zoneinfo.zip` in `$out/share/grafana/tools` that was installed from `srcStatic`. This only seems to be used on Windows[6] and that's not supported by this package, so I decided to drop it. [1] https://github.com/NixOS/nixpkgs/pull/251479 [2] https://github.com/NixOS/nixpkgs/pull/130201 [3] https://github.com/NixOS/nixpkgs/pull/104794 [4] https://github.com/grafana/grafana/blob/v10.3.1/Makefile [5] https://github.com/NixOS/nixpkgs/pull/245170 [6] https://github.com/grafana/grafana/blob/v10.3.1/pkg/setting/setting.go#L1012-L1014 --- pkgs/servers/monitoring/grafana/default.nix | 92 ++++++++++++++++----- pkgs/servers/monitoring/grafana/update.sh | 40 --------- 2 files changed, 72 insertions(+), 60 deletions(-) delete mode 100755 pkgs/servers/monitoring/grafana/update.sh diff --git a/pkgs/servers/monitoring/grafana/default.nix b/pkgs/servers/monitoring/grafana/default.nix index 9dc861c75866..5df8737eb2af 100644 --- a/pkgs/servers/monitoring/grafana/default.nix +++ b/pkgs/servers/monitoring/grafana/default.nix @@ -1,5 +1,35 @@ -{ lib, buildGoModule, fetchurl, fetchFromGitHub, nixosTests, tzdata, wire }: +{ lib, stdenv, buildGoModule, fetchFromGitHub +, tzdata, wire +, yarn, nodejs, cacert +, jq, moreutils +, nix-update-script, nixosTests +}: +let + # We need dev dependencies to run webpack, but patch away + # `cypress` (and @grafana/e2e which has a direct dependency on cypress). + # This attempts to download random blobs from the Internet in + # postInstall. Also, it's just a testing framework, so not worth the hassle. + patchAwayGrafanaE2E = '' + find . -name package.json | while IFS=$'\n' read -r pkg_json; do + <"$pkg_json" jq '. + { + "devDependencies": .devDependencies | del(."@grafana/e2e") | del(.cypress) + }' | sponge "$pkg_json" + done + rm -r packages/grafana-e2e + ''; + + # Injects a `t.Skip()` into a given test since + # there's apparently no other way to skip tests here. + skipTest = lineOffset: testCase: file: + let + jumpAndAppend = lib.concatStringsSep ";" (lib.replicate (lineOffset - 1) "n" ++ [ "a" ]); + in '' + sed -i -e '/${testCase}/{ + ${jumpAndAppend} t.Skip(); + }' ${file} + ''; +in buildGoModule rec { pname = "grafana"; version = "10.2.3"; @@ -13,25 +43,38 @@ buildGoModule rec { hash = "sha256-F61RtPEjQ4uFVcJLG04CD4//w8X7uJinxzYyoW/MosA="; }; - srcStatic = fetchurl { - url = "https://dl.grafana.com/oss/release/grafana-${version}.linux-amd64.tar.gz"; - hash = "sha256-xoZgaml1SB9PEI3kTE3zRlJR5O4tog58bua2blvc8to="; + offlineCache = stdenv.mkDerivation { + name = "${pname}-${version}-yarn-offline-cache"; + inherit src; + nativeBuildInputs = [ + yarn nodejs cacert + jq moreutils + ]; + buildPhase = '' + ${patchAwayGrafanaE2E} + export HOME="$(mktemp -d)" + yarn config set enableTelemetry 0 + yarn config set cacheFolder $out + yarn config set --json supportedArchitectures.os '[ "linux" ]' + yarn config set --json supportedArchitectures.cpu '["arm", "arm64", "ia32", "x64"]' + yarn + ''; + dontConfigure = true; + dontInstall = true; + dontFixup = true; + outputHashMode = "recursive"; + outputHash = "sha256-cD9Y72OWSj7zwhhAcrZouLpEFFwURSsWgDFRjwMQAxI="; }; vendorHash = "sha256-rQOnuh6t+cUqyAAnUhGgxMaW88pawnauAGQd6w0T57Q="; - nativeBuildInputs = [ wire ]; + nativeBuildInputs = [ wire yarn jq moreutils ]; - postConfigure = let - skipTest = lineOffset: testCase: file: - let - jumpAndAppend = lib.concatStringsSep ";" (lib.replicate (lineOffset - 1) "n" ++ [ "a" ]); - in '' - sed -i -e '/${testCase}/{ - ${jumpAndAppend} t.Skip(); - }' ${file} - ''; - in '' + postPatch = '' + ${patchAwayGrafanaE2E} + ''; + + postConfigure = '' # Generate DI code that's required to compile the package. # From https://github.com/grafana/grafana/blob/v8.2.3/Makefile#L33-L35 wire gen -tags oss ./pkg/server @@ -69,6 +112,18 @@ buildGoModule rec { # grafana> 2023/08/24 08:30:23 failed to copy objects, err: Post "https://storage.googleapis.com/upload/storage/v1/b/grafana-testing-repo/o?alt=json&name=test-path%2Fbuild%2FTestCopyLocalDir2194093976%2F001%2Ffile2.txt&prettyPrint=false&projection=full&uploadType=multipart": dial tcp: lookup storage.googleapis.com on [::1]:53: read udp [::1]:36436->[::1]:53: read: connection refused # grafana> panic: test timed out after 10m0s rm pkg/build/gcloud/storage/gsutil_test.go + + # Setup node_modules + export HOME="$(mktemp -d)" + yarn config set enableTelemetry 0 + yarn config set cacheFolder $offlineCache + yarn --immutable-cache + ''; + + postBuild = '' + # After having built all the Go code, run the JS builders now. + yarn run build + yarn run plugins:build-bundled ''; ldflags = [ @@ -86,16 +141,13 @@ buildGoModule rec { ''; postInstall = '' - tar -xvf $srcStatic mkdir -p $out/share/grafana - mv grafana-*/{public,conf,tools} $out/share/grafana/ - - cp ./conf/defaults.ini $out/share/grafana/conf/ + cp -r public conf $out/share/grafana/ ''; passthru = { tests = { inherit (nixosTests) grafana; }; - updateScript = ./update.sh; + updateScript = nix-update-script { }; }; meta = with lib; { diff --git a/pkgs/servers/monitoring/grafana/update.sh b/pkgs/servers/monitoring/grafana/update.sh deleted file mode 100755 index 3fd4fd56be7c..000000000000 --- a/pkgs/servers/monitoring/grafana/update.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -p curl common-updater-scripts jq nix nix-prefetch-scripts moreutils - -set -euxo pipefail - -FILE="$(nix-instantiate --eval -E 'with import ./. {}; (builtins.unsafeGetAttrPos "version" grafana).file' | tr -d '"')" -replaceHash() { - old="${1?old hash missing}" - new="${2?new hash missing}" - awk -v OLD="$old" -v NEW="$new" '{ - if (i=index($0, OLD)) { - $0 = substr($0, 1, i-1) NEW substr($0, i+length(OLD)); - } - print $0; - }' "$FILE" | sponge "$FILE" -} -extractVendorHash() { - original="${1?original hash missing}" - result="$(nix-build -A grafana.goModules 2>&1 | tail -n3 | grep 'got:' | cut -d: -f2- | xargs echo || true)" - [ -z "$result" ] && { echo "$original"; } || { echo "$result"; } -} - -oldVersion="$(nix-instantiate --eval -E "with import ./. {}; lib.getVersion grafana" | tr -d '"')" -latest="$(curl https://api.github.com/repos/grafana/grafana/releases/latest | jq '.tag_name' -r | tr -d 'v')" - -targetVersion="${1:-$latest}" -if [ ! "${oldVersion}" = "${targetVersion}" ]; then - update-source-version grafana "${targetVersion#v}" - oldStaticHash="$(nix-instantiate --eval -A grafana.srcStatic.outputHash | tr -d '"')" - newStaticHash="$(nix-prefetch-url "https://dl.grafana.com/oss/release/grafana-${targetVersion#v}.linux-amd64.tar.gz")" - newStaticHash="$(nix hash to-sri --type sha256 $newStaticHash)" - replaceHash "$oldStaticHash" "$newStaticHash" - goHash="$(nix-instantiate --eval -A grafana.vendorHash | tr -d '"')" - emptyHash="$(nix-instantiate --eval -A lib.fakeHash | tr -d '"')" - replaceHash "$goHash" "$emptyHash" - replaceHash "$emptyHash" "$(extractVendorHash "$goHash")" - nix-build -A grafana -else - echo "grafana is already up-to-date" -fi From b1c52266c24001439f8fda3217cf41fbb98b118d Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 23 Jan 2024 17:08:33 +0100 Subject: [PATCH 2/3] grafana: 10.2.3 -> 10.3.1 ChangeLog: https://github.com/grafana/grafana/releases/tag/v10.3.0 ChangeLog: https://github.com/grafana/grafana/releases/tag/v10.3.1 --- pkgs/servers/monitoring/grafana/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/servers/monitoring/grafana/default.nix b/pkgs/servers/monitoring/grafana/default.nix index 5df8737eb2af..4c2784e5ff01 100644 --- a/pkgs/servers/monitoring/grafana/default.nix +++ b/pkgs/servers/monitoring/grafana/default.nix @@ -32,7 +32,7 @@ let in buildGoModule rec { pname = "grafana"; - version = "10.2.3"; + version = "10.3.1"; excludedPackages = [ "alert_webhook_listener" "clean-swagger" "release_publisher" "slow_proxy" "slow_proxy_mac" "macaron" "devenv" "modowners" ]; @@ -40,7 +40,7 @@ buildGoModule rec { owner = "grafana"; repo = "grafana"; rev = "v${version}"; - hash = "sha256-F61RtPEjQ4uFVcJLG04CD4//w8X7uJinxzYyoW/MosA="; + hash = "sha256-UPIq7BWTlT0omt/SM5+vkfOHvsdcx/ikkjcW9X8pcw0="; }; offlineCache = stdenv.mkDerivation { @@ -63,10 +63,10 @@ buildGoModule rec { dontInstall = true; dontFixup = true; outputHashMode = "recursive"; - outputHash = "sha256-cD9Y72OWSj7zwhhAcrZouLpEFFwURSsWgDFRjwMQAxI="; + outputHash = "sha256-70eMa8E483f/Bz7iy+4Seap1EfIdjD5krnt6W9CUows="; }; - vendorHash = "sha256-rQOnuh6t+cUqyAAnUhGgxMaW88pawnauAGQd6w0T57Q="; + vendorHash = "sha256-Gf2A22d7/8xU/ld7kveqGonVKGFCArGNansPRGhfyXM="; nativeBuildInputs = [ wire yarn jq moreutils ]; From 4787b13af9e9bde7775465597201285f0cd02b5e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 23 Jan 2024 23:10:52 +0100 Subject: [PATCH 3/3] grafana: ensure that final derivation doesn't depend on yarn cache This saves 350M of runtime closure size. Also move the patching part of the offline cache into the patch phase. --- pkgs/servers/monitoring/grafana/default.nix | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkgs/servers/monitoring/grafana/default.nix b/pkgs/servers/monitoring/grafana/default.nix index 4c2784e5ff01..3b4d930d4537 100644 --- a/pkgs/servers/monitoring/grafana/default.nix +++ b/pkgs/servers/monitoring/grafana/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, buildGoModule, fetchFromGitHub +{ lib, stdenv, buildGoModule, fetchFromGitHub, removeReferencesTo , tzdata, wire , yarn, nodejs, cacert , jq, moreutils @@ -50,14 +50,18 @@ buildGoModule rec { yarn nodejs cacert jq moreutils ]; - buildPhase = '' + postPatch = '' ${patchAwayGrafanaE2E} + ''; + buildPhase = '' + runHook preBuild export HOME="$(mktemp -d)" yarn config set enableTelemetry 0 yarn config set cacheFolder $out yarn config set --json supportedArchitectures.os '[ "linux" ]' yarn config set --json supportedArchitectures.cpu '["arm", "arm64", "ia32", "x64"]' yarn + runHook postBuild ''; dontConfigure = true; dontInstall = true; @@ -66,9 +70,11 @@ buildGoModule rec { outputHash = "sha256-70eMa8E483f/Bz7iy+4Seap1EfIdjD5krnt6W9CUows="; }; + disallowedRequisites = [ offlineCache ]; + vendorHash = "sha256-Gf2A22d7/8xU/ld7kveqGonVKGFCArGNansPRGhfyXM="; - nativeBuildInputs = [ wire yarn jq moreutils ]; + nativeBuildInputs = [ wire yarn jq moreutils removeReferencesTo ]; postPatch = '' ${patchAwayGrafanaE2E} @@ -145,6 +151,12 @@ buildGoModule rec { cp -r public conf $out/share/grafana/ ''; + postFixup = '' + while read line; do + remove-references-to -t $offlineCache "$line" + done < <(find $out -type f -name '*.js.map' -or -name '*.js') + ''; + passthru = { tests = { inherit (nixosTests) grafana; }; updateScript = nix-update-script { };