diff --git a/pkgs/top-level/metrics.nix b/pkgs/top-level/metrics.nix index c6ff0f349313..c6380b74a667 100644 --- a/pkgs/top-level/metrics.nix +++ b/pkgs/top-level/metrics.nix @@ -1,27 +1,49 @@ { nixpkgs, pkgs }: -with pkgs; +let + inherit (pkgs) lib stdenvNoCC; + evalSystem = "x86_64-linux"; +in + +stdenvNoCC.mkDerivation { + name = "nixpkgs-metrics"; + + # Use structured attrs to pass in relevant information. + __structuredAttrs = true; + inherit evalSystem nixpkgs; + + outputs = [ + "out" + "raw" + ]; + + nativeBuildInputs = map lib.getBin [ + pkgs.nixVersions.latest + pkgs.time + pkgs.jq + ]; + + # see https://github.com/NixOS/nixpkgs/issues/52436 + #requiredSystemFeatures = [ "benchmark" ]; # dedicated `t2a` machine, by @vcunat + + # Required because this derivation doesn't have a `src`. + dontUnpack = true; + + configurePhase = '' + runHook preConfigure -runCommand "nixpkgs-metrics" - { - nativeBuildInputs = - with pkgs.lib; - map getBin [ - nix - time - jq - ]; - # see https://github.com/NixOS/nixpkgs/issues/52436 - #requiredSystemFeatures = [ "benchmark" ]; # dedicated `t2a` machine, by @vcunat - } - '' export NIX_STORE_DIR=$TMPDIR/store export NIX_STATE_DIR=$TMPDIR/state export NIX_PAGER= nix-store --init - mkdir -p $out/nix-support - touch $out/nix-support/hydra-build-products + runHook postConfigure + ''; + + buildPhase = '' + runHook preBuild + + release="$nixpkgs/nixos/release.nix" run() { local name="$1" @@ -29,58 +51,172 @@ runCommand "nixpkgs-metrics" echo "running $@" - case "$name" in - # Redirect stdout to /dev/null to avoid hitting "Output Limit - # Exceeded" on Hydra. - nix-env.qaDrv|nix-env.qaDrvAggressive) - NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=stats-nix time -o stats-time "$@" >/dev/null ;; - *) - NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=stats-nix time -o stats-time "$@" ;; - esac + mkdir -p "metrics/$name" + local output="metrics/$name/output" + local nix_stats="metrics/$name/nix-stats.json" + local time_stats="metrics/$name/time-stats.json" - cat stats-nix; echo; cat stats-time; echo + NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH="$nix_stats" command time -o "$time_stats" -- "$@" > "$output" - x=$(jq '.cpuTime' < stats-nix) - [[ -n $x ]] || exit 1 - echo "$name.time $x s" >> $out/nix-support/hydra-metrics + # Show the Nix statistics and the `time` statistics. + echo "Nix statistics for $@" + jq . "$nix_stats" + echo + echo "Time statistics for $@" + jq . "$time_stats" + echo - x=$(sed -e 's/.* \([0-9]\+\)maxresident.*/\1/ ; t ; d' < stats-time) - [[ -n $x ]] || exit 1 - echo "$name.maxresident $x KiB" >> $out/nix-support/hydra-metrics + cpuTime="$(jq '.cpuTime' < "$nix_stats")" + [[ -n $cpuTime ]] || exit 1 + echo "$name.time $cpuTime s" >> hydra-metrics - # nix-2.2 also outputs .symbols.bytes but that wasn't summed originally - # https://github.com/NixOS/nix/pull/2392/files#diff-8e6ba8c21672fc1a5f6f606e1e101c74L1762 - x=$(jq '[.envs,.list,.values,.sets] | map(.bytes) | add' < stats-nix) - [[ -n $x ]] || exit 1 - echo "$name.allocations $x B" >> $out/nix-support/hydra-metrics + maxresident="$(jq '.max_resident_set_kb' < "$time_stats")" + [[ -n $maxresident ]] || exit 1 + echo "$name.maxresident $maxresident KiB" >> hydra-metrics - x=$(jq '.values.number' < stats-nix) - [[ -n $x ]] || exit 1 - echo "$name.values $x" >> $out/nix-support/hydra-metrics + # Nix also outputs `.symbols.bytes` but since that wasn't summed originally, we don't count it here. + allocations="$(jq '[.envs,.list,.values,.sets] | map(.bytes) | add' < "$nix_stats")" + [[ -n $allocations ]] || exit 1 + echo "$name.allocations $allocations B" >> hydra-metrics + + values="$(jq '.values.number' < "$nix_stats")" + [[ -n $values ]] || exit 1 + echo "$name.values $values" >> hydra-metrics } - run nixos.smallContainer nix-instantiate --dry-run ${nixpkgs}/nixos/release.nix \ - -A closures.smallContainer.x86_64-linux --show-trace - run nixos.kde nix-instantiate --dry-run ${nixpkgs}/nixos/release.nix \ - -A closures.kde.x86_64-linux --show-trace - run nixos.lapp nix-instantiate --dry-run ${nixpkgs}/nixos/release.nix \ - -A closures.lapp.x86_64-linux --show-trace - run nix-env.qa nix-env -f ${nixpkgs} -qa - run nix-env.qaDrv nix-env -f ${nixpkgs} -qa --drv-path --meta --xml + run nixos.smallContainer nix-instantiate --option eval-system "$evalSystem" --dry-run "$release" -A closures.smallContainer.x86_64-linux --show-trace --no-gc-warning + run nixos.kde nix-instantiate --option eval-system "$evalSystem" --dry-run "$release" -A closures.kde.x86_64-linux --show-trace --no-gc-warning + run nixos.lapp nix-instantiate --option eval-system "$evalSystem" --dry-run "$release" -A closures.lapp.x86_64-linux --show-trace --no-gc-warning + run nix-env.qa nix-env --option eval-system "$evalSystem" -f "$nixpkgs" -qa + run nix-env.qaDrv nix-env --option eval-system "$evalSystem" -f "$nixpkgs" -qa --drv-path --meta --json # It's slightly unclear which of the set to track: qaCount, qaCountDrv, qaCountBroken. - num=$(nix-env -f ${nixpkgs} -qa | wc -l) - echo "nix-env.qaCount $num" >> $out/nix-support/hydra-metrics - qaCountDrv=$(nix-env -f ${nixpkgs} -qa --drv-path | wc -l) - num=$((num - $qaCountDrv)) - echo "nix-env.qaCountBroken $num" >> $out/nix-support/hydra-metrics + num="$(wc -l < metrics/nix-env.qa/output)" + echo "nix-env.qaCount $num" >> hydra-metrics + qaCountDrv="$(jq -r 'reduce .[].drvPath as $d (0; .+1)' metrics/nix-env.qaDrv/output)" + numBroken="$((num - $qaCountDrv))" + echo "nix-env.qaCountBroken $numBroken" >> hydra-metrics - # TODO: this has been ignored for some time - # GC Warning: Bad initial heap size 128k - ignoring it. - #export GC_INITIAL_HEAP_SIZE=128k - run nix-env.qaAggressive nix-env -f ${nixpkgs} -qa - run nix-env.qaDrvAggressive nix-env -f ${nixpkgs} -qa --drv-path --meta --xml + lines="$(find "$nixpkgs" -name "*.nix" -type f -print0 | xargs -0 cat | wc -l)" + echo "loc $lines" >> hydra-metrics - lines=$(find ${nixpkgs} -name "*.nix" -type f | xargs cat | wc -l) - echo "loc $lines" >> $out/nix-support/hydra-metrics - '' + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out/nix-support + touch $out/nix-support/hydra-build-products + mv hydra-metrics $out/nix-support/hydra-metrics + + # Save and compress the raw output + mv metrics $raw + xz -v $raw/*/output + + runHook postInstall + ''; + + meta = { + description = "Metrics tracked by Hydra about Nixpkgs"; + homepage = "https://hydra.nixos.org/job/nixpkgs/trunk/metrics"; + longDescription = '' + View the metrics for Nixpkgs evaluation over time at these URLs. + + These are all produced from running `nix` with `NIX_SHOW_STATS=1`. + See `EvalState::printStatistics` in the Nix source code for the implementation. + None of these metrics are inherently meaningful on their own. + Exercise caution in interpreting them as "bad" or "good". + + # Total repository statistics + + - [Lines of code in Nixpkgs](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/loc) + - [Count of broken packages using `nix-env -qa`](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qaCountBroken) + - [Count of packages using `nix-env -qa`](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qaCount) + + # Statistics about representative commands + + These are statistics gathered by running commands against Nixpkgs. + + | Name | Command | + |------|---------| + | `nix-env.qaDrv` | `nix-env -f ${nixpkgs} -qa --drv-path --meta --json` | + | `nix-env.qa` | `nix-env -f ${nixpkgs} -qa` | + | `nixos.kde` | `nix-instantiate --dry-run ${nixpkgs}/nixos/release.nix -A closures.kde.x86_64-linux --show-trace` | + | `nixos.lapp` | `nix-instantiate --dry-run ${nixpkgs}/nixos/release.nix -A closures.lapp.x86_64-linux --show-trace` | + | `nixos.smallContainer` | `nix-instantiate --dry-run ${nixpkgs}/nixos/release.nix -A closures.smallContainer.x86_64-linux --show-trace`| + + ## Allocations performed (in bytes) + + This counts `envs.bytes`, `list.bytes`, `values.bytes`, and `sets.bytes` from the Nix statistics. + + - [nix-env.qa.allocations](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qa.allocations) + - [nix-env.qaDrv.allocations](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qaDrv.allocations) + - [nixos.kde.allocations](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.kde.allocations) + - [nixos.lapp.allocations](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.lapp.allocations) + - [nixos.smallContainer.allocations](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.smallContainer.allocations) + + ## Maximum resident size (in number of KiB) + + This counts `maxresident` KiB (`%M`) from the `time` command on Linux. + + - [nix-env.qa.maxresident](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qa.maxresident) + - [nix-env.qaDrv.maxresident](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qaDrv.maxresident) + - [nixos.kde.maxresident](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.kde.maxresident) + - [nixos.lapp.maxresident](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.lapp.maxresident) + - [nixos.smallContainer.maxresident](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.smallContainer.maxresident) + + ## Time taken (in seconds) + + This counts `cpuTime` as reported in the Nix statistics. On Linux, this resolves to [`getrusage(RUSAGE_SELF)`](https://man7.org/linux/man-pages/man2/getrusage.2.html). + + - [nix-env.qa.time](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qa.time) + - [nix-env.qaDrv.time](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qaDrv.time) + - [nixos.kde.time](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.kde.time) + - [nixos.lapp.time](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.lapp.time) + - [nixos.smallContainer.time](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.smallContainer.time) + + ## Number of values + + This counts the total number of values allocated in Nix (see `EvalState::allocValue` in the Nix source code). + + - [nix-env.qa.values](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qa.values) + - [nix-env.qaDrv.values](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nix-env.qaDrv.values) + - [nixos.kde.values](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.kde.values) + - [nixos.lapp.values](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.lapp.values) + - [nixos.smallContainer.values](https://hydra.nixos.org/job/nixpkgs/trunk/metrics/metric/nixos.smallContainer.values) + ''; + }; + + # Convince `time` to output in JSON + env.TIME = builtins.toJSON { + real_time = "%e"; + user_time = "%U"; + sys_time = "%S"; + cpu_percent = "%P"; + max_resident_set_kb = "%M"; + avg_resident_set_kb = "%t"; + avg_total_mem_kb = "%K"; + avg_data_kb = "%D"; + avg_stack_kb = "%p"; + avg_unshared_data_kb = "%X"; + avg_shared_text_kb = "%Z"; + page_faults_major = "%F"; + page_faults_minor = "%R"; + swaps = "%W"; + context_switches_voluntary = "%c"; + context_switches_involuntary = "%w"; + io_reads = "%I"; + io_writes = "%O"; + signals_received = "%k"; + exit_status = "%x"; + command = "%C"; + }; + + # Don't allow aliases anywhere in Nixpkgs for the metrics. + env.NIXPKGS_CONFIG = builtins.toFile "nixpkgs-config.nix" '' + { + allowAliases = false; + } + ''; +}