From 80e011146bf92416ce58ffd62c191875f566810c Mon Sep 17 00:00:00 2001
From: Silvan Mosberger <silvan.mosberger@tweag.io>
Date: Mon, 27 Jan 2025 22:44:51 +0100
Subject: [PATCH] ci/eval/compare: Improve performance and avoid large stacks

Various improvements such as:
1. Avoiding deduplications when there can't be any duplicates
2. Avoiding O(n^2) deduplications
3. Using builtins.any to avoid list allocations
4. Using builtins.concatMap instead of lib.flatten when it's known that there's only one level of nesting
5. Using builtins.groupBy instead of folding with an accumulator

In particular 5. should fix CI exceeding the stack size on staging: https://github.com/NixOS/nixpkgs/actions/runs/12989244871/job/36240781244?pr=377253

While 2. in particular should make CI a lot faster.
---
 ci/eval/compare/default.nix     |  5 ++--
 ci/eval/compare/maintainers.nix | 43 +++++++++++----------------------
 2 files changed, 16 insertions(+), 32 deletions(-)

diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix
index 0c55b01c7160..b844d45f5902 100644
--- a/ci/eval/compare/default.nix
+++ b/ci/eval/compare/default.nix
@@ -69,7 +69,6 @@ let
     groupByPlatform
     extractPackageNames
     getLabels
-    uniqueStrings
     ;
 
   getAttrs = dir: builtins.fromJSON (builtins.readFile "${dir}/outpaths.json");
@@ -81,7 +80,7 @@ let
   # - values: lists of `packagePlatformPath`s
   diffAttrs = diff beforeAttrs afterAttrs;
 
-  rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed);
+  rebuilds = diffAttrs.added ++ diffAttrs.changed;
   rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds;
 
   changed-paths =
@@ -110,7 +109,7 @@ let
     );
 
   maintainers = import ./maintainers.nix {
-    changedattrs = lib.unique (map (a: a.packagePath) rebuildsPackagePlatformAttrs);
+    changedattrs = lib.attrNames (lib.groupBy (a: a.name) rebuildsPackagePlatformAttrs);
     changedpathsjson = touchedFilesJson;
   };
 in
diff --git a/ci/eval/compare/maintainers.nix b/ci/eval/compare/maintainers.nix
index 188d8b635729..64025c066f96 100644
--- a/ci/eval/compare/maintainers.nix
+++ b/ci/eval/compare/maintainers.nix
@@ -11,17 +11,13 @@ let
   changedpaths = builtins.fromJSON (builtins.readFile changedpathsjson);
 
   anyMatchingFile =
-    filename:
-    let
-      matching = builtins.filter (changed: lib.strings.hasSuffix changed filename) changedpaths;
-    in
-    (builtins.length matching) > 0;
+    filename: builtins.any (changed: lib.strings.hasSuffix changed filename) changedpaths;
 
-  anyMatchingFiles = files: (builtins.length (builtins.filter anyMatchingFile files)) > 0;
+  anyMatchingFiles = files: builtins.any anyMatchingFile files;
 
-  enrichedAttrs = builtins.map (path: {
-    path = path;
-    name = builtins.concatStringsSep "." path;
+  enrichedAttrs = builtins.map (name: {
+    path = lib.splitString "." name;
+    name = name;
   }) changedattrs;
 
   validPackageAttributes = builtins.filter (
@@ -80,27 +76,16 @@ let
 
   attrsWithModifiedFiles = builtins.filter (pkg: anyMatchingFiles pkg.filenames) attrsWithFilenames;
 
-  listToPing = lib.lists.flatten (
-    builtins.map (
-      pkg:
-      builtins.map (maintainer: {
-        id = maintainer.githubId;
-        packageName = pkg.name;
-        dueToFiles = pkg.filenames;
-      }) pkg.maintainers
-    ) attrsWithModifiedFiles
-  );
-
-  byMaintainer = lib.lists.foldr (
-    ping: collector:
-    collector
-    // {
-      "${toString ping.id}" = [
-        { inherit (ping) packageName dueToFiles; }
-      ] ++ (collector."${toString ping.id}" or [ ]);
-    }
-  ) { } listToPing;
+  listToPing = lib.concatMap (
+    pkg:
+    builtins.map (maintainer: {
+      id = maintainer.githubId;
+      packageName = pkg.name;
+      dueToFiles = pkg.filenames;
+    }) pkg.maintainers
+  ) attrsWithModifiedFiles;
 
+  byMaintainer = lib.groupBy (ping: toString ping.id) listToPing;
 
   packagesPerMaintainer = lib.attrsets.mapAttrs (
     maintainer: packages: builtins.map (pkg: pkg.packageName) packages