diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix
index 293877036e2f..1c83ea891630 100644
--- a/ci/eval/compare/default.nix
+++ b/ci/eval/compare/default.nix
@@ -13,7 +13,12 @@
   byName ? false,
 }:
 let
-  combined = builtins.storePath combinedDir;
+  # Usually we expect a derivation, but when evaluating in multiple separate steps, we pass
+  # nix store paths around. These need to be turned into (fake) derivations again to track
+  # dependencies properly.
+  # We use two steps for evaluation, because we compare results from two different checkouts.
+  # CI additionalls spreads evaluation across multiple workers.
+  combined = if lib.isDerivation combinedDir then combinedDir else lib.toDerivation combinedDir;
 
   /*
     Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs.
diff --git a/ci/eval/diff.nix b/ci/eval/diff.nix
index d22090601d30..692e2ec60194 100644
--- a/ci/eval/diff.nix
+++ b/ci/eval/diff.nix
@@ -11,8 +11,13 @@
 }:
 
 let
-  before = builtins.storePath beforeDir;
-  after = builtins.storePath afterDir;
+  # Usually we expect a derivation, but when evaluating in multiple separate steps, we pass
+  # nix store paths around. These need to be turned into (fake) derivations again to track
+  # dependencies properly.
+  # We use two steps for evaluation, because we compare results from two different checkouts.
+  # CI additionalls spreads evaluation across multiple workers.
+  before = if lib.isDerivation beforeDir then beforeDir else lib.toDerivation beforeDir;
+  after = if lib.isDerivation afterDir then afterDir else lib.toDerivation afterDir;
 
   /*
     Computes the key difference between two attrs