diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml
index c318acad7bca..3a061961a0cd 100644
--- a/.github/workflows/eval.yml
+++ b/.github/workflows/eval.yml
@@ -210,7 +210,7 @@ jobs:
             --arg beforeResultDir ./baseResult \
             --arg afterResultDir ./prResult \
             -o comparison
-
+          cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
           # TODO: Request reviews from maintainers for packages whose files are modified in the PR
 
       - name: Upload the combined results
diff --git a/ci/eval/default.nix b/ci/eval/default.nix
index efd8d6637a2d..ef107d4ce517 100644
--- a/ci/eval/default.nix
+++ b/ci/eval/default.nix
@@ -261,6 +261,7 @@ let
           --slurpfile after ${afterResultDir}/outpaths.json \
           > $out/changed-paths.json
 
+        jq -r -f ${./generate-step-summary.jq} < $out/changed-paths.json > $out/step-summary.md
         # TODO: Compare eval stats
       '';
 
diff --git a/ci/eval/generate-step-summary.jq b/ci/eval/generate-step-summary.jq
new file mode 100644
index 000000000000..28597eaec371
--- /dev/null
+++ b/ci/eval/generate-step-summary.jq
@@ -0,0 +1,15 @@
+def truncate(xs; n):
+  if xs | length > n then xs[:n] + ["..."]
+  else xs
+  end;
+
+def itemize_packages(xs):
+  # we truncate the list to stay below the GitHub limit of 1MB per step summary.
+  truncate(xs; 3000) | map("- [\(.)](https://search.nixos.org/packages?channel=unstable&show=\(.)&from=0&size=50&sort=relevance&type=packages&query=\(.))") | join("\n");
+
+def section(title; xs):
+  "<details> <summary>" + title + " (" + (xs | length | tostring) + ")</summary>\n\n" + itemize_packages(xs) + "</details>";
+
+section("Added packages"; .attrdiff.added) + "\n\n" +
+section("Removed packages"; .attrdiff.removed) + "\n\n" +
+section("Changed packages"; .attrdiff.changed)