diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml
index 8ad860a1aed8..dee31e1f5328 100644
--- a/.github/workflows/eval.yml
+++ b/.github/workflows/eval.yml
@@ -279,12 +279,11 @@ jobs:
                         const diff = JSON.parse(
                           readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
                         )
-                        const attrs = [].concat(
-                          diff.added,
-                          diff.removed,
-                          diff.changed,
-                          diff.rebuilds
-                        )
+                        const attrs = []
+                          .concat(diff.added, diff.removed, diff.changed, diff.rebuilds)
+                          // There are some special attributes, which are ignored for rebuilds.
+                          // These only have a single path component, because they lack the `.<system>` suffix.
+                          .filter((attr) => attr.split('.').length > 1)
                         if (attrs.length > 0) {
                           core.setFailed(
                             `${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,