diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.nix b/pkgs/test/nixpkgs-check-by-name/src/eval.nix
index b605a35a322a..7707dc732b70 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/eval.nix
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.nix
@@ -9,6 +9,9 @@
 let
   attrs = builtins.fromJSON (builtins.readFile attrsPath);
 
+  nixpkgsPathLength = builtins.stringLength (toString nixpkgsPath) + 1;
+  removeNixpkgsPrefix = builtins.substring nixpkgsPathLength (-1);
+
   # We need access to the `callPackage` arguments of each attribute.
   # The only way to do so is to override `callPackage` with our own version that adds this information to the result,
   # and then try to access this information.
@@ -20,7 +23,7 @@ let
         Manual = {
           path =
             if builtins.isPath fn then
-              toString fn
+              removeNixpkgsPrefix (toString fn)
             else
               null;
           empty_arg =
diff --git a/pkgs/test/nixpkgs-check-by-name/src/eval.rs b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
index 4004841c9430..7680af185bdb 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/eval.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/eval.rs
@@ -127,7 +127,6 @@ pub fn check_values(
     let check_result = validation::sequence(attributes.into_iter().map(
         |(attribute_name, attribute_value)| {
             let relative_package_file = structure::relative_file_for_package(&attribute_name);
-            let absolute_package_file = nixpkgs_path.join(&relative_package_file);
 
             use ratchet::RatchetState::*;
             use AttributeInfo::*;
@@ -170,7 +169,7 @@ pub fn check_values(
                         }),
                         Manual { path, empty_arg } => {
                             let correct_file = if let Some(call_package_path) = path {
-                                absolute_package_file == *call_package_path
+                                relative_package_file == *call_package_path
                             } else {
                                 false
                             };