diff --git a/pkgs/development/python-modules/ffcv/default.nix b/pkgs/development/python-modules/ffcv/default.nix
index 2aee132e294f..809a9ca696d8 100644
--- a/pkgs/development/python-modules/ffcv/default.nix
+++ b/pkgs/development/python-modules/ffcv/default.nix
@@ -35,7 +35,8 @@ buildPythonPackage rec {
       --replace "'webdataset'," ""
   '';
 
-  buildInputs = [ libjpeg pkgconfig ];
+  nativeBuildInputs = [ pkgconfig ];
+  buildInputs = [ libjpeg ];
   propagatedBuildInputs = [ opencv4 numba pandas pytorch-pfn-extras terminaltables tqdm ];
 
   # `ffcv._libffcv*.so` cannot be loaded in the nix build environment for some