diff --git a/pkgs/by-name/on/onnx/package.nix b/pkgs/by-name/on/onnx/package.nix new file mode 100644 index 000000000000..f911fa24b56e --- /dev/null +++ b/pkgs/by-name/on/onnx/package.nix @@ -0,0 +1,162 @@ +{ + cmake, + fetchFromGitHub, + gtest, + lib, + ninja, + protobuf, + python3Packages, + stdenv, +}: +let + inherit (lib) + cmakeFeature + licenses + maintainers + mapAttrsToList + ; + inherit (python3Packages) + build + pybind11 + python + setuptools + ; +in +stdenv.mkDerivation (finalAttrs: { + __structuredAttrs = true; + strictDeps = true; + + pname = "onnx"; + version = "1.19.1"; + + src = fetchFromGitHub { + owner = "onnx"; + repo = "onnx"; + tag = "v${finalAttrs.version}"; + hash = "sha256-lIw65HAaNE+pOiHPpbOxzX+gk8ueJfuAHxs+DV99Jh0="; + }; + + outputs = [ + "out" + "dist" # Python wheel output + ]; + + nativeBuildInputs = [ + build + cmake + ninja + pybind11 + python + setuptools + ]; + + # NOTE: Most of these can be removed after bumping to 1.20. + # See: + # - https://github.com/onnx/onnx/issues/7472 + # - https://github.com/pytorch/pytorch/issues/167525 + # NOTE: Darwin requires a static build, so this patch is unnecessary on that platform. + prePatch = lib.optionalString (!stdenv.hostPlatform.isDarwin) '' + nixLog "patching $PWD/CMakeLists.txt to allow tests to build with shared libraries" + substituteInPlace "$PWD/CMakeLists.txt" \ + --replace-fail \ + 'message(FATAL_ERROR "Tests requires static build")' \ + "" + + nixLog "patching $PWD/CMakeLists.txt to correctly link onnx_cpp2py_export with shared libraries" + substituteInPlace "$PWD/CMakeLists.txt" \ + --replace-fail \ + 'target_link_libraries(onnx_cpp2py_export PRIVATE $)' \ + 'target_link_libraries(onnx_cpp2py_export PRIVATE $ $)' + + nixLog "patching $PWD/CMakeLists.txt to fix symbol visibility" + substituteInPlace "$PWD/CMakeLists.txt" \ + --replace-fail \ + 'set_target_properties(onnx_proto PROPERTIES CXX_VISIBILITY_PRESET hidden)' \ + '# set_target_properties(onnx_proto PROPERTIES CXX_VISIBILITY_PRESET hidden)' \ + --replace-fail \ + 'set_target_properties(onnx_proto PROPERTIES VISIBILITY_INLINES_HIDDEN 1)' \ + '# set_target_properties(onnx_proto PROPERTIES VISIBILITY_INLINES_HIDDEN 1)' \ + --replace-fail \ + 'set_target_properties(onnx PROPERTIES CXX_VISIBILITY_PRESET hidden)' \ + '# set_target_properties(onnx PROPERTIES CXX_VISIBILITY_PRESET hidden)' \ + --replace-fail \ + 'set_target_properties(onnx PROPERTIES VISIBILITY_INLINES_HIDDEN ON)' \ + '# set_target_properties(onnx PROPERTIES VISIBILITY_INLINES_HIDDEN ON)' + + nixLog "patching $PWD/cmake/ONNXConfig.cmake.in to fix CMake discovery" + substituteInPlace "$PWD/cmake/ONNXConfig.cmake.in" \ + --replace-fail \ + 'if((NOT @@ONNX_USE_PROTOBUF_SHARED_LIBS@@) AND @@Build_Protobuf@@)' \ + 'if(NOT @ONNX_USE_PROTOBUF_SHARED_LIBS@)' + ''; + + # NOTE: python3Packages.protobuf does not propagate a dependency on protobuf's dev output, so we must bring it in + # for the CMake files. + buildInputs = [ + protobuf + python3Packages.protobuf + ]; + + # Declared in setup.py + cmakeBuildDir = ".setuptools-cmake-build"; + + # Python setup.py just takes stuff from the environment. + env = { + # NOTE: Darwin requires a static build; otherwise, tests fail in the Python package. + BUILD_SHARED_LIBS = if stdenv.hostPlatform.isDarwin then "0" else "1"; + ONNX_BUILD_PYTHON = "1"; + ONNX_BUILD_TESTS = if finalAttrs.doCheck then "1" else "0"; + # ONNX_ML is enabled by default, so we must explicitly disable it. + # See: https://github.com/onnx/onnx/blob/b751946c3d59a3c8358abcc0569b59e6ddb08cdd/CMakeLists.txt#L66-L73 + # NOTE: If this is `true`, onnx-tensorrt fails to build due to missing protobuf files. + ONNX_ML = "0"; + ONNX_NAMESPACE = "onnx"; + ONNX_USE_PROTOBUF_SHARED_LIBS = "1"; + }; + + cmakeFlags = mapAttrsToList cmakeFeature finalAttrs.env; + + preConfigure = '' + export MAX_JOBS=$NIX_BUILD_CORES + ''; + + # Leave the CMake bulid directory, export the `cmakeFlags` environment variable as CMAKE_ARGS so setup.py will pick + # them up, do the python build from the top-level, then resume the C++ build. + preBuild = '' + pushd .. + nixLog "exporting cmakeFlags as CMAKE_ARGS for Python build" + export CMAKE_ARGS="''${cmakeFlags[*]}" + nixLog "building Python wheel" + pyproject-build \ + --no-isolation \ + --outdir "$dist/" \ + --wheel + popd >/dev/null + ''; + + # NOTE: Python specific tests happen in the python package. + doCheck = true; + + checkInputs = [ gtest ]; + + preCheck = '' + nixLog "running C++ tests with $PWD/onnx_gtests" + "$PWD/onnx_gtests" + ''; + + postInstall = '' + nixLog "removing empty directories in $out/include/onnx" + find "$out/include/onnx" -type d -empty -delete + ''; + + meta = { + description = "Open Neural Network Exchange"; + homepage = "https://onnx.ai"; + license = licenses.asl20; + changelog = "https://github.com/onnx/onnx/releases/tag/v${finalAttrs.version}"; + maintainers = with maintainers; [ + acairncross + connorbaker + ]; + }; +}) diff --git a/pkgs/development/python-modules/onnx/default.nix b/pkgs/development/python-modules/onnx/default.nix index 45cc91854ec7..4ba0c3baf3c9 100644 --- a/pkgs/development/python-modules/onnx/default.nix +++ b/pkgs/development/python-modules/onnx/default.nix @@ -1,21 +1,11 @@ { - lib, buildPythonPackage, - fetchFromGitHub, - - # build-system - cmake, - pybind11, - setuptools, - - # buildInputs - abseil-cpp, - protobuf, - gtest, + onnx, # pkgs.onnx # dependencies ml-dtypes, numpy, + protobuf, typing-extensions, # tests @@ -24,32 +14,26 @@ pytestCheckHook, writableTmpDirAsHomeHook, }: +buildPythonPackage { + inherit (onnx) + pname + src # Needed for testing. + version + ; -let - gtestStatic = gtest.override { static = true; }; -in -buildPythonPackage rec { - pname = "onnx"; - version = "1.19.0"; - pyproject = true; + format = "wheel"; - src = fetchFromGitHub { - owner = "onnx"; - repo = "onnx"; - tag = "v${version}"; - hash = "sha256-dDc7ugzQHcArf9TRcF9Ofv16jc3gqhMWCZrYKJ7Udfw="; - }; + dontUseWheelUnpack = true; - build-system = [ - cmake - protobuf - setuptools - ]; + postUnpack = '' + cp -rv "${onnx.dist}" "$sourceRoot/dist" + chmod +w "$sourceRoot/dist" + ''; buildInputs = [ - abseil-cpp - gtestStatic - pybind11 + # onnx must be included to avoid shrinking during fixupPhase removing the RUNPATH entry on + # onnx_cpp2py_export.cpython-*.so. + onnx ]; dependencies = [ @@ -66,26 +50,11 @@ buildPythonPackage rec { writableTmpDirAsHomeHook ]; - preConfigure = '' - # Set CMAKE_INSTALL_LIBDIR to lib explicitly, because otherwise it gets set - # to lib64 and cmake incorrectly looks for the protobuf library in lib64 - export CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib -DONNX_USE_PROTOBUF_SHARED_LIBS=ON" - export CMAKE_ARGS+=" -Dgoogletest_STATIC_LIBRARIES=${gtestStatic}/lib/libgtest.a" - export ONNX_BUILD_TESTS=1 - ''; - - preBuild = '' - export MAX_JOBS=$NIX_BUILD_CORES - ''; - # The executables are just utility scripts that aren't too important postInstall = '' - rm -r $out/bin + rm -rv $out/bin ''; - # The setup.py does all the configuration - dontUseCmakeConfigure = true; - # detecting source dir as a python package confuses pytest preCheck = '' rm onnx/__init__.py @@ -98,18 +67,16 @@ buildPythonPackage rec { __darwinAllowLocalNetworking = true; - postCheck = '' - # run "cpp" tests - .setuptools-cmake-build/onnx_gtests - ''; - pythonImportsCheck = [ "onnx" ]; meta = { - description = "Open Neural Network Exchange"; - homepage = "https://onnx.ai"; - changelog = "https://github.com/onnx/onnx/releases/tag/v${version}"; - license = lib.licenses.asl20; - maintainers = with lib.maintainers; [ acairncross ]; + # Explicitly inherit from ONNX's meta to avoid pulling in attributes added by stdenv.mkDerivation. + inherit (onnx.meta) + changelog + description + homepage + license + maintainers + ; }; } diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 6051daa09f6a..4bbc973724f6 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -11087,7 +11087,11 @@ self: super: with self; { onlykey-solo-python = callPackage ../development/python-modules/onlykey-solo-python { }; - onnx = callPackage ../development/python-modules/onnx { }; + onnx = callPackage ../development/python-modules/onnx { + onnx = pkgs.onnx.override { + python3Packages = self; + }; + }; onnxconverter-common = callPackage ../development/python-modules/onnxconverter-common { };