diff --git a/pkgs/by-name/da/datafusion-cli/package.nix b/pkgs/by-name/da/datafusion-cli/package.nix index 39546f2cb7a7..189f5cf6fb9e 100644 --- a/pkgs/by-name/da/datafusion-cli/package.nix +++ b/pkgs/by-name/da/datafusion-cli/package.nix @@ -2,6 +2,8 @@ lib, rustPlatform, fetchFromGitHub, + versionCheckHook, + nix-update-script, }: rustPlatform.buildRustPackage (finalAttrs: { @@ -11,7 +13,7 @@ rustPlatform.buildRustPackage (finalAttrs: { src = fetchFromGitHub { name = "datafusion-cli-source"; owner = "apache"; - repo = "arrow-datafusion"; + repo = "datafusion"; tag = finalAttrs.version; hash = "sha256-DYNKYE8+rh/hkHpWnBl9C7licTst7WxNOV812vPXiQs="; }; @@ -37,11 +39,18 @@ rustPlatform.buildRustPackage (finalAttrs: { # timeout doCheck = false; + nativeInstallCheckInputs = [ + versionCheckHook + ]; + doInstallCheck = true; + + passthru.updateScript = nix-update-script { }; + meta = { description = "CLI for Apache Arrow DataFusion"; mainProgram = "datafusion-cli"; homepage = "https://arrow.apache.org/datafusion"; - changelog = "https://github.com/apache/arrow-datafusion/blob/${finalAttrs.version}/datafusion/CHANGELOG.md"; + changelog = "https://github.com/apache/datafusion/blob/${finalAttrs.src.tag}/datafusion/CHANGELOG.md"; license = lib.licenses.asl20; maintainers = with lib.maintainers; [ happysalada ]; }; diff --git a/pkgs/by-name/re/rerun/package.nix b/pkgs/by-name/re/rerun/package.nix index ea99724ea891..7f65d118d985 100644 --- a/pkgs/by-name/re/rerun/package.nix +++ b/pkgs/by-name/re/rerun/package.nix @@ -35,7 +35,7 @@ }: rustPlatform.buildRustPackage (finalAttrs: { pname = "rerun"; - version = "0.30.2"; + version = "0.31.1"; outputs = [ "out" @@ -46,7 +46,7 @@ rustPlatform.buildRustPackage (finalAttrs: { owner = "rerun-io"; repo = "rerun"; tag = finalAttrs.version; - hash = "sha256-KDtDX7qt+wOJDUVkqjJQPFp2umH7vYja8WyYven5emM="; + hash = "sha256-NnjX7vtVPz1rquTFL7qgAcoZ96RnOy5r+nI/bA6p1Es="; }; # The path in `build.rs` is wrong for some reason, so we patch it to make the passthru tests work @@ -55,7 +55,7 @@ rustPlatform.buildRustPackage (finalAttrs: { --replace-fail '"rerun_sdk/rerun_cli/rerun"' '"rerun_sdk/rerun"' ''; - cargoHash = "sha256-/VAPPz1KYy8kbNRyPq198x3uyX2/vbEqWJxY18+b7BI="; + cargoHash = "sha256-LoR/pTL1ZsBgVT4DBApiUtvbHIhsWhVg62T0hkSbCNo="; cargoBuildFlags = [ "--package" diff --git a/pkgs/development/python-modules/datafusion/default.nix b/pkgs/development/python-modules/datafusion/default.nix index a9ca4e859d89..6118e3d8589f 100644 --- a/pkgs/development/python-modules/datafusion/default.nix +++ b/pkgs/development/python-modules/datafusion/default.nix @@ -16,29 +16,31 @@ typing-extensions, # tests + arro3-core, + nanoarrow, numpy, pytest-asyncio, pytestCheckHook, }: -buildPythonPackage rec { +buildPythonPackage (finalAttrs: { pname = "datafusion"; - version = "50.1.0"; + version = "52.3.0"; pyproject = true; src = fetchFromGitHub { name = "datafusion-source"; owner = "apache"; - repo = "arrow-datafusion-python"; - tag = version; + repo = "datafusion-python"; + tag = finalAttrs.version; # Fetch arrow-testing and parquet-testing (tests assets) fetchSubmodules = true; - hash = "sha256-+r3msFc9yu3aJBDRI66A/AIctCbLxfZB3Ur/raDV3x8="; + hash = "sha256-kyJoG65XKSF+RElZlsdfVTZp/ufWiUw0YdCpQ8Qcg78="; }; cargoDeps = rustPlatform.fetchCargoVendor { - inherit pname src version; - hash = "sha256-XJ2x/EtMZu/fdS6XB/IydMfHmlaxEWJ3XJPY73WoGqs="; + inherit (finalAttrs) pname src version; + hash = "sha256-7/YWJORUjhhZSLyyBT6NFD0RzARJ3SKd11gn4kJ7aYw="; }; nativeBuildInputs = with rustPlatform; [ @@ -57,6 +59,8 @@ buildPythonPackage rec { ]; nativeCheckInputs = [ + arro3-core + nanoarrow numpy pytest-asyncio pytestCheckHook @@ -87,8 +91,8 @@ buildPythonPackage rec { that uses Apache Arrow as its in-memory format. ''; homepage = "https://arrow.apache.org/datafusion/"; - changelog = "https://github.com/apache/arrow-datafusion-python/blob/${version}/CHANGELOG.md"; + changelog = "https://github.com/apache/datafusion-python/blob/${finalAttrs.src.tag}/CHANGELOG.md"; license = with lib.licenses; [ asl20 ]; maintainers = with lib.maintainers; [ cpcloud ]; }; -} +}) diff --git a/pkgs/development/python-modules/lerobot/default.nix b/pkgs/development/python-modules/lerobot/default.nix index 187fac569c01..6887f60a7673 100644 --- a/pkgs/development/python-modules/lerobot/default.nix +++ b/pkgs/development/python-modules/lerobot/default.nix @@ -1,5 +1,6 @@ { lib, + stdenv, buildPythonPackage, fetchFromGitHub, @@ -20,6 +21,7 @@ huggingface-hub, imageio, jsonlines, + numpy, opencv-python-headless, packaging, pynput, @@ -39,14 +41,14 @@ buildPythonPackage (finalAttrs: { pname = "lerobot"; - version = "0.4.3"; + version = "0.5.0"; pyproject = true; src = fetchFromGitHub { owner = "huggingface"; repo = "lerobot"; tag = "v${finalAttrs.version}"; - hash = "sha256-3z8gyK9bx5GpFXM/kLbxume/e8F2U84yUTUhmn57mLs="; + hash = "sha256-i4szKM8b766mAA0AxCHB4s1wEEZbGeBKGeGpLICVsrM="; }; build-system = [ @@ -57,18 +59,16 @@ buildPythonPackage (finalAttrs: { pythonRelaxDeps = [ "av" - "datasets" "diffusers" "draccus" - "gymnasium" - "huggingface-hub" - "opencv" + "numpy" + "opencv-python-headless" "rerun-sdk" "torch" + "torchcodec" "torchvision" "wandb" ]; - dependencies = [ accelerate av @@ -83,6 +83,7 @@ buildPythonPackage (finalAttrs: { huggingface-hub imageio jsonlines + numpy opencv-python-headless packaging pynput @@ -124,17 +125,25 @@ buildPythonPackage (finalAttrs: { # Require internet access "test_act_backbone_lr" + "test_all_items_have_subtask" "test_backward_compatibility" "test_convert_image_to_video_dataset" "test_convert_image_to_video_dataset_subset_episodes" "test_dataset_initialization" "test_factory" "test_from_pretrained_nonexistent_path" + "test_getitem_has_subtask_index" + "test_getitem_returns_subtask_string" "test_load_config_nonexistent_path_tries_hub" "test_make_env_from_hub_async" "test_make_env_from_hub_with_trust" "test_policy_defaults" "test_save_and_load_pretrained" + "test_subtask_dataset_loads" + "test_subtask_index_in_features" + "test_subtask_index_maps_to_valid_subtask" + "test_subtask_metadata_loaded" + "test_task_and_subtask_coexist" # TypeError: stack(): argument 'tensors' (position 1) must be tuple of Tensors, not Column "test_check_timestamps_sync_slightly_off" @@ -161,6 +170,12 @@ buildPythonPackage (finalAttrs: { # TypeError: 'NoneType' object is not subscriptable "test_pi0_rtc_inference_with_prev_chunk" + ] + ++ lib.optionals (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) [ + # RuntimeError: Failed to initialize cpuinfo! + "test_complementary_data_float_dtype_conversion" + "test_float_dtype_conversion" + "test_float_dtype_with_mixed_tensors" ]; disabledTestPaths = [ diff --git a/pkgs/development/python-modules/nanoarrow/default.nix b/pkgs/development/python-modules/nanoarrow/default.nix new file mode 100644 index 000000000000..7993c3e2668e --- /dev/null +++ b/pkgs/development/python-modules/nanoarrow/default.nix @@ -0,0 +1,89 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + + # build-system + cython, + meson, + meson-python, + + # nativeBuildInputs + pkg-config, + + # buildInputs + zlib, + zstd-c, + + pytestCheckHook, +}: + +let + # Nanoarrow requires a specific post-0.6.1 flatcc commit that adds `_with_size` API variants not + # present in the upstream 0.6.1 release. + flatcc-src = fetchFromGitHub { + owner = "dvidelabs"; + repo = "flatcc"; + rev = "fd3c4ae5cd39f0651eda6a3a1a374278070135d6"; + hash = "sha256-8MqazKuwfFWVJ/yjT5fNrRzexFQ2ky4YTcZqOYjk9Qc="; + }; +in +buildPythonPackage (finalAttrs: { + pname = "nanoarrow"; + version = "0.8.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "apache"; + repo = "arrow-nanoarrow"; + tag = "apache-arrow-nanoarrow-${finalAttrs.version}"; + hash = "sha256-1iLbT1eeyZaoB75uYTgg4qns+C7b4DErqMwJ9nQPRls="; + }; + + # Pre-populate the meson subproject with the flatcc source so meson doesn't try to download it. + # The wrap's patch_directory overlay (meson.build) must also be applied. + postPatch = + let + flatcc-src-dest = "subprojects/flatcc-${flatcc-src.rev}"; + in + '' + cp -r --no-preserve=mode ${flatcc-src} ${flatcc-src-dest} + cp subprojects/packagefiles/flatcc/meson.build ${flatcc-src-dest}/ + ''; + + sourceRoot = "${finalAttrs.src.name}/python"; + + build-system = [ + cython + meson + meson-python + ]; + + nativeBuildInputs = [ + pkg-config + ]; + + buildInputs = [ + zlib + zstd-c + ]; + + mesonFlags = [ + # Use system zstd instead of the meson wrap + (lib.mesonOption "force_fallback_for" "flatcc") + ]; + + pythonImportsCheck = [ "nanoarrow" ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + meta = { + description = "Python bindings to the nanoarrow C library"; + homepage = "https://github.com/apache/arrow-nanoarrow/tree/main/python"; + changelog = "https://github.com/apache/arrow-nanoarrow/blob/${finalAttrs.src.tag}/CHANGELOG.md"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ GaetanLepage ]; + }; +}) diff --git a/pkgs/development/python-modules/rerun-sdk/default.nix b/pkgs/development/python-modules/rerun-sdk/default.nix index f5875901b576..0bae3f20b92c 100644 --- a/pkgs/development/python-modules/rerun-sdk/default.nix +++ b/pkgs/development/python-modules/rerun-sdk/default.nix @@ -102,17 +102,20 @@ buildPythonPackage { "test_isolated_streams" "test_send_dataframe_roundtrip" "test_server_failed_table_creation_does_not_leak_entry" + "test_server_version_info" "test_server_with_dataset_files" "test_server_with_dataset_prefix" "test_server_with_multiple_datasets" # TypeError: 'Snapshot' object is not callable + "test_chunk_record_batch" "test_schema_recording" ]; disabledTestPaths = [ # "fixture 'benchmark' not found" "tests/python/log_benchmark/test_log_benchmark.py" + "tests/python/log_benchmark/test_micro_benchmark.py" # ValueError: Failed to start Rerun server: Error loading RRD: couldn't decode "/build/source/tests/assets/rrd/dataset/file4.rrd" "rerun_py/tests/e2e_redap_tests" diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index e5c7e5fc7a5b..e1748f59d1b4 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -10787,6 +10787,8 @@ self: super: with self; { nampa = callPackage ../development/python-modules/nampa { }; + nanoarrow = callPackage ../development/python-modules/nanoarrow { zstd-c = pkgs.zstd; }; + nanobind = callPackage ../development/python-modules/nanobind { }; nanoeigenpy = callPackage ../development/python-modules/nanoeigenpy { };