rerun: 0.30.2 -> 0.31.1 (#506916)

pkgs/by-name/da/datafusion-cli/package.nix

@@ -2,6 +2,8 @@
   lib,
   rustPlatform,
   fetchFromGitHub,
+  versionCheckHook,
+  nix-update-script,
 }:
 
 rustPlatform.buildRustPackage (finalAttrs: {
@@ -11,7 +13,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
   src = fetchFromGitHub {
     name = "datafusion-cli-source";
     owner = "apache";
-    repo = "arrow-datafusion";
+    repo = "datafusion";
     tag = finalAttrs.version;
     hash = "sha256-DYNKYE8+rh/hkHpWnBl9C7licTst7WxNOV812vPXiQs=";
   };
@@ -37,11 +39,18 @@ rustPlatform.buildRustPackage (finalAttrs: {
   # timeout
   doCheck = false;
 
+  nativeInstallCheckInputs = [
+    versionCheckHook
+  ];
+  doInstallCheck = true;
+
+  passthru.updateScript = nix-update-script { };
+
   meta = {
     description = "CLI for Apache Arrow DataFusion";
     mainProgram = "datafusion-cli";
     homepage = "https://arrow.apache.org/datafusion";
-    changelog = "https://github.com/apache/arrow-datafusion/blob/${finalAttrs.version}/datafusion/CHANGELOG.md";
+    changelog = "https://github.com/apache/datafusion/blob/${finalAttrs.src.tag}/datafusion/CHANGELOG.md";
     license = lib.licenses.asl20;
     maintainers = with lib.maintainers; [ happysalada ];
   };

pkgs/by-name/re/rerun/package.nix

@@ -35,7 +35,7 @@
 }:
 rustPlatform.buildRustPackage (finalAttrs: {
   pname = "rerun";
-  version = "0.30.2";
+  version = "0.31.1";
 
   outputs = [
     "out"
@@ -46,7 +46,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
     owner = "rerun-io";
     repo = "rerun";
     tag = finalAttrs.version;
-    hash = "sha256-KDtDX7qt+wOJDUVkqjJQPFp2umH7vYja8WyYven5emM=";
+    hash = "sha256-NnjX7vtVPz1rquTFL7qgAcoZ96RnOy5r+nI/bA6p1Es=";
   };
 
   # The path in `build.rs` is wrong for some reason, so we patch it to make the passthru tests work
@@ -55,7 +55,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
       --replace-fail '"rerun_sdk/rerun_cli/rerun"' '"rerun_sdk/rerun"'
   '';
 
-  cargoHash = "sha256-/VAPPz1KYy8kbNRyPq198x3uyX2/vbEqWJxY18+b7BI=";
+  cargoHash = "sha256-LoR/pTL1ZsBgVT4DBApiUtvbHIhsWhVg62T0hkSbCNo=";
 
   cargoBuildFlags = [
     "--package"

pkgs/development/python-modules/datafusion/default.nix

@@ -16,29 +16,31 @@
   typing-extensions,
 
   # tests
+  arro3-core,
+  nanoarrow,
   numpy,
   pytest-asyncio,
   pytestCheckHook,
 }:
 
-buildPythonPackage rec {
+buildPythonPackage (finalAttrs: {
   pname = "datafusion";
-  version = "50.1.0";
+  version = "52.3.0";
   pyproject = true;
 
   src = fetchFromGitHub {
     name = "datafusion-source";
     owner = "apache";
-    repo = "arrow-datafusion-python";
-    tag = version;
+    repo = "datafusion-python";
+    tag = finalAttrs.version;
     # Fetch arrow-testing and parquet-testing (tests assets)
     fetchSubmodules = true;
-    hash = "sha256-+r3msFc9yu3aJBDRI66A/AIctCbLxfZB3Ur/raDV3x8=";
+    hash = "sha256-kyJoG65XKSF+RElZlsdfVTZp/ufWiUw0YdCpQ8Qcg78=";
   };
 
   cargoDeps = rustPlatform.fetchCargoVendor {
-    inherit pname src version;
-    hash = "sha256-XJ2x/EtMZu/fdS6XB/IydMfHmlaxEWJ3XJPY73WoGqs=";
+    inherit (finalAttrs) pname src version;
+    hash = "sha256-7/YWJORUjhhZSLyyBT6NFD0RzARJ3SKd11gn4kJ7aYw=";
   };
 
   nativeBuildInputs = with rustPlatform; [
@@ -57,6 +59,8 @@ buildPythonPackage rec {
   ];
 
   nativeCheckInputs = [
+    arro3-core
+    nanoarrow
     numpy
     pytest-asyncio
     pytestCheckHook
@@ -87,8 +91,8 @@ buildPythonPackage rec {
       that uses Apache Arrow as its in-memory format.
     '';
     homepage = "https://arrow.apache.org/datafusion/";
-    changelog = "https://github.com/apache/arrow-datafusion-python/blob/${version}/CHANGELOG.md";
+    changelog = "https://github.com/apache/datafusion-python/blob/${finalAttrs.src.tag}/CHANGELOG.md";
     license = with lib.licenses; [ asl20 ];
     maintainers = with lib.maintainers; [ cpcloud ];
   };
-}
+})

pkgs/development/python-modules/lerobot/default.nix

@@ -1,5 +1,6 @@
 {
   lib,
+  stdenv,
   buildPythonPackage,
   fetchFromGitHub,
 
@@ -20,6 +21,7 @@
   huggingface-hub,
   imageio,
   jsonlines,
+  numpy,
   opencv-python-headless,
   packaging,
   pynput,
@@ -39,14 +41,14 @@
 
 buildPythonPackage (finalAttrs: {
   pname = "lerobot";
-  version = "0.4.3";
+  version = "0.5.0";
   pyproject = true;
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = "lerobot";
     tag = "v${finalAttrs.version}";
-    hash = "sha256-3z8gyK9bx5GpFXM/kLbxume/e8F2U84yUTUhmn57mLs=";
+    hash = "sha256-i4szKM8b766mAA0AxCHB4s1wEEZbGeBKGeGpLICVsrM=";
   };
 
   build-system = [
@@ -57,18 +59,16 @@ buildPythonPackage (finalAttrs: {
 
   pythonRelaxDeps = [
     "av"
-    "datasets"
     "diffusers"
     "draccus"
-    "gymnasium"
-    "huggingface-hub"
-    "opencv"
+    "numpy"
+    "opencv-python-headless"
     "rerun-sdk"
     "torch"
+    "torchcodec"
     "torchvision"
     "wandb"
   ];
-
   dependencies = [
     accelerate
     av
@@ -83,6 +83,7 @@ buildPythonPackage (finalAttrs: {
     huggingface-hub
     imageio
     jsonlines
+    numpy
     opencv-python-headless
     packaging
     pynput
@@ -124,17 +125,25 @@ buildPythonPackage (finalAttrs: {
 
     # Require internet access
     "test_act_backbone_lr"
+    "test_all_items_have_subtask"
     "test_backward_compatibility"
     "test_convert_image_to_video_dataset"
     "test_convert_image_to_video_dataset_subset_episodes"
     "test_dataset_initialization"
     "test_factory"
     "test_from_pretrained_nonexistent_path"
+    "test_getitem_has_subtask_index"
+    "test_getitem_returns_subtask_string"
     "test_load_config_nonexistent_path_tries_hub"
     "test_make_env_from_hub_async"
     "test_make_env_from_hub_with_trust"
     "test_policy_defaults"
     "test_save_and_load_pretrained"
+    "test_subtask_dataset_loads"
+    "test_subtask_index_in_features"
+    "test_subtask_index_maps_to_valid_subtask"
+    "test_subtask_metadata_loaded"
+    "test_task_and_subtask_coexist"
 
     # TypeError: stack(): argument 'tensors' (position 1) must be tuple of Tensors, not Column
     "test_check_timestamps_sync_slightly_off"
@@ -161,6 +170,12 @@ buildPythonPackage (finalAttrs: {
 
     # TypeError: 'NoneType' object is not subscriptable
     "test_pi0_rtc_inference_with_prev_chunk"
+  ]
+  ++ lib.optionals (stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isAarch64) [
+    # RuntimeError: Failed to initialize cpuinfo!
+    "test_complementary_data_float_dtype_conversion"
+    "test_float_dtype_conversion"
+    "test_float_dtype_with_mixed_tensors"
   ];
 
   disabledTestPaths = [

pkgs/development/python-modules/nanoarrow/default.nix

@@ -0,0 +1,89 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  cython,
+  meson,
+  meson-python,
+
+  # nativeBuildInputs
+  pkg-config,
+
+  # buildInputs
+  zlib,
+  zstd-c,
+
+  pytestCheckHook,
+}:
+
+let
+  # Nanoarrow requires a specific post-0.6.1 flatcc commit that adds `_with_size` API variants not
+  # present in the upstream 0.6.1 release.
+  flatcc-src = fetchFromGitHub {
+    owner = "dvidelabs";
+    repo = "flatcc";
+    rev = "fd3c4ae5cd39f0651eda6a3a1a374278070135d6";
+    hash = "sha256-8MqazKuwfFWVJ/yjT5fNrRzexFQ2ky4YTcZqOYjk9Qc=";
+  };
+in
+buildPythonPackage (finalAttrs: {
+  pname = "nanoarrow";
+  version = "0.8.0";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "apache";
+    repo = "arrow-nanoarrow";
+    tag = "apache-arrow-nanoarrow-${finalAttrs.version}";
+    hash = "sha256-1iLbT1eeyZaoB75uYTgg4qns+C7b4DErqMwJ9nQPRls=";
+  };
+
+  # Pre-populate the meson subproject with the flatcc source so meson doesn't try to download it.
+  # The wrap's patch_directory overlay (meson.build) must also be applied.
+  postPatch =
+    let
+      flatcc-src-dest = "subprojects/flatcc-${flatcc-src.rev}";
+    in
+    ''
+      cp -r --no-preserve=mode ${flatcc-src} ${flatcc-src-dest}
+      cp subprojects/packagefiles/flatcc/meson.build ${flatcc-src-dest}/
+    '';
+
+  sourceRoot = "${finalAttrs.src.name}/python";
+
+  build-system = [
+    cython
+    meson
+    meson-python
+  ];
+
+  nativeBuildInputs = [
+    pkg-config
+  ];
+
+  buildInputs = [
+    zlib
+    zstd-c
+  ];
+
+  mesonFlags = [
+    # Use system zstd instead of the meson wrap
+    (lib.mesonOption "force_fallback_for" "flatcc")
+  ];
+
+  pythonImportsCheck = [ "nanoarrow" ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  meta = {
+    description = "Python bindings to the nanoarrow C library";
+    homepage = "https://github.com/apache/arrow-nanoarrow/tree/main/python";
+    changelog = "https://github.com/apache/arrow-nanoarrow/blob/${finalAttrs.src.tag}/CHANGELOG.md";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ GaetanLepage ];
+  };
+})

pkgs/development/python-modules/rerun-sdk/default.nix

@@ -102,17 +102,20 @@ buildPythonPackage {
     "test_isolated_streams"
     "test_send_dataframe_roundtrip"
     "test_server_failed_table_creation_does_not_leak_entry"
+    "test_server_version_info"
     "test_server_with_dataset_files"
     "test_server_with_dataset_prefix"
     "test_server_with_multiple_datasets"
 
     # TypeError: 'Snapshot' object is not callable
+    "test_chunk_record_batch"
     "test_schema_recording"
   ];
 
   disabledTestPaths = [
     # "fixture 'benchmark' not found"
     "tests/python/log_benchmark/test_log_benchmark.py"
+    "tests/python/log_benchmark/test_micro_benchmark.py"
 
     # ValueError: Failed to start Rerun server: Error loading RRD: couldn't decode "/build/source/tests/assets/rrd/dataset/file4.rrd"
     "rerun_py/tests/e2e_redap_tests"

pkgs/top-level/python-packages.nix

@@ -10787,6 +10787,8 @@ self: super: with self; {
 
   nampa = callPackage ../development/python-modules/nampa { };
 
+  nanoarrow = callPackage ../development/python-modules/nanoarrow { zstd-c = pkgs.zstd; };
+
   nanobind = callPackage ../development/python-modules/nanobind { };
 
   nanoeigenpy = callPackage ../development/python-modules/nanoeigenpy { };