python312Packages.mlcroissant: init at 1.0.17 (#401172)

2025-04-23 18:44:21 +02:00
parent cee01c3878 cf0af74237
commit daf7140b65
2 changed files with 99 additions and 0 deletions
@@ -0,0 +1,97 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  # build-system
+  setuptools,
+
+  # dependencies
+  absl-py,
+  etils,
+  jsonpath-rw,
+  networkx,
+  pandas,
+  pandas-stubs,
+  python-dateutil,
+  rdflib,
+  requests,
+  scipy,
+  tqdm,
+
+  # tests
+  apache-beam,
+  gitpython,
+  pillow,
+  pytestCheckHook,
+  pyyaml,
+  writableTmpDirAsHomeHook,
+}:
+
+buildPythonPackage rec {
+  pname = "mlcroissant";
+  version = "1.0.17";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "mlcommons";
+    repo = "croissant";
+    tag = "v${version}";
+    hash = "sha256-jiyr8x+YRSsRwOVxDPaWemPqglTKVb5jg4rRzUXd3BE=";
+  };
+
+  sourceRoot = "${src.name}/python/mlcroissant";
+
+  build-system = [
+    setuptools
+  ];
+
+  dependencies = [
+    absl-py
+    etils
+    jsonpath-rw
+    networkx
+    pandas
+    pandas-stubs
+    python-dateutil
+    rdflib
+    requests
+    scipy
+    tqdm
+  ] ++ etils.optional-dependencies.epath;
+
+  pythonImportsCheck = [ "mlcroissant" ];
+
+  nativeCheckInputs = [
+    apache-beam
+    gitpython
+    pillow
+    pytestCheckHook
+    pyyaml
+    writableTmpDirAsHomeHook
+  ];
+
+  disabledTests = [
+    # Requires internet access
+    "test_hermetic_loading_1_1"
+    "test_load_from_huggingface"
+    "test_nonhermetic_loading"
+    "test_nonhermetic_loading_1_0"
+
+    # AssertionError: assert {'records/aud...t32), 22050)'} == {'records/aud...t32), 22050)'}
+    "test_hermetic_loading"
+
+    # AttributeError: 'MaybeReshuffle' object has no attribute 'side_inputs'
+    "test_beam_hermetic_loading"
+  ];
+
+  meta = {
+    description = "High-level format for machine learning datasets that brings together four rich layers";
+    homepage = "https://github.com/mlcommons/croissant";
+    changelog = "https://github.com/mlcommons/croissant/releases/tag/v${version}";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ GaetanLepage ];
+    platforms = lib.platforms.all;
+    mainProgram = "mlcroissant";
+  };
+}
@@ -8902,6 +8902,8 @@ self: super: with self; {

  ml-dtypes = callPackage ../development/python-modules/ml-dtypes { };

+  mlcroissant = callPackage ../development/python-modules/mlcroissant { };
+
  mlflow = callPackage ../development/python-modules/mlflow { };

  mlrose = callPackage ../development/python-modules/mlrose { };