diff --git a/pkgs/by-name/ap/apache-orc/package.nix b/pkgs/by-name/ap/apache-orc/package.nix new file mode 100644 index 000000000000..b64b6683c2b1 --- /dev/null +++ b/pkgs/by-name/ap/apache-orc/package.nix @@ -0,0 +1,81 @@ +{ + lib, + stdenv, + fetchFromGitHub, + fetchurl, + fetchpatch, + cmake, + gtest, + lz4, + protobuf, + snappy, + zlib, + zstd, +}: + +let + orc-format = fetchurl { + name = "orc-format-1.0.0.tar.gz"; + url = "https://www.apache.org/dyn/closer.lua/orc/orc-format-1.0.0/orc-format-1.0.0.tar.gz?action=download"; + hash = "sha256-c5+uX/lLH4ErQTB3KANhBFv5LlEO8Es0phDiOpRdjNU="; + }; +in +stdenv.mkDerivation (finalAttrs: { + pname = "apache-orc"; + version = "2.1.1"; + + src = fetchFromGitHub { + owner = "apache"; + repo = "orc"; + tag = "v${finalAttrs.version}"; + hash = "sha256-Ib02mIUQaLEVwIBv5xdKDyX+FeI8vhh9+5hM7miKwHo="; + }; + + patches = [ + # Patch that adds 2 missing imports in source files + # To be removed this PR land: https://github.com/apache/orc/pull/2175 + (fetchpatch { + url = "https://github.com/apache/orc/commit/fb20db2440226da6b92d38ce260e5b850d2f0092.patch"; + hash = "sha256-rHGECXJoBPgZ62yZciYdSMq4pGnVt75lxkHyO46IiyQ="; + }) + ]; + + nativeBuildInputs = [ + cmake + ]; + + buildInputs = [ + gtest + lz4 + protobuf + snappy + zlib + zstd + ]; + + cmakeFlags = [ + (lib.cmakeFeature "CMAKE_BUILD_TYPE" "Release") + (lib.cmakeBool "BUILD_JAVA" false) + (lib.cmakeBool "STOP_BUILD_ON_WARNING" true) + (lib.cmakeBool "INSTALL_VENDORED_LIBS" false) + ]; + + env = { + GTEST_HOME = gtest.dev; + LZ4_ROOT = lz4; + ORC_FORMAT_URL = orc-format; + PROTOBUF_HOME = protobuf; + SNAPPY_ROOT = snappy.dev; + ZLIB_ROOT = zlib.dev; + ZSTD_ROOT = zstd.dev; + }; + + meta = { + changelog = "https://github.com/apache/orc/releases/tag/v${finalAttrs.version}"; + description = "The smallest, fastest columnar storage for Hadoop workloads"; + homepage = "https://github.com/apache/orc/"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ drupol ]; + platforms = lib.platforms.all; + }; +}) diff --git a/pkgs/by-name/sw/swh/package.nix b/pkgs/by-name/sw/swh/package.nix index 96561b1c703b..95828a6665d4 100644 --- a/pkgs/by-name/sw/swh/package.nix +++ b/pkgs/by-name/sw/swh/package.nix @@ -3,6 +3,7 @@ writeShellApplication, withSwhPythonPackages ? [ python3Packages.swh-auth + python3Packages.swh-export python3Packages.swh-model python3Packages.swh-objstorage python3Packages.swh-scanner diff --git a/pkgs/development/python-modules/pyorc/default.nix b/pkgs/development/python-modules/pyorc/default.nix new file mode 100644 index 000000000000..bd2935ef7853 --- /dev/null +++ b/pkgs/development/python-modules/pyorc/default.nix @@ -0,0 +1,73 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + pybind11, + setuptools, + pytestCheckHook, + tzdata, + python, + pkgs, +}: + +buildPythonPackage rec { + pname = "pyorc"; + version = "0.10.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "noirello"; + repo = "pyorc"; + tag = "v${version}"; + hash = "sha256-2w3Qh6g+Yg+D10kTow9YR6B6FhQ+z2DvgDy5GtYxH4g="; + }; + + build-system = [ + pybind11 + setuptools + ]; + + env = { + PYORC_SKIP_ORC_BUILD = "true"; + }; + + buildInputs = [ + pkgs.lz4 + pkgs.protobuf + pkgs.snappy + pkgs.zlib + pkgs.zstd + pkgs.apache-orc + ]; + + preCheck = '' + # provide timezone data, works only on linux + export TZDIR=${tzdata}/${python.sitePackages}/tzdata/zoneinfo + + substituteInPlace "tests/compare/test_writer_cmp.py" \ + --replace-fail "deps/bin/orc-contents" "orc-contents" + + substituteInPlace "tests/compare/test_reader_cmp.py" \ + --replace-fail "deps/bin/orc-metadata" "orc-metadata" + + mkdir -p deps + ln -s "${pkgs.apache-orc.src}/examples" "deps/" + ''; + + pythonImportsCheck = [ + "pyorc" + ]; + + nativeCheckInputs = [ + pytestCheckHook + pkgs.apache-orc + ]; + + meta = { + changelog = "https://github.com/noirello/pyorc/blob/${version}/CHANGELOG.rst"; + description = "Python module for Apache ORC file format"; + homepage = "https://github.com/noirello/pyorc"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ drupol ]; + }; +} diff --git a/pkgs/development/python-modules/pytest-kafka/default.nix b/pkgs/development/python-modules/pytest-kafka/default.nix new file mode 100644 index 000000000000..ee4c323e8cef --- /dev/null +++ b/pkgs/development/python-modules/pytest-kafka/default.nix @@ -0,0 +1,42 @@ +{ + lib, + buildPythonPackage, + fetchFromGitLab, + setuptools, + kafka-python-ng, + port-for, + pytest, +}: + +buildPythonPackage rec { + pname = "pytest-kafka"; + version = "0.8.1"; + pyproject = true; + + src = fetchFromGitLab { + owner = "karolinepauls"; + repo = "pytest-kafka"; + tag = "v${version}"; + hash = "sha256-OR8SpNswbPOVtAcFuZgrZJR5K6wPb1TS5leybKWr3zY="; + }; + + build-system = [ setuptools ]; + + dependencies = [ + kafka-python-ng + port-for + pytest + ]; + + pythonImportsCheck = [ "pytest_kafka" ]; + + # Tests depends on a kafka server running + doCheck = false; + + meta = { + description = "Pytest fixture factories for Zookeeper, Kafka server and Kafka consumer"; + homepage = "https://gitlab.com/karolinepauls/pytest-kafka"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ drupol ]; + }; +} diff --git a/pkgs/development/python-modules/swh-export/default.nix b/pkgs/development/python-modules/swh-export/default.nix new file mode 100644 index 000000000000..6429bd8b475e --- /dev/null +++ b/pkgs/development/python-modules/swh-export/default.nix @@ -0,0 +1,90 @@ +{ + lib, + buildPythonPackage, + fetchFromGitLab, + setuptools, + setuptools-scm, + boto3, + click, + tqdm, + pyorc, + plyvel, + python, + types-requests, + swh-core, + swh-journal, + swh-model, + swh-storage, + pytestCheckHook, + pytest-kafka, + pytest-mock, + tzdata, + pkgs, +}: + +buildPythonPackage rec { + pname = "swh-export"; + version = "1.8.0"; + pyproject = true; + + src = fetchFromGitLab { + domain = "gitlab.softwareheritage.org"; + group = "swh"; + owner = "devel"; + repo = "swh-export"; + tag = "v${version}"; + hash = "sha256-n97MMYn7EmTrv411YSxUD1+zfbFB8KOSns44N3NqqV8="; + }; + + build-system = [ + setuptools + setuptools-scm + ]; + + dependencies = [ + boto3 + click + tqdm + pyorc + plyvel + types-requests + swh-core + swh-journal + swh-model + swh-storage + ]; + + preCheck = '' + # provide timezone data, works only on linux + export TZDIR=${tzdata}/${python.sitePackages}/tzdata/zoneinfo + ''; + + pythonImportsCheck = [ "swh.export" ]; + + nativeCheckInputs = [ + pytestCheckHook + pytest-kafka + pytest-mock + pkgs.zstd + pkgs.pv + ]; + + disabledTests = [ + # I don't know how to fix the following error + # E fixture 'kafka_server' not found + "test_parallel_journal_processor" + "test_parallel_journal_processor_origin" + "test_parallel_journal_processor_origin_visit_status" + "test_parallel_journal_processor_offsets" + "test_parallel_journal_processor_masked" + "test_parallel_journal_processor_masked_origin" + "test_parallel_journal_processor_masked_origin_visit_statuses" + ]; + + meta = { + description = "Software Heritage dataset tools"; + homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-export"; + license = lib.licenses.gpl3Only; + maintainers = with lib.maintainers; [ drupol ]; + }; +} diff --git a/pkgs/development/python-modules/swh-objstorage/default.nix b/pkgs/development/python-modules/swh-objstorage/default.nix index 478bc7e07bdf..6672a3054cdb 100644 --- a/pkgs/development/python-modules/swh-objstorage/default.nix +++ b/pkgs/development/python-modules/swh-objstorage/default.nix @@ -1,5 +1,6 @@ { lib, + stdenv, buildPythonPackage, fetchFromGitLab, setuptools, @@ -102,6 +103,11 @@ buildPythonPackage rec { util-linux ] ++ psycopg.optional-dependencies.pool; + disabledTests = lib.optionals (stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isLinux) [ + # FAILED swh/objstorage/tests/test_objstorage_winery.py::test_winery_leaky_bucket_tick - assert 1 == 0 + "test_winery_leaky_bucket_tick" + ]; + meta = { description = "Content-addressable object storage for the Software Heritage project"; homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-objstorage"; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index f1d6ddb041a0..356c4ba47c9b 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -12772,6 +12772,8 @@ self: super: with self; { pyoppleio = callPackage ../development/python-modules/pyoppleio { }; + pyorc = callPackage ../development/python-modules/pyorc { }; + pyorthanc = callPackage ../development/python-modules/pyorthanc { }; pyosf = callPackage ../development/python-modules/pyosf { }; @@ -13541,6 +13543,8 @@ self: super: with self; { pytest-jupyter = callPackage ../development/python-modules/pytest-jupyter { }; + pytest-kafka = callPackage ../development/python-modules/pytest-kafka { }; + pytest-lazy-fixture = callPackage ../development/python-modules/pytest-lazy-fixture { }; pytest-lazy-fixtures = callPackage ../development/python-modules/pytest-lazy-fixtures { }; @@ -16669,6 +16673,8 @@ self: super: with self; { swh-core = callPackage ../development/python-modules/swh-core { }; + swh-export = callPackage ../development/python-modules/swh-export { }; + swh-journal = callPackage ../development/python-modules/swh-journal { }; swh-model = callPackage ../development/python-modules/swh-model { };