python312Packages.swh-export: init at 1.8.0 (#395541)
This commit is contained in:
Gaétan Lepage
@@ -0,0 +1,81 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
fetchFromGitHub,
|
||||
fetchurl,
|
||||
fetchpatch,
|
||||
cmake,
|
||||
gtest,
|
||||
lz4,
|
||||
protobuf,
|
||||
snappy,
|
||||
zlib,
|
||||
zstd,
|
||||
}:
|
||||
|
||||
let
|
||||
orc-format = fetchurl {
|
||||
name = "orc-format-1.0.0.tar.gz";
|
||||
url = "https://www.apache.org/dyn/closer.lua/orc/orc-format-1.0.0/orc-format-1.0.0.tar.gz?action=download";
|
||||
hash = "sha256-c5+uX/lLH4ErQTB3KANhBFv5LlEO8Es0phDiOpRdjNU=";
|
||||
};
|
||||
in
|
||||
stdenv.mkDerivation (finalAttrs: {
|
||||
pname = "apache-orc";
|
||||
version = "2.1.1";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "apache";
|
||||
repo = "orc";
|
||||
tag = "v${finalAttrs.version}";
|
||||
hash = "sha256-Ib02mIUQaLEVwIBv5xdKDyX+FeI8vhh9+5hM7miKwHo=";
|
||||
};
|
||||
|
||||
patches = [
|
||||
# Patch that adds 2 missing imports in source files
|
||||
# To be removed this PR land: https://github.com/apache/orc/pull/2175
|
||||
(fetchpatch {
|
||||
url = "https://github.com/apache/orc/commit/fb20db2440226da6b92d38ce260e5b850d2f0092.patch";
|
||||
hash = "sha256-rHGECXJoBPgZ62yZciYdSMq4pGnVt75lxkHyO46IiyQ=";
|
||||
})
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
cmake
|
||||
];
|
||||
|
||||
buildInputs = [
|
||||
gtest
|
||||
lz4
|
||||
protobuf
|
||||
snappy
|
||||
zlib
|
||||
zstd
|
||||
];
|
||||
|
||||
cmakeFlags = [
|
||||
(lib.cmakeFeature "CMAKE_BUILD_TYPE" "Release")
|
||||
(lib.cmakeBool "BUILD_JAVA" false)
|
||||
(lib.cmakeBool "STOP_BUILD_ON_WARNING" true)
|
||||
(lib.cmakeBool "INSTALL_VENDORED_LIBS" false)
|
||||
];
|
||||
|
||||
env = {
|
||||
GTEST_HOME = gtest.dev;
|
||||
LZ4_ROOT = lz4;
|
||||
ORC_FORMAT_URL = orc-format;
|
||||
PROTOBUF_HOME = protobuf;
|
||||
SNAPPY_ROOT = snappy.dev;
|
||||
ZLIB_ROOT = zlib.dev;
|
||||
ZSTD_ROOT = zstd.dev;
|
||||
};
|
||||
|
||||
meta = {
|
||||
changelog = "https://github.com/apache/orc/releases/tag/v${finalAttrs.version}";
|
||||
description = "The smallest, fastest columnar storage for Hadoop workloads";
|
||||
homepage = "https://github.com/apache/orc/";
|
||||
license = lib.licenses.asl20;
|
||||
maintainers = with lib.maintainers; [ drupol ];
|
||||
platforms = lib.platforms.all;
|
||||
};
|
||||
})
|
||||
@@ -3,6 +3,7 @@
|
||||
writeShellApplication,
|
||||
withSwhPythonPackages ? [
|
||||
python3Packages.swh-auth
|
||||
python3Packages.swh-export
|
||||
python3Packages.swh-model
|
||||
python3Packages.swh-objstorage
|
||||
python3Packages.swh-scanner
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
fetchFromGitHub,
|
||||
pybind11,
|
||||
setuptools,
|
||||
pytestCheckHook,
|
||||
tzdata,
|
||||
python,
|
||||
pkgs,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "pyorc";
|
||||
version = "0.10.0";
|
||||
pyproject = true;
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "noirello";
|
||||
repo = "pyorc";
|
||||
tag = "v${version}";
|
||||
hash = "sha256-2w3Qh6g+Yg+D10kTow9YR6B6FhQ+z2DvgDy5GtYxH4g=";
|
||||
};
|
||||
|
||||
build-system = [
|
||||
pybind11
|
||||
setuptools
|
||||
];
|
||||
|
||||
env = {
|
||||
PYORC_SKIP_ORC_BUILD = "true";
|
||||
};
|
||||
|
||||
buildInputs = [
|
||||
pkgs.lz4
|
||||
pkgs.protobuf
|
||||
pkgs.snappy
|
||||
pkgs.zlib
|
||||
pkgs.zstd
|
||||
pkgs.apache-orc
|
||||
];
|
||||
|
||||
preCheck = ''
|
||||
# provide timezone data, works only on linux
|
||||
export TZDIR=${tzdata}/${python.sitePackages}/tzdata/zoneinfo
|
||||
|
||||
substituteInPlace "tests/compare/test_writer_cmp.py" \
|
||||
--replace-fail "deps/bin/orc-contents" "orc-contents"
|
||||
|
||||
substituteInPlace "tests/compare/test_reader_cmp.py" \
|
||||
--replace-fail "deps/bin/orc-metadata" "orc-metadata"
|
||||
|
||||
mkdir -p deps
|
||||
ln -s "${pkgs.apache-orc.src}/examples" "deps/"
|
||||
'';
|
||||
|
||||
pythonImportsCheck = [
|
||||
"pyorc"
|
||||
];
|
||||
|
||||
nativeCheckInputs = [
|
||||
pytestCheckHook
|
||||
pkgs.apache-orc
|
||||
];
|
||||
|
||||
meta = {
|
||||
changelog = "https://github.com/noirello/pyorc/blob/${version}/CHANGELOG.rst";
|
||||
description = "Python module for Apache ORC file format";
|
||||
homepage = "https://github.com/noirello/pyorc";
|
||||
license = lib.licenses.asl20;
|
||||
maintainers = with lib.maintainers; [ drupol ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
fetchFromGitLab,
|
||||
setuptools,
|
||||
kafka-python-ng,
|
||||
port-for,
|
||||
pytest,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "pytest-kafka";
|
||||
version = "0.8.1";
|
||||
pyproject = true;
|
||||
|
||||
src = fetchFromGitLab {
|
||||
owner = "karolinepauls";
|
||||
repo = "pytest-kafka";
|
||||
tag = "v${version}";
|
||||
hash = "sha256-OR8SpNswbPOVtAcFuZgrZJR5K6wPb1TS5leybKWr3zY=";
|
||||
};
|
||||
|
||||
build-system = [ setuptools ];
|
||||
|
||||
dependencies = [
|
||||
kafka-python-ng
|
||||
port-for
|
||||
pytest
|
||||
];
|
||||
|
||||
pythonImportsCheck = [ "pytest_kafka" ];
|
||||
|
||||
# Tests depends on a kafka server running
|
||||
doCheck = false;
|
||||
|
||||
meta = {
|
||||
description = "Pytest fixture factories for Zookeeper, Kafka server and Kafka consumer";
|
||||
homepage = "https://gitlab.com/karolinepauls/pytest-kafka";
|
||||
license = lib.licenses.mit;
|
||||
maintainers = with lib.maintainers; [ drupol ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
fetchFromGitLab,
|
||||
setuptools,
|
||||
setuptools-scm,
|
||||
boto3,
|
||||
click,
|
||||
tqdm,
|
||||
pyorc,
|
||||
plyvel,
|
||||
python,
|
||||
types-requests,
|
||||
swh-core,
|
||||
swh-journal,
|
||||
swh-model,
|
||||
swh-storage,
|
||||
pytestCheckHook,
|
||||
pytest-kafka,
|
||||
pytest-mock,
|
||||
tzdata,
|
||||
pkgs,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "swh-export";
|
||||
version = "1.8.0";
|
||||
pyproject = true;
|
||||
|
||||
src = fetchFromGitLab {
|
||||
domain = "gitlab.softwareheritage.org";
|
||||
group = "swh";
|
||||
owner = "devel";
|
||||
repo = "swh-export";
|
||||
tag = "v${version}";
|
||||
hash = "sha256-n97MMYn7EmTrv411YSxUD1+zfbFB8KOSns44N3NqqV8=";
|
||||
};
|
||||
|
||||
build-system = [
|
||||
setuptools
|
||||
setuptools-scm
|
||||
];
|
||||
|
||||
dependencies = [
|
||||
boto3
|
||||
click
|
||||
tqdm
|
||||
pyorc
|
||||
plyvel
|
||||
types-requests
|
||||
swh-core
|
||||
swh-journal
|
||||
swh-model
|
||||
swh-storage
|
||||
];
|
||||
|
||||
preCheck = ''
|
||||
# provide timezone data, works only on linux
|
||||
export TZDIR=${tzdata}/${python.sitePackages}/tzdata/zoneinfo
|
||||
'';
|
||||
|
||||
pythonImportsCheck = [ "swh.export" ];
|
||||
|
||||
nativeCheckInputs = [
|
||||
pytestCheckHook
|
||||
pytest-kafka
|
||||
pytest-mock
|
||||
pkgs.zstd
|
||||
pkgs.pv
|
||||
];
|
||||
|
||||
disabledTests = [
|
||||
# I don't know how to fix the following error
|
||||
# E fixture 'kafka_server' not found
|
||||
"test_parallel_journal_processor"
|
||||
"test_parallel_journal_processor_origin"
|
||||
"test_parallel_journal_processor_origin_visit_status"
|
||||
"test_parallel_journal_processor_offsets"
|
||||
"test_parallel_journal_processor_masked"
|
||||
"test_parallel_journal_processor_masked_origin"
|
||||
"test_parallel_journal_processor_masked_origin_visit_statuses"
|
||||
];
|
||||
|
||||
meta = {
|
||||
description = "Software Heritage dataset tools";
|
||||
homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-export";
|
||||
license = lib.licenses.gpl3Only;
|
||||
maintainers = with lib.maintainers; [ drupol ];
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
buildPythonPackage,
|
||||
fetchFromGitLab,
|
||||
setuptools,
|
||||
@@ -102,6 +103,11 @@ buildPythonPackage rec {
|
||||
util-linux
|
||||
] ++ psycopg.optional-dependencies.pool;
|
||||
|
||||
disabledTests = lib.optionals (stdenv.hostPlatform.isAarch64 && stdenv.hostPlatform.isLinux) [
|
||||
# FAILED swh/objstorage/tests/test_objstorage_winery.py::test_winery_leaky_bucket_tick - assert 1 == 0
|
||||
"test_winery_leaky_bucket_tick"
|
||||
];
|
||||
|
||||
meta = {
|
||||
description = "Content-addressable object storage for the Software Heritage project";
|
||||
homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-objstorage";
|
||||
|
||||
@@ -12772,6 +12772,8 @@ self: super: with self; {
|
||||
|
||||
pyoppleio = callPackage ../development/python-modules/pyoppleio { };
|
||||
|
||||
pyorc = callPackage ../development/python-modules/pyorc { };
|
||||
|
||||
pyorthanc = callPackage ../development/python-modules/pyorthanc { };
|
||||
|
||||
pyosf = callPackage ../development/python-modules/pyosf { };
|
||||
@@ -13541,6 +13543,8 @@ self: super: with self; {
|
||||
|
||||
pytest-jupyter = callPackage ../development/python-modules/pytest-jupyter { };
|
||||
|
||||
pytest-kafka = callPackage ../development/python-modules/pytest-kafka { };
|
||||
|
||||
pytest-lazy-fixture = callPackage ../development/python-modules/pytest-lazy-fixture { };
|
||||
|
||||
pytest-lazy-fixtures = callPackage ../development/python-modules/pytest-lazy-fixtures { };
|
||||
@@ -16669,6 +16673,8 @@ self: super: with self; {
|
||||
|
||||
swh-core = callPackage ../development/python-modules/swh-core { };
|
||||
|
||||
swh-export = callPackage ../development/python-modules/swh-export { };
|
||||
|
||||
swh-journal = callPackage ../development/python-modules/swh-journal { };
|
||||
|
||||
swh-model = callPackage ../development/python-modules/swh-model { };
|
||||
|
||||
Reference in New Issue
Block a user