From 34f81670e057311015e9ddb5448630248c6a1428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Fri, 7 Nov 2025 10:05:49 -0800 Subject: [PATCH 1/9] python3Packages.click: fix SystemExit(1) when using click.File's stdin support This fixes upstream issue https://github.com/pallets/click/issues/2939 which e.g. breaks swh. --- pkgs/development/python-modules/click/default.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkgs/development/python-modules/click/default.nix b/pkgs/development/python-modules/click/default.nix index e028343fbb65..368589cac6ef 100644 --- a/pkgs/development/python-modules/click/default.nix +++ b/pkgs/development/python-modules/click/default.nix @@ -2,6 +2,7 @@ lib, buildPythonPackage, fetchFromGitHub, + fetchpatch, importlib-metadata, pytestCheckHook, @@ -26,6 +27,15 @@ buildPythonPackage rec { hash = "sha256-3FfLKwpfkiGfY2+H2fQoZwLBqfPlV46xw2Bc4YEsyps="; }; + patches = [ + # https://github.com/pallets/click/pull/2940 + (fetchpatch { + name = "fix-SystemExit-when-using-stdin.patch"; + url = "https://github.com/pallets/click/commit/93c6966eb3a575c2b600434d1cc9f4b3aee505ac.patch"; + hash = "sha256-DkVF0JnKbcsdAhgVjWJEDZZ8vr2sf6wba8P3SyRUy6o="; + }) + ]; + build-system = [ flit-core ]; nativeCheckInputs = [ pytestCheckHook ]; From 26a07c82b73c3f57a234dd05ec42a45ac926dd5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Fri, 7 Nov 2025 10:06:51 -0800 Subject: [PATCH 2/9] python3Packages.swh-core: 4.5.2 -> 4.6.0 Diff: https://gitlab.softwareheritage.org/swh/devel/swh-core/-/compare/v4.5.2...v4.6.0 Changelog: https://gitlab.softwareheritage.org/swh/devel/swh-core/-/tags/v4.6.0 --- .../python-modules/swh-core/default.nix | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/swh-core/default.nix b/pkgs/development/python-modules/swh-core/default.nix index abe4979eb1f1..3697adfb5894 100644 --- a/pkgs/development/python-modules/swh-core/default.nix +++ b/pkgs/development/python-modules/swh-core/default.nix @@ -18,6 +18,7 @@ hypothesis, iso8601, lzip, + moto, msgpack, postgresql, postgresqlTestHook, @@ -29,7 +30,9 @@ pytest-postgresql, pytz, requests-mock, + swh-model, systemd, + tqdm, types-deprecated, types-psycopg2, types-pytz, @@ -41,7 +44,7 @@ buildPythonPackage rec { pname = "swh-core"; - version = "4.5.2"; + version = "4.6.0"; pyproject = true; src = fetchFromGitLab { @@ -50,7 +53,7 @@ buildPythonPackage rec { owner = "devel"; repo = "swh-core"; tag = "v${version}"; - hash = "sha256-yNWij9GclQCysQe9Bukr4cHlZgSQqLAuX1KwGWzAK+0="; + hash = "sha256-dI+xfj0DnUbBdYIVycyJQg3B/jnH/eg/Ju8YX2k8Qkc="; }; build-system = [ @@ -58,6 +61,11 @@ buildPythonPackage rec { setuptools-scm ]; + pythonRelaxDeps = [ + # we patched click 8.2.1 + "click" + ]; + dependencies = [ backports-entry-points-selectable click @@ -79,6 +87,7 @@ buildPythonPackage rec { hypothesis iso8601 lzip + moto msgpack postgresql postgresqlTestHook @@ -90,7 +99,9 @@ buildPythonPackage rec { pytest-postgresql pytz requests-mock + swh-model systemd + tqdm types-deprecated types-psycopg2 types-pytz @@ -111,6 +122,7 @@ buildPythonPackage rec { ]; meta = { + changelog = "https://gitlab.softwareheritage.org/swh/devel/swh-core/-/tags/${src.tag}"; description = "Low-level utilities and helpers used by almost all other modules in the stack"; homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-core"; license = lib.licenses.gpl3Only; From c71c20308a5a0c6b8f6e766b90cc73e9601b8cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Fri, 7 Nov 2025 10:15:59 -0800 Subject: [PATCH 3/9] python3Packages.swh-web-client: unpin click --- pkgs/development/python-modules/swh-web-client/default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/development/python-modules/swh-web-client/default.nix b/pkgs/development/python-modules/swh-web-client/default.nix index b7ec8be2d090..deeba3f3a3b1 100644 --- a/pkgs/development/python-modules/swh-web-client/default.nix +++ b/pkgs/development/python-modules/swh-web-client/default.nix @@ -37,6 +37,11 @@ buildPythonPackage rec { setuptools-scm ]; + pythonRelaxDeps = [ + # we patched click 8.2.1 + "click" + ]; + dependencies = [ click dateutils From 7313cf858d36ded9b981e4cfa0cf101c54a683fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Fri, 7 Nov 2025 12:38:02 -0800 Subject: [PATCH 4/9] python3Packages.swh-scanner: fix meta --- pkgs/development/python-modules/swh-scanner/default.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/swh-scanner/default.nix b/pkgs/development/python-modules/swh-scanner/default.nix index 03fef49e9878..e49ba7eb207e 100644 --- a/pkgs/development/python-modules/swh-scanner/default.nix +++ b/pkgs/development/python-modules/swh-scanner/default.nix @@ -81,8 +81,9 @@ buildPythonPackage rec { ]; meta = { - description = "Implementation of the Data model of the Software Heritage project, used to archive source code artifacts"; - homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-model"; + changelog = "https://gitlab.softwareheritage.org/swh/devel/swh-scanner/-/tags/${src.tag}"; + description = "Source code scanner to analyze code bases and compare them with source code artifacts archived by Software Heritage"; + homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-scanner"; license = lib.licenses.gpl3Only; maintainers = [ ]; }; From 204fe9a3775a16fe22d255225cd083dd2c3a0fa9 Mon Sep 17 00:00:00 2001 From: Pol Dellaiera Date: Wed, 5 Nov 2025 18:48:12 +0100 Subject: [PATCH 5/9] python3Packages.swh-scanner: disable tests failing on click >= 8.2.0 The patch is no longer needed because the respective test is now disabled. --- .../python-modules/swh-scanner/default.nix | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/pkgs/development/python-modules/swh-scanner/default.nix b/pkgs/development/python-modules/swh-scanner/default.nix index e49ba7eb207e..5dca94bb53d2 100644 --- a/pkgs/development/python-modules/swh-scanner/default.nix +++ b/pkgs/development/python-modules/swh-scanner/default.nix @@ -2,7 +2,6 @@ lib, buildPythonPackage, fetchFromGitLab, - fetchpatch, setuptools, setuptools-scm, requests, @@ -36,15 +35,6 @@ buildPythonPackage rec { hash = "sha256-baUUuYFapBD7iuDaDP8CSR9f4glVZcS5qBpZddVf7z8="; }; - patches = [ - # To be removed at the next release - # See https://gitlab.softwareheritage.org/swh/devel/swh-scanner/-/merge_requests/160 - (fetchpatch { - url = "https://gitlab.softwareheritage.org/swh/devel/swh-scanner/-/commit/0eb273475826b0074844c7619b767c052562cfe4.patch"; - hash = "sha256-i3hpaQJmHpIYgix+/npICQGtJ/IKVRXcCTm2O1VsR9M="; - }) - ]; - build-system = [ setuptools setuptools-scm @@ -78,6 +68,8 @@ buildPythonPackage rec { disabledTestPaths = [ # pytestRemoveBytecodePhase fails with: "error (ignored): error: opening directory "/tmp/nix-build-python3.12-swh-scanner-0.8.3.drv-5/build/pytest-of-nixbld/pytest-0/test_randomdir_policy_info_cal0/big-directory/dir/dir/dir/ ......" "swh/scanner/tests/test_policy.py" + # TypeError: CliRunner.__init__() got an unexpected keyword argument 'mix_stderr' + "swh/scanner/tests/test_cli.py" ]; meta = { From c7349ac27b92228d69a152fc2f89b25ffd3d8fc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Fri, 7 Nov 2025 13:32:09 -0800 Subject: [PATCH 6/9] python3Packages.swh-shard: init at 2.2.0 --- .../python-modules/swh-shard/default.nix | 72 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 74 insertions(+) create mode 100644 pkgs/development/python-modules/swh-shard/default.nix diff --git a/pkgs/development/python-modules/swh-shard/default.nix b/pkgs/development/python-modules/swh-shard/default.nix new file mode 100644 index 000000000000..a07c344b04a2 --- /dev/null +++ b/pkgs/development/python-modules/swh-shard/default.nix @@ -0,0 +1,72 @@ +{ + buildPythonPackage, + click, + cmake, + cmph, + fetchFromGitLab, + lib, + ninja, + pkg-config, + pybind11, + pytest-mock, + pytestCheckHook, + scikit-build-core, + setuptools-scm, +}: + +buildPythonPackage rec { + pname = "swh-shard"; + version = "2.2.0"; + pyproject = true; + + src = fetchFromGitLab { + domain = "gitlab.softwareheritage.org"; + group = "swh"; + owner = "devel"; + repo = "swh-shard"; + tag = "v${version}"; + hash = "sha256-97oZ+Wa8GmyL2V4CnlSvaTbQZJ+mPbg6uVmWd0oxv1Q="; + }; + + build-system = [ + cmake + ninja + pybind11 + scikit-build-core + setuptools-scm + ]; + + nativeBuildInputs = [ + pkg-config + ]; + + dontUseCmakeConfigure = true; + + buildInputs = [ + cmph + ]; + + dependencies = [ + click + ]; + + pythonImportsCheck = [ "swh.shard" ]; + + nativeCheckInputs = [ + pytest-mock + pytestCheckHook + ]; + + preCheck = '' + # import from $out + rm src/swh/shard/*.py + ''; + + meta = { + changelog = "https://gitlab.softwareheritage.org/swh/devel/swh-shard/-/tags/v2.2.0"; + description = "Shard File Format for the Software Heritage Object Storage"; + homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-shard"; + license = lib.licenses.gpl3Only; + maintainers = [ lib.maintainers.dotlambda ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 451502c1d2ee..d089c2fcb4bb 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -18077,6 +18077,8 @@ self: super: with self; { swh-scheduler = callPackage ../development/python-modules/swh-scheduler { }; + swh-shard = callPackage ../development/python-modules/swh-shard { }; + swh-storage = callPackage ../development/python-modules/swh-storage { }; swh-web-client = callPackage ../development/python-modules/swh-web-client { }; From 2bd851a243ffc1d7e94bf4304b16199a1201a243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Fri, 7 Nov 2025 13:11:21 -0800 Subject: [PATCH 7/9] python3Packages.swh-objstorage: 4.0.0 -> 5.1.0 Diff: https://gitlab.softwareheritage.org/swh/devel/swh-objstorage/-/compare/v4.0.0...v5.1.0 Changelog: https://gitlab.softwareheritage.org/swh/devel/swh-objstorage/-/tags/v4.1.0 https://gitlab.softwareheritage.org/swh/devel/swh-objstorage/-/tags/v4.2.0 https://gitlab.softwareheritage.org/swh/devel/swh-objstorage/-/tags/v5.0.0 https://gitlab.softwareheritage.org/swh/devel/swh-objstorage/-/tags/v5.0.1 https://gitlab.softwareheritage.org/swh/devel/swh-objstorage/-/tags/v5.1.0 --- .../python-modules/swh-objstorage/default.nix | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkgs/development/python-modules/swh-objstorage/default.nix b/pkgs/development/python-modules/swh-objstorage/default.nix index 5dda6b224fa5..aac86b8de896 100644 --- a/pkgs/development/python-modules/swh-objstorage/default.nix +++ b/pkgs/development/python-modules/swh-objstorage/default.nix @@ -18,7 +18,7 @@ tenacity, swh-core, swh-model, - swh-perfecthash, + swh-shard, aiohttp, azure-core, azure-storage-blob, @@ -40,7 +40,7 @@ buildPythonPackage rec { pname = "swh-objstorage"; - version = "4.0.0"; + version = "5.1.0"; pyproject = true; src = fetchFromGitLab { @@ -49,7 +49,7 @@ buildPythonPackage rec { owner = "devel"; repo = "swh-objstorage"; tag = "v${version}"; - hash = "sha256-c0ZH2PMT9DVnpTV5PDyX0Yw4iHiJSolEgq/bMXEwXG8="; + hash = "sha256-NnNT9Lt/LGDIJpUmfkfPn6JnF3k8Usf2UVa88zHPKlg="; }; build-system = [ @@ -71,7 +71,7 @@ buildPythonPackage rec { tenacity swh-core swh-model - swh-perfecthash + swh-shard ]; preCheck = '' @@ -110,6 +110,7 @@ buildPythonPackage rec { ]; meta = { + changelog = "https://gitlab.softwareheritage.org/swh/devel/swh-objstorage/-/tags/${src.tag}"; description = "Content-addressable object storage for the Software Heritage project"; homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-objstorage"; license = lib.licenses.gpl3Only; From f5bdeee8163cd702fc4d5472f929f45a1de1982f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Fri, 7 Nov 2025 12:55:44 -0800 Subject: [PATCH 8/9] python3Packages.swh-storage: 3.1.0 -> 4.1.0 Diff: https://gitlab.softwareheritage.org/swh/devel/swh-storage/-/compare/v3.1.0...v4.1.0 Changelog: https://gitlab.softwareheritage.org/swh/devel/swh-storage/-/tags/v3.1.1 https://gitlab.softwareheritage.org/swh/devel/swh-storage/-/tags/v3.1.2 https://gitlab.softwareheritage.org/swh/devel/swh-storage/-/tags/v3.2.0 https://gitlab.softwareheritage.org/swh/devel/swh-storage/-/tags/v3.3.0 https://gitlab.softwareheritage.org/swh/devel/swh-storage/-/tags/v4.0.0 https://gitlab.softwareheritage.org/swh/devel/swh-storage/-/tags/v4.1.0 --- .../python-modules/swh-storage/default.nix | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pkgs/development/python-modules/swh-storage/default.nix b/pkgs/development/python-modules/swh-storage/default.nix index 74e312d07606..dcf09bf27cf4 100644 --- a/pkgs/development/python-modules/swh-storage/default.nix +++ b/pkgs/development/python-modules/swh-storage/default.nix @@ -12,6 +12,8 @@ iso8601, mypy-extensions, psycopg, + psycopg-pool, + pyasyncore, redis, tenacity, swh-core, @@ -30,7 +32,7 @@ buildPythonPackage rec { pname = "swh-storage"; - version = "3.1.0"; + version = "4.1.0"; pyproject = true; src = fetchFromGitLab { @@ -39,7 +41,7 @@ buildPythonPackage rec { owner = "devel"; repo = "swh-storage"; tag = "v${version}"; - hash = "sha256-Bxwc8OccmqadLjHtmhToHBYHGkD7Fw3Cl3go9VLV/Bs="; + hash = "sha256-JXeHE/wZ3Wcf1I1eGyCNlCGsreiQH6kCYZoDXV1h0A0="; }; build-system = [ @@ -47,6 +49,11 @@ buildPythonPackage rec { setuptools-scm ]; + pythonRelaxDeps = [ + # we patched click 8.2.1 + "click" + ]; + dependencies = [ backports-entry-points-selectable cassandra-driver @@ -56,13 +63,14 @@ buildPythonPackage rec { iso8601 mypy-extensions psycopg + psycopg-pool + pyasyncore redis tenacity swh-core swh-model swh-objstorage - ] - ++ psycopg.optional-dependencies.pool; + ]; pythonImportsCheck = [ "swh.storage" ]; @@ -88,11 +96,13 @@ buildPythonPackage rec { "swh/storage/tests/test_cassandra_migration.py" "swh/storage/tests/test_cassandra_ttl.py" "swh/storage/tests/test_cli_cassandra.py" + "swh/storage/tests/test_cli_object_references_cassandra.py" # Failing tests "swh/storage/tests/test_cli_object_references.py" ]; meta = { + changelog = "https://gitlab.softwareheritage.org/swh/devel/swh-storage/-/tags/${src.tag}"; description = "Abstraction layer over the archive, allowing to access all stored source code artifacts as well as their metadata"; homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-storage"; license = lib.licenses.gpl3Only; From 5e4c2ad71ffd4b580066211c964c4522fb7f03d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Fri, 7 Nov 2025 13:37:21 -0800 Subject: [PATCH 9/9] python3Packages.swh-scheduler: 3.1.0 -> 3.3.0 Diff: https://gitlab.softwareheritage.org/swh/devel/swh-scheduler/-/compare/v3.1.0...v3.3.0 Changelog: https://gitlab.softwareheritage.org/swh/devel/swh-scheduler/-/tags/v3.2.0 https://gitlab.softwareheritage.org/swh/devel/swh-scheduler/-/tags/v3.3.0 --- .../python-modules/swh-scheduler/default.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/swh-scheduler/default.nix b/pkgs/development/python-modules/swh-scheduler/default.nix index ba2f4e10d1b3..7bb37313ed3c 100644 --- a/pkgs/development/python-modules/swh-scheduler/default.nix +++ b/pkgs/development/python-modules/swh-scheduler/default.nix @@ -30,7 +30,7 @@ buildPythonPackage rec { pname = "swh-scheduler"; - version = "3.1.0"; + version = "3.3.0"; pyproject = true; src = fetchFromGitLab { @@ -39,7 +39,7 @@ buildPythonPackage rec { owner = "devel"; repo = "swh-scheduler"; tag = "v${version}"; - hash = "sha256-YpMHeZVHK8IPIiuBaPNR0D/yB9lIQ3DK7NEAiBmjWpA="; + hash = "sha256-Kv5QH3sj/InKOSjxGtwVxtoAluHx5eIxO5GqcbOs0NY="; }; build-system = [ @@ -47,6 +47,11 @@ buildPythonPackage rec { setuptools-scm ]; + pythonRelaxDeps = [ + # we patched click 8.2.1 + "click" + ]; + dependencies = [ celery flask @@ -80,6 +85,7 @@ buildPythonPackage rec { disabledTests = [ "test_setup_log_handler_with_env_configuration" ]; meta = { + changelog = "https://gitlab.softwareheritage.org/swh/devel/swh-scheduler/-/tags/${src.tag}"; description = "Job scheduler for the Software Heritage project"; homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-scheduler"; license = lib.licenses.gpl3Only;