From faaff6e3665ae07fc73abf0c76071f5275044874 Mon Sep 17 00:00:00 2001 From: jokatzke Date: Mon, 16 Oct 2023 10:48:07 +0200 Subject: [PATCH 1/6] maintainers: add jokatzke --- maintainers/maintainer-list.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index 471d97cb1bc0..f6afbd852103 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -8978,6 +8978,12 @@ fingerprint = "7249 70E6 A661 D84E 8B47 678A 0590 93B1 A278 BCD0"; }]; }; + jokatzke = { + email = "jokatzke@fastmail.com"; + github = "jokatzke"; + githubId = 46931073; + name = "Jonas Katzke"; + }; joko = { email = "ioannis.koutras@gmail.com"; github = "jokogr"; From 5731572c47b948cc9d1c1bb89f21c740b5cf0010 Mon Sep 17 00:00:00 2001 From: jokatzke Date: Fri, 5 Jan 2024 07:57:49 +0100 Subject: [PATCH 2/6] python3Packages.courlan: init at 0.9.5 --- .../python-modules/courlan/default.nix | 54 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 56 insertions(+) create mode 100644 pkgs/development/python-modules/courlan/default.nix diff --git a/pkgs/development/python-modules/courlan/default.nix b/pkgs/development/python-modules/courlan/default.nix new file mode 100644 index 000000000000..d906f44c0f16 --- /dev/null +++ b/pkgs/development/python-modules/courlan/default.nix @@ -0,0 +1,54 @@ +{ lib +, buildPythonPackage +, fetchPypi +, langcodes +, pytestCheckHook +, tld +, urllib3 +, pythonOlder +}: + +buildPythonPackage rec { + pname = "courlan"; + version = "0.9.5"; + format = "setuptools"; + + disabled = pythonOlder "3.6"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-ONw1suO/H11RbQDVGsEuveVD40F8a+b2oic8D8W1s1M="; + }; + + propagatedBuildInputs = [ + langcodes + tld + urllib3 + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + # disable tests that require an internet connection + disabledTests = [ + "test_urlcheck" + ]; + + # nixify path to the courlan binary in the test suite + postPatch = '' + substituteInPlace tests/unit_tests.py \ + --replace "\"courlan --help\"" "\"$out/bin/courlan --help\"" \ + --replace "courlan_bin = \"courlan\"" "courlan_bin = \"$out/bin/courlan\"" + ''; + + pythonImportsCheck = [ "courlan" ]; + + meta = with lib; { + description = "Clean, filter and sample URLs to optimize data collection"; + homepage = "https://github.com/adbar/courlan"; + changelog = "https://github.com/adbar/courlan/blob/v${version}/HISTORY.md"; + license = licenses.gpl3Plus; + maintainers = with maintainers; [ jokatzke ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 6d38976064ce..8b869860ebfd 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2372,6 +2372,8 @@ self: super: with self; { cot = callPackage ../development/python-modules/cot { }; + courlan = callPackage ../development/python-modules/courlan { }; + cov-core = callPackage ../development/python-modules/cov-core { }; coverage = callPackage ../development/python-modules/coverage { }; From b4edc2b435382440c6f99e5bad545a54e7e921e6 Mon Sep 17 00:00:00 2001 From: jokatzke Date: Mon, 16 Oct 2023 10:48:07 +0200 Subject: [PATCH 3/6] python3Packages.htmldate: init at 1.6.0 --- .../python-modules/htmldate/default.nix | 56 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 58 insertions(+) create mode 100644 pkgs/development/python-modules/htmldate/default.nix diff --git a/pkgs/development/python-modules/htmldate/default.nix b/pkgs/development/python-modules/htmldate/default.nix new file mode 100644 index 000000000000..464db47f03ed --- /dev/null +++ b/pkgs/development/python-modules/htmldate/default.nix @@ -0,0 +1,56 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pythonOlder +, charset-normalizer +, dateparser +, lxml +, pytestCheckHook +, python-dateutil +, urllib3 +, backports-datetime-fromisoformat +}: + +buildPythonPackage rec { + pname = "htmldate"; + version = "1.6.0"; + format = "setuptools"; + + disabled = pythonOlder "3.6"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-WCfI9iahaACinlfoGIo9MtCwjKTHvWYlN7c7u/IsRaY="; + }; + + propagatedBuildInputs = [ + charset-normalizer + dateparser + lxml + python-dateutil + urllib3 + ] ++ lib.optionals (pythonOlder "3.7") [ + backports-datetime-fromisoformat + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + # disable tests that require an internet connection + disabledTests = [ + "test_input" + "test_cli" + "test_download" + ]; + + pythonImportsCheck = [ "htmldate" ]; + + meta = with lib; { + description = "Fast and robust extraction of original and updated publication dates from URLs and web pages"; + homepage = "https://htmldate.readthedocs.io"; + changelog = "https://github.com/adbar/htmldate/blob/v${version}/CHANGELOG.md"; + license = licenses.gpl3Plus; + maintainers = with maintainers; [ jokatzke ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 8b869860ebfd..ed437fa524f9 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -5229,6 +5229,8 @@ self: super: with self; { html5-parser = callPackage ../development/python-modules/html5-parser { }; + htmldate = callPackage ../development/python-modules/htmldate { }; + htmllaundry = callPackage ../development/python-modules/htmllaundry { }; htmllistparse = callPackage ../development/python-modules/htmllistparse { }; From c0980c450417dcd95b7d5e047ce7a06d354e419d Mon Sep 17 00:00:00 2001 From: jokatzke Date: Mon, 16 Oct 2023 10:48:07 +0200 Subject: [PATCH 4/6] python3Packages.justext: init at 3.0.0 --- .../python-modules/justext/default.nix | 43 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 45 insertions(+) create mode 100644 pkgs/development/python-modules/justext/default.nix diff --git a/pkgs/development/python-modules/justext/default.nix b/pkgs/development/python-modules/justext/default.nix new file mode 100644 index 000000000000..82f0aa804565 --- /dev/null +++ b/pkgs/development/python-modules/justext/default.nix @@ -0,0 +1,43 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, pytestCheckHook +, lxml +}: + +buildPythonPackage rec { + pname = "justext"; + version = "3.0.0"; + format = "setuptools"; + + src = fetchFromGitHub { + owner = "miso-belica"; + repo = "jusText"; + rev = "refs/tags/v${version}"; + hash = "sha256-WNxDoM5666tEHS9pMl5dOoig4S7dSYaCLZq71tehWqw="; + }; + + propagatedBuildInputs = [ + lxml + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + # patch out coverage report + postPatch = '' + substituteInPlace setup.cfg \ + --replace " --cov=justext --cov-report=term-missing --no-cov-on-fail" "" + ''; + + pythonImportsCheck = [ "justext" ]; + + meta = with lib; { + description = "Heuristic based boilerplate removal tool"; + homepage = "https://github.com/miso-belica/jusText"; + changelog = "https://github.com/miso-belica/jusText/blob/v${version}/CHANGELOG.rst"; + license = licenses.bsd2; + maintainers = with maintainers; [ jokatzke ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index ed437fa524f9..bc1f54dd6ddb 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6002,6 +6002,8 @@ self: super: with self; { justbytes = callPackage ../development/python-modules/justbytes { }; + justext = callPackage ../development/python-modules/justext { }; + justnimbus = callPackage ../development/python-modules/justnimbus { }; jwcrypto = callPackage ../development/python-modules/jwcrypto { }; From 16d9eff4673f67ff74b9e42109b512f17cee7808 Mon Sep 17 00:00:00 2001 From: jokatzke Date: Mon, 16 Oct 2023 10:48:07 +0200 Subject: [PATCH 5/6] python3Packages.py3langid: init at 0.2.2 --- .../python-modules/py3langid/default.nix | 43 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 45 insertions(+) create mode 100644 pkgs/development/python-modules/py3langid/default.nix diff --git a/pkgs/development/python-modules/py3langid/default.nix b/pkgs/development/python-modules/py3langid/default.nix new file mode 100644 index 000000000000..02631320c4af --- /dev/null +++ b/pkgs/development/python-modules/py3langid/default.nix @@ -0,0 +1,43 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pythonOlder +, numpy +, pytestCheckHook +}: + +buildPythonPackage rec { + pname = "py3langid"; + version = "0.2.2"; + format = "setuptools"; + + disabled = pythonOlder "3.6"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-tN4B2tfnAfKdIWoJNeheCWzIZ1kD0j6oRFsrtfCQuW8="; + }; + + propagatedBuildInputs = [ + numpy + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + # nixify path to the courlan binary in the test suite + postPatch = '' + substituteInPlace tests/test_langid.py --replace "'langid'" "'$out/bin/langid'" + ''; + + pythonImportsCheck = [ "py3langid" ]; + + meta = with lib; { + description = "Fork of the language identification tool langid.py, featuring a modernized codebase and faster execution times"; + homepage = "https://github.com/adbar/py3langid"; + changelog = "https://github.com/adbar/py3langid/blob/v${version}/HISTORY.rst"; + license = licenses.bsd3; + maintainers = with maintainers; [ jokatzke ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index bc1f54dd6ddb..b908cbe0bfc3 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -9982,6 +9982,8 @@ self: super: with self; { py3exiv2 = callPackage ../development/python-modules/py3exiv2 { }; + py3langid = callPackage ../development/python-modules/py3langid { }; + py3nvml = callPackage ../development/python-modules/py3nvml { }; py3rijndael = callPackage ../development/python-modules/py3rijndael { }; From 8200b0b5a0997903a3d7c2ec4d90b27fe4b60d31 Mon Sep 17 00:00:00 2001 From: jokatzke Date: Mon, 16 Oct 2023 10:48:08 +0200 Subject: [PATCH 6/6] python3Packages.trafilatura: init at 1.6.3 --- .../python-modules/trafilatura/default.nix | 67 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 69 insertions(+) create mode 100644 pkgs/development/python-modules/trafilatura/default.nix diff --git a/pkgs/development/python-modules/trafilatura/default.nix b/pkgs/development/python-modules/trafilatura/default.nix new file mode 100644 index 000000000000..57042627c968 --- /dev/null +++ b/pkgs/development/python-modules/trafilatura/default.nix @@ -0,0 +1,67 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pytestCheckHook +, pythonOlder +, certifi +, charset-normalizer +, courlan +, htmldate +, justext +, lxml +, urllib3 +}: + +buildPythonPackage rec { + pname = "trafilatura"; + version = "1.6.3"; + format = "setuptools"; + + disabled = pythonOlder "3.6"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-Zx3W4AAOEBxLzo1w9ECLy3n8vyJ17iVZHv4z4sihYA0="; + }; + + propagatedBuildInputs = [ + certifi + charset-normalizer + courlan + htmldate + justext + lxml + urllib3 + ]; + + nativeCheckInputs = [ pytestCheckHook ]; + + # disable tests that require an internet connection + disabledTests = [ + "test_download" + "test_fetch" + "test_redirection" + "test_meta_redirections" + "test_crawl_page" + "test_whole" + "test_probing" + "test_cli_pipeline" + ]; + + # patch out gui cli because it is not supported in this packaging + # nixify path to the trafilatura binary in the test suite + postPatch = '' + substituteInPlace setup.py --replace '"trafilatura_gui=trafilatura.gui:main",' "" + substituteInPlace tests/cli_tests.py --replace "trafilatura_bin = 'trafilatura'" "trafilatura_bin = '$out/bin/trafilatura'" + ''; + + pythonImportsCheck = [ "trafilatura" ]; + + meta = with lib; { + description = "Python package and command-line tool designed to gather text on the Web"; + homepage = "https://trafilatura.readthedocs.io"; + changelog = "https://github.com/adbar/trafilatura/blob/v${version}/HISTORY.md"; + license = licenses.gpl3Plus; + maintainers = with maintainers; [ jokatzke ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index b908cbe0bfc3..8a76ceba9994 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -14493,6 +14493,8 @@ self: super: with self; { trackpy = callPackage ../development/python-modules/trackpy { }; + trafilatura = callPackage ../development/python-modules/trafilatura { }; + trailrunner = callPackage ../development/python-modules/trailrunner {}; trainer = callPackage ../development/python-modules/trainer {};