python3Packages.trafilatura: 2.0.0 -> 2.1.0 (#529231)

pkgs/development/python-modules/courlan/default.nix

@@ -1,32 +1,49 @@
 {
   lib,
-  babel,
   buildPythonPackage,
-  fetchPypi,
-  langcodes,
-  pytestCheckHook,
+  fetchFromGitHub,
+
+  # build-system,
   setuptools,
+
+  # dependencies
+  babel,
+  langcodes,
   tld,
   urllib3,
+
+  # tests
+  pytest-httpserver,
+  pytestCheckHook,
 }:
 
-buildPythonPackage rec {
+buildPythonPackage (finalAttrs: {
   pname = "courlan";
-  version = "1.3.2";
+  version = "1.4.0";
   pyproject = true;
+  __structuredAttrs = true;
 
-  src = fetchPypi {
-    inherit pname version;
-    hash = "sha256-C2b02zqcOabiLdJHxyz6pX1o6mYOlLsshOx9uHEq8ZA=";
+  src = fetchFromGitHub {
+    owner = "adbar";
+    repo = "courlan";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-hDbeAd/gil6S8zggBjmrDmnW3DMMdRvG1vULuhYA1zc=";
   };
 
   # Tests try to write to /tmp directly. use $TMPDIR instead.
-  postPatch = ''
-    substituteInPlace tests/unit_tests.py \
-      --replace-fail "\"courlan --help\"" "\"$out/bin/courlan --help\"" \
-      --replace-fail "courlan_bin = \"courlan\"" "courlan_bin = \"$out/bin/courlan\"" \
-      --replace-fail "/tmp" "$TMPDIR"
-  '';
+  postPatch =
+    let
+      courlanBinPath = "${placeholder "out"}/bin/courlan";
+    in
+    ''
+      substituteInPlace tests/unit_tests.py \
+        --replace-fail \
+          'assert os.system("courlan --help") == 0' \
+          'assert os.system("${courlanBinPath} --help") == 0' \
+        --replace-fail \
+          'courlan_bin = "courlan"' \
+          'courlan_bin = "${courlanBinPath}"'
+    '';
 
   build-system = [ setuptools ];
 
@@ -37,19 +54,24 @@ buildPythonPackage rec {
     urllib3
   ];
 
-  nativeCheckInputs = [ pytestCheckHook ];
+  nativeCheckInputs = [
+    pytest-httpserver
+    pytestCheckHook
+  ];
 
   # disable tests that require an internet connection
   disabledTests = [ "test_urlcheck" ];
 
   pythonImportsCheck = [ "courlan" ];
 
+  __darwinAllowLocalNetworking = true;
+
   meta = {
     description = "Clean, filter and sample URLs to optimize data collection";
     homepage = "https://github.com/adbar/courlan";
-    changelog = "https://github.com/adbar/courlan/blob/v${version}/HISTORY.md";
+    changelog = "https://github.com/adbar/courlan/blob/${finalAttrs.src.tag}/HISTORY.md";
     license = lib.licenses.asl20;
     maintainers = with lib.maintainers; [ jokatzke ];
     mainProgram = "courlan";
   };
-}
+})

pkgs/development/python-modules/trafilatura/default.nix

@@ -19,27 +19,33 @@
   pytestCheckHook,
 }:
 
-buildPythonPackage rec {
+buildPythonPackage (finalAttrs: {
   pname = "trafilatura";
-  version = "2.0.0";
+  version = "2.1.0";
   pyproject = true;
+  __structuredAttrs = true;
 
   src = fetchFromGitHub {
     owner = "adbar";
     repo = "trafilatura";
-    tag = "v${version}";
-    hash = "sha256-Cf1W3JEGSMkVmRZVTXYsXzZK/Nt/aDG890Sf0/0OZAA=";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-hSeJH+8JX8QC3zHMZ3+M2H0C3xI+BCvLnSo/Ih1wUQw=";
   };
 
-  postPatch = ''
+  postPatch =
     # nixify path to the trafilatura binary in the test suite
-    substituteInPlace tests/cli_tests.py \
-      --replace-fail 'trafilatura_bin = "trafilatura"' \
-                     'trafilatura_bin = "${placeholder "out"}/bin/trafilatura"'
-  '';
+    ''
+      substituteInPlace tests/cli_tests.py \
+        --replace-fail \
+          'trafilatura_bin = "trafilatura"' \
+          'trafilatura_bin = "${placeholder "out"}/bin/trafilatura"'
+    '';
 
   build-system = [ setuptools ];
 
+  pythonRelaxDeps = [
+    "lxml"
+  ];
   dependencies = [
     certifi
     charset-normalizer
@@ -53,15 +59,6 @@ buildPythonPackage rec {
   nativeCheckInputs = [ pytestCheckHook ];
 
   disabledTests = [
-    # TypeError: argument of type 'NoneType' is not iterable
-    # https://github.com/adbar/trafilatura/issues/805
-    "test_external"
-    "test_extract"
-
-    # AttributeError: 'NoneType' object has no attribute 'find'
-    # https://github.com/adbar/trafilatura/issues/805
-    "test_table_processing"
-
     # Disable tests that require an internet connection
     "test_cli_pipeline"
     "test_crawl_page"
@@ -75,10 +72,6 @@ buildPythonPackage rec {
     "test_queue"
     "test_redirection"
     "test_whole"
-
-    # AssertionError: assert ['deflate', 'gzip', 'zstd'] == ['deflate', 'gzip']
-    # https://github.com/adbar/trafilatura/issues/823
-    "test_config"
   ];
 
   pythonImportsCheck = [ "trafilatura" ];
@@ -86,10 +79,10 @@ buildPythonPackage rec {
   meta = {
     description = "Python package and command-line tool designed to gather text on the Web";
     homepage = "https://trafilatura.readthedocs.io";
-    changelog = "https://github.com/adbar/trafilatura/blob/v${version}/HISTORY.md";
+    changelog = "https://github.com/adbar/trafilatura/blob/${finalAttrs.src.tag}/HISTORY.md";
     downloadPage = "https://github.com/adbar/trafilatura";
     license = lib.licenses.asl20;
     maintainers = with lib.maintainers; [ jokatzke ];
     mainProgram = "trafilatura";
   };
-}
+})