python3Packages.scrapy: 2.13.4 -> 2.14.1 (#480177)

pkgs/development/python-modules/scrapy-fake-useragent/default.nix

@@ -1,60 +0,0 @@
-{
-  lib,
-  buildPythonPackage,
-  setuptools,
-  fake-useragent,
-  faker,
-  fetchFromGitHub,
-  pytest-cov-stub,
-  pytest-mock,
-  pytestCheckHook,
-  scrapy,
-}:
-
-buildPythonPackage {
-  pname = "scrapy-fake-useragent";
-  version = "1.4.4";
-  pyproject = true;
-
-  # PyPi tarball is corrupted
-  src = fetchFromGitHub {
-    owner = "alecxe";
-    repo = "scrapy-fake-useragent";
-    rev = "59c20d38c58c76618164760d546aa5b989a79b8b"; # no tags
-    hash = "sha256-khQMOQrrdHokvNqfaMWqXV7AzwGxTuxaLsZoLkNpZ3k=";
-  };
-
-  postPatch = ''
-    substituteInPlace pytest.ini \
-      --replace-fail " --fulltrace" ""
-  '';
-
-  build-system = [ setuptools ];
-
-  dependencies = [
-    fake-useragent
-    faker
-  ];
-
-  nativeCheckInputs = [
-    pytest-cov-stub
-    pytest-mock
-    pytestCheckHook
-    scrapy
-  ];
-
-  pythonImportsCheck = [ "scrapy_fake_useragent" ];
-
-  disabledTests = [
-    # AttributeError: 'RetryUserAgentMiddleware' object has no attribute 'EXCEPTIONS_TO_RETRY'
-    "test_random_ua_set_on_exception"
-  ];
-
-  meta = {
-    description = "Random User-Agent middleware based on fake-useragent";
-    homepage = "https://github.com/alecxe/scrapy-fake-useragent";
-    changelog = "https://github.com/alecxe/scrapy-fake-useragent/blob/master/CHANGELOG.rst";
-    license = lib.licenses.mit;
-    maintainers = [ ];
-  };
-}

pkgs/development/python-modules/scrapy-splash/default.nix

@@ -43,5 +43,8 @@ buildPythonPackage rec {
     homepage = "https://github.com/scrapy-plugins/scrapy-splash";
     license = lib.licenses.bsd3;
     maintainers = with lib.maintainers; [ evanjs ];
+    # incompatible with scrapy >= 2.14
+    # also deprecated by scrapy committers, see https://github.com/scrapy-plugins/scrapy-splash/issues/327
+    broken = true;
   };
 }

pkgs/development/python-modules/scrapy/default.nix

@@ -19,9 +19,13 @@
   pexpect,
   protego,
   pydispatcher,
+  pyftpdlib,
   pyopenssl,
+  pytest-asyncio,
+  pytest-twisted,
   pytest-xdist,
   pytestCheckHook,
+  pythonAtLeast,
   queuelib,
   service-identity,
   setuptools,
@@ -36,14 +40,14 @@
 
 buildPythonPackage rec {
   pname = "scrapy";
-  version = "2.13.4";
+  version = "2.14.1";
   pyproject = true;
 
   src = fetchFromGitHub {
     owner = "scrapy";
     repo = "scrapy";
     tag = version;
-    hash = "sha256-ZWiJmve1EhtwP9xo39N4f24g0KQMTPKJ43sDSfzi6r8=";
+    hash = "sha256-KDci1Z5TZ+3svotYXkEG1s+bPWtxzIfQQwOgvI0k8w0=";
   };
 
   pythonRelaxDeps = [
@@ -59,7 +63,7 @@ buildPythonPackage rec {
     setuptools
   ];
 
-  propagatedBuildInputs = [
+  dependencies = [
     cryptography
     cssselect
     defusedxml
@@ -84,7 +88,10 @@ buildPythonPackage rec {
     glibcLocales
     jmespath
     pexpect
+    pytest-asyncio
+    pytest-twisted
     pytest-xdist
+    pyftpdlib
     pytestCheckHook
     sybil
     testfixtures
@@ -93,6 +100,11 @@ buildPythonPackage rec {
 
   LC_ALL = "en_US.UTF-8";
 
+  pytestFlags = [
+    # DeprecationWarning: There is no current event loop
+    "-Wignore::DeprecationWarning"
+  ];
+
   disabledTestPaths = [
     "tests/test_proxy_connect.py"
     "tests/test_utils_display.py"
@@ -111,6 +123,9 @@ buildPythonPackage rec {
 
     # Don't test the documentation
     "docs"
+  ]
+  ++ lib.optionals (pythonAtLeast "3.14") [
+    "tests/test_feedexport.py"
   ];
 
   disabledTests = [
@@ -121,14 +136,20 @@ buildPythonPackage rec {
     "test_custom_asyncio_loop_enabled_true"
     "test_custom_loop_asyncio"
     "test_custom_loop_asyncio_deferred_signal"
-    "FileFeedStoragePreFeedOptionsTest" # https://github.com/scrapy/scrapy/issues/5157
+    # "FileFeedStoragePreFeedOptionsTest" # https://github.com/scrapy/scrapy/issues/5157
     "test_persist"
     "test_timeout_download_from_spider_nodata_rcvd"
     "test_timeout_download_from_spider_server_hangs"
     "test_unbounded_response"
     "CookiesMiddlewareTest"
+    "test_asyncio_enabled_reactor_same_loop"
+    "test_response_ip_address"
     # Test fails on Hydra
     "test_start_requests_laziness"
+
+    # Fails due to different path structure on NixOS
+    "test_start_deprecated_super"
+    "test_file_path"
   ]
   ++ lib.optionals stdenv.hostPlatform.isDarwin [
     "test_xmliter_encoding"
@@ -139,6 +160,9 @@ buildPythonPackage rec {
     # flaky on darwin-aarch64
     "test_fixed_delay"
     "test_start_requests_laziness"
+  ]
+  ++ lib.optionals (pythonAtLeast "3.14") [
+    "test_non_pickable_object"
   ];
 
   postInstall = ''

pkgs/top-level/python-aliases.nix

@@ -451,6 +451,7 @@ mapAliases {
   schemainspect = throw "schemainspect has been removed because it has transitively been marked broken since May 2024, and is unmaintained upstream."; # Added 2025-10-11
   scikitimage = throw "'scikitimage' has been renamed to/replaced by 'scikit-image'"; # Converted to throw 2025-10-29
   scikitlearn = throw "'scikitlearn' has been renamed to/replaced by 'scikit-learn'"; # Converted to throw 2025-10-29
+  scrapy-fake-useragent = throw "scrapy-fake-useragent has been removed because it is incompatible with scrapy >= 2.14"; # Added 2026-01-14
   seaserv = throw "seaserv has been removed as it is unmaintained"; # Added 2025-08-21
   sentry-sdk_2 = throw "'sentry-sdk_2' has been renamed to/replaced by 'sentry-sdk'"; # Converted to throw 2025-10-29
   setuptools_dso = throw "'setuptools_dso' has been renamed to/replaced by 'setuptools-dso'"; # Converted to throw 2025-10-29

pkgs/top-level/python-packages.nix

@@ -17003,8 +17003,6 @@ self: super: with self; {
 
   scrapy-deltafetch = callPackage ../development/python-modules/scrapy-deltafetch { };
 
-  scrapy-fake-useragent = callPackage ../development/python-modules/scrapy-fake-useragent { };
-
   scrapy-splash = callPackage ../development/python-modules/scrapy-splash { };
 
   screed = callPackage ../development/python-modules/screed { };