diff --git a/pkgs/development/python-modules/scrapy-fake-useragent/default.nix b/pkgs/development/python-modules/scrapy-fake-useragent/default.nix deleted file mode 100644 index cd34f8771591..000000000000 --- a/pkgs/development/python-modules/scrapy-fake-useragent/default.nix +++ /dev/null @@ -1,60 +0,0 @@ -{ - lib, - buildPythonPackage, - setuptools, - fake-useragent, - faker, - fetchFromGitHub, - pytest-cov-stub, - pytest-mock, - pytestCheckHook, - scrapy, -}: - -buildPythonPackage { - pname = "scrapy-fake-useragent"; - version = "1.4.4"; - pyproject = true; - - # PyPi tarball is corrupted - src = fetchFromGitHub { - owner = "alecxe"; - repo = "scrapy-fake-useragent"; - rev = "59c20d38c58c76618164760d546aa5b989a79b8b"; # no tags - hash = "sha256-khQMOQrrdHokvNqfaMWqXV7AzwGxTuxaLsZoLkNpZ3k="; - }; - - postPatch = '' - substituteInPlace pytest.ini \ - --replace-fail " --fulltrace" "" - ''; - - build-system = [ setuptools ]; - - dependencies = [ - fake-useragent - faker - ]; - - nativeCheckInputs = [ - pytest-cov-stub - pytest-mock - pytestCheckHook - scrapy - ]; - - pythonImportsCheck = [ "scrapy_fake_useragent" ]; - - disabledTests = [ - # AttributeError: 'RetryUserAgentMiddleware' object has no attribute 'EXCEPTIONS_TO_RETRY' - "test_random_ua_set_on_exception" - ]; - - meta = { - description = "Random User-Agent middleware based on fake-useragent"; - homepage = "https://github.com/alecxe/scrapy-fake-useragent"; - changelog = "https://github.com/alecxe/scrapy-fake-useragent/blob/master/CHANGELOG.rst"; - license = lib.licenses.mit; - maintainers = [ ]; - }; -} diff --git a/pkgs/development/python-modules/scrapy-splash/default.nix b/pkgs/development/python-modules/scrapy-splash/default.nix index a6491a8d3d17..2d060b70fc12 100644 --- a/pkgs/development/python-modules/scrapy-splash/default.nix +++ b/pkgs/development/python-modules/scrapy-splash/default.nix @@ -43,5 +43,8 @@ buildPythonPackage rec { homepage = "https://github.com/scrapy-plugins/scrapy-splash"; license = lib.licenses.bsd3; maintainers = with lib.maintainers; [ evanjs ]; + # incompatible with scrapy >= 2.14 + # also deprecated by scrapy committers, see https://github.com/scrapy-plugins/scrapy-splash/issues/327 + broken = true; }; } diff --git a/pkgs/development/python-modules/scrapy/default.nix b/pkgs/development/python-modules/scrapy/default.nix index 09389ba51f2b..0bc10158095b 100644 --- a/pkgs/development/python-modules/scrapy/default.nix +++ b/pkgs/development/python-modules/scrapy/default.nix @@ -19,9 +19,13 @@ pexpect, protego, pydispatcher, + pyftpdlib, pyopenssl, + pytest-asyncio, + pytest-twisted, pytest-xdist, pytestCheckHook, + pythonAtLeast, queuelib, service-identity, setuptools, @@ -36,14 +40,14 @@ buildPythonPackage rec { pname = "scrapy"; - version = "2.13.4"; + version = "2.14.1"; pyproject = true; src = fetchFromGitHub { owner = "scrapy"; repo = "scrapy"; tag = version; - hash = "sha256-ZWiJmve1EhtwP9xo39N4f24g0KQMTPKJ43sDSfzi6r8="; + hash = "sha256-KDci1Z5TZ+3svotYXkEG1s+bPWtxzIfQQwOgvI0k8w0="; }; pythonRelaxDeps = [ @@ -59,7 +63,7 @@ buildPythonPackage rec { setuptools ]; - propagatedBuildInputs = [ + dependencies = [ cryptography cssselect defusedxml @@ -84,7 +88,10 @@ buildPythonPackage rec { glibcLocales jmespath pexpect + pytest-asyncio + pytest-twisted pytest-xdist + pyftpdlib pytestCheckHook sybil testfixtures @@ -93,6 +100,11 @@ buildPythonPackage rec { LC_ALL = "en_US.UTF-8"; + pytestFlags = [ + # DeprecationWarning: There is no current event loop + "-Wignore::DeprecationWarning" + ]; + disabledTestPaths = [ "tests/test_proxy_connect.py" "tests/test_utils_display.py" @@ -111,6 +123,9 @@ buildPythonPackage rec { # Don't test the documentation "docs" + ] + ++ lib.optionals (pythonAtLeast "3.14") [ + "tests/test_feedexport.py" ]; disabledTests = [ @@ -121,14 +136,20 @@ buildPythonPackage rec { "test_custom_asyncio_loop_enabled_true" "test_custom_loop_asyncio" "test_custom_loop_asyncio_deferred_signal" - "FileFeedStoragePreFeedOptionsTest" # https://github.com/scrapy/scrapy/issues/5157 + # "FileFeedStoragePreFeedOptionsTest" # https://github.com/scrapy/scrapy/issues/5157 "test_persist" "test_timeout_download_from_spider_nodata_rcvd" "test_timeout_download_from_spider_server_hangs" "test_unbounded_response" "CookiesMiddlewareTest" + "test_asyncio_enabled_reactor_same_loop" + "test_response_ip_address" # Test fails on Hydra "test_start_requests_laziness" + + # Fails due to different path structure on NixOS + "test_start_deprecated_super" + "test_file_path" ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ "test_xmliter_encoding" @@ -139,6 +160,9 @@ buildPythonPackage rec { # flaky on darwin-aarch64 "test_fixed_delay" "test_start_requests_laziness" + ] + ++ lib.optionals (pythonAtLeast "3.14") [ + "test_non_pickable_object" ]; postInstall = '' diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index 8b4b7a891309..43013f9c378c 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -451,6 +451,7 @@ mapAliases { schemainspect = throw "schemainspect has been removed because it has transitively been marked broken since May 2024, and is unmaintained upstream."; # Added 2025-10-11 scikitimage = throw "'scikitimage' has been renamed to/replaced by 'scikit-image'"; # Converted to throw 2025-10-29 scikitlearn = throw "'scikitlearn' has been renamed to/replaced by 'scikit-learn'"; # Converted to throw 2025-10-29 + scrapy-fake-useragent = throw "scrapy-fake-useragent has been removed because it is incompatible with scrapy >= 2.14"; # Added 2026-01-14 seaserv = throw "seaserv has been removed as it is unmaintained"; # Added 2025-08-21 sentry-sdk_2 = throw "'sentry-sdk_2' has been renamed to/replaced by 'sentry-sdk'"; # Converted to throw 2025-10-29 setuptools_dso = throw "'setuptools_dso' has been renamed to/replaced by 'setuptools-dso'"; # Converted to throw 2025-10-29 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 6aaa368d228e..89b9164eecfd 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -17003,8 +17003,6 @@ self: super: with self; { scrapy-deltafetch = callPackage ../development/python-modules/scrapy-deltafetch { }; - scrapy-fake-useragent = callPackage ../development/python-modules/scrapy-fake-useragent { }; - scrapy-splash = callPackage ../development/python-modules/scrapy-splash { }; screed = callPackage ../development/python-modules/screed { };