python3Packages.crawl4ai: init at 0.9.0 (#538059)

2026-07-04 08:47:25 +00:00
parent 81f67be25a 2fa7554065
commit b7bc837f69
7 changed files with 551 additions and 0 deletions
@@ -0,0 +1,65 @@
+{
+  lib,
+  buildPythonPackage,
+  click,
+  click-log,
+  fetchFromGitHub,
+  geopandas,
+  networkx,
+  numpy,
+  pytestCheckHook,
+  rtree,
+  scipy,
+  setuptools,
+  shapely,
+  trimesh,
+}:
+
+buildPythonPackage (finalAttrs: {
+  pname = "alphashape";
+  version = "1.3.1";
+  pyproject = true;
+
+  __structuredAttrs = true;
+
+  src = fetchFromGitHub {
+    owner = "bellockk";
+    repo = "alphashape";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-T2wyU6fpiYRA1+9n//5EtOLhO1fzccQsie+gQj729Vs=";
+  };
+
+  build-system = [ setuptools ];
+
+  dependencies = [
+    click
+    click-log
+    shapely
+    numpy
+    trimesh
+    networkx
+    rtree
+    scipy
+  ];
+
+  nativeCheckInputs = [
+    geopandas
+    pytestCheckHook
+  ];
+
+  disabledTests = [
+    # TypeError
+    "test_given_a_four_point_polygon_with_no_alpha_return_input"
+    "test_given_a_point_return_a_point"
+  ];
+
+  pythonImportsCheck = [ "alphashape" ];
+
+  meta = with lib; {
+    description = "Toolbox for generating n-dimensional alpha shapes";
+    homepage = "https://github.com/bellockk/alphashape";
+    changelog = "https://github.com/bellockk/alphashape/releases/tag/${finalAttrs.src.tag}";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+})
@@ -0,0 +1,269 @@
+{
+  lib,
+  aiofiles,
+  aiohttp,
+  aiosqlite,
+  alphashape,
+  anyio,
+  beautifulsoup4,
+  brotli,
+  buildPythonPackage,
+  chardet,
+  click,
+  cssselect,
+  fake-useragent,
+  fetchFromGitHub,
+  h2,
+  httpx,
+  humanize,
+  lark,
+  litellm,
+  lxml,
+  nltk,
+  numpy,
+  patchright,
+  pillow,
+  playwright-stealth,
+  playwright,
+  psutil,
+  pydantic,
+  pyopenssl,
+  pypdf,
+  pytest-asyncio,
+  pytestCheckHook,
+  python-dotenv,
+  pyyaml,
+  rank-bm25,
+  requests,
+  rich,
+  scikit-learn,
+  selenium,
+  sentence-transformers,
+  setuptools,
+  shapely,
+  snowballstemmer,
+  tokenizers,
+  torch,
+  transformers,
+  unclecode-litellm,
+  writableTmpDirAsHomeHook,
+  xxhash,
+}:
+
+buildPythonPackage (finalAttrs: {
+  pname = "crawl4ai";
+  version = "0.9.0";
+  pyproject = true;
+
+  __structuredAttrs = true;
+
+  src = fetchFromGitHub {
+    owner = "unclecode";
+    repo = "crawl4ai";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-MHCnhp7itaJ6y++uUCnObBjiJ9Jw/aFBtd35WzDBz1A=";
+  };
+
+  pythonRelaxDeps = [
+    "lxml"
+    "snowballstemmer"
+  ];
+
+  pythonRemoveDeps = [ "patchright" ];
+
+  build-system = [
+    setuptools
+    writableTmpDirAsHomeHook
+  ];
+
+  dependencies = [
+    aiofiles
+    aiohttp
+    aiosqlite
+    alphashape
+    anyio
+    beautifulsoup4
+    brotli
+    chardet
+    click
+    cssselect
+    fake-useragent
+    httpx
+    humanize
+    lark
+    litellm
+    lxml
+    nltk
+    numpy
+    patchright
+    pillow
+    playwright
+    playwright-stealth
+    psutil
+    pydantic
+    pyopenssl
+    python-dotenv
+    pyyaml
+    rank-bm25
+    requests
+    rich
+    shapely
+    snowballstemmer
+    unclecode-litellm
+    xxhash
+  ];
+
+  optional-dependencies = {
+    all = [
+      nltk
+      pypdf
+      scikit-learn
+      selenium
+      sentence-transformers
+      tokenizers
+      torch
+      transformers
+    ];
+    cosine = [
+      nltk
+      sentence-transformers
+      torch
+      transformers
+    ];
+    pdf = [ pypdf ];
+    sync = [ selenium ];
+    torch = [
+      nltk
+      scikit-learn
+      torch
+    ];
+    transformer = [
+      sentence-transformers
+      tokenizers
+      transformers
+    ];
+  };
+
+  nativeCheckInputs = [
+    h2
+    pytest-asyncio
+    pytestCheckHook
+  ]
+  ++ lib.flatten (builtins.attrValues finalAttrs.passthru.optional-dependencies);
+
+  pytestFlags = [ "--continue-on-collection-errors" ];
+
+  disabledTestPaths = [
+    # sys.exit(1) that crashes pytest
+    "tests/test_cloud_bugs_batch.py"
+    # Require docker, browser, LLM API, network and parts
+    "deploy/docker/"
+    "docs/examples/"
+    "tests/adaptive/"
+    "tests/async/"
+    "tests/browser/"
+    "tests/cache_validation/test_end_to_end.py"
+    "tests/cache_validation/test_real_domains.py"
+    "tests/deep_crawling/test_deep_crawl_resume_integration.py"
+    "tests/docker/test_config_object.py"
+    "tests/docker/test_docker.py"
+    "tests/docker/test_filter_deep_crawl.py"
+    "tests/docker/test_hooks_client.py"
+    "tests/docker/test_hooks_comprehensive.py"
+    "tests/docker/test_hooks_utility.py"
+    "tests/docker/test_llm_params.py"
+    "tests/docker/test_rest_api_deep_crawl.py"
+    "tests/docker/test_server.py"
+    "tests/docker/test_server_requests.py"
+    "tests/docker/test_server_token.py"
+    "tests/general/test_async_crawler_strategy.py"
+    "tests/general/test_async_url_seeder_bm25.py"
+    "tests/general/test_async_webcrawler.py"
+    "tests/general/test_bff_scoring.py"
+    "tests/general/test_cache_context.py"
+    "tests/general/test_crawlers.py"
+    "tests/general/test_deep_crawl_filters.py"
+    "tests/general/test_download_file.py"
+    "tests/general/test_http_crawler_strategy.py"
+    "tests/general/test_llm_filter.py"
+    "tests/general/test_max_scroll.py"
+    "tests/general/test_mhtml.py"
+    "tests/general/test_schema_builder.py"
+    "tests/mcp/"
+    "tests/memory/test_docker_config_gen.py"
+    "tests/integration/test_domain_mapper_e2e.py"
+    "tests/general/test_network_console_capture.py"
+    "tests/general/test_robot_parser.py"
+    "tests/general/test_stream.py"
+    "tests/general/test_stream_dispatch.py"
+    "tests/proxy/"
+    "tests/regression/"
+    "tests/releases/"
+    "tests/test_arun_many.py"
+    "tests/test_cdp_changes.py"
+    "tests/test_cli_docs.py"
+    "tests/test_config_selection.py"
+    "tests/test_docker.py"
+    "tests/test_docker_api_with_llm_provider.py"
+    "tests/test_eval_security_adversarial.py"
+    "tests/test_issue_1594_mcp_sse.py"
+    "tests/test_issue_1611_llm_provider.py"
+    "tests/test_issue_1748_screenshot_scroll_delay.py"
+    "tests/test_issue_1750_screenshot_scan_full_page.py"
+    "tests/test_issue_1850_mcp_sse.py"
+    "tests/test_link_extractor.py"
+    "tests/test_llm_extraction_parallel_issue_1055.py"
+    "tests/test_llm_simple_url.py"
+    "tests/test_llmtxt.py"
+    "tests/test_main.py"
+    "tests/test_multi_config.py"
+    "tests/test_prefetch_integration.py"
+    "tests/test_prefetch_regression.py"
+    "tests/test_pr_1435_redirected_status_code.py"
+    "tests/test_pr_1463_device_scale_factor.py"
+    "tests/test_pyopenssl_update.py"
+    "tests/test_raw_html_browser.py"
+    "tests/test_raw_html_edge_cases.py"
+    "tests/test_raw_html_redirected_url.py"
+    "tests/test_scraping_strategy.py"
+    "tests/test_virtual_scroll.py"
+    "tests/test_web_crawler.py"
+  ];
+
+  disabledTests = [
+    # Tests require network access
+    "test_client_is_built_once_and_reused"
+    "test_reused_client_extracts_from_multiple_urls"
+    "test_clones_share_one_client"
+    "test_reused_client_checks_multiple_urls"
+    "test_concurrent_host_scanning"
+    "test_domain_with_scheme"
+    "test_probe_without_crt"
+    "test_examples"
+    "test_missing_url"
+    "test_basic_crawl"
+    # Tests require LLM API
+    "test_map_query_uses_query_config"
+    "test_legacy_single_config_for_query"
+    # Assertions Errors
+    "allowed_types5"
+    "allowed_types16"
+    "test_empty_href"
+    "test_href_is_only_fragment"
+    "test_href_with_fragment"
+    "test_invalid_base_url_netloc"
+    "test_invalid_base_url_scheme"
+    # Test checks GitHub Actions version
+    "test_no_v4_or_v5_checkout_remaining"
+  ];
+
+  pythonImportsCheck = [ "crawl4ai" ];
+
+  meta = {
+    description = "LLM Friendly Web Crawler & Scraper";
+    homepage = "https://github.com/unclecode/crawl4ai";
+    changelog = "https://github.com/unclecode/crawl4ai/blob/${finalAttrs.src.rev}/CHANGELOG.md";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ fab ];
+  };
+})
@@ -0,0 +1,35 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  pytestCheckHook,
+  setuptools,
+}:
+
+buildPythonPackage (finalAttrs: {
+  pname = "fake-http-header";
+  version = "0.3.5-unstable-2025-07-09";
+  pyproject = true;
+
+  __structuredAttrs = true;
+
+  src = fetchFromGitHub {
+    owner = "MichaelTatarski";
+    repo = "fake-http-header";
+    # https://github.com/MichaelTatarski/fake-http-header/issues/4
+    rev = "0f110477d3ecae916e88608a123360227bfb6109";
+    hash = "sha256-CznvDjzUeA0THsiIhuzvEDb4gXsP8IujpSCjt857qSo=";
+  };
+  build-system = [ setuptools ];
+
+  nativeCheckInputs = [ pytestCheckHook ];
+
+  pythonImportsCheck = [ "fake_http_header" ];
+
+  meta = with lib; {
+    description = "Generates random request fields for a http request header";
+    homepage = "https://github.com/MichaelTatarski/fake-http-header";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+})
@@ -0,0 +1,44 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  playwright,
+  setuptools,
+  toml,
+}:
+
+buildPythonPackage (finalAttrs: {
+  pname = "patchright";
+  version = "1.58.0";
+  pyproject = true;
+
+  __structuredAttrs = true;
+
+  src = fetchFromGitHub {
+    owner = "Kaliiiiiiiiii-Vinyzu";
+    repo = "patchright-python";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-msV0CbVbTDSAB1BgxkUOpuzQDr8vMK2/wxJy1SSUU80=";
+  };
+
+  build-system = [ setuptools ];
+
+  dependencies = [
+    playwright
+    toml
+  ];
+
+  # Module has no tests
+  doCheck = false;
+
+  # pythonImportsCheck disabled: module attempts filesystem writes at import time
+  pythonImportsCheck = [ ];
+
+  meta = {
+    description = "Undetected Python version of the Playwright testing and automation library";
+    homepage = "https://github.com/Kaliiiiiiiiii-Vinyzu/patchright-python";
+    changelog = "https://github.com/Kaliiiiiiiiii-Vinyzu/patchright-python/releases/tag/v${finalAttrs.src.tag}";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [ fab ];
+  };
+})
@@ -0,0 +1,62 @@
+{
+  lib,
+  buildPythonPackage,
+  fake-http-header,
+  fetchFromGitHub,
+  idna,
+  mockito,
+  playwright,
+  poetry-core,
+  pytestCheckHook,
+  urllib3,
+}:
+
+buildPythonPackage (finalAttrs: {
+  pname = "tf-playwright-stealth";
+  version = "1.2.2-unstable-2026-06-30";
+  pyproject = true;
+
+  __structuredAttrs = true;
+
+  src = fetchFromGitHub {
+    owner = "tinyfish-io";
+    repo = "tf-playwright-stealth";
+    rev = "01b1b7a436edee4363a5d6f78f626a602550138b";
+    hash = "sha256-dUU0JqnEda0s6nVzJNmU1l536YZ05SI3DYhb7onuBwY=";
+  };
+
+  pythonRelaxDeps = [
+    "idna"
+    "urllib3"
+  ];
+
+  pythonRemoveDeps = [ "agentql" ];
+
+  build-system = [ poetry-core ];
+
+  dependencies = [
+    fake-http-header
+    idna
+    playwright
+    urllib3
+  ];
+
+  nativeCheckInputs = [
+    mockito
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [ "playwright_stealth" ];
+
+  disabledTestPaths = [
+    # Requires agentql
+    "tests/e2e/"
+  ];
+
+  meta = with lib; {
+    description = "Module for using playwright stealthy";
+    homepage = "https://github.com/tinyfish-io/tf-playwright-stealth";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+})
@@ -0,0 +1,64 @@
+{
+  lib,
+  aiohttp,
+  buildPythonPackage,
+  click,
+  fastuuid,
+  fetchPypi,
+  httpx,
+  importlib-metadata,
+  jinja2,
+  jsonschema,
+  nix-update-script,
+  openai,
+  pydantic,
+  python-dotenv,
+  setuptools,
+  tiktoken,
+  tokenizers,
+}:
+
+buildPythonPackage (finalAttrs: {
+  pname = "unclecode-litellm";
+  version = "1.81.13";
+  pyproject = true;
+
+  __structuredAttrs = true;
+
+  src = fetchPypi {
+    pname = "unclecode_litellm";
+    inherit (finalAttrs) version;
+    hash = "sha256-23DjTj6FnAoH8CywLqpkT4+ktOzF4vO+mli9fRw/7tw=";
+  };
+
+  build-system = [ setuptools ];
+
+  dependencies = [
+    aiohttp
+    click
+    fastuuid
+    httpx
+    importlib-metadata
+    jinja2
+    jsonschema
+    openai
+    pydantic
+    python-dotenv
+    tiktoken
+    tokenizers
+  ];
+
+  # tests require access network
+  doCheck = false;
+
+  pythonImportsCheck = [ "litellm" ];
+
+  passthru.updateScript = nix-update-script { };
+
+  meta = {
+    description = "Pre-compromise fork of litellm to interface with LLM API providers";
+    homepage = "https://pypi.org/project/unclecode-litellm";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [ fab ];
+  };
+})
@@ -778,6 +778,8 @@ self: super: with self; {

  alphaessopenapi = callPackage ../development/python-modules/alphaessopenapi { };

+  alphashape = callPackage ../development/python-modules/alphashape { };
+
  altair = callPackage ../development/python-modules/altair { };

  altcha = callPackage ../development/python-modules/altcha { };
@@ -3402,6 +3404,8 @@ self: super: with self; {

  crate = callPackage ../development/python-modules/crate { };

+  crawl4ai = callPackage ../development/python-modules/crawl4ai { };
+
  crayons = callPackage ../development/python-modules/crayons { };

  crc = callPackage ../development/python-modules/crc { };
@@ -5505,6 +5509,8 @@ self: super: with self; {
    };
  };

+  fake-http-header = callPackage ../development/python-modules/fake-http-header { };
+
  fake-useragent = callPackage ../development/python-modules/fake-useragent { };

  faker = callPackage ../development/python-modules/faker { };
@@ -12567,6 +12573,8 @@ self: super: with self; {

  patchpy = callPackage ../development/python-modules/patchpy { };

+  patchright = callPackage ../development/python-modules/patchright { };
+
  path = callPackage ../development/python-modules/path { };

  path-and-address = callPackage ../development/python-modules/path-and-address { };
@@ -19890,6 +19898,8 @@ self: super: with self; {

  tf-keras = callPackage ../development/python-modules/tf-keras { };

+  tf-playwright-stealth = callPackage ../development/python-modules/tf-playwright-stealth { };
+
  tf2onnx = callPackage ../development/python-modules/tf2onnx { };

  tflearn = callPackage ../development/python-modules/tflearn { };
@@ -21142,6 +21152,8 @@ self: super: with self; {

  uncertainties = callPackage ../development/python-modules/uncertainties { };

+  unclecode-litellm = callPackage ../development/python-modules/unclecode-litellm { };
+
  uncompresspy = callPackage ../development/python-modules/uncompresspy { };

  uncompyle6 = callPackage ../development/python-modules/uncompyle6 { };