spider: init at 2.33.11

2025-03-18 17:51:49 +02:00
parent 8330994032
commit 86e400b0d2
1 changed files with 87 additions and 0 deletions
@@ -0,0 +1,87 @@
+{
+  fetchFromGitHub,
+  lib,
+  nix-update-script,
+  openssl,
+  pkg-config,
+  rust-jemalloc-sys,
+  rustPlatform,
+  sqlite,
+  stdenv,
+  versionCheckHook,
+  zstd,
+}:
+
+rustPlatform.buildRustPackage (finalAttrs: {
+  pname = "spider";
+  version = "2.33.11";
+
+  src = fetchFromGitHub {
+    owner = "spider-rs";
+    repo = "spider";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-wrtkBqq1wJWg+0K4+PILPdMQe1AFOhJ1dJHvwq2irQo=";
+  };
+
+  useFetchCargoVendor = true;
+  cargoHash = "sha256-QxfnpiGHYYk8v0rR/eE6KHW1KdZQGuJPz1jGwtl27rs=";
+
+  nativeBuildInputs = [
+    pkg-config
+    rustPlatform.bindgenHook
+  ];
+
+  buildInputs = [
+    openssl
+    rust-jemalloc-sys
+    sqlite
+    zstd
+  ];
+
+  env = {
+    OPENSSL_NO_VENDOR = true;
+    ZSTD_SYS_USE_PKG_CONFIG = true;
+  };
+
+  checkFlags =
+    [
+      # Sandbox limitation: no network or io_uring
+      "--skip=website::crawl"
+      "--skip=website::scrape"
+      "--skip=website::test_crawl_subdomains"
+      "--skip=website::test_crawl_tld"
+      "--skip=website::test_respect_robots_txt"
+      "--skip=page::parse_links"
+      "--skip=page::test_status_code"
+      "--skip=pdl_is_fresh"
+      "--skip=verify_revision_available"
+    ]
+    ++ lib.optionals stdenv.isDarwin [
+      # Sandbox limitation: attempted to create a NULL object
+      "--skip=website::test_link_duplicates"
+      "--skip=website::not_crawl_blacklist"
+      "--skip=website::test_crawl_budget"
+      "--skip=website::test_crawl_subscription"
+      "--skip=website::Website::subscribe_guard"
+      "--skip=website::Website::subscribe"
+    ];
+
+  doInstallCheck = true;
+  nativeInstallCheckInputs = [ versionCheckHook ];
+  versionCheckProgramArg = "--version";
+
+  passthru.updateScript = nix-update-script { };
+
+  meta = {
+    changelog = "https://github.com/spider-rs/spider/releases/tag/v${finalAttrs.version}";
+    description = "Web crawler and scraper, building blocks for data curation workloads";
+    homepage = "https://github.com/spider-rs/spider";
+    license = lib.licenses.mit;
+    mainProgram = "spider";
+    maintainers = with lib.maintainers; [
+      j-mendez
+      KSJ2000
+    ];
+    platforms = lib.platforms.unix;
+  };
+})