diff --git a/doc/build-helpers/testers.chapter.md b/doc/build-helpers/testers.chapter.md index b734cbbbd4e2..80d293cf1c70 100644 --- a/doc/build-helpers/testers.chapter.md +++ b/doc/build-helpers/testers.chapter.md @@ -40,6 +40,82 @@ If the `moduleNames` argument is omitted, `hasPkgConfigModules` will use `meta.p ::: +## `lycheeLinkCheck` {#tester-lycheeLinkCheck} + +Check a packaged static site's links with the [`lychee` package](https://search.nixos.org/packages?show=lychee&type=packages&query=lychee). + +You may use Nix to reproducibly build static websites, such as for software documentation. +Some packages will install documentation in their `out` or `doc` outputs, or maybe you have dedicated package where you've made your static site reproducible by running a generator, such as [Hugo](https://gohugo.io/) or [mdBook](https://rust-lang.github.io/mdBook/), in a derivation. + +If you have a static site that can be built with Nix, you can use `lycheeLinkCheck` to check that the hyperlinks in your site are correct, and do so as part of your Nix workflow and CI. + +:::{.example #ex-lycheelinkcheck} + +# Check hyperlinks in the `nix` documentation + +```nix +testers.lycheeLinkCheck { + site = nix.doc + "/share/doc/nix/manual"; +} +``` + +::: + +### Return value {#tester-lycheeLinkCheck-return} + +This tester produces a package that does not produce useful outputs, but only succeeds if the hyperlinks in your site are correct. The build log will list the broken links. + +It has two modes: + +- Build the returned derivation; its build process will check that internal hyperlinks are correct. This runs in the sandbox, so it will not check external hyperlinks, but it is quick and reliable. + +- Invoke the `.online` attribute with [`nix run`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-run) ([experimental](https://nixos.org/manual/nix/stable/contributing/experimental-features#xp-feature-nix-command)). This runs outside the sandbox, and checks that both internal and external hyperlinks are correct. + Example: + + ```shell + nix run nixpkgs#lychee.tests.ok.online + ``` + +### Inputs {#tester-lycheeLinkCheck-inputs} + +`site` (path or derivation) {#tester-lycheeLinkCheck-param-site} + +: The path to the files to check. + +`remap` (attribe set, optional) {#tester-lycheeLinkCheck-param-remap} + +: An attribute set where the attribute names are regular expressions. + The values should be strings, derivations, or path values. + + In the returned check's default configuration, external URLs are only checked when you run the `.online` attribute. + + By adding remappings, you can check offline that URLs to external resources are correct, by providing a stand-in from the file system. + + Before checking the existence of a URL, the regular expressions are matched and replaced by their corresponding values. + + Example: + + ```nix + { + "https://nix\\.dev/manual/nix/[a-z0-9.-]*" = "${nix.doc}/share/doc/nix/manual"; + "https://nixos\\.org/manual/nix/(un)?stable" = "${emptyDirectory}/placeholder-to-disallow-old-nix-docs-urls"; + } + ``` + + Store paths in the attribute values are automatically prefixed with `file://`, because lychee requires this for paths in the file system. + If this is a problem, or if you need to control the order in which replacements are performed, use `extraConfig.remap` instead. + +`extraConfig` (attribute set) {#tester-lycheeLinkCheck-param-extraConfig} + +: Extra configuration to pass to `lychee` in its [configuration file](https://github.com/lycheeverse/lychee/blob/master/lychee.example.toml). + It is automatically [translated](https://nixos.org/manual/nixos/stable/index.html#sec-settings-nix-representable) to TOML. + + Example: `{ "include_verbatim" = true; }` + +`lychee` (derivation, optional) {#tester-lycheeLinkCheck-param-lychee} + +: The `lychee` package to use. + ## `testVersion` {#tester-testVersion} Checks that the output from running a command contains the specified version string in it as a whole word. diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 362622d9b7ee..dbf9a6d6cb05 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -1,6 +1,10 @@ { pkgs, pkgsLinux, buildPackages, lib, callPackage, runCommand, stdenv, substituteAll, testers }: # Documentation is in doc/builders/testers.chapter.md { + # See https://nixos.org/manual/nixpkgs/unstable/#tester-lycheeLinkCheck + # or doc/builders/testers.chapter.md + inherit (callPackage ./lychee.nix {}) lycheeLinkCheck; + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailure # or doc/builders/testers.chapter.md testBuildFailure = drv: drv.overrideAttrs (orig: { diff --git a/pkgs/build-support/testers/lychee.nix b/pkgs/build-support/testers/lychee.nix new file mode 100644 index 000000000000..80088dc4f963 --- /dev/null +++ b/pkgs/build-support/testers/lychee.nix @@ -0,0 +1,69 @@ +deps@{ formats, lib, lychee, stdenv, writeShellApplication }: +let + inherit (lib) mapAttrsToList throwIf; + inherit (lib.strings) hasInfix hasPrefix escapeNixString; + + toURL = v: + let s = "${v}"; + in if hasPrefix builtins.storeDir s + then # lychee requires that paths on the file system are prefixed with file:// + "file://${s}" + else s; + + withCheckedName = name: + throwIf (hasInfix " " name) '' + lycheeLinkCheck: remap patterns must not contain spaces. + A space marks the end of the regex in lychee.toml. + + Please change attribute name 'remap.${escapeNixString name}' + ''; + + # See https://nixos.org/manual/nixpkgs/unstable/#tester-lycheeLinkCheck + # or doc/builders/testers.chapter.md + lycheeLinkCheck = { + site, + remap ? { }, + lychee ? deps.lychee, + extraConfig ? { }, + }: + stdenv.mkDerivation (finalAttrs: { + name = "lychee-link-check"; + inherit site; + nativeBuildInputs = [ finalAttrs.passthru.lychee ]; + configFile = (formats.toml {}).generate "lychee.toml" finalAttrs.passthru.config; + + # These can be overriden with overrideAttrs if needed. + passthru = { + inherit lychee remap; + config = { + include_fragments = true; + } // lib.optionalAttrs (finalAttrs.passthru.remap != { }) { + remap = + mapAttrsToList + (name: value: withCheckedName name "${name} ${toURL value}") + finalAttrs.passthru.remap; + } // extraConfig; + online = writeShellApplication { + name = "run-lychee-online"; + runtimeInputs = [ finalAttrs.passthru.lychee ]; + # Comment out to run shellcheck: + checkPhase = ""; + text = '' + site=${finalAttrs.site} + configFile=${finalAttrs.configFile} + echo Checking links on $site + exec lychee --config $configFile $site "$@" + ''; + }; + }; + buildCommand = '' + echo Checking internal links on $site + lychee --offline --config $configFile $site + touch $out + ''; + }); + +in +{ + inherit lycheeLinkCheck; +} diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index da67711156be..a815fe63e416 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -12,6 +12,8 @@ let in lib.recurseIntoAttrs { + lycheeLinkCheck = lib.recurseIntoAttrs pkgs.lychee.tests; + hasPkgConfigModules = pkgs.callPackage ../hasPkgConfigModules/tests.nix { }; runNixOSTest-example = pkgs-with-overlay.testers.runNixOSTest ({ lib, ... }: { diff --git a/pkgs/tools/networking/lychee/default.nix b/pkgs/tools/networking/lychee/default.nix index 610869897b7b..0763ee302109 100644 --- a/pkgs/tools/networking/lychee/default.nix +++ b/pkgs/tools/networking/lychee/default.nix @@ -1,4 +1,5 @@ -{ lib +{ callPackage +, lib , stdenv , rustPlatform , fetchFromGitHub @@ -6,6 +7,7 @@ , openssl , Security , SystemConfiguration +, testers }: rustPlatform.buildRustPackage rec { @@ -41,6 +43,15 @@ rustPlatform.buildRustPackage rec { "--skip=src/lib.rs" ]; + passthru.tests = { + # NOTE: These assume that testers.lycheeLinkCheck uses this exact derivation. + # Which is true most of the time, but not necessarily after overriding. + ok = callPackage ./tests/ok.nix { }; + fail = callPackage ./tests/fail.nix { }; + fail-emptyDirectory = callPackage ./tests/fail-emptyDirectory.nix { }; + network = testers.runNixOSTest ./tests/network.nix; + }; + meta = with lib; { description = "A fast, async, stream-based link checker written in Rust"; homepage = "https://github.com/lycheeverse/lychee"; diff --git a/pkgs/tools/networking/lychee/tests/fail-emptyDirectory.nix b/pkgs/tools/networking/lychee/tests/fail-emptyDirectory.nix new file mode 100644 index 000000000000..9db4b23b1bc9 --- /dev/null +++ b/pkgs/tools/networking/lychee/tests/fail-emptyDirectory.nix @@ -0,0 +1,28 @@ +{ runCommand, testers, emptyDirectory }: +let + sitePkg = runCommand "site" { } '' + dist=$out/dist + mkdir -p $dist + echo "foo" > $dist/index.html + echo "" > $dist/foo.html + ''; + check = testers.lycheeLinkCheck { + site = sitePkg + "/dist"; + remap = { + # Normally would recommend to append a subpath that hints why it's forbidden; see example in docs. + # However, we also want to test that a package is converted to a string *before* + # it's tested whether it's a store path. Mistake made during development caused: + # cannot check URI: InvalidUrlRemap("The remapping pattern must produce a valid URL, but it is not: /nix/store/4d0ix...empty-directory/foo + "https://example.com" = emptyDirectory; + }; + }; + + failure = testers.testBuildFailure check; +in + runCommand "link-check-fail" { inherit failure; } '' + # The details of the message might change, but we have to make sure the + # correct error is reported, so that we know it's not something else that + # went wrong. + grep 'empty-directory/foo.*Cannot find file' $failure/testBuildFailure.log >/dev/null + touch $out + '' diff --git a/pkgs/tools/networking/lychee/tests/fail.nix b/pkgs/tools/networking/lychee/tests/fail.nix new file mode 100644 index 000000000000..5a83814d3c44 --- /dev/null +++ b/pkgs/tools/networking/lychee/tests/fail.nix @@ -0,0 +1,21 @@ +{ runCommand, testers }: +let + sitePkg = runCommand "site" { } '' + dist=$out/dist + mkdir -p $dist + echo "foo" > $dist/index.html + echo "index" > $dist/foo.html + ''; + + linkCheck = testers.lycheeLinkCheck rec { + site = sitePkg + "/dist"; + remap = { "https://exampl[e]\\.com" = site; }; + }; + + failure = testers.testBuildFailure linkCheck; + +in + runCommand "link-check-fail" { inherit failure; } '' + grep -F foos-missing-anchor $failure/testBuildFailure.log >/dev/null + touch $out + '' diff --git a/pkgs/tools/networking/lychee/tests/network.nix b/pkgs/tools/networking/lychee/tests/network.nix new file mode 100644 index 000000000000..8d7f908a5805 --- /dev/null +++ b/pkgs/tools/networking/lychee/tests/network.nix @@ -0,0 +1,66 @@ +{ config, hostPkgs, lib, ... }: +let + sitePkg = hostPkgs.runCommand "site" { } '' + dist=$out/dist + mkdir -p $dist + echo "foo" > $dist/index.html + echo "index" > $dist/foo.html + ''; + check = config.node.pkgs.testers.lycheeLinkCheck { + site = sitePkg; + }; +in +{ + name = "testers-lychee-link-check-run"; + nodes.client = { ... }: { }; + nodes.example = { + networking.firewall.allowedTCPPorts = [ 80 ]; + services.nginx = { + enable = true; + virtualHosts."example" = { + locations."/" = { + root = "/var/www/example"; + index = "index.html"; + }; + }; + }; + + }; + testScript = '' + start_all() + + # SETUP + + example.succeed(""" + mkdir -p /var/www/example + echo '

hi

' > /var/www/example/index.html + """) + client.wait_until_succeeds(""" + curl --fail -v http://example + """) + + # FAILURE CASE + + client.succeed(""" + exec 1>&2 + r=0 + ${lib.getExe check.online} || { + r=$? + } + if [[ $r -ne 2 ]]; then + echo "lycheeLinkCheck unexpected exit code $r" + exit 1 + fi + """) + + # SUCCESS CASE + + example.succeed(""" + echo '

foo

' > /var/www/example/foo.html + """) + + client.succeed(""" + ${lib.getExe check.online} + """) + ''; +} diff --git a/pkgs/tools/networking/lychee/tests/ok.nix b/pkgs/tools/networking/lychee/tests/ok.nix new file mode 100644 index 000000000000..6a6c0b9c87b4 --- /dev/null +++ b/pkgs/tools/networking/lychee/tests/ok.nix @@ -0,0 +1,12 @@ +{ runCommand, testers }: +let + sitePkg = runCommand "site" { } '' + dist=$out/dist + mkdir -p $dist + echo "foobar" > $dist/index.html + echo "index" > $dist/foo.html + ''; +in testers.lycheeLinkCheck rec { + site = sitePkg + "/dist"; + remap = { "https://example.com" = site; }; +}