testers.lycheeLinkCheck: init (#298665)
* testers.lycheeLinkCheck: init Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
This commit is contained in:
Robert Hensing
@@ -40,6 +40,82 @@ If the `moduleNames` argument is omitted, `hasPkgConfigModules` will use `meta.p
|
||||
|
||||
:::
|
||||
|
||||
## `lycheeLinkCheck` {#tester-lycheeLinkCheck}
|
||||
|
||||
Check a packaged static site's links with the [`lychee` package](https://search.nixos.org/packages?show=lychee&type=packages&query=lychee).
|
||||
|
||||
You may use Nix to reproducibly build static websites, such as for software documentation.
|
||||
Some packages will install documentation in their `out` or `doc` outputs, or maybe you have dedicated package where you've made your static site reproducible by running a generator, such as [Hugo](https://gohugo.io/) or [mdBook](https://rust-lang.github.io/mdBook/), in a derivation.
|
||||
|
||||
If you have a static site that can be built with Nix, you can use `lycheeLinkCheck` to check that the hyperlinks in your site are correct, and do so as part of your Nix workflow and CI.
|
||||
|
||||
:::{.example #ex-lycheelinkcheck}
|
||||
|
||||
# Check hyperlinks in the `nix` documentation
|
||||
|
||||
```nix
|
||||
testers.lycheeLinkCheck {
|
||||
site = nix.doc + "/share/doc/nix/manual";
|
||||
}
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
### Return value {#tester-lycheeLinkCheck-return}
|
||||
|
||||
This tester produces a package that does not produce useful outputs, but only succeeds if the hyperlinks in your site are correct. The build log will list the broken links.
|
||||
|
||||
It has two modes:
|
||||
|
||||
- Build the returned derivation; its build process will check that internal hyperlinks are correct. This runs in the sandbox, so it will not check external hyperlinks, but it is quick and reliable.
|
||||
|
||||
- Invoke the `.online` attribute with [`nix run`](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-run) ([experimental](https://nixos.org/manual/nix/stable/contributing/experimental-features#xp-feature-nix-command)). This runs outside the sandbox, and checks that both internal and external hyperlinks are correct.
|
||||
Example:
|
||||
|
||||
```shell
|
||||
nix run nixpkgs#lychee.tests.ok.online
|
||||
```
|
||||
|
||||
### Inputs {#tester-lycheeLinkCheck-inputs}
|
||||
|
||||
`site` (path or derivation) {#tester-lycheeLinkCheck-param-site}
|
||||
|
||||
: The path to the files to check.
|
||||
|
||||
`remap` (attribe set, optional) {#tester-lycheeLinkCheck-param-remap}
|
||||
|
||||
: An attribute set where the attribute names are regular expressions.
|
||||
The values should be strings, derivations, or path values.
|
||||
|
||||
In the returned check's default configuration, external URLs are only checked when you run the `.online` attribute.
|
||||
|
||||
By adding remappings, you can check offline that URLs to external resources are correct, by providing a stand-in from the file system.
|
||||
|
||||
Before checking the existence of a URL, the regular expressions are matched and replaced by their corresponding values.
|
||||
|
||||
Example:
|
||||
|
||||
```nix
|
||||
{
|
||||
"https://nix\\.dev/manual/nix/[a-z0-9.-]*" = "${nix.doc}/share/doc/nix/manual";
|
||||
"https://nixos\\.org/manual/nix/(un)?stable" = "${emptyDirectory}/placeholder-to-disallow-old-nix-docs-urls";
|
||||
}
|
||||
```
|
||||
|
||||
Store paths in the attribute values are automatically prefixed with `file://`, because lychee requires this for paths in the file system.
|
||||
If this is a problem, or if you need to control the order in which replacements are performed, use `extraConfig.remap` instead.
|
||||
|
||||
`extraConfig` (attribute set) {#tester-lycheeLinkCheck-param-extraConfig}
|
||||
|
||||
: Extra configuration to pass to `lychee` in its [configuration file](https://github.com/lycheeverse/lychee/blob/master/lychee.example.toml).
|
||||
It is automatically [translated](https://nixos.org/manual/nixos/stable/index.html#sec-settings-nix-representable) to TOML.
|
||||
|
||||
Example: `{ "include_verbatim" = true; }`
|
||||
|
||||
`lychee` (derivation, optional) {#tester-lycheeLinkCheck-param-lychee}
|
||||
|
||||
: The `lychee` package to use.
|
||||
|
||||
## `testVersion` {#tester-testVersion}
|
||||
|
||||
Checks that the output from running a command contains the specified version string in it as a whole word.
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
{ pkgs, pkgsLinux, buildPackages, lib, callPackage, runCommand, stdenv, substituteAll, testers }:
|
||||
# Documentation is in doc/builders/testers.chapter.md
|
||||
{
|
||||
# See https://nixos.org/manual/nixpkgs/unstable/#tester-lycheeLinkCheck
|
||||
# or doc/builders/testers.chapter.md
|
||||
inherit (callPackage ./lychee.nix {}) lycheeLinkCheck;
|
||||
|
||||
# See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailure
|
||||
# or doc/builders/testers.chapter.md
|
||||
testBuildFailure = drv: drv.overrideAttrs (orig: {
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
deps@{ formats, lib, lychee, stdenv, writeShellApplication }:
|
||||
let
|
||||
inherit (lib) mapAttrsToList throwIf;
|
||||
inherit (lib.strings) hasInfix hasPrefix escapeNixString;
|
||||
|
||||
toURL = v:
|
||||
let s = "${v}";
|
||||
in if hasPrefix builtins.storeDir s
|
||||
then # lychee requires that paths on the file system are prefixed with file://
|
||||
"file://${s}"
|
||||
else s;
|
||||
|
||||
withCheckedName = name:
|
||||
throwIf (hasInfix " " name) ''
|
||||
lycheeLinkCheck: remap patterns must not contain spaces.
|
||||
A space marks the end of the regex in lychee.toml.
|
||||
|
||||
Please change attribute name 'remap.${escapeNixString name}'
|
||||
'';
|
||||
|
||||
# See https://nixos.org/manual/nixpkgs/unstable/#tester-lycheeLinkCheck
|
||||
# or doc/builders/testers.chapter.md
|
||||
lycheeLinkCheck = {
|
||||
site,
|
||||
remap ? { },
|
||||
lychee ? deps.lychee,
|
||||
extraConfig ? { },
|
||||
}:
|
||||
stdenv.mkDerivation (finalAttrs: {
|
||||
name = "lychee-link-check";
|
||||
inherit site;
|
||||
nativeBuildInputs = [ finalAttrs.passthru.lychee ];
|
||||
configFile = (formats.toml {}).generate "lychee.toml" finalAttrs.passthru.config;
|
||||
|
||||
# These can be overriden with overrideAttrs if needed.
|
||||
passthru = {
|
||||
inherit lychee remap;
|
||||
config = {
|
||||
include_fragments = true;
|
||||
} // lib.optionalAttrs (finalAttrs.passthru.remap != { }) {
|
||||
remap =
|
||||
mapAttrsToList
|
||||
(name: value: withCheckedName name "${name} ${toURL value}")
|
||||
finalAttrs.passthru.remap;
|
||||
} // extraConfig;
|
||||
online = writeShellApplication {
|
||||
name = "run-lychee-online";
|
||||
runtimeInputs = [ finalAttrs.passthru.lychee ];
|
||||
# Comment out to run shellcheck:
|
||||
checkPhase = "";
|
||||
text = ''
|
||||
site=${finalAttrs.site}
|
||||
configFile=${finalAttrs.configFile}
|
||||
echo Checking links on $site
|
||||
exec lychee --config $configFile $site "$@"
|
||||
'';
|
||||
};
|
||||
};
|
||||
buildCommand = ''
|
||||
echo Checking internal links on $site
|
||||
lychee --offline --config $configFile $site
|
||||
touch $out
|
||||
'';
|
||||
});
|
||||
|
||||
in
|
||||
{
|
||||
inherit lycheeLinkCheck;
|
||||
}
|
||||
@@ -12,6 +12,8 @@ let
|
||||
|
||||
in
|
||||
lib.recurseIntoAttrs {
|
||||
lycheeLinkCheck = lib.recurseIntoAttrs pkgs.lychee.tests;
|
||||
|
||||
hasPkgConfigModules = pkgs.callPackage ../hasPkgConfigModules/tests.nix { };
|
||||
|
||||
runNixOSTest-example = pkgs-with-overlay.testers.runNixOSTest ({ lib, ... }: {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{ lib
|
||||
{ callPackage
|
||||
, lib
|
||||
, stdenv
|
||||
, rustPlatform
|
||||
, fetchFromGitHub
|
||||
@@ -6,6 +7,7 @@
|
||||
, openssl
|
||||
, Security
|
||||
, SystemConfiguration
|
||||
, testers
|
||||
}:
|
||||
|
||||
rustPlatform.buildRustPackage rec {
|
||||
@@ -41,6 +43,15 @@ rustPlatform.buildRustPackage rec {
|
||||
"--skip=src/lib.rs"
|
||||
];
|
||||
|
||||
passthru.tests = {
|
||||
# NOTE: These assume that testers.lycheeLinkCheck uses this exact derivation.
|
||||
# Which is true most of the time, but not necessarily after overriding.
|
||||
ok = callPackage ./tests/ok.nix { };
|
||||
fail = callPackage ./tests/fail.nix { };
|
||||
fail-emptyDirectory = callPackage ./tests/fail-emptyDirectory.nix { };
|
||||
network = testers.runNixOSTest ./tests/network.nix;
|
||||
};
|
||||
|
||||
meta = with lib; {
|
||||
description = "A fast, async, stream-based link checker written in Rust";
|
||||
homepage = "https://github.com/lycheeverse/lychee";
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
{ runCommand, testers, emptyDirectory }:
|
||||
let
|
||||
sitePkg = runCommand "site" { } ''
|
||||
dist=$out/dist
|
||||
mkdir -p $dist
|
||||
echo "<html><body><a href=\"https://example.com/foo\">foo</a></body></html>" > $dist/index.html
|
||||
echo "<html><body></body></html>" > $dist/foo.html
|
||||
'';
|
||||
check = testers.lycheeLinkCheck {
|
||||
site = sitePkg + "/dist";
|
||||
remap = {
|
||||
# Normally would recommend to append a subpath that hints why it's forbidden; see example in docs.
|
||||
# However, we also want to test that a package is converted to a string *before*
|
||||
# it's tested whether it's a store path. Mistake made during development caused:
|
||||
# cannot check URI: InvalidUrlRemap("The remapping pattern must produce a valid URL, but it is not: /nix/store/4d0ix...empty-directory/foo
|
||||
"https://example.com" = emptyDirectory;
|
||||
};
|
||||
};
|
||||
|
||||
failure = testers.testBuildFailure check;
|
||||
in
|
||||
runCommand "link-check-fail" { inherit failure; } ''
|
||||
# The details of the message might change, but we have to make sure the
|
||||
# correct error is reported, so that we know it's not something else that
|
||||
# went wrong.
|
||||
grep 'empty-directory/foo.*Cannot find file' $failure/testBuildFailure.log >/dev/null
|
||||
touch $out
|
||||
''
|
||||
@@ -0,0 +1,21 @@
|
||||
{ runCommand, testers }:
|
||||
let
|
||||
sitePkg = runCommand "site" { } ''
|
||||
dist=$out/dist
|
||||
mkdir -p $dist
|
||||
echo "<html><body><a href=\"https://example.com/foo.html#foos-missing-anchor\">foo</a></body></html>" > $dist/index.html
|
||||
echo "<html><body><a href=\".\">index</a></body></html>" > $dist/foo.html
|
||||
'';
|
||||
|
||||
linkCheck = testers.lycheeLinkCheck rec {
|
||||
site = sitePkg + "/dist";
|
||||
remap = { "https://exampl[e]\\.com" = site; };
|
||||
};
|
||||
|
||||
failure = testers.testBuildFailure linkCheck;
|
||||
|
||||
in
|
||||
runCommand "link-check-fail" { inherit failure; } ''
|
||||
grep -F foos-missing-anchor $failure/testBuildFailure.log >/dev/null
|
||||
touch $out
|
||||
''
|
||||
@@ -0,0 +1,66 @@
|
||||
{ config, hostPkgs, lib, ... }:
|
||||
let
|
||||
sitePkg = hostPkgs.runCommand "site" { } ''
|
||||
dist=$out/dist
|
||||
mkdir -p $dist
|
||||
echo "<html><body><a href=\"http://example/foo.html\">foo</a></body></html>" > $dist/index.html
|
||||
echo "<html><body><a href=\".\">index</a></body></html>" > $dist/foo.html
|
||||
'';
|
||||
check = config.node.pkgs.testers.lycheeLinkCheck {
|
||||
site = sitePkg;
|
||||
};
|
||||
in
|
||||
{
|
||||
name = "testers-lychee-link-check-run";
|
||||
nodes.client = { ... }: { };
|
||||
nodes.example = {
|
||||
networking.firewall.allowedTCPPorts = [ 80 ];
|
||||
services.nginx = {
|
||||
enable = true;
|
||||
virtualHosts."example" = {
|
||||
locations."/" = {
|
||||
root = "/var/www/example";
|
||||
index = "index.html";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
};
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
# SETUP
|
||||
|
||||
example.succeed("""
|
||||
mkdir -p /var/www/example
|
||||
echo '<h1>hi</h1>' > /var/www/example/index.html
|
||||
""")
|
||||
client.wait_until_succeeds("""
|
||||
curl --fail -v http://example
|
||||
""")
|
||||
|
||||
# FAILURE CASE
|
||||
|
||||
client.succeed("""
|
||||
exec 1>&2
|
||||
r=0
|
||||
${lib.getExe check.online} || {
|
||||
r=$?
|
||||
}
|
||||
if [[ $r -ne 2 ]]; then
|
||||
echo "lycheeLinkCheck unexpected exit code $r"
|
||||
exit 1
|
||||
fi
|
||||
""")
|
||||
|
||||
# SUCCESS CASE
|
||||
|
||||
example.succeed("""
|
||||
echo '<h1>foo</h1>' > /var/www/example/foo.html
|
||||
""")
|
||||
|
||||
client.succeed("""
|
||||
${lib.getExe check.online}
|
||||
""")
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{ runCommand, testers }:
|
||||
let
|
||||
sitePkg = runCommand "site" { } ''
|
||||
dist=$out/dist
|
||||
mkdir -p $dist
|
||||
echo "<html><body><a href=\"https://example.com/foo.html\">foo</a><a href=\"https://nixos.org/this-is-ignored.html\">bar</a></body></html>" > $dist/index.html
|
||||
echo "<html><body><a href=\".\">index</a></body></html>" > $dist/foo.html
|
||||
'';
|
||||
in testers.lycheeLinkCheck rec {
|
||||
site = sitePkg + "/dist";
|
||||
remap = { "https://example.com" = site; };
|
||||
}
|
||||
Reference in New Issue
Block a user