From 89979c9c5bef4cfc89e66662b1238a0bf7617702 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Tue, 12 Oct 2021 12:31:02 -0700 Subject: [PATCH 01/10] writeShellApplication: init --- doc/builders/trivial-builders.chapter.md | 22 +++++++++++ pkgs/build-support/trivial-builders.nix | 47 +++++++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/doc/builders/trivial-builders.chapter.md b/doc/builders/trivial-builders.chapter.md index 46620e1b459c..4c0318a2541a 100644 --- a/doc/builders/trivial-builders.chapter.md +++ b/doc/builders/trivial-builders.chapter.md @@ -47,6 +47,28 @@ These functions write `text` to the Nix store. This is useful for creating scrip Many more commands wrap `writeTextFile` including `writeText`, `writeTextDir`, `writeScript`, and `writeScriptBin`. These are convenience functions over `writeTextFile`. +## `writeShellApplication` {#trivial-builder-writeShellApplication} + +This can be used to easily produce a shell script that has some dependencies (`buildInputs`). It automatically sets the `PATH` of the script to contain all of the listed inputs, sets some sanity shellopts (`errexit`, `nounset`, `pipefail`), and checks the resulting script with [`shellcheck`](https://github.com/koalaman/shellcheck). + +For example, look at the following code: + +```nix +writeShellApplication { + name = "show-nixos-org"; + + buildInputs = [ curl w3m ]; + + text = '' + curl -s 'https://nixos.org' | w3m -dump -T text/html + ''; +} +``` + +Unlike with normal `writeShellScriptBin`, there is no need to manually write out `${curl}/bin/curl`, setting the PATH +was handled by `writeShellApplication`. Moreover, the script is being checked with `shellcheck` for more strict +validation. + ## `symlinkJoin` {#trivial-builder-symlinkJoin} This can be used to put many derivations into the same directory structure. It works by creating a new derivation and adding symlinks to each of the paths listed. It expects two arguments, `name`, and `paths`. `name` is the name used in the Nix store path for the created derivation. `paths` is a list of paths that will be symlinked. These paths can be to Nix store derivations or any other subdirectory contained within. diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 58cdeb269d58..506f84310160 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, stdenvNoCC, lndir, runtimeShell }: +{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, makeBinPath, shellcheck }: rec { @@ -249,6 +249,51 @@ rec { ''; }; + /* + * Similar to writeShellScriptBin and writeScriptBin. + * Writes an executable Shell script to /nix/store//bin/ and + * checks its syntax with shellcheck and the shell's -n option. + * Automatically includes sane set of shellopts (errexit, nounset, pipefail) + * and handles creation of PATH based on buildInputs + * + * Example: + * # Writes my-file to /nix/store//bin/my-file and makes executable. + * writeShellApplication { + * name = "my-file"; + * buildInputs = [ curl w3m ]; + * text = '' + * curl -s 'https://nixos.org' | w3m -dump -T text/html + * ''; + * } + */ + writeShellApplication = + { name + , text + , buildInputs ? [ ] + , checkPhase ? null + }: + writeTextFile { + inherit name; + executable = true; + destination = "/bin/${name}"; + text = '' + #!${runtimeShell} + set -o errexit + set -o nounset + set- o pipefail + + export PATH="${makeBinPath buildInputs}:$PATH" + + ${text} + ''; + + checkPhase = if checkPhase == null then '' + ${stdenv.shell} -n $out/bin/${name} + ${shellcheck}/bin/shellcheck $out/bin/${name} + '' + else checkPhase; + }; + # Create a C binary writeCBin = name: code: runCommandCC name From 0e4f04b74c3ec5948e2d860540409cf02306cb33 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Tue, 12 Oct 2021 12:48:27 -0700 Subject: [PATCH 02/10] writeShellApplication: buildInputs -> runtimeInputs --- doc/builders/trivial-builders.chapter.md | 4 ++-- pkgs/build-support/trivial-builders.nix | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/builders/trivial-builders.chapter.md b/doc/builders/trivial-builders.chapter.md index 4c0318a2541a..c3a3572cd9f4 100644 --- a/doc/builders/trivial-builders.chapter.md +++ b/doc/builders/trivial-builders.chapter.md @@ -49,7 +49,7 @@ Many more commands wrap `writeTextFile` including `writeText`, `writeTextDir`, ` ## `writeShellApplication` {#trivial-builder-writeShellApplication} -This can be used to easily produce a shell script that has some dependencies (`buildInputs`). It automatically sets the `PATH` of the script to contain all of the listed inputs, sets some sanity shellopts (`errexit`, `nounset`, `pipefail`), and checks the resulting script with [`shellcheck`](https://github.com/koalaman/shellcheck). +This can be used to easily produce a shell script that has some dependencies (`runtimeInputs`). It automatically sets the `PATH` of the script to contain all of the listed inputs, sets some sanity shellopts (`errexit`, `nounset`, `pipefail`), and checks the resulting script with [`shellcheck`](https://github.com/koalaman/shellcheck). For example, look at the following code: @@ -57,7 +57,7 @@ For example, look at the following code: writeShellApplication { name = "show-nixos-org"; - buildInputs = [ curl w3m ]; + runtimeInputs = [ curl w3m ]; text = '' curl -s 'https://nixos.org' | w3m -dump -T text/html diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 506f84310160..e0571632e760 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -260,7 +260,7 @@ rec { * # Writes my-file to /nix/store//bin/my-file and makes executable. * writeShellApplication { * name = "my-file"; - * buildInputs = [ curl w3m ]; + * runtimeInputs = [ curl w3m ]; * text = '' * curl -s 'https://nixos.org' | w3m -dump -T text/html * ''; @@ -269,7 +269,7 @@ rec { writeShellApplication = { name , text - , buildInputs ? [ ] + , runtimeInputs ? [ ] , checkPhase ? null }: writeTextFile { @@ -282,7 +282,7 @@ rec { set -o nounset set- o pipefail - export PATH="${makeBinPath buildInputs}:$PATH" + export PATH="${makeBinPath runtimeInputs}:$PATH" ${text} ''; From 21c299f077d6614176dcb12c5899cca79549dc78 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Tue, 12 Oct 2021 12:53:02 -0700 Subject: [PATCH 03/10] writeShellApplication: get shellcheck from pkgsBuildHost --- pkgs/build-support/trivial-builders.nix | 4 ++-- pkgs/top-level/stage.nix | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index e0571632e760..707c5da0931f 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, makeBinPath, shellcheck }: +{ lib, stdenv, stdenvNoCC, lndir, runtimeShell, shellcheck }: rec { @@ -282,7 +282,7 @@ rec { set -o nounset set- o pipefail - export PATH="${makeBinPath runtimeInputs}:$PATH" + export PATH="${lib.makeBinPath runtimeInputs}:$PATH" ${text} ''; diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 7db79276d899..af1ae7871e57 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -76,7 +76,9 @@ let trivialBuilders = self: super: import ../build-support/trivial-builders.nix { - inherit lib; inherit (self) stdenv stdenvNoCC; inherit (self.pkgsBuildHost.xorg) lndir; + inherit lib; inherit (self) stdenv stdenvNoCC; + inherit (self.pkgsBuildHost) shellcheck; + inherit (self.pkgsBuildHost.xorg) lndir; inherit (self) runtimeShell; }; From d13430a8da4f476cb9772f45ebf307cb9e778d4d Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Tue, 12 Oct 2021 12:57:23 -0700 Subject: [PATCH 04/10] pkgs/top-level/stage: format trivialBuilders import --- pkgs/top-level/stage.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index af1ae7871e57..9d34ddb3685f 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -76,10 +76,10 @@ let trivialBuilders = self: super: import ../build-support/trivial-builders.nix { - inherit lib; inherit (self) stdenv stdenvNoCC; + inherit lib; + inherit (self) runtimeShell stdenv stdenvNoCC; inherit (self.pkgsBuildHost) shellcheck; inherit (self.pkgsBuildHost.xorg) lndir; - inherit (self) runtimeShell; }; stdenvBootstappingAndPlatforms = self: super: let From 97de845b60fbc5e6c074999c7158b2200810191f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Tue, 12 Oct 2021 13:13:58 -0700 Subject: [PATCH 05/10] writeShellApplication: fix setting pipefail --- pkgs/build-support/trivial-builders.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 707c5da0931f..692ac6801dee 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -280,7 +280,7 @@ rec { #!${runtimeShell} set -o errexit set -o nounset - set- o pipefail + set -o pipefail export PATH="${lib.makeBinPath runtimeInputs}:$PATH" From 014b23232f4cf6815b14292f50edd0d4fce7cc02 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Wed, 13 Oct 2021 12:48:28 -0700 Subject: [PATCH 06/10] writeShellApplication: fix typo in inline docs --- pkgs/build-support/trivial-builders.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 692ac6801dee..5f2d845e1b81 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -254,7 +254,7 @@ rec { * Writes an executable Shell script to /nix/store//bin/ and * checks its syntax with shellcheck and the shell's -n option. * Automatically includes sane set of shellopts (errexit, nounset, pipefail) - * and handles creation of PATH based on buildInputs + * and handles creation of PATH based on runtimeInputs * * Example: * # Writes my-file to /nix/store//bin/my-file and makes executable. From 949182a4a243f9884a02c0dc5c2667f312f8bbc2 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Sun, 7 Nov 2021 20:22:03 -0800 Subject: [PATCH 07/10] writeShellApplication: document runtimeShell == stdenv.shell assumption --- pkgs/build-support/trivial-builders.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 5f2d845e1b81..3d5eb64bc821 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -256,6 +256,10 @@ rec { * Automatically includes sane set of shellopts (errexit, nounset, pipefail) * and handles creation of PATH based on runtimeInputs * + * Note that the checkPhase uses stdenv.shell for the test run of the script, + * while the generated shebang uses runtimeShell. If, for whatever reason, + * those were to mismatch you might lose fidelity in the default checks. + * * Example: * # Writes my-file to /nix/store//bin/my-file and makes executable. * writeShellApplication { From d8ec0eeab7c631f9700d2fc66063f9fa06dbf693 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 8 Nov 2021 09:19:06 -0800 Subject: [PATCH 08/10] writeShellApplication: run pre/postCheck hooks --- pkgs/build-support/trivial-builders.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 3d5eb64bc821..beea1250b5f3 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -292,8 +292,10 @@ rec { ''; checkPhase = if checkPhase == null then '' + runHook preCheck ${stdenv.shell} -n $out/bin/${name} ${shellcheck}/bin/shellcheck $out/bin/${name} + runHook postCheck '' else checkPhase; }; From ba7b7357ac48c98d367c1aadf2d5124914a47a6a Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 8 Nov 2021 09:28:58 -0800 Subject: [PATCH 09/10] writeTextFile: allow passing extra derivationArgs --- pkgs/build-support/trivial-builders.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index beea1250b5f3..090986dda5f8 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -111,9 +111,10 @@ rec { , executable ? false # run chmod +x ? , destination ? "" # relative path appended to $out eg "/bin/foo" , checkPhase ? "" # syntax checks, e.g. for scripts + , meta ? { } }: runCommand name - { inherit text executable checkPhase; + { inherit text executable checkPhase meta; passAsFile = [ "text" ]; # Pointless to do this on a remote machine. preferLocalBuild = true; From 174995d14f680db4d820ee8daeeb7cd7840b0147 Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Mon, 8 Nov 2021 09:29:14 -0800 Subject: [PATCH 10/10] writeShellApplication: set meta.mainProgram --- pkgs/build-support/trivial-builders.nix | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkgs/build-support/trivial-builders.nix b/pkgs/build-support/trivial-builders.nix index 090986dda5f8..2966675be7ca 100644 --- a/pkgs/build-support/trivial-builders.nix +++ b/pkgs/build-support/trivial-builders.nix @@ -292,13 +292,16 @@ rec { ${text} ''; - checkPhase = if checkPhase == null then '' - runHook preCheck - ${stdenv.shell} -n $out/bin/${name} - ${shellcheck}/bin/shellcheck $out/bin/${name} - runHook postCheck - '' - else checkPhase; + checkPhase = + if checkPhase == null then '' + runHook preCheck + ${stdenv.shell} -n $out/bin/${name} + ${shellcheck}/bin/shellcheck $out/bin/${name} + runHook postCheck + '' + else checkPhase; + + meta.mainProgram = name; }; # Create a C binary