Merge pull request #311811 from abathur/move_lore_overrides_to_passthru_staging

binlore: migrate override lore to package passthru

pkgs/by-name/pd/pdf2odt/package.nix

@@ -49,6 +49,10 @@ resholve.mkDerivation rec {
       imagemagick
       zip
     ];
+    execer = [
+      # zip can exec; confirmed 2 invocations in pdf2odt don't
+      "cannot:${zip}/bin/zip"
+    ];
   };
 
   meta = with lib; {

pkgs/development/libraries/libarchive/default.nix

@@ -23,6 +23,9 @@
 , cmake
 , nix
 , samba
+
+# for passthru.lore
+, binlore
 }:
 
 assert xarSupport -> libxml2 != null;
@@ -125,4 +128,11 @@ stdenv.mkDerivation (finalAttrs: {
   passthru.tests = {
     inherit cmake nix samba;
   };
+
+  # bsdtar is detected as "cannot" because its exec is internal to
+  # calls it makes into libarchive itself. If binlore gains support
+  # for detecting another layer down into libraries, this can be cut.
+  passthru.binlore.out = binlore.synthesize finalAttrs.finalPackage ''
+    execer can bin/bsdtar
+  '';
 })

pkgs/development/libraries/ncurses/default.nix

@@ -11,6 +11,7 @@
 , mouseSupport ? false, gpm
 , unicodeSupport ? true
 , testers
+, binlore
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -180,6 +181,17 @@ stdenv.mkDerivation (finalAttrs: {
     rm "$out"/lib/*.a
   '';
 
+  # I'm not very familiar with ncurses, but it looks like most of the
+  # exec here will run hard-coded executables. There's one that is
+  # dynamic, but it looks like it only comes from executing a terminfo
+  # file, so I think it isn't going to be under user control via CLI?
+  # Happy to have someone help nail this down in either direction!
+  # The "capability" is 'iprog', and I could only find 1 real example:
+  # https://invisible-island.net/ncurses/terminfo.ti.html#tic-linux-s
+  passthru.binlore.out = binlore.synthesize ncurses ''
+    execer cannot bin/{reset,tput,tset}
+  '';
+
   meta = with lib; {
     homepage = "https://www.gnu.org/software/ncurses/";
     description = "Free software emulation of curses in SVR4 and more";

pkgs/development/tools/analysis/binlore/default.nix

@@ -56,58 +56,169 @@ let
     #   in here, but I'm erring on the side of flexibility
     #   since this form will make it easier to pilot other
     #   uses of binlore.
-    callback = lore: drv: overrides: ''
+    callback = lore: drv: ''
       if [[ -d "${drv}/bin" ]] || [[ -d "${drv}/lib" ]] || [[ -d "${drv}/libexec" ]]; then
         echo generating binlore for $drv by running:
         echo "${yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${yallback}/bin/yallback ${lore.yallback}"
       else
         echo "failed to generate binlore for $drv (none of ${drv}/{bin,lib,libexec} exist)"
       fi
-    '' +
-    /*
-    Override lore for some packages. Unsure, but for now:
-    1. start with the ~name (pname-version)
-    2. remove characters from the end until we find a match
-       in overrides/
-    3. execute the override script with the list of expected
-       lore types
-    */
-    ''
-      i=''${#identifier}
-      filter=
-      while [[ $i > 0 ]] && [[ -z "$filter" ]]; do
-        if [[ -f "${overrides}/''${identifier:0:$i}" ]]; then
-          filter="${overrides}/''${identifier:0:$i}"
-          echo using "${overrides}/''${identifier:0:$i}" to generate overriden binlore for $drv
-          break
-        fi
-        ((i--)) || true # don't break build
-      done # || true # don't break build
+
       if [[ -d "${drv}/bin" ]] || [[ -d "${drv}/lib" ]] || [[ -d "${drv}/libexec" ]]; then
-        ${yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${yallback}/bin/yallback ${lore.yallback} "$filter"
+        ${yara}/bin/yara --scan-list --recursive ${lore.rules} <(printf '%s\n' ${drv}/{bin,lib,libexec}) | ${yallback}/bin/yallback ${lore.yallback}
       fi
     '';
   };
-  overrides = (src + "/overrides");
 
 in rec {
+  /*
+    Output a directory containing lore for multiple drvs.
+
+    This will `make` lore for drv in drvs and then combine lore
+    of the same type across all packages into a single file.
+
+    When drvs are also specified in the strip argument, corresponding
+    lore is made relative by stripping the path of each drv from
+    matching entries. (This is mainly useful in a build process that
+    uses a chain of two or more derivations where the output of one
+    is the source for the next. See resholve for an example.)
+  */
   collect = { lore ? loreDef, drvs, strip ? [ ] }: (runCommand "more-binlore" { } ''
     mkdir $out
     for lorefile in ${toString lore.types}; do
       cat ${lib.concatMapStrings (x: x + "/$lorefile ") (map (make lore) (map lib.getBin (builtins.filter lib.isDerivation drvs)))} > $out/$lorefile
-      substituteInPlace $out/$lorefile ${lib.concatMapStrings (x: "--replace '${x}/' '' ") strip}
+      substituteInPlace $out/$lorefile ${lib.concatMapStrings (x: "--replace-quiet '${x}/' '' ") strip}
     done
   '');
-  # TODO: echo for debug, can be removed at some point
+
+  /*
+    Output a directory containing lore for a single drv.
+
+    This produces lore for the derivation (via lore.callback) and
+    appends any lore that the derivation itself wrote to nix-support
+    or which was overridden in drv.binlore.<outputName> (passthru).
+
+    > *Note*: Since the passthru is attached to all outputs, binlore
+    > is an attrset namespaced by outputName to support packages with
+    > executables in more than one output.
+
+    Since the last entry wins, the effective priority is:
+    drv.binlore.<outputName> > $drv/nix-support > lore generated here by callback
+  */
   make = lore: drv: runCommand "${drv.name}-binlore" {
-      identifier = drv.name;
       drv = drv;
     } (''
     mkdir $out
     touch $out/{${builtins.concatStringsSep "," lore.types}}
 
-    ${lore.callback lore drv overrides}
+    ${lore.callback lore drv}
+    '' +
+    # append lore from package's $out and drv.binlore.${drv.outputName} (last entry wins)
+    ''
+    for lore_type in ${builtins.toString lore.types}; do
+      if [[ -f "${drv}/nix-support/$lore_type" ]]; then
+        cat "${drv}/nix-support/$lore_type" >> "$out/$lore_type"
+      fi
+      '' + lib.optionalString (builtins.hasAttr "binlore" drv && builtins.hasAttr drv.outputName drv.binlore) ''
+      if [[ -f "${drv.binlore."${drv.outputName}"}/$lore_type" ]]; then
+        cat "${drv.binlore."${drv.outputName}"}/$lore_type" >> "$out/$lore_type"
+      fi
+      '' + ''
+    done
 
     echo binlore for $drv written to $out
   '');
+
+  /*
+    Utility function for creating override lore for drv.
+
+    We normally attach this lore to `drv.passthru.binlore.<outputName>`.
+
+    > *Notes*:
+    > - Since the passthru is attached to all outputs, binlore is an
+    >   attrset namespaced by outputName to support packages with
+    >   executables in more than one output. You'll generally just use
+    >   `out` or `bin`.
+    > - We can reconsider the passthru attr name if someone adds
+    >   a new lore provider. We settled on `.binlore` for now to make it
+    >   easier for people to figure out what this is for.
+
+    The lore argument should be a Shell script (string) that generates
+    the necessary lore. You can use arbitrary Shell, but this function
+    includes a shell DSL you can use to declare/generate lore in most
+    cases. It has the following functions:
+
+    - `execer <verdict> [<path>...]`
+    - `wrapper <wrapper_path> <original_path>`
+
+    Writing every override explicitly in a Nix list would be tedious
+    for large packages, but this small shell DSL enables us to express
+    many overrides efficiently via pathname expansion/globbing.
+
+    Here's a very general example of both functions:
+
+    passthru.binlore.out = binlore.synthesize finalAttrs.finalPackage ''
+      execer can bin/hello bin/{a,b,c}
+      wrapper bin/hello bin/.hello-wrapped
+    '';
+
+    And here's a specific example of how pathname expansion enables us
+    to express lore for the single-binary variant of coreutils while
+    being both explicit and (somewhat) efficient:
+
+    passthru = {} // optionalAttrs (singleBinary != false) {
+      binlore.out = binlore.synthesize coreutils ''
+        execer can bin/{chroot,env,install,nice,nohup,runcon,sort,split,stdbuf,timeout}
+        execer cannot bin/{[,b2sum,base32,base64,basename,basenc,cat,chcon,chgrp,chmod,chown,cksum,comm,cp,csplit,cut,date,dd,df,dir,dircolors,dirname,du,echo,expand,expr,factor,false,fmt,fold,groups,head,hostid,id,join,kill,link,ln,logname,ls,md5sum,mkdir,mkfifo,mknod,mktemp,mv,nl,nproc,numfmt,od,paste,pathchk,pinky,pr,printenv,printf,ptx,pwd,readlink,realpath,rm,rmdir,seq,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,shred,shuf,sleep,stat,stty,sum,sync,tac,tail,tee,test,touch,tr,true,truncate,tsort,tty,uname,unexpand,uniq,unlink,uptime,users,vdir,wc,who,whoami,yes}
+      '';
+    };
+
+    Caution: Be thoughtful about using a bare wildcard (*) glob here.
+    We should generally override lore only when a human understands if
+    the executable will exec arbitrary user-passed executables. A bare
+    glob can match new executables added in future package versions
+    before anyone can audit them.
+  */
+  synthesize = drv: loreSynthesizingScript: runCommand "${drv.name}-lore-override" {
+    drv = drv;
+  } (''
+    execer(){
+      local verdict="$1"
+
+      shift
+
+      for path in "$@"; do
+        if [[ -f "$PWD/$path" ]]; then
+          echo "$verdict:$PWD/$path"
+        else
+          echo "error: Tried to synthesize execer lore for missing file: $PWD/$path" >&2
+          exit 2
+        fi
+      done
+    } >> $out/execers
+
+    wrapper(){
+      local wrapper="$1"
+      local original="$2"
+
+      if [[ ! -f "$wrapper" ]]; then
+        echo "error: Tried to synthesize wrapper lore for missing wrapper: $PWD/$wrapper" >&2
+        exit 2
+      fi
+
+      if [[ ! -f "$original" ]]; then
+        echo "error: Tried to synthesize wrapper lore for missing original: $PWD/$original" >&2
+        exit 2
+      fi
+
+      echo "$PWD/$wrapper:$PWD/$original"
+
+    } >> $out/wrappers
+
+    mkdir $out
+
+    # lore override commands are relative to the drv root
+    cd $drv
+
+  '' + loreSynthesizingScript);
 }

pkgs/os-specific/linux/nixos-rebuild/default.nix

@@ -10,6 +10,8 @@
 , lib
 , nixosTests
 , installShellFiles
+, binlore
+, nixos-rebuild
 }:
 let
   fallback = import ./../../../../nixos/modules/installer/tools/nix-fallback-paths.nix;
@@ -49,6 +51,13 @@ substitute {
     target-host = nixosTests.nixos-rebuild-target-host;
   };
 
+  # nixos-rebuild can’t execute its arguments
+  # (but it can run ssh with the with the options stored in $NIX_SSHOPTS,
+  # and ssh can execute its arguments...)
+  passthru.binlore.out = binlore.synthesize nixos-rebuild ''
+    execer cannot bin/nixos-rebuild
+  '';
+
   meta = {
     description = "Rebuild your NixOS configuration and switch to it, on local hosts and remote";
     homepage = "https://github.com/NixOS/nixpkgs/tree/master/pkgs/os-specific/linux/nixos-rebuild";

pkgs/os-specific/linux/procps-ng/default.nix

@@ -15,6 +15,9 @@
   # exception is ‘watch’ which is portable enough to run on pretty much
   # any UNIX-compatible system.
 , watchOnly ? !(stdenv.isLinux || stdenv.isCygwin)
+
+, binlore
+, procps
 }:
 
 stdenv.mkDerivation rec {
@@ -61,6 +64,12 @@ stdenv.mkDerivation rec {
     install -m 0644 -D watch.1 $out/share/man/man1/watch.1
   '';
 
+  # no obvious exec in documented arguments; haven't trawled source
+  # to figure out what exec binlore hits on
+  passthru.binlore.out = binlore.synthesize procps ''
+    execer cannot bin/{ps,top,free}
+  '';
+
   meta = with lib; {
     homepage = "https://gitlab.com/procps-ng/procps";
     description = "Utilities that give information about processes using the /proc filesystem";

pkgs/tools/misc/coreutils/default.nix

@@ -7,6 +7,8 @@
 , perl
 , texinfo
 , xz
+, binlore
+, coreutils
 , gmpSupport ? true, gmp
 , aclSupport ? stdenv.isLinux, acl
 , attrSupport ? stdenv.isLinux, attr
@@ -27,7 +29,7 @@ assert aclSupport -> acl != null;
 assert selinuxSupport -> libselinux != null && libsepol != null;
 
 let
-  inherit (lib) concatStringsSep isString optional optionals optionalString;
+  inherit (lib) concatStringsSep isString optional optionalAttrs optionals optionalString;
   isCross = (stdenv.hostPlatform != stdenv.buildPlatform);
 in
 stdenv.mkDerivation rec {
@@ -181,6 +183,26 @@ stdenv.mkDerivation rec {
     rm -r "$out/share"
   '';
 
+  passthru = {} // optionalAttrs (singleBinary != false) {
+    # everything in the single binary gets the same verdict, so we
+    # override _that case_ with verdicts from separate binaries.
+    #
+    # binlore only spots exec in runcon on some platforms (i.e., not
+    # darwin; see comment on inverse case below)
+    binlore.out = binlore.synthesize coreutils ''
+      execer can bin/{chroot,env,install,nice,nohup,runcon,sort,split,stdbuf,timeout}
+      execer cannot bin/{[,b2sum,base32,base64,basename,basenc,cat,chcon,chgrp,chmod,chown,cksum,comm,cp,csplit,cut,date,dd,df,dir,dircolors,dirname,du,echo,expand,expr,factor,false,fmt,fold,groups,head,hostid,id,join,kill,link,ln,logname,ls,md5sum,mkdir,mkfifo,mknod,mktemp,mv,nl,nproc,numfmt,od,paste,pathchk,pinky,pr,printenv,printf,ptx,pwd,readlink,realpath,rm,rmdir,seq,sha1sum,sha224sum,sha256sum,sha384sum,sha512sum,shred,shuf,sleep,stat,stty,sum,sync,tac,tail,tee,test,touch,tr,true,truncate,tsort,tty,uname,unexpand,uniq,unlink,uptime,users,vdir,wc,who,whoami,yes}
+    '';
+  } // optionalAttrs (singleBinary == false) {
+    # binlore only spots exec in runcon on some platforms (i.e., not
+    # darwin; I have a note that the behavior may need selinux?).
+    # hard-set it so people working on macOS don't miss cases of
+    # runcon until ofBorg fails.
+    binlore.out = binlore.synthesize coreutils ''
+      execer can bin/runcon
+    '';
+  };
+
   meta = with lib; {
     homepage = "https://www.gnu.org/software/coreutils/";
     description = "GNU Core Utilities";

pkgs/tools/misc/linuxquota/default.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, fetchurl, e2fsprogs, openldap, pkg-config }:
+{ lib, stdenv, fetchurl, e2fsprogs, openldap, pkg-config, binlore, linuxquota }:
 
 stdenv.mkDerivation rec {
   version = "4.09";
@@ -14,6 +14,10 @@ stdenv.mkDerivation rec {
   nativeBuildInputs = [ pkg-config ];
   buildInputs = [ e2fsprogs openldap ];
 
+  passthru.binlore.out = binlore.synthesize linuxquota ''
+    execer cannot bin/quota
+  '';
+
   meta = with lib; {
     description = "Tools to manage kernel-level quotas in Linux";
     homepage = "https://sourceforge.net/projects/linuxquota/";

pkgs/tools/nix/nixos-install-tools/default.nix

@@ -8,6 +8,7 @@
   nixos-install-tools,
   runCommand,
   nixosTests,
+  binlore,
 }:
 let
   inherit (nixos {}) config;
@@ -62,6 +63,12 @@ in
       touch $out
     '';
   };
+
+  # no documented flags show signs of exec; skim of source suggests
+  # it's just --help execing man
+  passthru.binlore.out = binlore.synthesize nixos-install-tools ''
+    execer cannot bin/nixos-generate-config
+  '';
 }).overrideAttrs {
   inherit version;
   pname = "nixos-install-tools";

pkgs/tools/text/esh/default.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, fetchFromGitHub, asciidoctor, gawk, gnused, runtimeShell }:
+{ lib, stdenv, fetchFromGitHub, asciidoctor, gawk, gnused, runtimeShell, binlore, esh }:
 
 stdenv.mkDerivation rec {
   pname = "esh";
@@ -30,6 +30,14 @@ stdenv.mkDerivation rec {
   doCheck = true;
   checkTarget = "test";
 
+  # working around a bug in file. Was fixed in
+  # file 5.41-5.43 but regressed in 5.44+
+  # see https://bugs.astron.com/view.php?id=276
+  # "can" verdict because of `-s SHELL` arg
+  passthru.binlore.out = binlore.synthesize esh ''
+    execer can bin/esh
+  '';
+
   meta = with lib; {
     description = "Simple templating engine based on shell";
     mainProgram = "esh";

pkgs/top-level/unixtools.nix

@@ -1,4 +1,4 @@
-{ pkgs, buildEnv, runCommand, lib, stdenv, freebsd }:
+{ pkgs, buildEnv, runCommand, lib, stdenv, freebsd, binlore }:
 
 # These are some unix tools that are commonly included in the /usr/bin
 # and /usr/sbin directory under more normal distributions. Along with
@@ -30,7 +30,9 @@ let
         priority = 10;
         platforms = platforms.${stdenv.hostPlatform.parsed.kernel.name} or platforms.all;
       };
-      passthru = { inherit provider; };
+      passthru = { inherit provider; } // lib.optionalAttrs (builtins.hasAttr "binlore" providers) {
+        binlore.out = (binlore.synthesize (getBin bins.${cmd}) providers.binlore);
+      };
       preferLocalBuild = true;
     } ''
       if ! [ -x ${bin} ]; then
@@ -76,6 +78,10 @@ let
       linux = if stdenv.hostPlatform.libc == "glibc" then pkgs.stdenv.cc.libc
               else pkgs.netbsd.getconf;
       darwin = pkgs.darwin.system_cmds;
+      # I don't see any obvious arg exec in the doc/manpage
+      binlore = ''
+        execer cannot bin/getconf
+      '';
     };
     getent = {
       linux = if stdenv.hostPlatform.libc == "glibc" then pkgs.stdenv.cc.libc.getent
@@ -118,6 +124,11 @@ let
       linux = pkgs.glibc;
       darwin = pkgs.darwin.adv_cmds;
       freebsd = pkgs.freebsd.locale;
+      # technically just targeting glibc version
+      # no obvious exec in manpage
+      binlore = ''
+        execer cannot bin/locale
+      '';
     };
     logger = {
       linux = pkgs.util-linux;
@@ -130,6 +141,13 @@ let
       linux = pkgs.util-linux;
       darwin = pkgs.darwin.diskdev_cmds;
       freebsd = freebsd.mount;
+      # technically just targeting the darwin version; binlore already
+      # ids the util-linux copy as 'cannot'
+      # no obvious exec in manpage args; I think binlore flags 'can'
+      # on the code to run `mount_<filesystem>` variants
+      binlore = ''
+        execer cannot bin/mount
+      '';
     };
     netstat = {
       linux = pkgs.nettools;
@@ -145,6 +163,12 @@ let
       linux = pkgs.procps;
       darwin = pkgs.darwin.ps;
       freebsd = pkgs.freebsd.bin;
+      # technically just targeting procps ps (which ids as can)
+      # but I don't see obvious exec in args; have yet to look
+      # for underlying cause in source
+      binlore = ''
+        execer cannot bin/ps
+      '';
     };
     quota = {
       linux = pkgs.linuxquota;
@@ -168,6 +192,13 @@ let
       linux = pkgs.procps;
       darwin = pkgs.darwin.top;
       freebsd = pkgs.freebsd.top;
+      # technically just targeting procps top; haven't needed this in
+      # any scripts so far, but overriding it for consistency with ps
+      # override above and in procps. (procps also overrides 'free',
+      # but it isn't included here.)
+      binlore = ''
+        execer cannot bin/top
+      '';
     };
     umount = {
       linux = pkgs.util-linux;