Merge pull request #218331 from xworld21/texlive-new-dependency-resolution

texlive.combine: move dependencies to attribute tlDeps, resolve them …

doc/languages-frameworks/texlive.section.md

@@ -40,17 +40,24 @@ Since release 15.09 there is a new TeX Live packaging that lives entirely under
 
 ## Custom packages {#sec-language-texlive-custom-packages}
 
+You may find that you need to use an external TeX package. A derivation for such package has to provide the contents of the "texmf" directory in its output and provide the appropriate `tlType` attribute (one of `"run"`, `"bin"`, `"doc"`, `"source"`). Dependencies on other TeX packages can be listed in the attribute `tlDeps`.
 
-You may find that you need to use an external TeX package. A derivation for such package has to provide contents of the "texmf" directory in its output and provide the `tlType` attribute. Here is a (very verbose) example:
+Such derivation must then be listed in the attribute `pkgs` of an attribute set passed to `texlive.combine`, for instance by passing `extraPkgs = { pkgs = [ custom_package ]; };`. Within Nixpkgs, `pkgs` should be part of the derivation itself, allowing users to call `texlive.combine { inherit (texlive) scheme-small; inherit some_tex_package; }`.
+
+Here is a (very verbose) example where the attribute `pkgs` is attached to the derivation itself, which requires creating a fixed point. See also the packages `auctex`, `eukleides`, `mftrace` for more examples.
 
 ```nix
 with import <nixpkgs> {};
 
 let
-  foiltex_run = stdenvNoCC.mkDerivation {
+  foiltex = stdenvNoCC.mkDerivation (finalAttrs: {
     pname = "latex-foiltex";
     version = "2.1.4b";
-    passthru.tlType = "run";
+    passthru = {
+      pkgs = [ finalAttrs.finalPackage ];
+      tlDeps = with texlive; [ latex ];
+      tlType = "run";
+    };
 
     srcs = [
       (fetchurl {
@@ -102,8 +109,7 @@ let
       maintainers = with maintainers; [ veprbl ];
       platforms = platforms.all;
     };
-  };
-  foiltex = { pkgs = [ foiltex_run ]; };
+  });
 
   latex_with_foiltex = texlive.combine {
     inherit (texlive) scheme-small;

pkgs/applications/misc/auto-multiple-choice/default.nix

@@ -23,14 +23,13 @@
 , poppler
 , auto-multiple-choice
 }:
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: rec {
   pname = "auto-multiple-choice";
   version = "1.5.2";
   src = fetchurl {
     url = "https://download.auto-multiple-choice.net/${pname}_${version}_precomp.tar.gz";
     sha256 = "sha256-AjonJOooSe53Fww3QU6Dft95ojNqWrTuPul3nkIbctM=";
   };
-  tlType = "run";
 
   # There's only the Makefile
   dontConfigure = true;
@@ -137,6 +136,11 @@ stdenv.mkDerivation rec {
     XMLWriter
   ]);
 
+  passthru = {
+    tlType = "run";
+    pkgs = [ finalAttrs.finalPackage ];
+  };
+
   meta = with lib; {
     description = "Create and manage multiple choice questionnaires with automated marking.";
     longDescription = ''
@@ -156,10 +160,7 @@ stdenv.mkDerivation rec {
         auto-multiple-choice
         (texlive.combine {
           inherit (pkgs.texlive) scheme-full;
-          extra =
-            {
-              pkgs = [ auto-multiple-choice ];
-            };
+          inherit auto-multiple-choice;
         })
       ];
       </screen>
@@ -172,4 +173,4 @@ stdenv.mkDerivation rec {
     maintainers = [ maintainers.thblt ];
     platforms = platforms.all;
   };
-}
+})

pkgs/applications/science/math/eukleides/default.nix

@@ -1,6 +1,6 @@
 { lib, stdenv, fetchurl, bison, flex, makeWrapper, texinfo4, getopt, readline, texlive }:
 
-lib.fix (eukleides: stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: rec {
   pname = "eukleides";
   version = "1.5.4";
 
@@ -49,10 +49,12 @@ lib.fix (eukleides: stdenv.mkDerivation rec {
 
   outputs = [ "out" "doc" "tex" ];
 
-  passthru.tlType = "run";
-  passthru.pkgs = [ eukleides.tex ]
+  passthru = {
+    tlType = "run";
     # packages needed by euktoeps, euktopdf and eukleides.sty
-    ++ (with texlive; collection-pstricks.pkgs ++ epstopdf.pkgs ++ iftex.pkgs ++ moreverb.pkgs);
+    tlDeps = with texlive; [ collection-pstricks epstopdf iftex moreverb ];
+    pkgs = [ finalAttrs.finalPackage.tex ];
+  };
 
   meta = {
     description = "Geometry Drawing Language";

pkgs/development/tools/literate-programming/noweb/default.nix

@@ -1,6 +1,6 @@
 { lib, stdenv, fetchFromGitHub, nawk, groff, icon-lang, useIcon ? true }:
 
-lib.fix (noweb: stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: rec {
   pname = "noweb";
   version = "2.12";
 
@@ -70,8 +70,10 @@ lib.fix (noweb: stdenv.mkDerivation rec {
 
   outputs = [ "out" "tex" ];
 
-  tlType = "run";
-  passthru.pkgs = [ noweb.tex ];
+  passthru = {
+    tlType = "run";
+    pkgs = [ finalAttrs.finalPackage.tex ];
+  };
 
   meta = with lib; {
     description = "A simple, extensible literate-programming tool";

pkgs/misc/sagetex/default.nix

@@ -4,10 +4,9 @@
 , texlive
 }:
 
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: rec {
   pname = "sagetex";
   version = "3.6.1";
-  passthru.tlType = "run";
 
   src = fetchFromGitHub {
     owner = "sagemath";
@@ -30,6 +29,11 @@ stdenv.mkDerivation rec {
     cp -va *.sty *.cfg *.def "$path/"
   '';
 
+  passthru = {
+    tlType = "run";
+    pkgs = [ finalAttrs.finalPackage ];
+  };
+
   meta = with lib; {
     description = "Embed code, results of computations, and plots from Sage into LaTeX documents";
     homepage = "https://github.com/sagemath/sagetex";
@@ -37,4 +41,4 @@ stdenv.mkDerivation rec {
     maintainers = with maintainers; [ alexnortung ];
     platforms = platforms.all;
   };
-}
+})

pkgs/tools/typesetting/tex/mftrace/default.nix

@@ -20,7 +20,7 @@
   - fontforge = null (limited functionality)
 */
 
-let self = stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: rec {
   pname = "mftrace";
   version = "1.2.20";
 
@@ -39,14 +39,16 @@ let self = stdenv.mkDerivation rec {
   buildInputs = [ fontforge potrace ];
 
   postInstall = ''
-    wrapProgram $out/bin/mftrace --prefix PATH : ${lib.makeBinPath buildInputs}
+    wrapProgram $out/bin/mftrace --prefix PATH : ${lib.makeBinPath finalAttrs.buildInputs}
   '';
 
   # experimental texlive.combine support
   # (note that only the bin/ folder will be combined into texlive)
-  passthru.tlType = "bin";
-  passthru.pkgs = [ self ] ++
-    (with texlive; kpathsea.pkgs ++ t1utils.pkgs ++ metafont.pkgs);
+  passthru = {
+    tlType = "bin";
+    tlDeps = with texlive; [ kpathsea t1utils metafont ];
+    pkgs = [ finalAttrs.finalPackage ];
+  };
 
   meta = with lib; {
     description = "Scalable PostScript Fonts for MetaFont";
@@ -60,4 +62,4 @@ let self = stdenv.mkDerivation rec {
     maintainers = with maintainers; [ xworld21 ];
     platforms = platforms.all;
   };
-}; in self
+})

pkgs/tools/typesetting/tex/texlive/combine.nix

@@ -10,18 +10,18 @@ let
   pkgSet = removeAttrs args [ "pkgFilter" "extraName" "extraVersion" ] // {
     # include a fake "core" package
     core.pkgs = [
-      (bin.core.out // { pname = "core"; tlType = "bin"; })
-      (bin.core.doc // { pname = "core"; tlType = "doc"; })
+      (bin.core.out // { pname = "core"; version = "0"; tlType = "bin"; })
+      (bin.core.doc // { pname = "core"; version = "0"; tlType = "doc"; })
     ];
   };
   pkgList = rec {
     all = lib.filter pkgFilter (combinePkgs (lib.attrValues pkgSet));
     splitBin = builtins.partition (p: p.tlType == "bin") all;
-    bin = mkUniqueOutPaths splitBin.right
+    bin = splitBin.right
       ++ lib.optional
           (lib.any (p: p.tlType == "run" && p.pname == "pdfcrop") splitBin.wrong)
           (lib.getBin ghostscript);
-    nonbin = mkUniqueOutPaths splitBin.wrong;
+    nonbin = splitBin.wrong;
 
     # extra interpreters needed for shebangs, based on 2015 schemes "medium" and "tetex"
     # (omitted tk needed in pname == "epspdf", bin/epspdftk)
@@ -33,17 +33,13 @@ let
       ++ lib.optional (lib.any pkgNeedsRuby splitBin.wrong) ruby;
   };
 
-  sortedUniqueStrings = list: lib.sort (a: b: a < b) (lib.unique list);
-
-  mkUniqueOutPaths = pkgs: lib.unique
-    (map (p: p.outPath) (builtins.filter lib.isDerivation pkgs));
-
   name = "texlive-${extraName}-${bin.texliveYear}${extraVersion}";
 
   texmf = (buildEnv {
     name = "${name}-texmf";
 
-    paths = pkgList.nonbin;
+    # remove fake derivations (without 'outPath') to avoid undesired build dependencies
+    paths = lib.catAttrs "outPath" pkgList.nonbin;
 
     nativeBuildInputs = [ perl bin.core.out ];
 
@@ -72,7 +68,9 @@ in (buildEnv {
   inherit name;
 
   ignoreCollisions = false;
-  paths = pkgList.bin ++ [ doc ];
+
+  # remove fake derivations (without 'outPath') to avoid undesired build dependencies
+  paths = lib.catAttrs "outPath" pkgList.bin ++ [ doc ];
   pathsToLink = [
     "/"
     "/bin" # ensure these are writeable directories
@@ -123,9 +121,9 @@ in (buildEnv {
     # now filter hyphenation patterns and formats
   (let
     hyphens = lib.filter (p: p.hasHyphens or false && p.tlType == "run") pkgList.splitBin.wrong;
-    hyphenPNames = sortedUniqueStrings (map (p: p.pname) hyphens);
+    hyphenPNames = map (p: p.pname) hyphens;
     formats = lib.filter (p: p.hasFormats or false && p.tlType == "run") pkgList.splitBin.wrong;
-    formatPNames = sortedUniqueStrings (map (p: p.pname) formats);
+    formatPNames = map (p: p.pname) formats;
     # sed expression that prints the lines in /start/,/end/ except for /end/
     section = start: end: "/${start}/,/${end}/{ /${start}/p; /${end}/!p; };\n";
     script =

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -31,10 +31,8 @@ let
   # the set of TeX Live packages, collections, and schemes; using upstream naming
   tl = let
     orig = import ./tlpdb.nix;
-    removeSelfDep = lib.mapAttrs
-      (n: p: if p ? deps then p // { deps = lib.filter (dn: n != dn) p.deps; }
-                         else p);
-    clean = removeSelfDep (orig // {
+
+    overridden = orig // {
       # overrides of texlive.tlpdb
 
       texlive-msg-translations = orig.texlive-msg-translations // {
@@ -57,11 +55,6 @@ let
         deps = orig.collection-plaingeneric.deps ++ [ "xdvi" ];
       };
 
-      # override cyclic dependency until #167226 is fixed
-      xecjk = orig.xecjk // {
-        deps = lib.remove "ctex" orig.xecjk.deps;
-      };
-
       texdoc = orig.texdoc // {
         # build Data.tlpdb.lua (part of the 'tlType == "run"' package)
         postUnpack = ''
@@ -80,19 +73,16 @@ let
           fi
         '';
       };
-    }); # overrides
+    }; # overrides
 
-    linkDeps = lib.mapAttrs (_: attrs: attrs // lib.optionalAttrs (attrs ? deps) {
-      deps = builtins.map (n: tl.${n}) attrs.deps;
-    }); # transform [ "dep1" "dep2" ... ] into [ tl."dep1" ... ]
-
-    in lib.mapAttrs flatDeps (linkDeps clean);
+    in lib.mapAttrs mkTLPkg overridden;
     # TODO: texlive.infra for web2c config?
 
 
-  flatDeps = pname: attrs:
+  # create a TeX package: an attribute set { pkgs = [ ... ]; ... } where pkgs is a list of derivations
+  mkTLPkg = pname: attrs:
     let
-      version = attrs.version or (builtins.toString attrs.revision);
+      version = attrs.version or (toString attrs.revision);
       mkPkgV = tlType: let
         pkg = attrs // {
           sha512 = attrs.sha512.${tlType};
@@ -110,13 +100,13 @@ let
             tlType = "run";
             hasFormats = attrs.hasFormats or false;
             hasHyphens = attrs.hasHyphens or false;
+            tlDeps = map (n: tl.${n}) (attrs.deps or []);
           }
         )]
         ++ lib.optional (attrs.sha512 ? doc) (mkPkgV "doc")
         ++ lib.optional (attrs.sha512 ? source) (mkPkgV "source")
         ++ lib.optional (bin ? ${pname})
-            ( bin.${pname} // { inherit pname; tlType = "bin"; } )
-        ++ combinePkgs (attrs.deps or []);
+            ( bin.${pname} // { tlType = "bin"; } );
     };
 
   # for daily snapshots
@@ -172,6 +162,9 @@ let
           # metadata for texlive.combine
           passthru = {
             inherit pname tlType version;
+          } // lib.optionalAttrs (tlType == "run" && args ? deps) {
+            tlDeps = map (n: tl.${n}) args.deps;
+          } // lib.optionalAttrs (tlType == "run") {
             hasFormats = args.hasFormats or false;
             hasHyphens = args.hasHyphens or false;
           };
@@ -190,8 +183,20 @@ let
       );
 
   # combine a set of TL packages into a single TL meta-package
-  combinePkgs = pkgList: lib.concatLists # uniqueness is handled in `combine`
-    (builtins.map (a: a.pkgs) pkgList);
+  combinePkgs = pkgList: lib.catAttrs "pkg" (
+    let
+      # a TeX package is an attribute set { pkgs = [ ... ]; ... } where pkgs is a list of derivations
+      # the derivations make up the TeX package and optionally (for backward compatibility) its dependencies
+      tlPkgToSets = { pkgs, ... }: map ({ pname, tlType, version, outputName ? "", ... }@pkg: {
+          # outputName required to distinguish among bin.core-big outputs
+          key = "${pname}.${tlType}-${version}-${outputName}";
+          inherit pkg;
+        }) pkgs;
+      pkgListToSets = lib.concatMap tlPkgToSets; in
+    builtins.genericClosure {
+      startSet = pkgListToSets pkgList;
+      operator = { pkg, ... }: pkgListToSets (pkg.tlDeps or []);
+    });
 
 in
   tl // {