Reapply "pkgs-lib/formats: Use .attrs.json where possible"

This expands on https://github.com/NixOS/nixpkgs/pull/498928 that
introduced __structuredAttrs here by actually using data in
`.attrs.json` when it makes sense, instead of relying on environment
variables.

This reverts commit bf6ada5d78.
This reapplies commit 691dc02df0.

pkgs/pkgs-lib/formats.nix

@@ -146,14 +146,14 @@ optionalAttrs allowAliases aliases
           runCommand name
             {
               nativeBuildInputs = [ jq ];
-              value = toJSON value;
+              inherit value;
               preferLocalBuild = true;
               __structuredAttrs = true;
             }
+            # NIX_ATTRS_JSON_FILE won't have `value` if it's null, but jq returns null for missing properties anyway
+            # jsonNull test keeps this in check
             ''
-              valuePath="$TMPDIR/value"
-              printf "%s" "$value" > "$valuePath"
-              jq . "$valuePath" > $out
+              jq .value "$NIX_ATTRS_JSON_FILE" > $out
             ''
         ) { };
 
@@ -171,14 +171,16 @@ optionalAttrs allowAliases aliases
           runCommand name
             {
               nativeBuildInputs = [ remarshal_0_17 ];
-              value = toJSON value;
+              inherit value;
               preferLocalBuild = true;
               __structuredAttrs = true;
             }
             ''
-              valuePath="$TMPDIR/value"
-              printf "%s" "$value" > "$valuePath"
-              json2yaml "$valuePath" "$out"
+              json2yaml ${
+                # attributes with null values are omitted from the JSON with structured attrs
+                # yaml_1_1Null test keeps this in check
+                if value == null then ''<(echo "null")'' else ''--unwrap value "$NIX_ATTRS_JSON_FILE"''
+              } "$out"
             ''
         ) { };
 
@@ -196,14 +198,16 @@ optionalAttrs allowAliases aliases
           runCommand name
             {
               nativeBuildInputs = [ remarshal ];
-              value = toJSON value;
+              inherit value;
               preferLocalBuild = true;
               __structuredAttrs = true;
             }
             ''
-              valuePath="$TMPDIR/value"
-              printf "%s" "$value" > "$valuePath"
-              json2yaml "$valuePath" "$out"
+              json2yaml ${
+                # attributes with null values are omitted from the JSON with structured attrs
+                # yaml_1_2Null test keeps this in check
+                if value == null then ''<(echo "null")'' else ''--unwrap value "$NIX_ATTRS_JSON_FILE"''
+              } "$out"
             ''
         ) { };
 
@@ -960,8 +964,10 @@ optionalAttrs allowAliases aliases
                 python3
                 black
               ];
-              imports = toJSON (value._imports or [ ]);
-              value = toJSON (removeAttrs value [ "_imports" ]);
+              imports = value._imports or [ ];
+              # value must be an attrset, type would verify that,
+              # otherwise removeAttrs will fail.
+              value = removeAttrs value [ "_imports" ];
               pythonGen = pkgs.writeText "pythonGen" ''
                 import json
                 import os
@@ -984,26 +990,20 @@ optionalAttrs allowAliases aliases
                     else:
                         return repr(value)
 
-                with open(os.environ["importsPath"], "r") as f:
-                    imports = json.load(f)
-                    if imports is not None:
-                        for i in imports:
+                with open(os.environ["NIX_ATTRS_JSON_FILE"], "r") as f:
+                    attrs = json.load(f)
+                    if attrs["imports"] is not None:
+                        for i in attrs["imports"]:
                             print(f"import {i}")
                         print()
 
-                with open(os.environ["valuePath"], "r") as f:
-                    for key, value in json.load(f).items():
+                    for key, value in attrs["value"].items():
                         print(f"{key} = {recursive_repr(value)}")
               '';
               preferLocalBuild = true;
               __structuredAttrs = true;
             }
             ''
-              export importsPath="$TMPDIR/imports"
-              printf "%s" "$imports" > "$importsPath"
-              export valuePath="$TMPDIR/value"
-              printf "%s" "$value" > "$valuePath"
-              cat "$valuePath"
               python3 "$pythonGen" > $out
               black $out
             ''
@@ -1039,14 +1039,16 @@ optionalAttrs allowAliases aliases
                   python3Packages.xmltodict
                   libxml2Python
                 ];
-                value = toJSON value;
+                inherit value;
                 pythonGen = pkgs.writeText "pythonGen" ''
                   import json
                   import os
                   import xmltodict
 
-                  with open(os.environ["valuePath"], "r") as f:
-                      print(xmltodict.unparse(json.load(f), full_document=${
+                  with open(os.environ["NIX_ATTRS_JSON_FILE"], "r") as f:
+                      value = json.load(f).get("value")
+                      assert type(value) is dict, "value must be an attrset"
+                      print(xmltodict.unparse(value, full_document=${
                         if withHeader then "True" else "False"
                       }, pretty=True, indent=" " * 2))
                 '';
@@ -1054,8 +1056,6 @@ optionalAttrs allowAliases aliases
                 __structuredAttrs = true;
               }
               ''
-                export valuePath="$TMPDIR/value"
-                printf "%s" "$value" > "$valuePath"
                 python3 "$pythonGen" > $out
                 xmllint $out > /dev/null
               ''

pkgs/pkgs-lib/formats/configobj/default.nix

@@ -3,10 +3,6 @@
   pkgs,
 }:
 let
-  inherit (lib)
-    toJSON
-    ;
-
   inherit (lib.types)
     serializableValueWith
     ;
@@ -25,12 +21,12 @@ in
               (pkgs.python3.withPackages (ps: [ ps.configobj ]))
             ];
 
-            valuesJSON = toJSON value;
+            inherit value;
             __structuredAttrs = true;
             strictDeps = true;
           }
           ''
-            printf "%s" "$valuesJSON" | python3 ${./generate.py} > "$out"
+            python3 ${./generate.py} > "$out"
           '';
     };
 }

pkgs/pkgs-lib/formats/configobj/generate.py

@@ -1,7 +1,10 @@
 import json
+import os
 import sys
 from configobj import ConfigObj
 
 config = ConfigObj(interpolation=False, encoding="UTF8")
-config.update(json.load(sys.stdin))
+with open(os.environ["NIX_ATTRS_JSON_FILE"]) as f:
+    value = json.load(f).get("value")
+config.update(value)
 config.write(sys.stdout.buffer)