From 7601f8a10854992b594fc85ecb4685a1a0ec3346 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 23 Jun 2026 15:42:14 +0200 Subject: [PATCH 1/2] pkgs-lib/formats: Add tests for serializing null values In the effort to pass values directly as structured values, we ran into issue with serializing null's. This adds tests for null for all formats that support it and negative tests for some others. See https://github.com/NixOS/nixpkgs/pull/524404#issuecomment-4665694636 --- pkgs/pkgs-lib/formats.nix | 8 +++- pkgs/pkgs-lib/tests/formats.nix | 83 +++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix index ef7d104f20e4..a184b0cbce53 100644 --- a/pkgs/pkgs-lib/formats.nix +++ b/pkgs/pkgs-lib/formats.nix @@ -1017,7 +1017,13 @@ optionalAttrs allowAliases aliases }: if format == "badgerfish" then { - type = serializableValueWith { typeName = "XML"; }; + type = + attrsOf (serializableValueWith { + typeName = "XML"; + }) + // { + description = "XML value"; + }; generate = name: value: diff --git a/pkgs/pkgs-lib/tests/formats.nix b/pkgs/pkgs-lib/tests/formats.nix index ee6d817287c2..242fd64bc70a 100644 --- a/pkgs/pkgs-lib/tests/formats.nix +++ b/pkgs/pkgs-lib/tests/formats.nix @@ -142,6 +142,14 @@ runBuildTests { ''; }; + jsonNull = shouldPass { + format = formats.json { }; + input = null; + expected = '' + null + ''; + }; + yaml_1_1Atoms = shouldPass { format = formats.yaml_1_1 { }; input = { @@ -176,6 +184,15 @@ runBuildTests { ''; }; + yaml_1_1Null = shouldPass { + format = formats.yaml_1_1 { }; + input = null; + expected = '' + null + ... + ''; + }; + yaml_1_2Atoms = shouldPass { format = formats.yaml_1_2 { }; input = { @@ -210,6 +227,16 @@ runBuildTests { ''; }; + yaml_1_2Null = shouldPass { + format = formats.yaml_1_2 { }; + input = null; + # nixfmt insists on removing indentation, so force it with ${" "} + expected = '' + + ${" "}null + ''; + }; + iniAtoms = shouldPass { format = formats.ini { }; input = { @@ -879,6 +906,14 @@ runBuildTests { ''; }; + cdnNull = shouldPass { + format = formats.cdn { }; + input = null; + expected = '' + null: null + ''; + }; + # This test is responsible for # 1. testing type coercions # 2. providing a more readable example test @@ -989,6 +1024,14 @@ runBuildTests { ''; }; + luaNull = shouldPass { + format = formats.lua { }; + input = null; + expected = '' + return nil + ''; + }; + nixConfAtoms = shouldPass { format = formats.nixConf { package = pkgs.nix; @@ -1014,6 +1057,15 @@ runBuildTests { ''; }; + nixConfNull = shouldFail { + format = formats.nixConf { + package = pkgs.nix; + version = pkgs.nix.version; + extraOptions = "ignore-try = false"; + }; + input = null; + }; + phpAtoms = shouldPass rec { format = formats.php { finalVariable = "config"; }; input = { @@ -1043,6 +1095,16 @@ runBuildTests { ''; }; + phpNull = shouldPass { + format = formats.php { finalVariable = "config"; }; + input = null; + expected = '' + ''; }; + PlistNull = shouldPass { + format = formats.plist { }; + input = null; + expected = '' + + + + + ''; + }; + hcl1Atoms = shouldPass { format = formats.hcl1 { }; input = { From efdeeebcb9127136cdb83f2212d785f0d33b1407 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 23 Jun 2026 14:40:23 +0200 Subject: [PATCH 2/2] Reapply "pkgs-lib/formats: Use .attrs.json where possible" This expands on https://github.com/NixOS/nixpkgs/pull/498928 that introduced __structuredAttrs here by actually using data in `.attrs.json` when it makes sense, instead of relying on environment variables. This reverts commit bf6ada5d786e8aa4faff128e7e73a4792c2e99fe. This reapplies commit 691dc02df0111c320cd6697aacd4b113ba632e40. --- pkgs/pkgs-lib/formats.nix | 60 ++++++++++----------- pkgs/pkgs-lib/formats/configobj/default.nix | 8 +-- pkgs/pkgs-lib/formats/configobj/generate.py | 5 +- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/pkgs/pkgs-lib/formats.nix b/pkgs/pkgs-lib/formats.nix index a184b0cbce53..f23f740ab7a9 100644 --- a/pkgs/pkgs-lib/formats.nix +++ b/pkgs/pkgs-lib/formats.nix @@ -146,14 +146,14 @@ optionalAttrs allowAliases aliases runCommand name { nativeBuildInputs = [ jq ]; - value = toJSON value; + inherit value; preferLocalBuild = true; __structuredAttrs = true; } + # NIX_ATTRS_JSON_FILE won't have `value` if it's null, but jq returns null for missing properties anyway + # jsonNull test keeps this in check '' - valuePath="$TMPDIR/value" - printf "%s" "$value" > "$valuePath" - jq . "$valuePath" > $out + jq .value "$NIX_ATTRS_JSON_FILE" > $out '' ) { }; @@ -171,14 +171,16 @@ optionalAttrs allowAliases aliases runCommand name { nativeBuildInputs = [ remarshal_0_17 ]; - value = toJSON value; + inherit value; preferLocalBuild = true; __structuredAttrs = true; } '' - valuePath="$TMPDIR/value" - printf "%s" "$value" > "$valuePath" - json2yaml "$valuePath" "$out" + json2yaml ${ + # attributes with null values are omitted from the JSON with structured attrs + # yaml_1_1Null test keeps this in check + if value == null then ''<(echo "null")'' else ''--unwrap value "$NIX_ATTRS_JSON_FILE"'' + } "$out" '' ) { }; @@ -196,14 +198,16 @@ optionalAttrs allowAliases aliases runCommand name { nativeBuildInputs = [ remarshal ]; - value = toJSON value; + inherit value; preferLocalBuild = true; __structuredAttrs = true; } '' - valuePath="$TMPDIR/value" - printf "%s" "$value" > "$valuePath" - json2yaml "$valuePath" "$out" + json2yaml ${ + # attributes with null values are omitted from the JSON with structured attrs + # yaml_1_2Null test keeps this in check + if value == null then ''<(echo "null")'' else ''--unwrap value "$NIX_ATTRS_JSON_FILE"'' + } "$out" '' ) { }; @@ -960,8 +964,10 @@ optionalAttrs allowAliases aliases python3 black ]; - imports = toJSON (value._imports or [ ]); - value = toJSON (removeAttrs value [ "_imports" ]); + imports = value._imports or [ ]; + # value must be an attrset, type would verify that, + # otherwise removeAttrs will fail. + value = removeAttrs value [ "_imports" ]; pythonGen = pkgs.writeText "pythonGen" '' import json import os @@ -984,26 +990,20 @@ optionalAttrs allowAliases aliases else: return repr(value) - with open(os.environ["importsPath"], "r") as f: - imports = json.load(f) - if imports is not None: - for i in imports: + with open(os.environ["NIX_ATTRS_JSON_FILE"], "r") as f: + attrs = json.load(f) + if attrs["imports"] is not None: + for i in attrs["imports"]: print(f"import {i}") print() - with open(os.environ["valuePath"], "r") as f: - for key, value in json.load(f).items(): + for key, value in attrs["value"].items(): print(f"{key} = {recursive_repr(value)}") ''; preferLocalBuild = true; __structuredAttrs = true; } '' - export importsPath="$TMPDIR/imports" - printf "%s" "$imports" > "$importsPath" - export valuePath="$TMPDIR/value" - printf "%s" "$value" > "$valuePath" - cat "$valuePath" python3 "$pythonGen" > $out black $out '' @@ -1039,14 +1039,16 @@ optionalAttrs allowAliases aliases python3Packages.xmltodict libxml2Python ]; - value = toJSON value; + inherit value; pythonGen = pkgs.writeText "pythonGen" '' import json import os import xmltodict - with open(os.environ["valuePath"], "r") as f: - print(xmltodict.unparse(json.load(f), full_document=${ + with open(os.environ["NIX_ATTRS_JSON_FILE"], "r") as f: + value = json.load(f).get("value") + assert type(value) is dict, "value must be an attrset" + print(xmltodict.unparse(value, full_document=${ if withHeader then "True" else "False" }, pretty=True, indent=" " * 2)) ''; @@ -1054,8 +1056,6 @@ optionalAttrs allowAliases aliases __structuredAttrs = true; } '' - export valuePath="$TMPDIR/value" - printf "%s" "$value" > "$valuePath" python3 "$pythonGen" > $out xmllint $out > /dev/null '' diff --git a/pkgs/pkgs-lib/formats/configobj/default.nix b/pkgs/pkgs-lib/formats/configobj/default.nix index 8179c4412e5f..227d8e7aa990 100644 --- a/pkgs/pkgs-lib/formats/configobj/default.nix +++ b/pkgs/pkgs-lib/formats/configobj/default.nix @@ -3,10 +3,6 @@ pkgs, }: let - inherit (lib) - toJSON - ; - inherit (lib.types) serializableValueWith ; @@ -25,12 +21,12 @@ in (pkgs.python3.withPackages (ps: [ ps.configobj ])) ]; - valuesJSON = toJSON value; + inherit value; __structuredAttrs = true; strictDeps = true; } '' - printf "%s" "$valuesJSON" | python3 ${./generate.py} > "$out" + python3 ${./generate.py} > "$out" ''; }; } diff --git a/pkgs/pkgs-lib/formats/configobj/generate.py b/pkgs/pkgs-lib/formats/configobj/generate.py index 5863396c120d..e531e68a4fbb 100644 --- a/pkgs/pkgs-lib/formats/configobj/generate.py +++ b/pkgs/pkgs-lib/formats/configobj/generate.py @@ -1,7 +1,10 @@ import json +import os import sys from configobj import ConfigObj config = ConfigObj(interpolation=False, encoding="UTF8") -config.update(json.load(sys.stdin)) +with open(os.environ["NIX_ATTRS_JSON_FILE"]) as f: + value = json.load(f).get("value") +config.update(value) config.write(sys.stdout.buffer)