From 4670400309f5fb898f4b63e7091d62cc531be46f Mon Sep 17 00:00:00 2001 From: pennae Date: Thu, 18 Nov 2021 20:21:53 +0100 Subject: [PATCH 1/4] nixos/lib/make-options-doc: generate asciidoc/md in derivations use the json file derivation we already have to also generate the asciidoc and md options docs instead of formatting the options in nix. docbook docs are already produced in derivations. the new script produce the exact same output as the old in-nix generation. --- nixos/lib/make-options-doc/default.nix | 74 +++---------------- .../lib/make-options-doc/generateAsciiDoc.py | 37 ++++++++++ .../make-options-doc/generateCommonMark.py | 27 +++++++ 3 files changed, 75 insertions(+), 63 deletions(-) create mode 100644 nixos/lib/make-options-doc/generateAsciiDoc.py create mode 100644 nixos/lib/make-options-doc/generateCommonMark.py diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index e058e70f3888..498a60134ab6 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -93,72 +93,20 @@ let optionsNix = builtins.listToAttrs (map (o: { name = o.name; value = removeAttrs o ["name" "visible" "internal"]; }) optionsList); - # TODO: declarations: link to github - singleAsciiDoc = name: value: '' - == ${name} - - ${value.description} - - [discrete] - === details - - Type:: ${value.type} - ${ if lib.hasAttr "default" value - then '' - Default:: - + - ---- - ${builtins.toJSON value.default} - ---- - '' - else "No Default:: {blank}" - } - ${ if value.readOnly - then "Read Only:: {blank}" - else "" - } - ${ if lib.hasAttr "example" value - then '' - Example:: - + - ---- - ${builtins.toJSON value.example} - ---- - '' - else "No Example:: {blank}" - } - ''; - - singleMDDoc = name: value: '' - ## ${lib.escape [ "<" ">" ] name} - ${value.description} - - ${lib.optionalString (value ? type) '' - *_Type_*: - ${value.type} - ''} - - ${lib.optionalString (value ? default) '' - *_Default_* - ``` - ${builtins.toJSON value.default} - ``` - ''} - - ${lib.optionalString (value ? example) '' - *_Example_* - ``` - ${builtins.toJSON value.example} - ``` - ''} - ''; - -in { +in rec { inherit optionsNix; - optionsAsciiDoc = lib.concatStringsSep "\n" (lib.mapAttrsToList singleAsciiDoc optionsNix); + optionsAsciiDoc = pkgs.runCommand "options.adoc" {} '' + ${pkgs.python3Minimal}/bin/python ${./generateAsciiDoc.py} \ + < ${optionsJSON}/share/doc/nixos/options.json \ + > $out + ''; - optionsMDDoc = lib.concatStringsSep "\n" (lib.mapAttrsToList singleMDDoc optionsNix); + optionsCommonMark = pkgs.runCommand "options.md" {} '' + ${pkgs.python3Minimal}/bin/python ${./generateCommonMark.py} \ + < ${optionsJSON}/share/doc/nixos/options.json \ + > $out + ''; optionsJSON = pkgs.runCommand "options.json" { meta.description = "List of NixOS options in JSON format"; diff --git a/nixos/lib/make-options-doc/generateAsciiDoc.py b/nixos/lib/make-options-doc/generateAsciiDoc.py new file mode 100644 index 000000000000..48eadd248c5a --- /dev/null +++ b/nixos/lib/make-options-doc/generateAsciiDoc.py @@ -0,0 +1,37 @@ +import json +import sys + +options = json.load(sys.stdin) +# TODO: declarations: link to github +for (name, value) in options.items(): + print(f'== {name}') + print() + print(value['description']) + print() + print('[discrete]') + print('=== details') + print() + print(f'Type:: {value["type"]}') + if 'default' in value: + print('Default::') + print('+') + print('----') + print(json.dumps(value['default'], ensure_ascii=False, separators=(',', ':'))) + print('----') + print() + else: + print('No Default:: {blank}') + if value['readOnly']: + print('Read Only:: {blank}') + else: + print() + if 'example' in value: + print('Example::') + print('+') + print('----') + print(json.dumps(value['example'], ensure_ascii=False, separators=(',', ':'))) + print('----') + print() + else: + print('No Example:: {blank}') + print() diff --git a/nixos/lib/make-options-doc/generateCommonMark.py b/nixos/lib/make-options-doc/generateCommonMark.py new file mode 100644 index 000000000000..404e53b0df9c --- /dev/null +++ b/nixos/lib/make-options-doc/generateCommonMark.py @@ -0,0 +1,27 @@ +import json +import sys + +options = json.load(sys.stdin) +for (name, value) in options.items(): + print('##', name.replace('<', '\\<').replace('>', '\\>')) + print(value['description']) + print() + if 'type' in value: + print('*_Type_*:') + print(value['type']) + print() + print() + if 'default' in value: + print('*_Default_*') + print('```') + print(json.dumps(value['default'], ensure_ascii=False, separators=(',', ':'))) + print('```') + print() + print() + if 'example' in value: + print('*_Example_*') + print('```') + print(json.dumps(value['example'], ensure_ascii=False, separators=(',', ':'))) + print('```') + print() + print() From c533b018631d68da543f20ff52775c562fa8b13d Mon Sep 17 00:00:00 2001 From: pennae Date: Thu, 18 Nov 2021 20:26:12 +0100 Subject: [PATCH 2/4] nixos/doc/manual: remove non-matching optionsXML inherit --- nixos/doc/manual/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix index 151743d9fb58..31b6da01c6bd 100644 --- a/nixos/doc/manual/default.nix +++ b/nixos/doc/manual/default.nix @@ -161,7 +161,7 @@ let in rec { inherit generatedSources; - inherit (optionsDoc) optionsJSON optionsXML optionsDocBook; + inherit (optionsDoc) optionsJSON optionsDocBook; # Generate the NixOS manual. manualHTML = runCommand "nixos-manual-html" From 9b97a2ea883a53f89a2c22d93edb7a218c194330 Mon Sep 17 00:00:00 2001 From: pennae Date: Thu, 18 Nov 2021 20:28:52 +0100 Subject: [PATCH 3/4] nix/lib/make-options-doc: remove nix-level sorting there are no remaining users of sorted option lists except the docbook build, which sorts its input separately. --- nixos/lib/make-options-doc/default.nix | 18 ++---------------- nixos/lib/make-options-doc/sortXML.py | 1 - 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 498a60134ab6..4e3f62544eb3 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -31,7 +31,7 @@ let else if lib.isFunction x then "" else x; - optionsListDesc = lib.flip map optionsListVisible + optionsList = lib.flip map optionsListVisible (opt: transformOptions opt // lib.optionalAttrs (opt ? example) { example = substFunction opt.example; } // lib.optionalAttrs (opt ? default) { default = substFunction opt.default; } @@ -69,27 +69,13 @@ let + ""; in "${lib.concatStringsSep "\n" (map (p: describe (unpack p)) packages)}"; - # Custom "less" that pushes up all the things ending in ".enable*" - # and ".package*" - optionLess = a: b: - let - ise = lib.hasPrefix "enable"; - isp = lib.hasPrefix "package"; - cmp = lib.splitByAndCompare ise lib.compare - (lib.splitByAndCompare isp lib.compare lib.compare); - in lib.compareLists cmp a.loc b.loc < 0; - # Remove invisible and internal options. optionsListVisible = lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList options); - # Customly sort option list for the man page. - # Always ensure that the sort order matches sortXML.py! - optionsList = lib.sort optionLess optionsListDesc; - # Convert the list of options into an XML file. # This file is *not* sorted sorted to save on eval time, since the docbook XML # and the manpage depend on it and thus we evaluate this on every system rebuild. - optionsXML = builtins.toFile "options.xml" (builtins.toXML optionsListDesc); + optionsXML = builtins.toFile "options.xml" (builtins.toXML optionsList); optionsNix = builtins.listToAttrs (map (o: { name = o.name; value = removeAttrs o ["name" "visible" "internal"]; }) optionsList); diff --git a/nixos/lib/make-options-doc/sortXML.py b/nixos/lib/make-options-doc/sortXML.py index 717820788c94..e63ff3538b3f 100644 --- a/nixos/lib/make-options-doc/sortXML.py +++ b/nixos/lib/make-options-doc/sortXML.py @@ -19,7 +19,6 @@ def sortKey(opt): for p in opt.findall('attr[@name="loc"]/list/string') ] -# always ensure that the sort order matches the order used in the nix expression! options.sort(key=sortKey) doc = ET.Element("expr") From 027f7e1b7f4552c9c9f0ff909a6863d5bf347a72 Mon Sep 17 00:00:00 2001 From: pennae Date: Thu, 18 Nov 2021 23:00:03 +0100 Subject: [PATCH 4/4] nixos/lib/make-options-doc: generate options.xml from options.json to do this we must replace derivations with attrsets in make-options-doc, since xml can represent derivations differently from attrset but json cannot. this also given asciidoc and mddoc the ability to handle derivation differently, which they previously didn't have. --- nixos/lib/make-options-doc/default.nix | 39 ++++++++++++------- .../make-options-doc/options-to-docbook.xsl | 2 +- .../lib/make-options-doc/optionsJSONtoXML.nix | 6 +++ 3 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 nixos/lib/make-options-doc/optionsJSONtoXML.nix diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index 4e3f62544eb3..493006c92e7f 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -24,18 +24,25 @@ }: let - # Replace functions by the string - substFunction = x: - if builtins.isAttrs x then lib.mapAttrs (name: substFunction) x - else if builtins.isList x then map substFunction x + # Make a value safe for JSON. Functions are replaced by the string "", + # derivations are replaced with an attrset + # { _type = "derivation"; name = ; }. + # We need to handle derivations specially because consumers want to know about them, + # but we can't easily use the type,name subset of keys (since type is often used as + # a module option and might cause confusion). Use _type,name instead to the same + # effect, since _type is already used by the module system. + substSpecial = x: + if lib.isDerivation x then { _type = "derivation"; name = x.name; } + else if builtins.isAttrs x then lib.mapAttrs (name: substSpecial) x + else if builtins.isList x then map substSpecial x else if lib.isFunction x then "" else x; optionsList = lib.flip map optionsListVisible (opt: transformOptions opt - // lib.optionalAttrs (opt ? example) { example = substFunction opt.example; } - // lib.optionalAttrs (opt ? default) { default = substFunction opt.default; } - // lib.optionalAttrs (opt ? type) { type = substFunction opt.type; } + // lib.optionalAttrs (opt ? example) { example = substSpecial opt.example; } + // lib.optionalAttrs (opt ? default) { default = substSpecial opt.default; } + // lib.optionalAttrs (opt ? type) { type = substSpecial opt.type; } // lib.optionalAttrs (opt ? relatedPackages && opt.relatedPackages != []) { relatedPackages = genRelatedPackages opt.relatedPackages opt.name; } ); @@ -72,11 +79,6 @@ let # Remove invisible and internal options. optionsListVisible = lib.filter (opt: opt.visible && !opt.internal) (lib.optionAttrSetToDocList options); - # Convert the list of options into an XML file. - # This file is *not* sorted sorted to save on eval time, since the docbook XML - # and the manpage depend on it and thus we evaluate this on every system rebuild. - optionsXML = builtins.toFile "options.xml" (builtins.toXML optionsList); - optionsNix = builtins.listToAttrs (map (o: { name = o.name; value = removeAttrs o ["name" "visible" "internal"]; }) optionsList); in rec { @@ -110,7 +112,18 @@ in rec { mkdir -p $out/nix-support echo "file json $dst/options.json" >> $out/nix-support/hydra-build-products echo "file json-br $dst/options.json.br" >> $out/nix-support/hydra-build-products - ''; # */ + ''; + + # Convert options.json into an XML file. + # The actual generation of the xml file is done in nix purely for the convenience + # of not having to generate the xml some other way + optionsXML = pkgs.runCommand "options.xml" {} '' + ${pkgs.nix}/bin/nix-instantiate \ + --store dummy:// \ + --eval --xml --strict ${./optionsJSONtoXML.nix} \ + --argstr file ${optionsJSON}/share/doc/nixos/options.json \ + > "$out" + ''; optionsDocBook = pkgs.runCommand "options-docbook.xml" {} '' optionsXML=${optionsXML} diff --git a/nixos/lib/make-options-doc/options-to-docbook.xsl b/nixos/lib/make-options-doc/options-to-docbook.xsl index da4cd164bf20..b9ac26450514 100644 --- a/nixos/lib/make-options-doc/options-to-docbook.xsl +++ b/nixos/lib/make-options-doc/options-to-docbook.xsl @@ -189,7 +189,7 @@ - + (build of ) diff --git a/nixos/lib/make-options-doc/optionsJSONtoXML.nix b/nixos/lib/make-options-doc/optionsJSONtoXML.nix new file mode 100644 index 000000000000..ba50c5f898b5 --- /dev/null +++ b/nixos/lib/make-options-doc/optionsJSONtoXML.nix @@ -0,0 +1,6 @@ +{ file }: + +builtins.attrValues + (builtins.mapAttrs + (name: def: def // { inherit name; }) + (builtins.fromJSON (builtins.readFile file)))