diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py index b8dc8e7f08eb..e28d516fed39 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/__init__.py @@ -16,129 +16,13 @@ from mdit_py_plugins.deflist import deflist_plugin from mdit_py_plugins.myst_role import myst_role_plugin from xml.sax.saxutils import escape, quoteattr -from .docbook import make_xml_id, DocBookRenderer -from .md import md_escape -from .options import option_is - -class Converter: - def __init__(self, manpage_urls: Dict[str, str]): - self._manpage_urls = frozendict(manpage_urls) - - self._md = markdown_it.MarkdownIt( - "commonmark", - { - 'maxNesting': 100, # default is 20 - 'html': False, # not useful since we target many formats - 'typographer': True, # required for smartquotes - }, - renderer_cls=lambda parser: DocBookRenderer(self._manpage_urls, parser) - ) - # TODO maybe fork the plugin and have only a single rule for all? - self._md.use(container_plugin, name="{.note}") - self._md.use(container_plugin, name="{.important}") - self._md.use(container_plugin, name="{.warning}") - self._md.use(deflist_plugin) - self._md.use(myst_role_plugin) - self._md.enable(["smartquotes", "replacements"]) - - def render(self, src: str) -> str: - return self._md.render(src) - -md = Converter(json.load(open(os.getenv('MANPAGE_URLS')))) - -# converts in-place! -def convertMD(options: Dict[str, Any]) -> str: - def convertCode(name: str, option: Dict[str, Any], key: str): - if option_is(option, key, 'literalMD'): - option[key] = md.render(f"*{key.capitalize()}:*\n{option[key]['text']}") - elif option_is(option, key, 'literalExpression'): - code = option[key]['text'] - # for multi-line code blocks we only have to count ` runs at the beginning - # of a line, but this is much easier. - multiline = '\n' in code - longest, current = (0, 0) - for c in code: - current = current + 1 if c == '`' else 0 - longest = max(current, longest) - # inline literals need a space to separate ticks from content, code blocks - # need newlines. inline literals need one extra tick, code blocks need three. - ticks, sep = ('`' * (longest + (3 if multiline else 1)), '\n' if multiline else ' ') - code = f"{ticks}{sep}{code}{sep}{ticks}" - option[key] = md.render(f"*{key.capitalize()}:*\n{code}") - elif option_is(option, key, 'literalDocBook'): - option[key] = f"{key.capitalize()}: {option[key]['text']}" - elif key in option: - raise Exception(f"{name} {key} has unrecognized type", option[key]) - - for (name, option) in options.items(): - try: - if option_is(option, 'description', 'mdDoc'): - option['description'] = md.render(option['description']['text']) - elif markdownByDefault: - option['description'] = md.render(option['description']) - else: - option['description'] = ("" + - option['description'] + - "") - - convertCode(name, option, 'example') - convertCode(name, option, 'default') - - if typ := option.get('type'): - ro = " *(read only)*" if option.get('readOnly', False) else "" - option['type'] = md.render(f'*Type:* {md_escape(typ)}{ro}') - - if 'relatedPackages' in option: - option['relatedPackages'] = md.render(option['relatedPackages']) - except Exception as e: - raise Exception(f"Failed to render option {name}") from e - - return options +from .options import DocBookConverter def need_env(n): if n not in os.environ: raise RuntimeError("required environment variable not set", n) return os.environ[n] -OTD_REVISION = need_env('OTD_REVISION') -OTD_DOCUMENT_TYPE = need_env('OTD_DOCUMENT_TYPE') -OTD_VARIABLE_LIST_ID = need_env('OTD_VARIABLE_LIST_ID') -OTD_OPTION_ID_PREFIX = need_env('OTD_OPTION_ID_PREFIX') - -def print_decl_def(header, locs): - print(f"""{header}:""") - print(f"""""") - for loc in locs: - # locations can be either plain strings (specific to nixpkgs), or attrsets - # { name = "foo/bar.nix"; url = "https://github.com/....."; } - if isinstance(loc, str): - # Hyperlink the filename either to the NixOS github - # repository (if it’s a module and we have a revision number), - # or to the local filesystem. - if not loc.startswith('/'): - if OTD_REVISION == 'local': - href = f"https://github.com/NixOS/nixpkgs/blob/master/{loc}" - else: - href = f"https://github.com/NixOS/nixpkgs/blob/{OTD_REVISION}/{loc}" - else: - href = f"file://{loc}" - # Print the filename and make it user-friendly by replacing the - # /nix/store/ prefix by the default location of nixos - # sources. - if not loc.startswith('/'): - name = f"" - elif loc.contains('nixops') and loc.contains('/nix/'): - name = f"" - else: - name = loc - print(f"""""") - print(escape(name)) - print(f"""""") - else: - href = f" xlink:href={quoteattr(loc['url'])}" if 'url' in loc else "" - print(f"""{escape(loc['name'])}""") - print(f"""""") - def main(): markdownByDefault = False optOffset = 0 @@ -147,49 +31,15 @@ def main(): optOffset += 1 markdownByDefault = True - options = convertMD(json.load(open(sys.argv[1 + optOffset], 'r'))) + md = DocBookConverter( + json.load(open(os.getenv('MANPAGE_URLS'))), + revision = need_env('OTD_REVISION'), + document_type = need_env('OTD_DOCUMENT_TYPE'), + varlist_id = need_env('OTD_VARIABLE_LIST_ID'), + id_prefix = need_env('OTD_OPTION_ID_PREFIX'), + markdown_by_default = markdownByDefault + ) - keys = list(options.keys()) - keys.sort(key=lambda opt: [ (0 if p.startswith("enable") else 1 if p.startswith("package") else 2, p) - for p in options[opt]['loc'] ]) - - print(f"""""") - if OTD_DOCUMENT_TYPE == 'appendix': - print("""""") - print(""" Configuration Options""") - print(f"""""") - - for name in keys: - opt = options[name] - id = OTD_OPTION_ID_PREFIX + make_xml_id(name) - print(f"""""") - # NOTE adding extra spaces here introduces spaces into xref link expansions - print(f"""""", end='') - print(f"""""", end='') - print(f"""""") - print(f"""""") - print(opt['description']) - if typ := opt.get('type'): - print(typ) - if default := opt.get('default'): - print(default) - if example := opt.get('example'): - print(example) - if related := opt.get('relatedPackages'): - print(f"""""") - print(f""" Related packages:""") - print(f"""""") - print(related) - if decl := opt.get('declarations'): - print_decl_def("Declared by", decl) - if defs := opt.get('definitions'): - print_decl_def("Defined by", defs) - print(f"""""") - print(f"""""") - - print("""""") - if OTD_DOCUMENT_TYPE == 'appendix': - print("""""") + options = json.load(open(sys.argv[1 + optOffset], 'r')) + md.add_options(options) + print(md.finalize()) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py index 01e9659ce27e..50e09fc6db4b 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py @@ -1,9 +1,13 @@ from collections.abc import Mapping, MutableMapping, Sequence -from typing import Any, Optional +from frozendict import frozendict # type: ignore[attr-defined] +from typing import Any, Callable, Optional import markdown_it from markdown_it.token import Token from markdown_it.utils import OptionsDict +from mdit_py_plugins.container import container_plugin # type: ignore[attr-defined] +from mdit_py_plugins.deflist import deflist_plugin # type: ignore[attr-defined] +from mdit_py_plugins.myst_role import myst_role_plugin # type: ignore[attr-defined] _md_escape_table = { ord('*'): '\\*', @@ -175,3 +179,29 @@ class Renderer(markdown_it.renderer.RendererProtocol): def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, env: MutableMapping[str, Any]) -> str: raise RuntimeError("md token not supported", token) + +class Converter: + __renderer__: Callable[[Mapping[str, str], markdown_it.MarkdownIt], Renderer] + + def __init__(self, manpage_urls: Mapping[str, str]): + self._manpage_urls = frozendict(manpage_urls) + + self._md = markdown_it.MarkdownIt( + "commonmark", + { + 'maxNesting': 100, # default is 20 + 'html': False, # not useful since we target many formats + 'typographer': True, # required for smartquotes + }, + renderer_cls=lambda parser: self.__renderer__(self._manpage_urls, parser) + ) + # TODO maybe fork the plugin and have only a single rule for all? + self._md.use(container_plugin, name="{.note}") + self._md.use(container_plugin, name="{.important}") + self._md.use(container_plugin, name="{.warning}") + self._md.use(deflist_plugin) + self._md.use(myst_role_plugin) + self._md.enable(["smartquotes", "replacements"]) + + def _render(self, src: str) -> str: + return self._md.render(src) # type: ignore[no-any-return] diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index 0bba89a1e9a0..218683d1591c 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -1,6 +1,9 @@ -from typing import Optional +from typing import Any, Optional +from xml.sax.saxutils import escape, quoteattr -from .types import Option +from .docbook import DocBookRenderer, make_xml_id +from .md import Converter, md_escape +from .types import OptionLoc, Option, RenderedOption def option_is(option: Option, key: str, typ: str) -> Optional[dict[str, str]]: if key not in option: @@ -10,3 +13,187 @@ def option_is(option: Option, key: str, typ: str) -> Optional[dict[str, str]]: if option[key].get('_type') != typ: # type: ignore[union-attr] return None return option[key] # type: ignore[return-value] + +class DocBookConverter(Converter): + __renderer__ = DocBookRenderer + _options: dict[str, RenderedOption] + + def __init__(self, manpage_urls: dict[str, str], + revision: str, + document_type: str, + varlist_id: str, + id_prefix: str, + markdown_by_default: bool): + super().__init__(manpage_urls) + self._options = {} + self._revision = revision + self._document_type = document_type + self._varlist_id = varlist_id + self._id_prefix = id_prefix + self._markdown_by_default = markdown_by_default + + def _format_decl_def_loc(self, loc: OptionLoc) -> tuple[Optional[str], str]: + # locations can be either plain strings (specific to nixpkgs), or attrsets + # { name = "foo/bar.nix"; url = "https://github.com/....."; } + if isinstance(loc, str): + # Hyperlink the filename either to the NixOS github + # repository (if it’s a module and we have a revision number), + # or to the local filesystem. + if not loc.startswith('/'): + if self._revision == 'local': + href = f"https://github.com/NixOS/nixpkgs/blob/master/{loc}" + else: + href = f"https://github.com/NixOS/nixpkgs/blob/{self._revision}/{loc}" + else: + href = f"file://{loc}" + # Print the filename and make it user-friendly by replacing the + # /nix/store/ prefix by the default location of nixos + # sources. + if not loc.startswith('/'): + name = f"" + elif 'nixops' in loc and '/nix/' in loc: + name = f"" + else: + name = loc + return (href, name) + else: + return (loc['url'] if 'url' in loc else None, loc['name']) + + def _render_decl_def(self, header: str, locs: list[OptionLoc]) -> list[str]: + result = [] + result += self._decl_def_header(header) + for loc in locs: + href, name = self._format_decl_def_loc(loc) + result += self._decl_def_entry(href, name) + result += self._decl_def_footer() + return result + + def _render_code(self, option: Option, key: str) -> list[str]: + if lit := option_is(option, key, 'literalDocBook'): + return [ f"{key.capitalize()}: {lit['text']}" ] + elif lit := option_is(option, key, 'literalMD'): + return [ self._render(f"*{key.capitalize()}:*\n{lit['text']}") ] + elif lit := option_is(option, key, 'literalExpression'): + code = lit['text'] + # for multi-line code blocks we only have to count ` runs at the beginning + # of a line, but this is much easier. + multiline = '\n' in code + longest, current = (0, 0) + for c in code: + current = current + 1 if c == '`' else 0 + longest = max(current, longest) + # inline literals need a space to separate ticks from content, code blocks + # need newlines. inline literals need one extra tick, code blocks need three. + ticks, sep = ('`' * (longest + (3 if multiline else 1)), '\n' if multiline else ' ') + code = f"{ticks}{sep}{code}{sep}{ticks}" + return [ self._render(f"*{key.capitalize()}:*\n{code}") ] + elif key in option: + raise Exception(f"{key} has unrecognized type", option[key]) + else: + return [] + + def _render_description(self, desc: str | dict[str, str]) -> list[str]: + if isinstance(desc, str) and not self._markdown_by_default: + return [ f"{desc}" ] + elif isinstance(desc, str) and self._markdown_by_default: + return [ self._render(desc) ] + elif isinstance(desc, dict) and desc.get('_type') == 'mdDoc': + return [ self._render(desc['text']) ] + else: + raise Exception("description has unrecognized type", desc) + + def _convert_one(self, option: dict[str, Any]) -> list[str]: + result = [] + + if desc := option.get('description'): + result += self._render_description(desc) + if typ := option.get('type'): + ro = " *(read only)*" if option.get('readOnly', False) else "" + result.append(self._render(f"*Type:* {md_escape(typ)}{ro}")) + + result += self._render_code(option, 'default') + result += self._render_code(option, 'example') + + if related := option.get('relatedPackages'): + result += self._related_packages_header() + result.append(self._render(related)) + if decl := option.get('declarations'): + result += self._render_decl_def("Declared by", decl) + if defs := option.get('definitions'): + result += self._render_decl_def("Defined by", defs) + + return result + + def add_options(self, options: dict[str, Any]) -> None: + for (name, option) in options.items(): + try: + self._options[name] = RenderedOption(option['loc'], self._convert_one(option)) + except Exception as e: + raise Exception(f"Failed to render option {name}") from e + + def _related_packages_header(self) -> list[str]: + return [ + "", + " Related packages:", + "", + ] + + def _decl_def_header(self, header: str) -> list[str]: + return [ + f"{header}:", + "" + ] + + def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]: + if href is not None: + href = " xlink:href=" + quoteattr(href) + return [ + f"", + escape(name), + "" + ] + + def _decl_def_footer(self) -> list[str]: + return [ "" ] + + def finalize(self) -> str: + keys = list(self._options.keys()) + keys.sort(key=lambda opt: [ (0 if p.startswith("enable") else 1 if p.startswith("package") else 2, p) + for p in self._options[opt].loc ]) + + result = [] + + result.append('') + if self._document_type == 'appendix': + result += [ + '', + ' Configuration Options', + ] + result += [ + f'', + ] + + for name in keys: + id = make_xml_id(self._id_prefix + name) + result += [ + "", + # NOTE adding extra spaces here introduces spaces into xref link expansions + (f"" + + f""), + "" + ] + result += self._options[name].lines + result += [ + "", + "" + ] + + result.append("") + if self._document_type == 'appendix': + result.append("") + + return "\n".join(result) diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/types.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/types.py index f6825ac936a9..9c1ec4f65bf5 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/types.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/types.py @@ -1,4 +1,7 @@ -from typing import Optional, Tuple +from typing import Optional, Tuple, NamedTuple OptionLoc = str | dict[str, str] Option = dict[str, str | dict[str, str] | list[OptionLoc]] + +RenderedOption = NamedTuple('RenderedOption', [('loc', list[str]), + ('lines', list[str])])