gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos-render-docs: don't use markdown-it RendererProtocol

Browse Source 
our renderers carry significantly more state than markdown-it wants to
easily cater for, and the html renderer will need even more state still.
relying on the markdown-it-provided rendering functions has already
proven to be a nuisance, and since parsing and rendering are split well
enough we can just replace the rendering part with our own stuff outright.

this also frees us from the tyranny of having to set instance variables
before calling super().__init__ just to make sure that the renderer
creation callback has access to everything it needs.
This commit is contained in:
pennae
2023-02-17 17:49:08 +01:00
parent 3794c04d79
commit 0236dcb59f
13 changed files with 101 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py
View File

@@ -5,7 +5,6 @@ from urllib.parse import quote
from .md import Renderer
import markdown_it
from markdown_it.token import Token
from markdown_it.utils import OptionsDict
@@ -59,8 +58,8 @@ class AsciiDocRenderer(Renderer):
_list_stack: list[List]
_attrspans: list[str]
def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):
super().__init__(manpage_urls, parser)
def __init__(self, manpage_urls: Mapping[str, str]):
super().__init__(manpage_urls)
self._parstack = [ Par("\n\n", "====") ]
self._list_stack = []
self._attrspans = []

5
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py
View File

@@ -4,7 +4,6 @@ from typing import Any, cast, Optional
from .md import md_escape, md_make_code, Renderer
import markdown_it
from markdown_it.token import Token
from markdown_it.utils import OptionsDict
@@ -26,8 +25,8 @@ class CommonMarkRenderer(Renderer):
_link_stack: list[str]
_list_stack: list[List]
def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):
super().__init__(manpage_urls, parser)
def __init__(self, manpage_urls: Mapping[str, str]):
super().__init__(manpage_urls)
self._parstack = [ Par("") ]
self._link_stack = []
self._list_stack = []

5
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/docbook.py
View File

@@ -32,14 +32,13 @@ class Heading(NamedTuple):
partintro_closed: bool = False
class DocBookRenderer(Renderer):
__output__ = "docbook"
_link_tags: list[str]
_deflists: list[Deflist]
_headings: list[Heading]
_attrspans: list[str]
def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):
super().__init__(manpage_urls, parser)
def __init__(self, manpage_urls: Mapping[str, str]):
super().__init__(manpage_urls)
self._link_tags = []
self._deflists = []
self._headings = []

7
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manpage.py
View File

@@ -75,8 +75,6 @@ class List:
# horizontal motion in a line) we do attempt to copy the style of mdoc(7) semantic requests
# as appropriate for each markup element.
class ManpageRenderer(Renderer):
__output__ = "man"
# whether to emit mdoc .Ql equivalents for inline code or just the contents. this is
# mainly used by the options manpage converter to not emit extra quotes in defaults
# and examples where it's already clear from context that the following text is code.
@@ -90,9 +88,8 @@ class ManpageRenderer(Renderer):
_list_stack: list[List]
_font_stack: list[str]
def __init__(self, manpage_urls: Mapping[str, str], href_targets: dict[str, str],
parser: Optional[markdown_it.MarkdownIt] = None):
super().__init__(manpage_urls, parser)
def __init__(self, manpage_urls: Mapping[str, str], href_targets: dict[str, str]):
super().__init__(manpage_urls)
self._href_targets = href_targets
self._link_stack = []
self._do_parbreak_stack = []

18
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/manual.py
View File

@@ -18,9 +18,8 @@ from .md import Converter
class ManualDocBookRenderer(DocBookRenderer):
_toplevel_tag: str
def __init__(self, toplevel_tag: str, manpage_urls: Mapping[str, str],
parser: Optional[markdown_it.MarkdownIt] = None):
super().__init__(manpage_urls, parser)
def __init__(self, toplevel_tag: str, manpage_urls: Mapping[str, str]):
super().__init__(manpage_urls)
self._toplevel_tag = toplevel_tag
self.rules |= {
'included_sections': lambda *args: self._included_thing("section", *args),
@@ -92,7 +91,7 @@ class ManualDocBookRenderer(DocBookRenderer):
self._headings[-1] = self._headings[-1]._replace(partintro_closed=True)
# must nest properly for structural includes. this requires saving at least
# the headings stack, but creating new renderers is cheap and much easier.
r = ManualDocBookRenderer(tag, self._manpage_urls, None)
r = ManualDocBookRenderer(tag, self._manpage_urls)
for (included, path) in token.meta['included']:
try:
result.append(r.render(included, options, env))
@@ -118,16 +117,13 @@ class ManualDocBookRenderer(DocBookRenderer):
info = f" language={quoteattr(token.info)}" if token.info != "" else ""
return f"<programlisting{info}>\n{escape(token.content)}</programlisting>"
class DocBookConverter(Converter):
def __renderer__(self, manpage_urls: Mapping[str, str],
parser: Optional[markdown_it.MarkdownIt]) -> ManualDocBookRenderer:
return ManualDocBookRenderer('book', manpage_urls, parser)
class DocBookConverter(Converter[ManualDocBookRenderer]):
_base_paths: list[Path]
_revision: str
def __init__(self, manpage_urls: Mapping[str, str], revision: str):
super().__init__(manpage_urls)
super().__init__()
self._renderer = ManualDocBookRenderer('book', manpage_urls)
self._revision = revision
def convert(self, file: Path) -> str:
@@ -195,7 +191,7 @@ class DocBookConverter(Converter):
try:
conv = options.DocBookConverter(
self._manpage_urls, self._revision, False, 'fragment', varlist_id, id_prefix)
self._renderer._manpage_urls, self._revision, False, 'fragment', varlist_id, id_prefix)
with open(self._base_paths[-1].parent / source, 'r') as f:
conv.add_options(json.load(f))
token.meta['rendered-options'] = conv.finalize(fragment=True)

32
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/md.py
View File

@@ -1,6 +1,6 @@
from abc import ABC
from collections.abc import Mapping, MutableMapping, Sequence
from typing import Any, Callable, cast, get_args, Iterable, Literal, NoReturn, Optional
from typing import Any, Callable, cast, Generic, get_args, Iterable, Literal, NoReturn, Optional, TypeVar
import dataclasses
import re
@@ -44,11 +44,11 @@ AttrBlockKind = Literal['admonition', 'example']
AdmonitionKind = Literal["note", "caution", "tip", "important", "warning"]
class Renderer(markdown_it.renderer.RendererProtocol):
class Renderer:
_admonitions: dict[AdmonitionKind, tuple[RenderFn, RenderFn]]
_admonition_stack: list[AdmonitionKind]
def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):
def __init__(self, manpage_urls: Mapping[str, str]):
self._manpage_urls = manpage_urls
self.rules = {
'text': self.text,
@@ -466,12 +466,26 @@ def _block_attr(md: markdown_it.MarkdownIt) -> None:
md.core.ruler.push("block_attr", block_attr)
class Converter(ABC):
__renderer__: Callable[[Mapping[str, str], markdown_it.MarkdownIt], Renderer]
TR = TypeVar('TR', bound='Renderer')
def __init__(self, manpage_urls: Mapping[str, str]):
self._manpage_urls = manpage_urls
class Converter(ABC, Generic[TR]):
# we explicitly disable markdown-it rendering support and use our own entirely.
# rendering is well separated from parsing and our renderers carry much more state than
# markdown-it easily acknowledges as 'good' (unless we used the untyped env args to
# shuttle that state around, which is very fragile)
class ForbiddenRenderer(markdown_it.renderer.RendererProtocol):
__output__ = "none"
def __init__(self, parser: Optional[markdown_it.MarkdownIt]):
pass
def render(self, tokens: Sequence[Token], options: OptionsDict,
env: MutableMapping[str, Any]) -> str:
raise NotImplementedError("do not use Converter._md.renderer. 'tis a silly place")
_renderer: TR
def __init__(self) -> None:
self._md = markdown_it.MarkdownIt(
"commonmark",
{
@@ -479,7 +493,7 @@ class Converter(ABC):
'html': False, # not useful since we target many formats
'typographer': True, # required for smartquotes
},
renderer_cls=lambda parser: self.__renderer__(self._manpage_urls, parser)
renderer_cls=self.ForbiddenRenderer
)
self._md.use(
container_plugin,
@@ -502,4 +516,4 @@ class Converter(ABC):
def _render(self, src: str, env: Optional[MutableMapping[str, Any]] = None) -> str:
env = {} if env is None else env
tokens = self._parse(src, env)
return self._md.renderer.render(tokens, self._md.options, env) # type: ignore[no-any-return]
return self._renderer.render(tokens, self._md.options, env)

60
pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py
View File

@@ -7,12 +7,13 @@ from abc import abstractmethod
from collections.abc import Mapping, MutableMapping, Sequence
from markdown_it.utils import OptionsDict
from markdown_it.token import Token
from typing import Any, Optional
from typing import Any, Generic, Optional
from urllib.parse import quote
from xml.sax.saxutils import escape, quoteattr
import markdown_it
from . import md
from . import parallel
from .asciidoc import AsciiDocRenderer, asciidoc_escape
from .commonmark import CommonMarkRenderer
@@ -30,15 +31,13 @@ def option_is(option: Option, key: str, typ: str) -> Optional[dict[str, str]]:
return None
return option[key] # type: ignore[return-value]
class BaseConverter(Converter):
class BaseConverter(Converter[md.TR], Generic[md.TR]):
__option_block_separator__: str
_options: dict[str, RenderedOption]
def __init__(self, manpage_urls: Mapping[str, str],
revision: str,
markdown_by_default: bool):
super().__init__(manpage_urls)
def __init__(self, revision: str, markdown_by_default: bool):
super().__init__()
self._options = {}
self._revision = revision
self._markdown_by_default = markdown_by_default
@@ -153,7 +152,7 @@ class BaseConverter(Converter):
# since it's good enough so far.
@classmethod
@abstractmethod
def _parallel_render_init_worker(cls, a: Any) -> BaseConverter: raise NotImplementedError()
def _parallel_render_init_worker(cls, a: Any) -> BaseConverter[md.TR]: raise NotImplementedError()
def _render_option(self, name: str, option: dict[str, Any]) -> RenderedOption:
try:
@@ -162,7 +161,7 @@ class BaseConverter(Converter):
raise Exception(f"Failed to render option {name}") from e
@classmethod
def _parallel_render_step(cls, s: BaseConverter, a: Any) -> RenderedOption:
def _parallel_render_step(cls, s: BaseConverter[md.TR], a: Any) -> RenderedOption:
return s._render_option(*a)
def add_options(self, options: dict[str, Any]) -> None:
@@ -199,8 +198,7 @@ class OptionsDocBookRenderer(OptionDocsRestrictions, DocBookRenderer):
token.meta['compact'] = False
return super().bullet_list_open(token, tokens, i, options, env)
class DocBookConverter(BaseConverter):
__renderer__ = OptionsDocBookRenderer
class DocBookConverter(BaseConverter[OptionsDocBookRenderer]):
__option_block_separator__ = ""
def __init__(self, manpage_urls: Mapping[str, str],
@@ -209,13 +207,14 @@ class DocBookConverter(BaseConverter):
document_type: str,
varlist_id: str,
id_prefix: str):
super().__init__(manpage_urls, revision, markdown_by_default)
super().__init__(revision, markdown_by_default)
self._renderer = OptionsDocBookRenderer(manpage_urls)
self._document_type = document_type
self._varlist_id = varlist_id
self._id_prefix = id_prefix
def _parallel_render_prepare(self) -> Any:
return (self._manpage_urls, self._revision, self._markdown_by_default, self._document_type,
return (self._renderer._manpage_urls, self._revision, self._markdown_by_default, self._document_type,
self._varlist_id, self._id_prefix)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> DocBookConverter:
@@ -300,11 +299,7 @@ class DocBookConverter(BaseConverter):
class OptionsManpageRenderer(OptionDocsRestrictions, ManpageRenderer):
pass
class ManpageConverter(BaseConverter):
def __renderer__(self, manpage_urls: Mapping[str, str],
parser: Optional[markdown_it.MarkdownIt] = None) -> OptionsManpageRenderer:
return OptionsManpageRenderer(manpage_urls, self._options_by_id, parser)
class ManpageConverter(BaseConverter[OptionsManpageRenderer]):
__option_block_separator__ = ".sp"
_options_by_id: dict[str, str]
@@ -314,8 +309,9 @@ class ManpageConverter(BaseConverter):
*,
# only for parallel rendering
_options_by_id: Optional[dict[str, str]] = None):
super().__init__(revision, markdown_by_default)
self._options_by_id = _options_by_id or {}
super().__init__({}, revision, markdown_by_default)
self._renderer = OptionsManpageRenderer({}, self._options_by_id)
def _parallel_render_prepare(self) -> Any:
return ((self._revision, self._markdown_by_default), { '_options_by_id': self._options_by_id })
@@ -324,10 +320,9 @@ class ManpageConverter(BaseConverter):
return cls(*a[0], **a[1])
def _render_option(self, name: str, option: dict[str, Any]) -> RenderedOption:
assert isinstance(self._md.renderer, OptionsManpageRenderer)
links = self._md.renderer.link_footnotes = []
links = self._renderer.link_footnotes = []
result = super()._render_option(name, option)
self._md.renderer.link_footnotes = None
self._renderer.link_footnotes = None
return result._replace(links=links)
def add_options(self, options: dict[str, Any]) -> None:
@@ -339,12 +334,11 @@ class ManpageConverter(BaseConverter):
if lit := option_is(option, key, 'literalDocBook'):
raise RuntimeError("can't render manpages in the presence of docbook")
else:
assert isinstance(self._md.renderer, OptionsManpageRenderer)
try:
self._md.renderer.inline_code_is_quoted = False
self._renderer.inline_code_is_quoted = False
return super()._render_code(option, key)
finally:
self._md.renderer.inline_code_is_quoted = True
self._renderer.inline_code_is_quoted = True
def _render_description(self, desc: str | dict[str, Any]) -> list[str]:
if isinstance(desc, str) and not self._markdown_by_default:
@@ -428,12 +422,15 @@ class ManpageConverter(BaseConverter):
class OptionsCommonMarkRenderer(OptionDocsRestrictions, CommonMarkRenderer):
pass
class CommonMarkConverter(BaseConverter):
__renderer__ = OptionsCommonMarkRenderer
class CommonMarkConverter(BaseConverter[OptionsCommonMarkRenderer]):
__option_block_separator__ = ""
def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool):
super().__init__(revision, markdown_by_default)
self._renderer = OptionsCommonMarkRenderer(manpage_urls)
def _parallel_render_prepare(self) -> Any:
return (self._manpage_urls, self._revision, self._markdown_by_default)
return (self._renderer._manpage_urls, self._revision, self._markdown_by_default)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter:
return cls(*a)
@@ -481,12 +478,15 @@ class CommonMarkConverter(BaseConverter):
class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer):
pass
class AsciiDocConverter(BaseConverter):
__renderer__ = AsciiDocRenderer
class AsciiDocConverter(BaseConverter[OptionsAsciiDocRenderer]):
__option_block_separator__ = ""
def __init__(self, manpage_urls: Mapping[str, str], revision: str, markdown_by_default: bool):
super().__init__(revision, markdown_by_default)
self._renderer = OptionsAsciiDocRenderer(manpage_urls)
def _parallel_render_prepare(self) -> Any:
return (self._manpage_urls, self._revision, self._markdown_by_default)
return (self._renderer._manpage_urls, self._revision, self._markdown_by_default)
@classmethod
def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter:
return cls(*a)