nixos-render-docs: add options commonmark converter (4d3aef76) · Commits · nix / nixpkgs

nixos/lib/make-options-doc/default.nix

+7 −4

Original line number	Diff line number	Diff line
		@@ -98,11 +98,14 @@ in rec {
		> $out
		'';

		optionsCommonMark = pkgs.runCommand "options.md" {} ''
		${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \
		--format commonmark \
		optionsCommonMark = pkgs.runCommand "options.md" {
		nativeBuildInputs = [ pkgs.nixos-render-docs ];
		} ''
		nixos-render-docs -j $NIX_BUILD_CORES options commonmark \
		--manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \
		--revision ${lib.escapeShellArg revision} \
		${optionsJSON}/share/doc/nixos/options.json \
		> $out
		$out
		'';

		optionsJSON = pkgs.runCommand "options.json"

nixos/lib/make-options-doc/generateDoc.py

+2 −31

Original line number	Diff line number	Diff line
		@@ -2,7 +2,7 @@ import argparse
		import json
		import sys

		formats = ['commonmark', 'asciidoc']
		formats = ['asciidoc']

		parser = argparse.ArgumentParser(
		description = 'Generate documentation for a set of JSON-formatted NixOS options'
		@@ -38,33 +38,6 @@ class OptionsEncoder(json.JSONEncoder):

		return super().encode(obj)

		def generate_commonmark(options):
		for (name, value) in options.items():
		print('##', name.replace('<', '<').replace('>', '>'))
		print(value['description'])
		print()
		if 'type' in value:
		print('_Type_')
		print ('```')
		print(value['type'])
		print ('```')
		print()
		print()
		if 'default' in value:
		print('_Default_')
		print('```')
		print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))
		print('```')
		print()
		print()
		if 'example' in value:
		print('_Example_')
		print('```')
		print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))
		print('```')
		print()
		print()

		# TODO: declarations: link to github
		def generate_asciidoc(options):
		for (name, value) in options.items():
		@@ -103,9 +76,7 @@ def generate_asciidoc(options):
		with open(args.nix_options_path) as nix_options_json:
		options = json.load(nix_options_json)

		if args.format == 'commonmark':
		generate_commonmark(options)
		elif args.format == 'asciidoc':
		if args.format == 'asciidoc':
		generate_asciidoc(options)
		else:
		raise Exception(f'Unsupported documentation format `--format {args.format}`')

pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/commonmark.py

0 → 100644

+231 −0

Original line number	Diff line number	Diff line
		from collections.abc import Mapping, MutableMapping, Sequence
		from dataclasses import dataclass
		from typing import Any, cast, Optional

		from .md import md_escape, md_make_code, Renderer

		import markdown_it
		from markdown_it.token import Token
		from markdown_it.utils import OptionsDict

		@dataclass(kw_only=True)
		class List:
		next_idx: Optional[int] = None
		compact: bool
		first_item_seen: bool = False

		@dataclass
		class Par:
		indent: str
		continuing: bool = False

		class CommonMarkRenderer(Renderer):
		__output__ = "commonmark"

		_parstack: list[Par]
		_link_stack: list[str]
		_list_stack: list[List]

		def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):
		super().__init__(manpage_urls, parser)
		self._parstack = [ Par("") ]
		self._link_stack = []
		self._list_stack = []

		def _enter_block(self, extra_indent: str) -> None:
		self._parstack.append(Par(self._parstack[-1].indent + extra_indent))
		def _leave_block(self) -> None:
		self._parstack.pop()
		self._parstack[-1].continuing = True
		def _break(self) -> str:
		self._parstack[-1].continuing = True
		return f"\n{self._parstack[-1].indent}"
		def _maybe_parbreak(self) -> str:
		result = f"\n{self._parstack[-1].indent}" * 2 if self._parstack[-1].continuing else ""
		self._parstack[-1].continuing = True
		return result

		def _admonition_open(self, kind: str) -> str:
		pbreak = self._maybe_parbreak()
		self._enter_block("")
		return f"{pbreak}{kind}: "
		def _admonition_close(self) -> str:
		self._leave_block()
		return ""

		def _indent_raw(self, s: str) -> str:
		if '\n' not in s:
		return s
		return f"\n{self._parstack[-1].indent}".join(s.splitlines())

		def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._parstack[-1].continuing = True
		return self._indent_raw(md_escape(token.content))
		def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._maybe_parbreak()
		def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return ""
		def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return f" {self._break()}"
		def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._break()
		def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._parstack[-1].continuing = True
		return md_make_code(token.content)
		def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self.fence(token, tokens, i, options, env)
		def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._parstack[-1].continuing = True
		self._link_stack.append(cast(str, token.attrs['href']))
		return "["
		def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return f"]({md_escape(self._link_stack.pop())})"
		def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		lst = self._list_stack[-1]
		lbreak = "" if not lst.first_item_seen else self._break() * (1 if lst.compact else 2)
		lst.first_item_seen = True
		head = " -"
		if lst.next_idx is not None:
		head = f" {lst.next_idx}."
		lst.next_idx += 1
		self._enter_block(" " * (len(head) + 1))
		return f'{lbreak}{head} '
		def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._leave_block()
		return ""
		def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._list_stack.append(List(compact=bool(token.meta['compact'])))
		return self._maybe_parbreak()
		def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._list_stack.pop()
		return ""
		def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return "*"
		def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return "*"
		def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return "**"
		def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return "**"
		def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		code = token.content
		if code.endswith('\n'):
		code = code[:-1]
		pbreak = self._maybe_parbreak()
		return pbreak + self._indent_raw(md_make_code(code, info=token.info, multiline=True))
		def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		pbreak = self._maybe_parbreak()
		self._enter_block("> ")
		return pbreak + "> "
		def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._leave_block()
		return ""
		def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_open("Note")
		def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_close()
		def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_open("Caution")
		def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_close()
		def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_open("Important")
		def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_close()
		def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_open("Tip")
		def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_close()
		def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_open("Warning")
		def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return self._admonition_close()
		def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._list_stack.append(List(compact=False))
		return ""
		def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._list_stack.pop()
		return ""
		def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		pbreak = self._maybe_parbreak()
		self._enter_block(" ")
		# add an opening zero-width non-joiner to separate our emphasis from possible
		# emphasis in the provided term
		return f'{pbreak} - *{chr(0x200C)}'
		def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return f"{chr(0x200C)}*"
		def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._parstack[-1].continuing = True
		return ""
		def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._leave_block()
		return ""
		def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._parstack[-1].continuing = True
		content = md_make_code(token.content)
		if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)):
		return f"[{content}]({url})"
		return content # no roles in regular commonmark
		def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		# there's no way we can emit attrspans correctly in all cases. we could use inline
		# html for ids, but that would not round-trip. same holds for classes. since this
		# renderer is only used for approximate options export and all of these things are
		# not allowed in options we can ignore them for now.
		return ""
		def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return ""
		def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return token.markup + " "
		def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		return "\n"
		def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._list_stack.append(
		List(next_idx = cast(int, token.attrs.get('start', 1)),
		compact = bool(token.meta['compact'])))
		return self._maybe_parbreak()
		def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,
		env: MutableMapping[str, Any]) -> str:
		self._list_stack.pop()
		return ""

pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py

+76 −0

Original line number	Diff line number	Diff line
		@@ -13,6 +13,7 @@ from xml.sax.saxutils import escape, quoteattr
		import markdown_it

		from . import parallel
		from .commonmark import CommonMarkRenderer
		from .docbook import DocBookRenderer, make_xml_id
		from .manpage import ManpageRenderer, man_escape
		from .md import Converter, md_escape, md_make_code
		@@ -422,6 +423,59 @@ class ManpageConverter(BaseConverter):

		return "\n".join(result)

		class OptionsCommonMarkRenderer(OptionDocsRestrictions, CommonMarkRenderer):
		pass

		class CommonMarkConverter(BaseConverter):
		__renderer__ = OptionsCommonMarkRenderer
		__option_block_separator__ = ""

		def _parallel_render_prepare(self) -> Any:
		return (self._manpage_urls, self._revision, self._markdown_by_default)
		@classmethod
		def _parallel_render_init_worker(cls, a: Any) -> CommonMarkConverter:
		return cls(*a)

		def _render_code(self, option: dict[str, Any], key: str) -> list[str]:
		# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
		# incorrect, since users rely on it.
		if lit := option_is(option, key, 'literalDocBook'):
		return [ f"{key.capitalize()}: {lit['text']}" ]
		else:
		return super()._render_code(option, key)

		def _render_description(self, desc: str \| dict[str, Any]) -> list[str]:
		# NOTE this duplicates the old direct-paste behavior, even if it is somewhat
		# incorrect, since users rely on it.
		if isinstance(desc, str) and not self._markdown_by_default:
		return [ desc ]
		else:
		return super()._render_description(desc)

		def _related_packages_header(self) -> list[str]:
		return [ "Related packages:" ]

		def _decl_def_header(self, header: str) -> list[str]:
		return [ f"{header}:" ]

		def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]:
		if href is not None:
		return [ f" - [{md_escape(name)}]({href})" ]
		return [ f" - {md_escape(name)}" ]

		def _decl_def_footer(self) -> list[str]:
		return []

		def finalize(self) -> str:
		result = []

		for (name, opt) in self._sorted_options():
		result.append(f"## {md_escape(name)}\n")
		result += opt.lines
		result.append("\n\n")

		return "\n".join(result)

		def _build_cli_db(p: argparse.ArgumentParser) -> None:
		p.add_argument('--manpage-urls', required=True)
		p.add_argument('--revision', required=True)
		@@ -437,6 +491,13 @@ def _build_cli_manpage(p: argparse.ArgumentParser) -> None:
		p.add_argument("infile")
		p.add_argument("outfile")

		def _build_cli_commonmark(p: argparse.ArgumentParser) -> None:
		p.add_argument('--manpage-urls', required=True)
		p.add_argument('--revision', required=True)
		p.add_argument('--markdown-by-default', default=False, action='store_true')
		p.add_argument("infile")
		p.add_argument("outfile")

		def _run_cli_db(args: argparse.Namespace) -> None:
		with open(args.manpage_urls, 'r') as manpage_urls:
		md = DocBookConverter(
		@@ -464,15 +525,30 @@ def _run_cli_manpage(args: argparse.Namespace) -> None:
		with open(args.outfile, 'w') as f:
		f.write(md.finalize())

		def _run_cli_commonmark(args: argparse.Namespace) -> None:
		with open(args.manpage_urls, 'r') as manpage_urls:
		md = CommonMarkConverter(
		json.load(manpage_urls),
		revision = args.revision,
		markdown_by_default = args.markdown_by_default)

		with open(args.infile, 'r') as f:
		md.add_options(json.load(f))
		with open(args.outfile, 'w') as f:
		f.write(md.finalize())

		def build_cli(p: argparse.ArgumentParser) -> None:
		formats = p.add_subparsers(dest='format', required=True)
		_build_cli_db(formats.add_parser('docbook'))
		_build_cli_manpage(formats.add_parser('manpage'))
		_build_cli_commonmark(formats.add_parser('commonmark'))

		def run_cli(args: argparse.Namespace) -> None:
		if args.format == 'docbook':
		_run_cli_db(args)
		elif args.format == 'manpage':
		_run_cli_manpage(args)
		elif args.format == 'commonmark':
		_run_cli_commonmark(args)
		else:
		raise RuntimeError('format not hooked up', args)

pkgs/tools/nix/nixos-render-docs/src/tests/test_commonmark.py

0 → 100644

+92 −0

Original line number	Diff line number	Diff line
		import nixos_render_docs

		from sample_md import sample1

		from typing import Mapping, Optional

		import markdown_it

		class Converter(nixos_render_docs.md.Converter):
		__renderer__ = nixos_render_docs.commonmark.CommonMarkRenderer

		# NOTE: in these tests we represent trailing spaces by ` ` and replace them with real space later,
		# since a number of editors will strip trailing whitespace on save and that would break the tests.

		def test_indented_fence() -> None:
		c = Converter({})
		s = """\
		> - ```foo
		> thing
		>
		> rest
		> ```\
		""".replace(' ', ' ')
		assert c._render(s) == s

		def test_full() -> None:
		c = Converter({ 'man(1)': 'http://example.org' })
		assert c._render(sample1) == f"""\
		Warning: foo

		Note: nested

		[
		multiline
		](link)

		[` man(1) `](http://example.org) reference

		some nested anchors

		emph strong nesting emph and strong* and ` code `*

		- wide bullet

		- list

		1. wide ordered

		2. list

		- narrow bullet
		- list

		1. narrow ordered
		2. list

		> quotes
		>
		> > with nesting
		> >
		> > ```
		> > nested code block
		> > ```
		>
		> - and lists
		> - ```
		> containing code
		> ```
		>
		> and more quote

		100. list starting at 100
		101. goes on

		- ‌deflist‌

		> with a quote
		> and stuff

		```
		code block
		```

		```
		fenced block
		```

		text

		- ‌more stuff in same deflist‌

		foo""".replace(' ', ' ')