Commit 417dd2ad authored by pennae's avatar pennae Committed by pennae
Browse files

nixos-render-docs: add options asciidoc converter 

same reasoning as for the earlier commonmark converter.
parent 4d3aef76

nixos/lib/make-options-doc/default.nix

+7 −4
Original line number Diff line number Diff line
@@ -91,11 +91,14 @@ let 
in rec {

  inherit optionsNix;



  optionsAsciiDoc = pkgs.runCommand "options.adoc" {} ''

    ${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \

      --format asciidoc \

  optionsAsciiDoc = pkgs.runCommand "options.adoc" {

    nativeBuildInputs = [ pkgs.nixos-render-docs ];

  } ''

    nixos-render-docs -j $NIX_BUILD_CORES options asciidoc \

      --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \

      --revision ${lib.escapeShellArg revision} \

      ${optionsJSON}/share/doc/nixos/options.json \

      > $out

      $out

  '';



  optionsCommonMark = pkgs.runCommand "options.md" {

nixos/lib/make-options-doc/generateDoc.py

deleted100644 → 0
+0 −83
Original line number Diff line number Diff line 
import argparse

import json

import sys



formats = ['asciidoc']



parser = argparse.ArgumentParser(

    description = 'Generate documentation for a set of JSON-formatted NixOS options'

)

parser.add_argument(

    'nix_options_path',

    help = 'a path to a JSON file containing the NixOS options'

)

parser.add_argument(

    '-f',

    '--format',

    choices = formats,

    required = True,

    help = f'the documentation format to generate'

)



args = parser.parse_args()



class OptionsEncoder(json.JSONEncoder):

    def encode(self, obj):

        # Unpack literal expressions and other Nix types.

        # Don't escape the strings: they were escaped when initially serialized to JSON.

        if isinstance(obj, dict):

            _type = obj.get('_type')

            if _type is not None:

                if _type == 'literalExpression' or _type == 'literalDocBook':

                    return obj['text']



                if _type == 'derivation':

                    return obj['name']



                raise Exception(f'Unexpected type `{_type}` in {json.dumps(obj)}')



        return super().encode(obj)



# TODO: declarations: link to github

def generate_asciidoc(options):

    for (name, value) in options.items():

        print(f'== {name}')

        print()

        print(value['description'])

        print()

        print('[discrete]')

        print('=== details')

        print()

        print(f'Type:: {value["type"]}')

        if 'default' in value:

            print('Default::')

            print('+')

            print('----')

            print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))

            print('----')

            print()

        else:

            print('No Default:: {blank}')

        if value['readOnly']:

            print('Read Only:: {blank}')

        else:

            print()

        if 'example' in value:

            print('Example::')

            print('+')

            print('----')

            print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':')))

            print('----')

            print()

        else:

            print('No Example:: {blank}')

        print()



with open(args.nix_options_path) as nix_options_json:

    options = json.load(nix_options_json)



    if args.format == 'asciidoc':

        generate_asciidoc(options)

    else:

        raise Exception(f'Unsupported documentation format `--format {args.format}`')

pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py

0 → 100644
+262 −0
Original line number Diff line number Diff line 
from collections.abc import Mapping, MutableMapping, Sequence

from dataclasses import dataclass

from typing import Any, cast, Optional

from urllib.parse import quote



from .md import Renderer



import markdown_it

from markdown_it.token import Token

from markdown_it.utils import OptionsDict



_asciidoc_escapes = {

    # escape all dots, just in case one is pasted at SOL

    ord('.'): "{zwsp}.",

    # may be replaced by typographic variants

    ord("'"): "{apos}",

    ord('"'): "{quot}",

    # passthrough character

    ord('+'): "{plus}",

    # table marker

    ord('|'): "{vbar}",

    # xml entity reference

    ord('&'): "{amp}",

    # crossrefs. < needs extra escaping because links break in odd ways if they start with it

    ord('<'): "{zwsp}+<+{zwsp}",

    ord('>'): "{gt}",

    # anchors, links, block attributes

    ord('['): "{startsb}",

    ord(']'): "{endsb}",

    # superscript, subscript

    ord('^'): "{caret}",

    ord('~'): "{tilde}",

    # bold

    ord('*'): "{asterisk}",

    # backslash

    ord('\\'): "{backslash}",

    # inline code

    ord('`'): "{backtick}",

}

def asciidoc_escape(s: str) -> str:

    s = s.translate(_asciidoc_escapes)

    # :: is deflist item, ;; is has a replacement but no idea why

    return s.replace("::", "{two-colons}").replace(";;", "{two-semicolons}")



@dataclass(kw_only=True)

class List:

    head: str



@dataclass()

class Par:

    sep: str

    block_delim: str

    continuing: bool = False



class AsciiDocRenderer(Renderer):

    __output__ = "asciidoc"



    _parstack: list[Par]

    _list_stack: list[List]

    _attrspans: list[str]



    def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None):

        super().__init__(manpage_urls, parser)

        self._parstack = [ Par("\n\n", "====") ]

        self._list_stack = []

        self._attrspans = []



    def _enter_block(self, is_list: bool) -> None:

        self._parstack.append(Par("\n+\n" if is_list else "\n\n", self._parstack[-1].block_delim + "="))

    def _leave_block(self) -> None:

        self._parstack.pop()

    def _break(self, force: bool = False) -> str:

        result = self._parstack[-1].sep if force or self._parstack[-1].continuing else ""

        self._parstack[-1].continuing = True

        return result



    def _admonition_open(self, kind: str) -> str:

        pbreak = self._break()

        self._enter_block(False)

        return f"{pbreak}[{kind}]\n{self._parstack[-2].block_delim}\n"

    def _admonition_close(self) -> str:

        self._leave_block()

        return f"\n{self._parstack[-1].block_delim}\n"



    def _list_open(self, token: Token, head: str) -> str:

        attrs = []

        if (idx := token.attrs.get('start')) is not None:

            attrs.append(f"start={idx}")

        if token.meta['compact']:

            attrs.append('options="compact"')

        if self._list_stack:

            head *= len(self._list_stack[0].head) + 1

        self._list_stack.append(List(head=head))

        return f"{self._break()}[{','.join(attrs)}]"

    def _list_close(self) -> str:

        self._list_stack.pop()

        return ""



    def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

             env: MutableMapping[str, Any]) -> str:

        self._parstack[-1].continuing = True

        return asciidoc_escape(token.content)

    def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                       env: MutableMapping[str, Any]) -> str:

        return self._break()

    def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                        env: MutableMapping[str, Any]) -> str:

        return ""

    def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                  env: MutableMapping[str, Any]) -> str:

        return " +\n"

    def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                  env: MutableMapping[str, Any]) -> str:

        return f" "

    def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                    env: MutableMapping[str, Any]) -> str:

        self._parstack[-1].continuing = True

        return f"``{asciidoc_escape(token.content)}``"

    def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                   env: MutableMapping[str, Any]) -> str:

        return self.fence(token, tokens, i, options, env)

    def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                  env: MutableMapping[str, Any]) -> str:

        self._parstack[-1].continuing = True

        return f"link:{quote(cast(str, token.attrs['href']), safe='/:')}["

    def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                   env: MutableMapping[str, Any]) -> str:

        return "]"

    def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                       env: MutableMapping[str, Any]) -> str:

        self._enter_block(True)

        # allow the next token to be a block or an inline.

        return f'\n{self._list_stack[-1].head} {{empty}}'

    def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                        env: MutableMapping[str, Any]) -> str:

        self._leave_block()

        return "\n"

    def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                         env: MutableMapping[str, Any]) -> str:

        return self._list_open(token, '*')

    def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                          env: MutableMapping[str, Any]) -> str:

        return self._list_close()

    def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                env: MutableMapping[str, Any]) -> str:

        return "__"

    def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                 env: MutableMapping[str, Any]) -> str:

        return "__"

    def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                    env: MutableMapping[str, Any]) -> str:

        return "**"

    def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                     env: MutableMapping[str, Any]) -> str:

        return "**"

    def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

              env: MutableMapping[str, Any]) -> str:

        attrs = f"[source,{token.info}]\n" if token.info else ""

        code = token.content

        if code.endswith('\n'):

            code = code[:-1]

        return f"{self._break(True)}{attrs}----\n{code}\n----"

    def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                        env: MutableMapping[str, Any]) -> str:

        pbreak = self._break(True)

        self._enter_block(False)

        return f"{pbreak}[quote]\n{self._parstack[-2].block_delim}\n"

    def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                         env: MutableMapping[str, Any]) -> str:

        self._leave_block()

        return f"\n{self._parstack[-1].block_delim}"

    def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                  env: MutableMapping[str, Any]) -> str:

        return self._admonition_open("NOTE")

    def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                   env: MutableMapping[str, Any]) -> str:

        return self._admonition_close()

    def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                     env: MutableMapping[str, Any]) -> str:

        return self._admonition_open("CAUTION")

    def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                      env: MutableMapping[str, Any]) -> str:

        return self._admonition_close()

    def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                       env: MutableMapping[str, Any]) -> str:

        return self._admonition_open("IMPORTANT")

    def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                        env: MutableMapping[str, Any]) -> str:

        return self._admonition_close()

    def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                 env: MutableMapping[str, Any]) -> str:

        return self._admonition_open("TIP")

    def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                  env: MutableMapping[str, Any]) -> str:

        return self._admonition_close()

    def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                     env: MutableMapping[str, Any]) -> str:

        return self._admonition_open("WARNING")

    def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                      env: MutableMapping[str, Any]) -> str:

        return self._admonition_close()

    def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                env: MutableMapping[str, Any]) -> str:

        return f"{self._break()}[]"

    def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                 env: MutableMapping[str, Any]) -> str:

        return ""

    def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                env: MutableMapping[str, Any]) -> str:

        return self._break()

    def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                 env: MutableMapping[str, Any]) -> str:

        self._enter_block(True)

        return ":: {empty}"

    def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                env: MutableMapping[str, Any]) -> str:

        return ""

    def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                 env: MutableMapping[str, Any]) -> str:

        self._leave_block()

        return "\n"

    def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                  env: MutableMapping[str, Any]) -> str:

        self._parstack[-1].continuing = True

        content = asciidoc_escape(token.content)

        if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)):

            return f"link:{quote(url, safe='/:')}[{content}]"

        return f"[.{token.meta['name']}]``{asciidoc_escape(token.content)}``"

    def inline_anchor(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                      env: MutableMapping[str, Any]) -> str:

        self._parstack[-1].continuing = True

        return f"[[{token.attrs['id']}]]"

    def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                        env: MutableMapping[str, Any]) -> str:

        self._parstack[-1].continuing = True

        (id_part, class_part) = ("", "")

        if id := token.attrs.get('id'):

            id_part = f"[[{id}]]"

        if s := token.attrs.get('class'):

            if s == 'keycap':

                class_part = "kbd:["

                self._attrspans.append("]")

            else:

                return super().attr_span_begin(token, tokens, i, options, env)

        else:

            self._attrspans.append("")

        return id_part + class_part

    def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                      env: MutableMapping[str, Any]) -> str:

        return self._attrspans.pop()

    def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                     env: MutableMapping[str, Any]) -> str:

        return token.markup.replace("#", "=") + " "

    def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                      env: MutableMapping[str, Any]) -> str:

        return "\n"

    def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                          env: MutableMapping[str, Any]) -> str:

        return self._list_open(token, '.')

    def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict,

                           env: MutableMapping[str, Any]) -> str:

        return self._list_close()

pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py

+77 −0
Original line number Diff line number Diff line
@@ -8,11 +8,13 @@ from collections.abc import Mapping, MutableMapping, Sequence 
from markdown_it.utils import OptionsDict

from markdown_it.token import Token

from typing import Any, Optional

from urllib.parse import quote

from xml.sax.saxutils import escape, quoteattr



import markdown_it



from . import parallel

from .asciidoc import AsciiDocRenderer, asciidoc_escape

from .commonmark import CommonMarkRenderer

from .docbook import DocBookRenderer, make_xml_id

from .manpage import ManpageRenderer, man_escape
@@ -476,6 +478,59 @@ class CommonMarkConverter(BaseConverter): 


        return "\n".join(result)



class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer):

    pass



class AsciiDocConverter(BaseConverter):

    __renderer__ = AsciiDocRenderer

    __option_block_separator__ = ""



    def _parallel_render_prepare(self) -> Any:

        return (self._manpage_urls, self._revision, self._markdown_by_default)

    @classmethod

    def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter:

        return cls(*a)



    def _render_code(self, option: dict[str, Any], key: str) -> list[str]:

        # NOTE this duplicates the old direct-paste behavior, even if it is somewhat

        # incorrect, since users rely on it.

        if lit := option_is(option, key, 'literalDocBook'):

            return [ f"*{key.capitalize()}:* {lit['text']}" ]

        else:

            return super()._render_code(option, key)



    def _render_description(self, desc: str | dict[str, Any]) -> list[str]:

        # NOTE this duplicates the old direct-paste behavior, even if it is somewhat

        # incorrect, since users rely on it.

        if isinstance(desc, str) and not self._markdown_by_default:

            return [ desc ]

        else:

            return super()._render_description(desc)



    def _related_packages_header(self) -> list[str]:

        return [ "__Related packages:__" ]



    def _decl_def_header(self, header: str) -> list[str]:

        return [ f"__{header}:__\n" ]



    def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]:

        if href is not None:

            return [ f"* link:{quote(href, safe='/:')}[{asciidoc_escape(name)}]" ]

        return [ f"* {asciidoc_escape(name)}" ]



    def _decl_def_footer(self) -> list[str]:

        return []



    def finalize(self) -> str:

        result = []



        for (name, opt) in self._sorted_options():

            result.append(f"== {asciidoc_escape(name)}\n")

            result += opt.lines

            result.append("\n\n")



        return "\n".join(result)



def _build_cli_db(p: argparse.ArgumentParser) -> None:

    p.add_argument('--manpage-urls', required=True)

    p.add_argument('--revision', required=True)
@@ -498,6 +553,13 @@ def _build_cli_commonmark(p: argparse.ArgumentParser) -> None: 
    p.add_argument("infile")

    p.add_argument("outfile")



def _build_cli_asciidoc(p: argparse.ArgumentParser) -> None:

    p.add_argument('--manpage-urls', required=True)

    p.add_argument('--revision', required=True)

    p.add_argument('--markdown-by-default', default=False, action='store_true')

    p.add_argument("infile")

    p.add_argument("outfile")



def _run_cli_db(args: argparse.Namespace) -> None:

    with open(args.manpage_urls, 'r') as manpage_urls:

        md = DocBookConverter(
@@ -537,11 +599,24 @@ def _run_cli_commonmark(args: argparse.Namespace) -> None: 
        with open(args.outfile, 'w') as f:

            f.write(md.finalize())



def _run_cli_asciidoc(args: argparse.Namespace) -> None:

    with open(args.manpage_urls, 'r') as manpage_urls:

        md = AsciiDocConverter(

            json.load(manpage_urls),

            revision = args.revision,

            markdown_by_default = args.markdown_by_default)



        with open(args.infile, 'r') as f:

            md.add_options(json.load(f))

        with open(args.outfile, 'w') as f:

            f.write(md.finalize())



def build_cli(p: argparse.ArgumentParser) -> None:

    formats = p.add_subparsers(dest='format', required=True)

    _build_cli_db(formats.add_parser('docbook'))

    _build_cli_manpage(formats.add_parser('manpage'))

    _build_cli_commonmark(formats.add_parser('commonmark'))

    _build_cli_asciidoc(formats.add_parser('asciidoc'))



def run_cli(args: argparse.Namespace) -> None:

    if args.format == 'docbook':
@@ -550,5 +625,7 @@ def run_cli(args: argparse.Namespace) -> None: 
        _run_cli_manpage(args)

    elif args.format == 'commonmark':

        _run_cli_commonmark(args)

    elif args.format == 'asciidoc':

        _run_cli_asciidoc(args)

    else:

        raise RuntimeError('format not hooked up', args)

pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py

0 → 100644
+143 −0
Original line number Diff line number Diff line 
import nixos_render_docs



from sample_md import sample1



class Converter(nixos_render_docs.md.Converter):

    __renderer__ = nixos_render_docs.asciidoc.AsciiDocRenderer



def test_lists() -> None:

    c = Converter({})

    # attaching to the nth ancestor list requires n newlines before the +

    assert c._render("""\

- a



  b

- c

  - d

    - e



      1



  f

""") == """\

[]

* {empty}a

+

b



* {empty}c

+

[options="compact"]

** {empty}d

+

[]

** {empty}e

+

1





+

f

"""



def test_full() -> None:

    c = Converter({ 'man(1)': 'http://example.org' })

    assert c._render(sample1) == """\

[WARNING]

====

foo



[NOTE]

=====

nested

=====



====





link:link[ multiline ]



link:http://example.org[man(1)] reference



[[b]]some [[a]]nested anchors



__emph__ **strong** __nesting emph **and strong** and ``code``__



[]

* {empty}wide bullet



* {empty}list





[]

. {empty}wide ordered



. {empty}list





[options="compact"]

* {empty}narrow bullet



* {empty}list





[options="compact"]

. {empty}narrow ordered



. {empty}list





[quote]

====

quotes



[quote]

=====

with __nesting__



----

nested code block

----

=====



[options="compact"]

* {empty}and lists



* {empty}

+

----

containing code

----





and more quote

====



[start=100,options="compact"]

. {empty}list starting at 100



. {empty}goes on





[]



deflist:: {empty}

+

[quote]

=====

with a quote and stuff

=====

+

----

code block

----

+

----

fenced block

----

+

text





more stuff in same deflist:: {empty}foo

"""