move nested tokens to core

Author: chrisjsewell

markdown_it/doc_renderer.py renamed to markdown_it/_doc_renderer.py

@@ -1,8 +1,8 @@
+"""NOTE: this will eventually be moved out of core"""
 from contextlib import contextmanager
 import json
-from typing import List, Optional, Union
+from typing import List
 
-import attr
 import yaml
 
 from docutils import nodes
@@ -17,54 +17,7 @@
 # from docutils.statemachine import StringList
 from docutils.utils import new_document, Reporter  # noqa
 
-from markdown_it.token import Token
-
-
-@attr.s(slots=True)
-class NestedTokens:
-    opening: Token = attr.ib()
-    closing: Optional[Token] = attr.ib()
-    children: List[Union[Token, "NestedTokens"]] = attr.ib(factory=list)
-
-    def __getattr__(self, name):
-        return getattr(self.opening, name)
-
-    def attrGet(self, name: str) -> str:
-        """ Get the value of attribute `name`, or null if it does not exist."""
-        return self.opening.attrGet(name)
-
-
-def get_nested(tokens: List[Token]) -> List[Union[Token, NestedTokens]]:
-    """
-    """
-    output = []
-
-    tokens = list(reversed(tokens))
-    while tokens:
-        token = tokens.pop()
-
-        if token.nesting == 0:
-            output.append(token)
-            if token.children:
-                token.children = get_nested(token.children)
-            continue
-
-        assert token.nesting == 1, token.nesting
-
-        nested_tokens = [token]
-        nesting = 1
-        while tokens and nesting != 0:
-            token = tokens.pop()
-            nested_tokens.append(token)
-            nesting += token.nesting
-        if nesting != 0:
-            raise ValueError(f"unclosed tokens starting {nested_tokens[0]}")
-
-        child = NestedTokens(nested_tokens[0], nested_tokens[-1])
-        output.append(child)
-        child.children = get_nested(nested_tokens[1:-1])
-
-    return output
+from markdown_it.token import Token, nest_tokens
 
 
 def make_document(source_path="notset") -> nodes.document:
@@ -90,7 +43,7 @@ def __init__(self, options=None, env=None):
         self._level_to_elem = {0: self.document}
 
     def run_render(self, tokens: List[Token]):
-        tokens = get_nested(tokens)
+        tokens = nest_tokens(tokens)
         for i, token in enumerate(tokens):
             if f"render_{token.type}" in self.rules:
                 self.rules[f"render_{token.type}"](self, token)
@@ -265,7 +218,7 @@ def render_heading_open(self, token):
         self.current_node = section
 
     def render_link_open(self, token):
-        # TODO I think this may be already handled?
+        # TODO I think this is maybe already handled at this point?
         # refuri = escape_url(token.target)
         refuri = target = token.attrGet("href")
         ref_node = nodes.reference(target, target, refuri=refuri)
@@ -338,6 +291,25 @@ def render_myst_role(self, token):
         self.add_line_and_source_path(node, token)
         self.current_node.append(node)
 
+    # def render_table_open(self, token):
+    #     # print(token)
+    #     # raise
+
+    #     table = nodes.table()
+    #     table["classes"] += ["colwidths-auto"]
+    #     self.add_line_and_source_path(table, token)
+
+    #     thead = nodes.thead()
+    #     # TODO there can never be more than one header row (at least in mardown-it)
+    #     header = token.children[0].children[0]
+    #     for hrow in header.children:
+    #         nodes.t
+    #         style = hrow.attrGet("style")
+
+    #     tgroup = nodes.tgroup(cols)
+    #     table += tgroup
+    #     tgroup += thead
+
 
 def dict_to_docinfo(data):
     """Render a key/val pair as a docutils field node."""

markdown_it/token.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Union
 
 import attr
 
@@ -95,3 +95,57 @@ def as_dict(self, children=True, filter=None, dict_factory=dict):
         return attr.asdict(
             self, recurse=children, filter=filter, dict_factory=dict_factory
         )
+
+
+@attr.s(slots=True)
+class NestedTokens:
+    """A class that closely resembles a Token,
+    but for a an opening/closing Token pair, and their containing children.
+    """
+
+    opening: Token = attr.ib()
+    closing: Optional[Token] = attr.ib()
+    children: List[Union[Token, "NestedTokens"]] = attr.ib(factory=list)
+
+    def __getattr__(self, name):
+        return getattr(self.opening, name)
+
+    def attrGet(self, name: str) -> str:
+        """ Get the value of attribute `name`, or null if it does not exist."""
+        return self.opening.attrGet(name)
+
+
+def nest_tokens(tokens: List[Token]) -> List[Union[Token, NestedTokens]]:
+    """Convert the token stream to a list of tokens and nested tokens.
+
+    ``NestedTokens`` contain the open and close tokens and a list of children
+    of all tokens in between (recursively nested)
+    """
+    output = []
+
+    tokens = list(reversed(tokens))
+    while tokens:
+        token = tokens.pop()
+
+        if token.nesting == 0:
+            output.append(token)
+            if token.children:
+                token.children = nest_tokens(token.children)
+            continue
+
+        assert token.nesting == 1, token.nesting
+
+        nested_tokens = [token]
+        nesting = 1
+        while tokens and nesting != 0:
+            token = tokens.pop()
+            nested_tokens.append(token)
+            nesting += token.nesting
+        if nesting != 0:
+            raise ValueError(f"unclosed tokens starting {nested_tokens[0]}")
+
+        child = NestedTokens(nested_tokens[0], nested_tokens[-1])
+        output.append(child)
+        child.children = nest_tokens(nested_tokens[1:-1])
+
+    return output

tests/test_api/test_token.py

@@ -0,0 +1,59 @@
+from markdown_it.token import Token, nest_tokens, NestedTokens
+
+
+def test_token():
+    token = Token("name", "tag", 0)
+    assert token.as_dict() == {
+        "type": "name",
+        "tag": "tag",
+        "nesting": 0,
+        "attrs": None,
+        "map": None,
+        "level": 0,
+        "children": None,
+        "content": "",
+        "markup": "",
+        "info": "",
+        "meta": None,
+        "block": False,
+        "hidden": False,
+    }
+    token.attrSet("a", "b")
+    assert token.attrGet("a") == "b"
+    token.attrJoin("a", "c")
+    assert token.attrGet("a") == "b c"
+    token.attrPush(["x", "y"])
+    assert token.attrGet("x") == "y"
+    assert token.attrIndex("a") == 0
+    assert token.attrIndex("x") == 1
+    assert token.attrIndex("j") == -1
+
+
+def test_nest_tokens():
+    tokens = nest_tokens(
+        [
+            Token("start", "", 0),
+            Token("open", "", 1),
+            Token("open_inner", "", 1),
+            Token("inner", "", 0),
+            Token("close_inner", "", -1),
+            Token("close", "", -1),
+            Token("end", "", 0),
+        ]
+    )
+    assert [t.type for t in tokens] == ["start", "open", "end"]
+    assert isinstance(tokens[0], Token)
+    assert isinstance(tokens[1], NestedTokens)
+    assert isinstance(tokens[2], Token)
+
+    nested = tokens[1]
+    assert nested.opening.type == "open"
+    assert nested.closing.type == "close"
+    assert len(nested.children) == 1
+    assert nested.children[0].type == "open_inner"
+
+    nested2 = nested.children[0]
+    assert nested2.opening.type == "open_inner"
+    assert nested2.closing.type == "close_inner"
+    assert len(nested2.children) == 1
+    assert nested2.children[0].type == "inner"