Impl code snippet (#37)

And various refactoring and bug fixes.

Author: SilverRainZ

src/sphinxnotes/snippet/cache.py

@@ -93,7 +93,7 @@ def post_purge(self, key: DocID, value: list[Item]) -> None:
     def get_by_index_id(self, key: IndexID) -> Item | None:
         """Like get(), but use IndexID as key."""
         doc_id, item_index = self.index_id_to_doc_id.get(key, (None, None))
-        if not doc_id or not item_index:
+        if not doc_id or item_index is None:
             return None
         return self[doc_id][item_index]
 
@@ -105,4 +105,4 @@ def gen_index_id(self) -> str:
 
     def stringify(self, key: DocID, value: list[Item]) -> str:
         """Overwrite PDict.stringify."""
-        return key[1]
+        return key[1]  # docname

src/sphinxnotes/snippet/cli.py

@@ -60,9 +60,9 @@ def main(argv: list[str] = sys.argv[1:]):
         formatter_class=HelpFormatter,
         epilog=dedent("""
                                      snippet tags:
-                                       d (document)          a reST document 
-                                       s (section)           a reST section
-                                       c (code)              snippet with code blocks
+                                       d (document)          a document
+                                       s (section)           a section
+                                       c (code)              a code block
                                        * (any)               wildcard for any snippet"""),
     )
     parser.add_argument(
@@ -140,7 +140,12 @@ def main(argv: list[str] = sys.argv[1:]):
         '--text',
         '-t',
         action='store_true',
-        help='get source reStructuredText of snippet',
+        help='get text representation of snippet',
+    )
+    getparser.add_argument(
+        '--src',
+        action='store_true',
+        help='get source text of snippet',
     )
     getparser.add_argument(
         '--url',
@@ -273,7 +278,9 @@ def p(*args, **opts):
             p('no such index ID', file=sys.stderr)
             sys.exit(1)
         if args.text:
-            p('\n'.join(item.snippet.rst))
+            p('\n'.join(item.snippet.text))
+        if args.src:
+            p('\n'.join(item.snippet.source))
         if args.docname:
             p(item.snippet.docname)
         if args.file:

src/sphinxnotes/snippet/ext.py

@@ -26,7 +26,7 @@
     from collections.abc import Iterator
 
 from .config import Config
-from .snippets import Snippet, WithTitle, Document, Section
+from .snippets import Snippet, WithTitle, Document, Section, Code
 from .picker import pick
 from .cache import Cache, Item
 from .keyword import Extractor
@@ -45,53 +45,38 @@ def extract_tags(s: Snippet) -> str:
         tags += 'd'
     elif isinstance(s, Section):
         tags += 's'
+    elif isinstance(s, Code):
+        tags += 'c'
     return tags
 
 
 def extract_excerpt(s: Snippet) -> str:
     if isinstance(s, Document) and s.title is not None:
-        return '<' + s.title.text + '>'
+        return '<' + s.title + '>'
     elif isinstance(s, Section) and s.title is not None:
-        return '[' + s.title.text + ']'
+        return '[' + s.title + ']'
+    elif isinstance(s, Code):
+        return '`' + (s.lang + ':').ljust(8, ' ') + ' ' + s.desc + '`'
     return ''
 
 
 def extract_keywords(s: Snippet) -> list[str]:
     keywords = [s.docname]
-    # TODO: Deal with more snippet
     if isinstance(s, WithTitle) and s.title is not None:
-        keywords.extend(extractor.extract(s.title.text, strip_stopwords=False))
+        keywords.extend(extractor.extract(s.title, strip_stopwords=False))
+    if isinstance(s, Code):
+        keywords.extend(extractor.extract(s.desc, strip_stopwords=False))
     return keywords
 
 
-def is_document_matched(
-    pats: dict[str, list[str]], docname: str
-) -> dict[str, list[str]]:
-    """Whether the docname matched by given patterns pats"""
-    new_pats = {}
-    for tag, ps in pats.items():
+def _get_document_allowed_tags(pats: dict[str, list[str]], docname: str) -> str:
+    """Return the tags of snippets that are allowed to be picked from the document."""
+    allowed_tags = ''
+    for tags, ps in pats.items():
         for pat in ps:
             if re.match(pat, docname):
-                new_pats.setdefault(tag, []).append(pat)
-    return new_pats
-
-
-def is_snippet_matched(pats: dict[str, list[str]], s: [Snippet], docname: str) -> bool:
-    """Whether the snippet's tags and docname matched by given patterns pats"""
-    if '*' in pats:  # Wildcard
-        for pat in pats['*']:
-            if re.match(pat, docname):
-                return True
-
-    not_in_pats = True
-    for k in extract_tags(s):
-        if k not in pats:
-            continue
-        not_in_pats = False
-        for pat in pats[k]:
-            if re.match(pat, docname):
-                return True
-    return not_in_pats
+                allowed_tags += tags
+    return allowed_tags
 
 
 def on_config_inited(app: Sphinx, appcfg: SphinxConfig) -> None:
@@ -113,6 +98,7 @@ def on_env_get_outdated(
     removed: set[str],
 ) -> list[str]:
     # Remove purged indexes and snippetes from db
+    assert cache is not None
     for docname in removed:
         del cache[(app.config.project, docname)]
     return []
@@ -126,15 +112,16 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N
         )
         return
 
-    pats = is_document_matched(app.config.snippet_patterns, docname)
-    if len(pats) == 0:
-        logger.debug('[snippet] skip picking because %s is not matched', docname)
+    allowed_tags = _get_document_allowed_tags(app.config.snippet_patterns, docname)
+    if not allowed_tags:
+        logger.debug('[snippet] skip picking: no tag allowed for document %s', docname)
         return
 
     doc = []
     snippets = pick(app, doctree, docname)
     for s, n in snippets:
-        if not is_snippet_matched(pats, s, docname):
+        # FIXME: Better filter logic.
+        if extract_tags(s) not in allowed_tags:
             continue
         tpath = [x.astext() for x in titlepath.resolve(app.env, docname, n)]
         if isinstance(s, Section):
@@ -162,6 +149,7 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N
 
 
 def on_builder_finished(app: Sphinx, exception) -> None:
+    assert cache is not None
     cache.dump()
 
 

src/sphinxnotes/snippet/integration/binding.nvim

@@ -11,7 +11,7 @@ function! g:SphinxNotesSnippetListAndView()
   function! s:CallView(selection)
     call g:SphinxNotesSnippetView(s:SplitID(a:selection))
   endfunction
-  call g:SphinxNotesSnippetList(function('s:CallView'), 'ds')
+  call g:SphinxNotesSnippetList(function('s:CallView'), '*')
 endfunction
 
 " https://github.com/anhmv/vim-float-window/blob/master/plugin/float-window.vim
@@ -40,7 +40,7 @@ function! g:SphinxNotesSnippetView(id)
   " Press enter to return
   nmap <buffer> <CR> :call nvim_win_close(g:sphinx_notes_snippet_win, v:true)<CR>
 
-  let cmd = [s:snippet, 'get', '--text', a:id]
+  let cmd = [s:snippet, 'get', '--src', a:id]
   call append(line('$'), ['.. hint:: Press <ENTER> to return'])
   execute '$read !' . '..'
   execute '$read !' . join(cmd, ' ')

src/sphinxnotes/snippet/integration/binding.sh

@@ -6,7 +6,7 @@
 # :Version: 20240828
 
 function snippet_view() {
-  selection=$(snippet_list --tags ds)
+  selection=$(snippet_list)
   [ -z "$selection" ] && return
 
   # Make sure we have $PAGER
@@ -18,7 +18,7 @@ function snippet_view() {
     fi
   fi
 
-  echo "$SNIPPET get --text $selection | $PAGER"
+  echo "$SNIPPET get --src $selection | $PAGER"
 }
 
 function snippet_edit() {

src/sphinxnotes/snippet/picker.py

@@ -15,7 +15,7 @@
 
 from sphinx.util import logging
 
-from .snippets import Snippet, Section, Document
+from .snippets import Snippet, Section, Document, Code
 
 if TYPE_CHECKING:
     from sphinx.application import Sphinx
@@ -25,81 +25,71 @@
 
 def pick(
     app: Sphinx, doctree: nodes.document, docname: str
-) -> list[tuple[Snippet, nodes.section]]:
+) -> list[tuple[Snippet, nodes.Element]]:
     """
-    Pick snippets from document, return a list of snippet and the section
-    it belongs to.
+    Pick snippets from document, return a list of snippet and the related node.
+
+    As :class:`Snippet` can not hold any refs to doctree, we additionly returns
+    the related nodes here. To ensure the caller can back reference to original
+    document node and do more things (e.g. generate title path).
     """
     # FIXME: Why doctree.source is always None?
     if not doctree.attributes.get('source'):
-        logger.debug('Skipped document without source')
+        logger.debug('Skip document without source')
         return []
 
     metadata = app.env.metadata.get(docname, {})
     if 'no-search' in metadata or 'nosearch' in metadata:
-        logger.debug('Skipped document with nosearch metadata')
+        logger.debug('Skip document with nosearch metadata')
         return []
 
-    snippets: list[tuple[Snippet, nodes.section]] = []
-
-    # Pick document
-    toplevel_section = doctree.next_node(nodes.section)
-    if toplevel_section:
-        snippets.append((Document(doctree), toplevel_section))
-    else:
-        logger.warning('can not pick document without child section: %s', doctree)
-
-    # Pick sections
-    section_picker = SectionPicker(doctree)
-    doctree.walkabout(section_picker)
-    snippets.extend(section_picker.sections)
+    # Walk doctree and pick snippets.
+    picker = SnippetPicker(doctree)
+    doctree.walkabout(picker)
 
-    return snippets
+    return picker.snippets
 
 
-class SectionPicker(nodes.SparseNodeVisitor):
+class SnippetPicker(nodes.SparseNodeVisitor):
     """Node visitor for picking snippets from document."""
 
-    #: Constant list of unsupported languages (:class:`pygments.lexers.Lexer`)
-    UNSUPPORTED_LANGUAGES: list[str] = ['default']
+    #: List of picked snippets and the section it belongs to
+    snippets: list[tuple[Snippet, nodes.Element]]
 
-    #: List of picked section snippets and the section it belongs to
-    sections: list[tuple[Section, nodes.section]]
+    #: Stack of nested sections.
+    _sections: list[nodes.section]
 
-    _section_has_code_block: bool
-    _section_level: int
-
-    def __init__(self, document: nodes.document) -> None:
-        super().__init__(document)
-        self.sections = []
-        self._section_has_code_block = False
-        self._section_level = 0
+    def __init__(self, doctree: nodes.document) -> None:
+        super().__init__(doctree)
+        self.snippets = []
+        self._sections = []
 
     ###################
     # Visitor methods #
     ###################
 
     def visit_literal_block(self, node: nodes.literal_block) -> None:
-        if node['language'] in self.UNSUPPORTED_LANGUAGES:
+        try:
+            code = Code(node)
+        except ValueError as e:
+            logger.debug(f'skip {node}: {e}')
             raise nodes.SkipNode
-        self._has_code_block = True
+        self.snippets.append((code, node))
 
     def visit_section(self, node: nodes.section) -> None:
-        self._section_level += 1
+        self._sections.append(node)
 
     def depart_section(self, node: nodes.section) -> None:
-        self._section_level -= 1
-        self._has_code_block = False
+        section = self._sections.pop()
+        assert section == node
 
         # Skip non-leaf section without content
         if self._is_empty_non_leaf_section(node):
             return
-        # Skip toplevel section, we generate :class:`Document` for it
-        if self._section_level == 0:
-            return
-
-        # TODO: code block
-        self.sections.append((Section(node), node))
+        if len(self._sections) == 0:
+            self.snippets.append((Document(self.document), node))
+        else:
+            self.snippets.append((Section(node), node))
 
     def unknown_visit(self, node: nodes.Node) -> None:
         pass  # Ignore any unknown node