From c24cbd3ad98ca294ceb57dbd5768f9ab99785537 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Wed, 28 Aug 2024 12:59:01 +0800 Subject: [PATCH 01/17] feat(cli): Add docname glob filter for list subcommand (#29) --- src/sphinxnotes/snippet/cli.py | 32 ++++++++++++++++--- .../snippet/integration/binding.sh | 8 ++--- src/sphinxnotes/snippet/integration/plugin.sh | 6 ++-- .../snippet/integration/plugin.vim | 1 + src/sphinxnotes/snippet/table.py | 10 +++--- 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/src/sphinxnotes/snippet/cli.py b/src/sphinxnotes/snippet/cli.py index dcc732f..ee9f741 100644 --- a/src/sphinxnotes/snippet/cli.py +++ b/src/sphinxnotes/snippet/cli.py @@ -9,17 +9,18 @@ from __future__ import annotations import sys import argparse -from typing import List +from typing import List, Iterable, Tuple from os import path from textwrap import dedent from shutil import get_terminal_size import posixpath from xdg.BaseDirectory import xdg_config_home +from sphinx.util.matching import patmatch from . import __version__ from .config import Config -from .cache import Cache +from .cache import Cache, IndexID, Index from .table import tablify, COLUMNS DEFAULT_CONFIG_FILE = path.join(xdg_config_home, 'sphinxnotes', 'snippet', 'conf.py') @@ -83,7 +84,14 @@ def main(argv: List[str] = sys.argv[1:]): help='list snippet indexes, columns of indexes: %s' % COLUMNS, ) listparser.add_argument( - '--tags', '-t', type=str, default='*', help='list specified tags only' + '--tags', '-t', type=str, default='*', help='list snippets with specified tags' + ) + listparser.add_argument( + '--docname', + '-d', + type=str, + default='**', + help='list snippets whose docname matches shell-style glob pattern', ) listparser.add_argument( '--width', @@ -205,9 +213,23 @@ def _on_command_stat(args: argparse.Namespace): print(f'\t {v} snippets(s)') +def _filter_list_items( + cache: Cache, tags: str, docname_glob: str +) -> Iterable[Tuple[IndexID, Index]]: + for index_id, index in cache.indexes.items(): + # Filter by tags. + if index[0] not in tags and '*' not in tags: + continue + # Filter by docname. + (_, docname), _ = cache.index_id_to_doc_id[index_id] + if not patmatch(docname, docname_glob): + continue + yield (index_id, index) + + def _on_command_list(args: argparse.Namespace): - rows = tablify(args.cache.indexes, args.tags, args.width) - for row in rows: + items = _filter_list_items(args.cache, args.tags, args.docname) + for row in tablify(items, args.width): print(row) diff --git a/src/sphinxnotes/snippet/integration/binding.sh b/src/sphinxnotes/snippet/integration/binding.sh index a07dd17..474b382 100644 --- a/src/sphinxnotes/snippet/integration/binding.sh +++ b/src/sphinxnotes/snippet/integration/binding.sh @@ -3,10 +3,10 @@ # # :Author: Shengyu Zhang # :Date: 2021-08-14 -# :Version: 20211114 +# :Version: 20240828 function snippet_view() { - selection=$(snippet_list ds) + selection=$(snippet_list --tags ds) [ -z "$selection" ] && return # Make sure we have $PAGER @@ -22,14 +22,14 @@ function snippet_view() { } function snippet_edit() { - selection=$(snippet_list ds) + selection=$(snippet_list --tags ds) [ -z "$selection" ] && return echo "vim +\$($SNIPPET get --line-start $selection) \$($SNIPPET get --file $selection)" } function snippet_url() { - selection=$(snippet_list ds) + selection=$(snippet_list --tags ds) [ -z "$selection" ] && return echo "xdg-open \$($SNIPPET get --url $selection)" diff --git a/src/sphinxnotes/snippet/integration/plugin.sh b/src/sphinxnotes/snippet/integration/plugin.sh index 5b75e0a..3d24a91 100644 --- a/src/sphinxnotes/snippet/integration/plugin.sh +++ b/src/sphinxnotes/snippet/integration/plugin.sh @@ -3,15 +3,15 @@ # # :Author: Shengyu Zhang # :Date: 2021-03-20 -# :Version: 20211114 +# :Version: 20240828 # Make sure we have $SNIPPET [ -z "$SNIPPET"] && SNIPPET='snippet' -# Arguments: $1: kinds +# Arguments: $*: Extra opts of ``snippet list`` # Returns: snippet_id function snippet_list() { - $SNIPPET list --tags $1 --width $(($(tput cols) - 2)) | \ + $SNIPPET list --width $(($(tput cols) - 2)) "$@" | \ fzf --with-nth 2.. \ --no-hscroll \ --header-lines 1 \ diff --git a/src/sphinxnotes/snippet/integration/plugin.vim b/src/sphinxnotes/snippet/integration/plugin.vim index cfe77f7..da79b48 100644 --- a/src/sphinxnotes/snippet/integration/plugin.vim +++ b/src/sphinxnotes/snippet/integration/plugin.vim @@ -13,6 +13,7 @@ function! s:SplitID(row) return split(a:row, ' ')[0] endfunction +" TODO: extra opts function! g:SphinxNotesSnippetList(callback, tags) let l:width = 0.9 let cmd = [s:snippet, 'list', diff --git a/src/sphinxnotes/snippet/table.py b/src/sphinxnotes/snippet/table.py index 26a8a70..f38f135 100644 --- a/src/sphinxnotes/snippet/table.py +++ b/src/sphinxnotes/snippet/table.py @@ -7,7 +7,7 @@ """ from __future__ import annotations -from typing import Iterator, Dict +from typing import Iterable, Tuple from .cache import Index, IndexID from .utils import ellipsis @@ -17,8 +17,8 @@ COLUMN_DELIMITER = ' ' -def tablify(indexes: Dict[IndexID, Index], tags: str, width: int) -> Iterator[str]: - """Create a table from sequence of cache.Index.""" +def tablify(indexes: Iterable[Tuple[IndexID, Index]], width: int) -> Iterable[str]: + """Create a table from sequence of indices""" # Calcuate width width = width @@ -41,10 +41,8 @@ def tablify(indexes: Dict[IndexID, Index], tags: str, width: int) -> Iterator[st yield header # Write rows - for index_id, index in indexes.items(): + for index_id, index in indexes: # TODO: assert index? - if index[0] not in tags and '*' not in tags: - continue row = COLUMN_DELIMITER.join( [ index_id, # ID From 730e885852ddbc8933534023df91eb5d2c40cb4c Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Wed, 28 Aug 2024 12:59:26 +0800 Subject: [PATCH 02/17] feat(ext): Add docname to keywords --- src/sphinxnotes/snippet/ext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sphinxnotes/snippet/ext.py b/src/sphinxnotes/snippet/ext.py index e37fd23..95cd9ea 100644 --- a/src/sphinxnotes/snippet/ext.py +++ b/src/sphinxnotes/snippet/ext.py @@ -57,7 +57,7 @@ def extract_excerpt(s: Snippet) -> str: def extract_keywords(s: Snippet) -> List[str]: - keywords = [] + keywords = [s.docname] # TODO: Deal with more snippet if isinstance(s, WithTitle) and s.title is not None: keywords.extend(extractor.extract(s.title.text, strip_stopwords=False)) From 278f624cd8594b7f15d1db713baa5c7b5a65b80b Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Wed, 9 Oct 2024 20:24:54 +0800 Subject: [PATCH 03/17] feat: Record document dependent files (#32) --- src/sphinxnotes/snippet/__init__.py | 14 +++++++++- src/sphinxnotes/snippet/cli.py | 42 +++++++++++++++++++++++------ src/sphinxnotes/snippet/picker.py | 2 +- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/sphinxnotes/snippet/__init__.py b/src/sphinxnotes/snippet/__init__.py index 042d354..0a6c945 100644 --- a/src/sphinxnotes/snippet/__init__.py +++ b/src/sphinxnotes/snippet/__init__.py @@ -9,6 +9,7 @@ from __future__ import annotations from typing import List, Tuple, Optional, TYPE_CHECKING import itertools +from os import path from docutils import nodes @@ -29,7 +30,7 @@ class Snippet(object): # :rst:role:`doc`. docname: str - #: Source file path of snippet + #: Absolute path of the source file. file: str #: Line number range of snippet, in the source file which is left closed @@ -136,10 +137,21 @@ def __init__(self, node: nodes.section) -> None: class Document(Section): + #: A set of absolute paths of dependent files for document. + #: Obtained from :attr:`BuildEnvironment.dependencies`. + deps: set[str] + def __init__(self, node: nodes.document) -> None: assert isinstance(node, nodes.document) super().__init__(node.next_node(nodes.section)) + # Record document's dependent files + self.deps = set() + env: BuildEnvironment = node.settings.env + for dep in env.dependencies[self.docname]: + # Relative to documentation root -> Absolute path of file system. + self.deps.add(path.join(env.srcdir, dep)) + ################ # Nodes helper # diff --git a/src/sphinxnotes/snippet/cli.py b/src/sphinxnotes/snippet/cli.py index ee9f741..9e3d81f 100644 --- a/src/sphinxnotes/snippet/cli.py +++ b/src/sphinxnotes/snippet/cli.py @@ -18,7 +18,7 @@ from xdg.BaseDirectory import xdg_config_home from sphinx.util.matching import patmatch -from . import __version__ +from . import __version__, Document from .config import Config from .cache import Cache, IndexID, Index from .table import tablify, COLUMNS @@ -114,6 +114,9 @@ def main(argv: List[str] = sys.argv[1:]): getparser.add_argument( '--file', '-f', action='store_true', help='get source file path of snippet' ) + getparser.add_argument( + '--deps', action='store_true', help='get dependent files of document' + ) getparser.add_argument( '--line-start', action='store_true', @@ -234,17 +237,36 @@ def _on_command_list(args: argparse.Namespace): def _on_command_get(args: argparse.Namespace): + # Wrapper for warning when nothing is printed + printed = False + + def p(*args, **opts): + nonlocal printed + printed = True + print(*args, **opts) + for index_id in args.index_id: item = args.cache.get_by_index_id(index_id) if not item: - print('no such index ID', file=sys.stderr) + p('no such index ID', file=sys.stderr) sys.exit(1) if args.text: - print('\n'.join(item.snippet.rst)) + p('\n'.join(item.snippet.rst)) if args.docname: - print(item.snippet.docname) + p(item.snippet.docname) if args.file: - print(item.snippet.file) + p(item.snippet.file) + if args.deps: + if not isinstance(item.snippet, Document): + print( + f'{type(item.snippet)} ({index_id}) is not a document', + file=sys.stderr, + ) + sys.exit(1) + if len(item.snippet.deps) == 0: + p('') # prevent print nothing warning + for dep in item.snippet.deps: + p(dep) if args.url: # HACK: get doc id in better way doc_id, _ = args.cache.index_id_to_doc_id.get(index_id) @@ -258,11 +280,15 @@ def _on_command_get(args: argparse.Namespace): url = posixpath.join(base_url, doc_id[1] + '.html') if item.snippet.refid: url += '#' + item.snippet.refid - print(url) + p(url) if args.line_start: - print(item.snippet.lineno[0]) + p(item.snippet.lineno[0]) if args.line_end: - print(item.snippet.lineno[1]) + p(item.snippet.lineno[1]) + + if not printed: + print('please specify at least one argument', file=sys.stderr) + sys.exit(1) def _on_command_integration(args: argparse.Namespace): diff --git a/src/sphinxnotes/snippet/picker.py b/src/sphinxnotes/snippet/picker.py index 9b2ac2a..d7e7383 100644 --- a/src/sphinxnotes/snippet/picker.py +++ b/src/sphinxnotes/snippet/picker.py @@ -40,7 +40,7 @@ def pick( logger.debug('Skipped document with nosearch metadata') return [] - snippets = [] + snippets: list[tuple[Snippet, nodes.section]] = [] # Pick document toplevel_section = doctree.next_node(nodes.section) From 264be701753ca5e2c2daa06b19156daf543c08ee Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sun, 13 Oct 2024 13:52:05 +0800 Subject: [PATCH 04/17] feat(integration): Use d to input docname in vim (#33) --- .../snippet/integration/binding.vim | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/sphinxnotes/snippet/integration/binding.vim b/src/sphinxnotes/snippet/integration/binding.vim index 3b4539d..5919f0b 100644 --- a/src/sphinxnotes/snippet/integration/binding.vim +++ b/src/sphinxnotes/snippet/integration/binding.vim @@ -10,7 +10,7 @@ function! g:SphinxNotesSnippetEdit(id) let file = system(join([s:snippet, 'get', '--file', a:id, '2>/dev/null'], ' ')) let line = system(join([s:snippet, 'get', '--line-start', a:id, '2>/dev/null'], ' ')) if &modified - execute 'tabedit ' . file + execute 'vsplit ' . file else execute 'edit ' . file endif @@ -38,7 +38,27 @@ function! g:SphinxNotesSnippetListAndUrl() call g:SphinxNotesSnippetList(function('s:CallUrl'), 'ds') endfunction +function! g:SphinxNotesSnippetInput(id, item) + let content = system(join([s:snippet, 'get', '--' . a:item, a:id, '2>/dev/null'], ' ')) + let content = substitute(content, '\n\+$', '', '') " skip trailing \n + if a:item == 'docname' + " Create doc reference. + let content = ':doc:`/' . content . '`' + endif + execute 'normal! i' . content +endfunction + +function! g:SphinxNotesSnippetListAndInputDocname() + function! s:InputDocname(selection) + call g:SphinxNotesSnippetInput(s:SplitID(a:selection), 'docname') + endfunction + call g:SphinxNotesSnippetList(function('s:InputDocname'), 'd') +endfunction + nmap e :call g:SphinxNotesSnippetListAndEdit() nmap u :call g:SphinxNotesSnippetListAndUrl() +nmap d :call g:SphinxNotesSnippetListAndInputDocname() +" FIXME: can't return to insert mode even use a/startinsert!/C-o +imap d :call g:SphinxNotesSnippetListAndInputDocname() " vim: set shiftwidth=2: From 8f8042513471a3783c47fca1db5dce86c2f1b7b4 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sun, 13 Oct 2024 13:58:59 +0800 Subject: [PATCH 05/17] refactor: __init__.py -> snippets.py --- src/sphinxnotes/snippet/cache.py | 2 +- src/sphinxnotes/snippet/cli.py | 8 ++++++-- src/sphinxnotes/snippet/ext.py | 2 +- src/sphinxnotes/snippet/picker.py | 2 +- .../snippet/{__init__.py => snippets.py} | 15 ++++++++------- 5 files changed, 17 insertions(+), 12 deletions(-) rename src/sphinxnotes/snippet/{__init__.py => snippets.py} (94%) diff --git a/src/sphinxnotes/snippet/cache.py b/src/sphinxnotes/snippet/cache.py index 5c5c4e3..c4acf0b 100644 --- a/src/sphinxnotes/snippet/cache.py +++ b/src/sphinxnotes/snippet/cache.py @@ -9,7 +9,7 @@ from typing import List, Tuple, Dict, Optional from dataclasses import dataclass -from . import Snippet +from .snippets import Snippet from .utils.pdict import PDict diff --git a/src/sphinxnotes/snippet/cli.py b/src/sphinxnotes/snippet/cli.py index 9e3d81f..99a69d2 100644 --- a/src/sphinxnotes/snippet/cli.py +++ b/src/sphinxnotes/snippet/cli.py @@ -14,11 +14,12 @@ from textwrap import dedent from shutil import get_terminal_size import posixpath +from importlib.metadata import version from xdg.BaseDirectory import xdg_config_home from sphinx.util.matching import patmatch -from . import __version__, Document +from .snippets import Document from .config import Config from .cache import Cache, IndexID, Index from .table import tablify, COLUMNS @@ -61,7 +62,10 @@ def main(argv: List[str] = sys.argv[1:]): * (any) wildcard for any snippet"""), ) parser.add_argument( - '-v', '--version', action='version', version='%(prog)s ' + __version__ + '-v', + '--version', + action='version', + version='%(prog)s ' + version('sphinxnotes.any'), ) parser.add_argument( '-c', '--config', default=DEFAULT_CONFIG_FILE, help='path to configuration file' diff --git a/src/sphinxnotes/snippet/ext.py b/src/sphinxnotes/snippet/ext.py index 95cd9ea..9dece86 100644 --- a/src/sphinxnotes/snippet/ext.py +++ b/src/sphinxnotes/snippet/ext.py @@ -26,7 +26,7 @@ from collections.abc import Iterator from .config import Config -from . import Snippet, WithTitle, Document, Section +from .snippets import Snippet, WithTitle, Document, Section from .picker import pick from .cache import Cache, Item from .keyword import Extractor diff --git a/src/sphinxnotes/snippet/picker.py b/src/sphinxnotes/snippet/picker.py index d7e7383..f29cb71 100644 --- a/src/sphinxnotes/snippet/picker.py +++ b/src/sphinxnotes/snippet/picker.py @@ -15,7 +15,7 @@ from sphinx.util import logging -from . import Snippet, Section, Document +from .snippets import Snippet, Section, Document if TYPE_CHECKING: from sphinx.application import Sphinx diff --git a/src/sphinxnotes/snippet/__init__.py b/src/sphinxnotes/snippet/snippets.py similarity index 94% rename from src/sphinxnotes/snippet/__init__.py rename to src/sphinxnotes/snippet/snippets.py index 0a6c945..988f076 100644 --- a/src/sphinxnotes/snippet/__init__.py +++ b/src/sphinxnotes/snippet/snippets.py @@ -1,8 +1,10 @@ """ -sphinxnotes.snippet -~~~~~~~~~~~~~~~~~~~ +sphinxnotes.snippet.snippets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -:copyright: Copyright 2020 Shengyu Zhang +Definitions of various snippets. + +:copyright: Copyright 2024 Shengyu Zhang :license: BSD, see LICENSE for details. """ @@ -16,14 +18,13 @@ if TYPE_CHECKING: from sphinx.environment import BuildEnvironment -__version__ = '1.1.1' - class Snippet(object): """ - Snippet is base class of reStructuredText snippet. + Snippet is structured fragments extracted from a single Sphinx document + (can also be said to be a reStructuredText file). - :param nodes: Document nodes that make up this snippet + :param nodes: nodes of doctree that make up this snippet. """ #: docname where the snippet is located, can be referenced by From 113979c7b2f5e36dfd4660c0da38b93c1de99a49 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sun, 13 Oct 2024 19:32:25 +0800 Subject: [PATCH 06/17] refactor: Move extension entrypoint from ext.py to __init__.py Then it can be consistency with other sphinxnotes projects. --- docs/conf.py | 2 +- src/sphinxnotes/snippet/__init__.py | 36 +++++++++++++++++++++++++++++ src/sphinxnotes/snippet/ext.py | 18 +++------------ 3 files changed, 40 insertions(+), 16 deletions(-) create mode 100644 src/sphinxnotes/snippet/__init__.py diff --git a/docs/conf.py b/docs/conf.py index c7d27b2..d6cdf88 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -152,7 +152,7 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('../src/sphinxnotes')) -extensions.append('snippet.ext') +extensions.append('snippet') # DOG FOOD CONFIGURATION START diff --git a/src/sphinxnotes/snippet/__init__.py b/src/sphinxnotes/snippet/__init__.py new file mode 100644 index 0000000..0fd7955 --- /dev/null +++ b/src/sphinxnotes/snippet/__init__.py @@ -0,0 +1,36 @@ +""" +sphinxnotes.snippet +~~~~~~~~~~~~~~~~~~~ + +Sphinx extension entrypoint. + +:copyright: Copyright 2024 Shengyu Zhang +:license: BSD, see LICENSE for details. +""" + + +def setup(app): + # **WARNING**: We don't import these packages globally, because the current + # package (sphinxnotes.snippet) is always resloved when importing + # sphinxnotes.snippet.*. If we import packages here, eventually we will + # load a lot of packages from the Sphinx. It will seriously **SLOW DOWN** + # the startup time of our CLI tool (sphinxnotes.snippet.cli). + # + # .. seealso:: https://github.com/sphinx-notes/snippet/pull/31 + from .ext import ( + SnippetBuilder, + on_config_inited, + on_env_get_outdated, + on_doctree_resolved, + on_builder_finished, + ) + + app.add_builder(SnippetBuilder) + + app.add_config_value('snippet_config', {}, '') + app.add_config_value('snippet_patterns', {'*': ['.*']}, '') + + app.connect('config-inited', on_config_inited) + app.connect('env-get-outdated', on_env_get_outdated) + app.connect('doctree-resolved', on_doctree_resolved) + app.connect('build-finished', on_builder_finished) diff --git a/src/sphinxnotes/snippet/ext.py b/src/sphinxnotes/snippet/ext.py index 9dece86..d1cd265 100644 --- a/src/sphinxnotes/snippet/ext.py +++ b/src/sphinxnotes/snippet/ext.py @@ -1,10 +1,10 @@ """ -sphinxnotes.ext.snippet +sphinxnotes.snippet.ext ~~~~~~~~~~~~~~~~~~~~~~~ -Sphinx extension for sphinxnotes.snippet. +Sphinx extension implementation, but the entrypoint is located at __init__.py. -:copyright: Copyright 2021 Shengyu Zhang +:copyright: Copyright 2024 Shengyu Zhang :license: BSD, see LICENSE for details. """ @@ -206,15 +206,3 @@ def _format_modified_time(timestamp: float) -> str: """Return an RFC 3339 formatted string representing the given timestamp.""" seconds, fraction = divmod(timestamp, 1) return time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(seconds)) + f'.{fraction:.3f}' - - -def setup(app: Sphinx): - app.add_builder(SnippetBuilder) - - app.add_config_value('snippet_config', {}, '') - app.add_config_value('snippet_patterns', {'*': ['.*']}, '') - - app.connect('config-inited', on_config_inited) - app.connect('env-get-outdated', on_env_get_outdated) - app.connect('doctree-resolved', on_doctree_resolved) - app.connect('build-finished', on_builder_finished) From 31a46cb379e20a711dc20cc6dc81d8a4ad16798f Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sun, 13 Oct 2024 19:33:03 +0800 Subject: [PATCH 07/17] improve(cli): Optimize startup time --- src/sphinxnotes/snippet/cli.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/sphinxnotes/snippet/cli.py b/src/sphinxnotes/snippet/cli.py index 99a69d2..0f28556 100644 --- a/src/sphinxnotes/snippet/cli.py +++ b/src/sphinxnotes/snippet/cli.py @@ -2,10 +2,15 @@ sphinxnotes.snippet.cli ~~~~~~~~~~~~~~~~~~~~~~~ -:copyright: Copyright 2020 Shengyu Zhang +Command line entrypoint. + +:copyright: Copyright 2024 Shengyu Zhang :license: BSD, see LICENSE for details. """ +# **NOTE**: Import new packages with caution: +# Importing complex packages (like sphinx.*) will directly slow down the +# startup of the CLI tool. from __future__ import annotations import sys import argparse @@ -14,10 +19,8 @@ from textwrap import dedent from shutil import get_terminal_size import posixpath -from importlib.metadata import version from xdg.BaseDirectory import xdg_config_home -from sphinx.util.matching import patmatch from .snippets import Document from .config import Config @@ -40,7 +43,7 @@ def get_integration_file(fn: str) -> str: .. seealso:: see ``[tool.setuptools.package-data]`` section of pyproject.toml to know - how files are included. + how files are included. """ # TODO: use https://docs.python.org/3/library/importlib.resources.html#importlib.resources.files prefix = path.abspath(path.dirname(__file__)) @@ -62,10 +65,11 @@ def main(argv: List[str] = sys.argv[1:]): * (any) wildcard for any snippet"""), ) parser.add_argument( - '-v', '--version', - action='version', - version='%(prog)s ' + version('sphinxnotes.any'), + # add_argument provides action='version', but it requires a version + # literal and doesn't support lazily obtaining version. + action='store_true', + help="show program's version number and exit", ) parser.add_argument( '-c', '--config', default=DEFAULT_CONFIG_FILE, help='path to configuration file' @@ -180,6 +184,16 @@ def main(argv: List[str] = sys.argv[1:]): # Parse command line arguments args = parser.parse_args(argv) + # Print version message. + # See parser.add_argument('--version', ...) for more detais. + if args.version: + # NOTE: Importing is slow, do it on demand. + from importlib.metadata import version + + pkgname = 'sphinxnotes.snippet' + print(pkgname, version(pkgname)) + parser.exit() + # Load config from file if args.config == DEFAULT_CONFIG_FILE and not path.isfile(DEFAULT_CONFIG_FILE): print( @@ -223,6 +237,9 @@ def _on_command_stat(args: argparse.Namespace): def _filter_list_items( cache: Cache, tags: str, docname_glob: str ) -> Iterable[Tuple[IndexID, Index]]: + # NOTE: Importing is slow, do it on demand. + from sphinx.util.matching import patmatch + for index_id, index in cache.indexes.items(): # Filter by tags. if index[0] not in tags and '*' not in tags: From a5e6fde64aad9fd15b18ca5b52c4ef7aa54a4139 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Thu, 17 Oct 2024 10:02:50 +0800 Subject: [PATCH 08/17] chore: Update project template to sphinx-notes/cookiecutter@c4f14dab --- .cruft.json | 2 +- Makefile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.cruft.json b/.cruft.json index e3324c6..7919327 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/sphinx-notes/template", - "commit": "80a61fa9abcd9474d8cfbc36d0bf5d41f99c916c", + "commit": "c4f14dab2840eeff6352647a923642b6377d1f49", "checkout": null, "context": { "cookiecutter": { diff --git a/Makefile b/Makefile index d1f5d08..c2786f4 100644 --- a/Makefile +++ b/Makefile @@ -21,8 +21,8 @@ view: .PHONY: clean clean: - $(MAKE) -C docs/ clean - $(RM) dist/ + $(MAKE) -C docs/ clean | true + $(RM) dist/ | true .PHONY: clean fmt: From b70dc37ce165c0d994d2e353d98e04ac0382d44a Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sat, 19 Oct 2024 09:27:25 +0800 Subject: [PATCH 09/17] fix: Handle broken pipe error (#36) --- src/sphinxnotes/snippet/cli.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/sphinxnotes/snippet/cli.py b/src/sphinxnotes/snippet/cli.py index 0f28556..9063abf 100644 --- a/src/sphinxnotes/snippet/cli.py +++ b/src/sphinxnotes/snippet/cli.py @@ -13,9 +13,10 @@ # startup of the CLI tool. from __future__ import annotations import sys +import os +from os import path import argparse from typing import List, Iterable, Tuple -from os import path from textwrap import dedent from shutil import get_terminal_size import posixpath @@ -344,4 +345,13 @@ def _on_command_integration(args: argparse.Namespace): if __name__ == '__main__': - sys.exit(main()) + # Prevent "[Errno 32] Broken pipe" error. + # https://docs.python.org/3/library/signal.html#note-on-sigpipe + try: + sys.exit(main()) + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown. + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE From 021df50afe5d0a3e8afe334226896ace6f77e06d Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sat, 19 Oct 2024 10:25:18 +0800 Subject: [PATCH 10/17] chore: Type hinting fixes and update --- pyproject.toml | 2 +- src/sphinxnotes/snippet/cache.py | 48 +++++++++++----------- src/sphinxnotes/snippet/cli.py | 6 +-- src/sphinxnotes/snippet/ext.py | 19 +++++---- src/sphinxnotes/snippet/keyword.py | 13 +++--- src/sphinxnotes/snippet/picker.py | 2 +- src/sphinxnotes/snippet/snippets.py | 25 ++++------- src/sphinxnotes/snippet/table.py | 4 +- src/sphinxnotes/snippet/utils/ellipsis.py | 5 +-- src/sphinxnotes/snippet/utils/pdict.py | 16 ++++---- src/sphinxnotes/snippet/utils/titlepath.py | 8 ++-- 11 files changed, 69 insertions(+), 79 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c64582d..9642379 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ "Topic :: Utilities", ] -requires-python = ">=3.8" +requires-python = ">=3.12" dependencies = [ "Sphinx >= 4", "langid", diff --git a/src/sphinxnotes/snippet/cache.py b/src/sphinxnotes/snippet/cache.py index c4acf0b..cc2ed0a 100644 --- a/src/sphinxnotes/snippet/cache.py +++ b/src/sphinxnotes/snippet/cache.py @@ -1,4 +1,5 @@ -"""sphinxnotes.snippet.cache +""" +sphinxnotes.snippet.cache ~~~~~~~~~~~~~~~~~~~~~~~~~ :copyright: Copyright 2021 Shengyu Zhang @@ -6,7 +7,6 @@ """ from __future__ import annotations -from typing import List, Tuple, Dict, Optional from dataclasses import dataclass from .snippets import Snippet @@ -18,25 +18,25 @@ class Item(object): """Item of snippet cache.""" snippet: Snippet - tags: List[str] + tags: str excerpt: str - titlepath: List[str] - keywords: List[str] + titlepath: list[str] + keywords: list[str] -DocID = Tuple[str, str] # (project, docname) +DocID = tuple[str, str] # (project, docname) IndexID = str # UUID -Index = Tuple[str, str, List[str], List[str]] # (tags, excerpt, titlepath, keywords) +Index = tuple[str, str, list[str], list[str]] # (tags, excerpt, titlepath, keywords) -class Cache(PDict): - """A DocID -> List[Item] Cache.""" +class Cache(PDict[DocID, list[Item]]): + """A DocID -> list[Item] Cache.""" - indexes: Dict[IndexID, Index] - index_id_to_doc_id: Dict[IndexID, Tuple[DocID, int]] - doc_id_to_index_ids: Dict[DocID, List[IndexID]] - num_snippets_by_project: Dict[str, int] - num_snippets_by_docid: Dict[DocID, int] + indexes: dict[IndexID, Index] + index_id_to_doc_id: dict[IndexID, tuple[DocID, int]] + doc_id_to_index_ids: dict[DocID, list[IndexID]] + num_snippets_by_project: dict[str, int] + num_snippets_by_docid: dict[DocID, int] def __init__(self, dirname: str) -> None: self.indexes = {} @@ -46,7 +46,7 @@ def __init__(self, dirname: str) -> None: self.num_snippets_by_docid = {} super().__init__(dirname) - def post_dump(self, key: DocID, items: List[Item]) -> None: + def post_dump(self, key: DocID, value: list[Item]) -> None: """Overwrite PDict.post_dump.""" # Remove old indexes and index IDs if exists @@ -55,7 +55,7 @@ def post_dump(self, key: DocID, items: List[Item]) -> None: del self.indexes[old_index_id] # Add new index to every where - for i, item in enumerate(items): + for i, item in enumerate(value): index_id = self.gen_index_id() self.indexes[index_id] = ( item.tags, @@ -69,12 +69,12 @@ def post_dump(self, key: DocID, items: List[Item]) -> None: # Update statistic if key[0] not in self.num_snippets_by_project: self.num_snippets_by_project[key[0]] = 0 - self.num_snippets_by_project[key[0]] += len(items) + self.num_snippets_by_project[key[0]] += len(value) if key not in self.num_snippets_by_docid: self.num_snippets_by_docid[key] = 0 - self.num_snippets_by_docid[key] += len(items) + self.num_snippets_by_docid[key] += len(value) - def post_purge(self, key: DocID, items: List[Item]) -> None: + def post_purge(self, key: DocID, value: list[Item]) -> None: """Overwrite PDict.post_purge.""" # Purge indexes @@ -83,17 +83,17 @@ def post_purge(self, key: DocID, items: List[Item]) -> None: del self.indexes[index_id] # Update statistic - self.num_snippets_by_project[key[0]] -= len(items) + self.num_snippets_by_project[key[0]] -= len(value) if self.num_snippets_by_project[key[0]] == 0: del self.num_snippets_by_project[key[0]] - self.num_snippets_by_docid[key] -= len(items) + self.num_snippets_by_docid[key] -= len(value) if self.num_snippets_by_docid[key] == 0: del self.num_snippets_by_docid[key] - def get_by_index_id(self, key: IndexID) -> Optional[Item]: + def get_by_index_id(self, key: IndexID) -> Item | None: """Like get(), but use IndexID as key.""" doc_id, item_index = self.index_id_to_doc_id.get(key, (None, None)) - if not doc_id: + if not doc_id or not item_index: return None return self[doc_id][item_index] @@ -103,6 +103,6 @@ def gen_index_id(self) -> str: return uuid.uuid4().hex[:7] - def stringify(self, key: DocID, items: List[Item]) -> str: + def stringify(self, key: DocID, value: list[Item]) -> str: """Overwrite PDict.stringify.""" return key[1] diff --git a/src/sphinxnotes/snippet/cli.py b/src/sphinxnotes/snippet/cli.py index 9063abf..c2f2d0e 100644 --- a/src/sphinxnotes/snippet/cli.py +++ b/src/sphinxnotes/snippet/cli.py @@ -16,7 +16,7 @@ import os from os import path import argparse -from typing import List, Iterable, Tuple +from typing import Iterable from textwrap import dedent from shutil import get_terminal_size import posixpath @@ -51,7 +51,7 @@ def get_integration_file(fn: str) -> str: return path.join(prefix, 'integration', fn) -def main(argv: List[str] = sys.argv[1:]): +def main(argv: list[str] = sys.argv[1:]): """Command line entrypoint.""" parser = argparse.ArgumentParser( @@ -237,7 +237,7 @@ def _on_command_stat(args: argparse.Namespace): def _filter_list_items( cache: Cache, tags: str, docname_glob: str -) -> Iterable[Tuple[IndexID, Index]]: +) -> Iterable[tuple[IndexID, Index]]: # NOTE: Importing is slow, do it on demand. from sphinx.util.matching import patmatch diff --git a/src/sphinxnotes/snippet/ext.py b/src/sphinxnotes/snippet/ext.py index d1cd265..b320ef5 100644 --- a/src/sphinxnotes/snippet/ext.py +++ b/src/sphinxnotes/snippet/ext.py @@ -9,7 +9,7 @@ """ from __future__ import annotations -from typing import List, Set, TYPE_CHECKING, Dict +from typing import TYPE_CHECKING import re from os import path import time @@ -56,7 +56,7 @@ def extract_excerpt(s: Snippet) -> str: return '' -def extract_keywords(s: Snippet) -> List[str]: +def extract_keywords(s: Snippet) -> list[str]: keywords = [s.docname] # TODO: Deal with more snippet if isinstance(s, WithTitle) and s.title is not None: @@ -65,8 +65,8 @@ def extract_keywords(s: Snippet) -> List[str]: def is_document_matched( - pats: Dict[str, List[str]], docname: str -) -> Dict[str, List[str]]: + pats: dict[str, list[str]], docname: str +) -> dict[str, list[str]]: """Whether the docname matched by given patterns pats""" new_pats = {} for tag, ps in pats.items(): @@ -76,7 +76,7 @@ def is_document_matched( return new_pats -def is_snippet_matched(pats: Dict[str, List[str]], s: [Snippet], docname: str) -> bool: +def is_snippet_matched(pats: dict[str, list[str]], s: [Snippet], docname: str) -> bool: """Whether the snippet's tags and docname matched by given patterns pats""" if '*' in pats: # Wildcard for pat in pats['*']: @@ -108,10 +108,10 @@ def on_config_inited(app: Sphinx, appcfg: SphinxConfig) -> None: def on_env_get_outdated( app: Sphinx, env: BuildEnvironment, - added: Set[str], - changed: Set[str], - removed: Set[str], -) -> List[str]: + added: set[str], + changed: set[str], + removed: set[str], +) -> list[str]: # Remove purged indexes and snippetes from db for docname in removed: del cache[(app.config.project, docname)] @@ -150,6 +150,7 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N ) cache_key = (app.config.project, docname) + assert cache is not None if len(doc) != 0: cache[cache_key] = doc elif cache_key in cache: diff --git a/src/sphinxnotes/snippet/keyword.py b/src/sphinxnotes/snippet/keyword.py index 6ff21d5..80d45f8 100644 --- a/src/sphinxnotes/snippet/keyword.py +++ b/src/sphinxnotes/snippet/keyword.py @@ -9,7 +9,6 @@ """ from __future__ import annotations -from typing import List, Optional import string from collections import Counter @@ -47,8 +46,8 @@ def __init__(self): ) def extract( - self, text: str, top_n: Optional[int] = None, strip_stopwords: bool = True - ) -> List[str]: + self, text: str, top_n: int | None = None, strip_stopwords: bool = True + ) -> list[str]: """Return keywords of given text.""" # TODO: zh -> en # Normalize @@ -87,7 +86,7 @@ def normalize(self, text: str) -> str: text = text.replace('\n', ' ') return text - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: # Get top most 5 langs langs = self._detect_langs(text)[:5] tokens = [text] @@ -104,10 +103,10 @@ def tokenize(self, text: str) -> List[str]: new_tokens = [] return tokens - def trans_to_pinyin(self, word: str) -> Optional[str]: + def trans_to_pinyin(self, word: str) -> str | None: return ' '.join(self._pinyin(word, errors='ignore')) - def strip_stopwords(self, words: List[str]) -> List[str]: + def strip_stopwords(self, words: list[str]) -> list[str]: stw = self._stopwords(['en', 'zh']) new_words = [] for word in words: @@ -115,5 +114,5 @@ def strip_stopwords(self, words: List[str]) -> List[str]: new_words.append(word) return new_words - def strip_invalid_token(self, tokens: List[str]) -> List[str]: + def strip_invalid_token(self, tokens: list[str]) -> list[str]: return [token for token in tokens if token != ''] diff --git a/src/sphinxnotes/snippet/picker.py b/src/sphinxnotes/snippet/picker.py index f29cb71..16b2941 100644 --- a/src/sphinxnotes/snippet/picker.py +++ b/src/sphinxnotes/snippet/picker.py @@ -58,7 +58,7 @@ def pick( class SectionPicker(nodes.SparseNodeVisitor): - """Node visitor for picking code snippet from document.""" + """Node visitor for picking snippets from document.""" #: Constant list of unsupported languages (:class:`pygments.lexers.Lexer`) UNSUPPORTED_LANGUAGES: list[str] = ['default'] diff --git a/src/sphinxnotes/snippet/snippets.py b/src/sphinxnotes/snippet/snippets.py index 988f076..133e91c 100644 --- a/src/sphinxnotes/snippet/snippets.py +++ b/src/sphinxnotes/snippet/snippets.py @@ -9,7 +9,7 @@ """ from __future__ import annotations -from typing import List, Tuple, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING import itertools from os import path @@ -36,16 +36,16 @@ class Snippet(object): #: Line number range of snippet, in the source file which is left closed #: and right opened. - lineno: Tuple[int, int] + lineno: tuple[int, int] #: The original reStructuredText of snippet - rst: List[str] + rst: list[str] #: The possible identifier key of snippet, which is picked from nodes' #: (or nodes' parent's) `ids attr`_. #: #: .. _ids attr: https://docutils.sourceforge.io/docs/ref/doctree.html#ids - refid: Optional[str] + refid: str | None def __init__(self, *nodes: nodes.Node) -> None: assert len(nodes) != 0 @@ -94,11 +94,11 @@ def __init__(self, node: nodes.Node) -> None: self.text = node.astext() -class CodeBlock(Text): +class Code(Text): #: Language of code block language: str #: Caption of code block - caption: Optional[str] + caption: str | None def __init__(self, node: nodes.literal_block) -> None: assert isinstance(node, nodes.literal_block) @@ -107,15 +107,6 @@ def __init__(self, node: nodes.literal_block) -> None: self.caption = node.get('caption') -class WithCodeBlock(object): - code_blocks: List[CodeBlock] - - def __init__(self, nodes: nodes.Nodes) -> None: - self.code_blocks = [] - for n in nodes.traverse(nodes.literal_block): - self.code_blocks.append(self.CodeBlock(n)) - - class Title(Text): def __init__(self, node: nodes.title) -> None: assert isinstance(node, nodes.title) @@ -123,7 +114,7 @@ def __init__(self, node: nodes.title) -> None: class WithTitle(object): - title: Optional[Title] + title: Title | None def __init__(self, node: nodes.Node) -> None: title_node = node.next_node(nodes.title) @@ -178,7 +169,7 @@ def _line_of_start(node: nodes.Node) -> int: return node.line -def _line_of_end(node: nodes.Node) -> Optional[int]: +def _line_of_end(node: nodes.Node) -> int | None: next_node = node.next_node(descend=False, siblings=True, ascend=True) while next_node: if next_node.line: diff --git a/src/sphinxnotes/snippet/table.py b/src/sphinxnotes/snippet/table.py index f38f135..8138573 100644 --- a/src/sphinxnotes/snippet/table.py +++ b/src/sphinxnotes/snippet/table.py @@ -7,7 +7,7 @@ """ from __future__ import annotations -from typing import Iterable, Tuple +from typing import Iterable from .cache import Index, IndexID from .utils import ellipsis @@ -17,7 +17,7 @@ COLUMN_DELIMITER = ' ' -def tablify(indexes: Iterable[Tuple[IndexID, Index]], width: int) -> Iterable[str]: +def tablify(indexes: Iterable[tuple[IndexID, Index]], width: int) -> Iterable[str]: """Create a table from sequence of indices""" # Calcuate width diff --git a/src/sphinxnotes/snippet/utils/ellipsis.py b/src/sphinxnotes/snippet/utils/ellipsis.py index 11bfcc5..d46bda8 100644 --- a/src/sphinxnotes/snippet/utils/ellipsis.py +++ b/src/sphinxnotes/snippet/utils/ellipsis.py @@ -9,12 +9,11 @@ """ from __future__ import annotations -from typing import List from wcwidth import wcswidth def ellipsis( - text: str, width: int, ellipsis_sym: str = '..', blank_sym: str = None + text: str, width: int, ellipsis_sym: str = '..', blank_sym: str | None = None ) -> str: text_width = wcswidth(text) if text_width <= width: @@ -34,7 +33,7 @@ def ellipsis( def join( - lst: List[str], + lst: list[str], total_width: int, title_width: int, separate_sym: str = '/', diff --git a/src/sphinxnotes/snippet/utils/pdict.py b/src/sphinxnotes/snippet/utils/pdict.py index 59d293f..ef93a38 100644 --- a/src/sphinxnotes/snippet/utils/pdict.py +++ b/src/sphinxnotes/snippet/utils/pdict.py @@ -11,7 +11,7 @@ from __future__ import annotations import os from os import path -from typing import Dict, Optional, Iterable, TypeVar +from typing import Iterator, TypeVar import pickle from collections.abc import MutableMapping from hashlib import sha1 @@ -21,16 +21,16 @@ # FIXME: PDict is buggy -class PDict(MutableMapping): +class PDict(MutableMapping[K, V]): """A persistent dict with event handlers.""" dirname: str # The real in memory store of values - _store: Dict[K, V] + _store: dict[K, V | None] # Items that need write back to store - _dirty_items: Dict[K, V] + _dirty_items: dict[K, V] # Items that need purge from store - _orphan_items: Dict[K, V] + _orphan_items: dict[K, V] def __init__(self, dirname: str) -> None: self.dirname = dirname @@ -38,7 +38,7 @@ def __init__(self, dirname: str) -> None: self._dirty_items = {} self._orphan_items = {} - def __getitem__(self, key: K) -> Optional[V]: + def __getitem__(self, key: K) -> V: if key not in self._store: raise KeyError value = self._store[key] @@ -65,7 +65,7 @@ def __delitem__(self, key: K) -> None: else: self._orphan_items[key] = value - def __iter__(self) -> Iterable: + def __iter__(self) -> Iterator: return iter(self._store) def __len__(self) -> int: @@ -142,4 +142,4 @@ def post_purge(self, key: K, value: V) -> None: pass def stringify(self, key: K, value: V) -> str: - return key + return str(key) diff --git a/src/sphinxnotes/snippet/utils/titlepath.py b/src/sphinxnotes/snippet/utils/titlepath.py index 9231537..1e827d3 100644 --- a/src/sphinxnotes/snippet/utils/titlepath.py +++ b/src/sphinxnotes/snippet/utils/titlepath.py @@ -9,7 +9,7 @@ """ from __future__ import annotations -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING from docutils import nodes @@ -19,11 +19,11 @@ def resolve( env: BuilderEnviornment, docname: str, node: nodes.Node -) -> List[nodes.title]: +) -> list[nodes.title]: return resolve_section(node) + resolve_document(env, docname) -def resolve_section(node: nodes.section) -> List[nodes.title]: +def resolve_section(node: nodes.section) -> list[nodes.title]: # FIXME: doc is None titlenodes = [] while node: @@ -33,7 +33,7 @@ def resolve_section(node: nodes.section) -> List[nodes.title]: return titlenodes -def resolve_document(env: BuilderEnviornment, docname: str) -> List[nodes.title]: +def resolve_document(env: BuilderEnviornment, docname: str) -> list[nodes.title]: """ .. note:: Title of document itself does not included in the returned list """ From deef7e45d0dde1c250e1c5148efa74504a24691e Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Mon, 21 Oct 2024 17:19:24 +0800 Subject: [PATCH 11/17] Impl code snippet (#37) And various refactoring and bug fixes. --- src/sphinxnotes/snippet/cache.py | 4 +- src/sphinxnotes/snippet/cli.py | 17 ++- src/sphinxnotes/snippet/ext.py | 58 +++----- .../snippet/integration/binding.nvim | 4 +- .../snippet/integration/binding.sh | 4 +- src/sphinxnotes/snippet/picker.py | 78 +++++----- src/sphinxnotes/snippet/snippets.py | 136 ++++++++++++------ src/sphinxnotes/snippet/utils/titlepath.py | 13 +- 8 files changed, 173 insertions(+), 141 deletions(-) diff --git a/src/sphinxnotes/snippet/cache.py b/src/sphinxnotes/snippet/cache.py index cc2ed0a..78a2efa 100644 --- a/src/sphinxnotes/snippet/cache.py +++ b/src/sphinxnotes/snippet/cache.py @@ -93,7 +93,7 @@ def post_purge(self, key: DocID, value: list[Item]) -> None: def get_by_index_id(self, key: IndexID) -> Item | None: """Like get(), but use IndexID as key.""" doc_id, item_index = self.index_id_to_doc_id.get(key, (None, None)) - if not doc_id or not item_index: + if not doc_id or item_index is None: return None return self[doc_id][item_index] @@ -105,4 +105,4 @@ def gen_index_id(self) -> str: def stringify(self, key: DocID, value: list[Item]) -> str: """Overwrite PDict.stringify.""" - return key[1] + return key[1] # docname diff --git a/src/sphinxnotes/snippet/cli.py b/src/sphinxnotes/snippet/cli.py index c2f2d0e..a52b17a 100644 --- a/src/sphinxnotes/snippet/cli.py +++ b/src/sphinxnotes/snippet/cli.py @@ -60,9 +60,9 @@ def main(argv: list[str] = sys.argv[1:]): formatter_class=HelpFormatter, epilog=dedent(""" snippet tags: - d (document) a reST document - s (section) a reST section - c (code) snippet with code blocks + d (document) a document + s (section) a section + c (code) a code block * (any) wildcard for any snippet"""), ) parser.add_argument( @@ -140,7 +140,12 @@ def main(argv: list[str] = sys.argv[1:]): '--text', '-t', action='store_true', - help='get source reStructuredText of snippet', + help='get text representation of snippet', + ) + getparser.add_argument( + '--src', + action='store_true', + help='get source text of snippet', ) getparser.add_argument( '--url', @@ -273,7 +278,9 @@ def p(*args, **opts): p('no such index ID', file=sys.stderr) sys.exit(1) if args.text: - p('\n'.join(item.snippet.rst)) + p('\n'.join(item.snippet.text)) + if args.src: + p('\n'.join(item.snippet.source)) if args.docname: p(item.snippet.docname) if args.file: diff --git a/src/sphinxnotes/snippet/ext.py b/src/sphinxnotes/snippet/ext.py index b320ef5..7a863cc 100644 --- a/src/sphinxnotes/snippet/ext.py +++ b/src/sphinxnotes/snippet/ext.py @@ -26,7 +26,7 @@ from collections.abc import Iterator from .config import Config -from .snippets import Snippet, WithTitle, Document, Section +from .snippets import Snippet, WithTitle, Document, Section, Code from .picker import pick from .cache import Cache, Item from .keyword import Extractor @@ -45,53 +45,38 @@ def extract_tags(s: Snippet) -> str: tags += 'd' elif isinstance(s, Section): tags += 's' + elif isinstance(s, Code): + tags += 'c' return tags def extract_excerpt(s: Snippet) -> str: if isinstance(s, Document) and s.title is not None: - return '<' + s.title.text + '>' + return '<' + s.title + '>' elif isinstance(s, Section) and s.title is not None: - return '[' + s.title.text + ']' + return '[' + s.title + ']' + elif isinstance(s, Code): + return '`' + (s.lang + ':').ljust(8, ' ') + ' ' + s.desc + '`' return '' def extract_keywords(s: Snippet) -> list[str]: keywords = [s.docname] - # TODO: Deal with more snippet if isinstance(s, WithTitle) and s.title is not None: - keywords.extend(extractor.extract(s.title.text, strip_stopwords=False)) + keywords.extend(extractor.extract(s.title, strip_stopwords=False)) + if isinstance(s, Code): + keywords.extend(extractor.extract(s.desc, strip_stopwords=False)) return keywords -def is_document_matched( - pats: dict[str, list[str]], docname: str -) -> dict[str, list[str]]: - """Whether the docname matched by given patterns pats""" - new_pats = {} - for tag, ps in pats.items(): +def _get_document_allowed_tags(pats: dict[str, list[str]], docname: str) -> str: + """Return the tags of snippets that are allowed to be picked from the document.""" + allowed_tags = '' + for tags, ps in pats.items(): for pat in ps: if re.match(pat, docname): - new_pats.setdefault(tag, []).append(pat) - return new_pats - - -def is_snippet_matched(pats: dict[str, list[str]], s: [Snippet], docname: str) -> bool: - """Whether the snippet's tags and docname matched by given patterns pats""" - if '*' in pats: # Wildcard - for pat in pats['*']: - if re.match(pat, docname): - return True - - not_in_pats = True - for k in extract_tags(s): - if k not in pats: - continue - not_in_pats = False - for pat in pats[k]: - if re.match(pat, docname): - return True - return not_in_pats + allowed_tags += tags + return allowed_tags def on_config_inited(app: Sphinx, appcfg: SphinxConfig) -> None: @@ -113,6 +98,7 @@ def on_env_get_outdated( removed: set[str], ) -> list[str]: # Remove purged indexes and snippetes from db + assert cache is not None for docname in removed: del cache[(app.config.project, docname)] return [] @@ -126,15 +112,16 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N ) return - pats = is_document_matched(app.config.snippet_patterns, docname) - if len(pats) == 0: - logger.debug('[snippet] skip picking because %s is not matched', docname) + allowed_tags = _get_document_allowed_tags(app.config.snippet_patterns, docname) + if not allowed_tags: + logger.debug('[snippet] skip picking: no tag allowed for document %s', docname) return doc = [] snippets = pick(app, doctree, docname) for s, n in snippets: - if not is_snippet_matched(pats, s, docname): + # FIXME: Better filter logic. + if extract_tags(s) not in allowed_tags: continue tpath = [x.astext() for x in titlepath.resolve(app.env, docname, n)] if isinstance(s, Section): @@ -162,6 +149,7 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N def on_builder_finished(app: Sphinx, exception) -> None: + assert cache is not None cache.dump() diff --git a/src/sphinxnotes/snippet/integration/binding.nvim b/src/sphinxnotes/snippet/integration/binding.nvim index 6f05c5f..0414214 100644 --- a/src/sphinxnotes/snippet/integration/binding.nvim +++ b/src/sphinxnotes/snippet/integration/binding.nvim @@ -11,7 +11,7 @@ function! g:SphinxNotesSnippetListAndView() function! s:CallView(selection) call g:SphinxNotesSnippetView(s:SplitID(a:selection)) endfunction - call g:SphinxNotesSnippetList(function('s:CallView'), 'ds') + call g:SphinxNotesSnippetList(function('s:CallView'), '*') endfunction " https://github.com/anhmv/vim-float-window/blob/master/plugin/float-window.vim @@ -40,7 +40,7 @@ function! g:SphinxNotesSnippetView(id) " Press enter to return nmap :call nvim_win_close(g:sphinx_notes_snippet_win, v:true) - let cmd = [s:snippet, 'get', '--text', a:id] + let cmd = [s:snippet, 'get', '--src', a:id] call append(line('$'), ['.. hint:: Press to return']) execute '$read !' . '..' execute '$read !' . join(cmd, ' ') diff --git a/src/sphinxnotes/snippet/integration/binding.sh b/src/sphinxnotes/snippet/integration/binding.sh index 474b382..f831d58 100644 --- a/src/sphinxnotes/snippet/integration/binding.sh +++ b/src/sphinxnotes/snippet/integration/binding.sh @@ -6,7 +6,7 @@ # :Version: 20240828 function snippet_view() { - selection=$(snippet_list --tags ds) + selection=$(snippet_list) [ -z "$selection" ] && return # Make sure we have $PAGER @@ -18,7 +18,7 @@ function snippet_view() { fi fi - echo "$SNIPPET get --text $selection | $PAGER" + echo "$SNIPPET get --src $selection | $PAGER" } function snippet_edit() { diff --git a/src/sphinxnotes/snippet/picker.py b/src/sphinxnotes/snippet/picker.py index 16b2941..667bcc1 100644 --- a/src/sphinxnotes/snippet/picker.py +++ b/src/sphinxnotes/snippet/picker.py @@ -15,7 +15,7 @@ from sphinx.util import logging -from .snippets import Snippet, Section, Document +from .snippets import Snippet, Section, Document, Code if TYPE_CHECKING: from sphinx.application import Sphinx @@ -25,81 +25,71 @@ def pick( app: Sphinx, doctree: nodes.document, docname: str -) -> list[tuple[Snippet, nodes.section]]: +) -> list[tuple[Snippet, nodes.Element]]: """ - Pick snippets from document, return a list of snippet and the section - it belongs to. + Pick snippets from document, return a list of snippet and the related node. + + As :class:`Snippet` can not hold any refs to doctree, we additionly returns + the related nodes here. To ensure the caller can back reference to original + document node and do more things (e.g. generate title path). """ # FIXME: Why doctree.source is always None? if not doctree.attributes.get('source'): - logger.debug('Skipped document without source') + logger.debug('Skip document without source') return [] metadata = app.env.metadata.get(docname, {}) if 'no-search' in metadata or 'nosearch' in metadata: - logger.debug('Skipped document with nosearch metadata') + logger.debug('Skip document with nosearch metadata') return [] - snippets: list[tuple[Snippet, nodes.section]] = [] - - # Pick document - toplevel_section = doctree.next_node(nodes.section) - if toplevel_section: - snippets.append((Document(doctree), toplevel_section)) - else: - logger.warning('can not pick document without child section: %s', doctree) - - # Pick sections - section_picker = SectionPicker(doctree) - doctree.walkabout(section_picker) - snippets.extend(section_picker.sections) + # Walk doctree and pick snippets. + picker = SnippetPicker(doctree) + doctree.walkabout(picker) - return snippets + return picker.snippets -class SectionPicker(nodes.SparseNodeVisitor): +class SnippetPicker(nodes.SparseNodeVisitor): """Node visitor for picking snippets from document.""" - #: Constant list of unsupported languages (:class:`pygments.lexers.Lexer`) - UNSUPPORTED_LANGUAGES: list[str] = ['default'] + #: List of picked snippets and the section it belongs to + snippets: list[tuple[Snippet, nodes.Element]] - #: List of picked section snippets and the section it belongs to - sections: list[tuple[Section, nodes.section]] + #: Stack of nested sections. + _sections: list[nodes.section] - _section_has_code_block: bool - _section_level: int - - def __init__(self, document: nodes.document) -> None: - super().__init__(document) - self.sections = [] - self._section_has_code_block = False - self._section_level = 0 + def __init__(self, doctree: nodes.document) -> None: + super().__init__(doctree) + self.snippets = [] + self._sections = [] ################### # Visitor methods # ################### def visit_literal_block(self, node: nodes.literal_block) -> None: - if node['language'] in self.UNSUPPORTED_LANGUAGES: + try: + code = Code(node) + except ValueError as e: + logger.debug(f'skip {node}: {e}') raise nodes.SkipNode - self._has_code_block = True + self.snippets.append((code, node)) def visit_section(self, node: nodes.section) -> None: - self._section_level += 1 + self._sections.append(node) def depart_section(self, node: nodes.section) -> None: - self._section_level -= 1 - self._has_code_block = False + section = self._sections.pop() + assert section == node # Skip non-leaf section without content if self._is_empty_non_leaf_section(node): return - # Skip toplevel section, we generate :class:`Document` for it - if self._section_level == 0: - return - - # TODO: code block - self.sections.append((Section(node), node)) + if len(self._sections) == 0: + self.snippets.append((Document(self.document), node)) + else: + self.snippets.append((Section(node), node)) def unknown_visit(self, node: nodes.Node) -> None: pass # Ignore any unknown node diff --git a/src/sphinxnotes/snippet/snippets.py b/src/sphinxnotes/snippet/snippets.py index 133e91c..f14661a 100644 --- a/src/sphinxnotes/snippet/snippets.py +++ b/src/sphinxnotes/snippet/snippets.py @@ -12,6 +12,8 @@ from typing import TYPE_CHECKING import itertools from os import path +import sys +from pygments.lexers.shell import BashSessionLexer from docutils import nodes @@ -22,24 +24,35 @@ class Snippet(object): """ Snippet is structured fragments extracted from a single Sphinx document - (can also be said to be a reStructuredText file). + (usually, also a single reStructuredText file). :param nodes: nodes of doctree that make up this snippet. + + .. warning:: + + Snippet will be persisted to disk via pickle, to keep it simple, + it CAN NOT holds reference to any doctree ``nodes`` + (or even any non-std module). """ #: docname where the snippet is located, can be referenced by # :rst:role:`doc`. docname: str - #: Absolute path of the source file. + #: Absolute path to the source file. file: str - #: Line number range of snippet, in the source file which is left closed - #: and right opened. + #: Line number range of source file (:attr:`Snippet.file`), + #: left closed and right opened. lineno: tuple[int, int] - #: The original reStructuredText of snippet - rst: list[str] + #: The source text read from source file (:attr:`Snippet.file`), + # in Markdown or reStructuredText. + source: list[str] + + #: Text representation of the snippet, usually generated form + # :meth:`nodes.Element.astext`. + text: list[str] #: The possible identifier key of snippet, which is picked from nodes' #: (or nodes' parent's) `ids attr`_. @@ -47,28 +60,42 @@ class Snippet(object): #: .. _ids attr: https://docutils.sourceforge.io/docs/ref/doctree.html#ids refid: str | None - def __init__(self, *nodes: nodes.Node) -> None: + def __init__(self, *nodes: nodes.Element) -> None: assert len(nodes) != 0 - env: BuildEnvironment = nodes[0].document.settings.env - self.file = nodes[0].source - self.docname = env.path2doc(self.file) + env: BuildEnvironment = nodes[0].document.settings.env # type: ignore - lineno = [float('inf'), -float('inf')] + file, docname = None, None + for node in nodes: + if (src := nodes[0].source) and path.exists(src): + file = src + docname = env.path2doc(file) + break + if not file or not docname: + raise ValueError(f'Nodes {nodes} lacks source file or docname') + self.file = file + self.docname = docname + + lineno = [sys.maxsize, -sys.maxsize] for node in nodes: if not node.line: continue # Skip node that have None line, I dont know why lineno[0] = min(lineno[0], _line_of_start(node)) lineno[1] = max(lineno[1], _line_of_end(node)) - self.lineno = lineno + self.lineno = (lineno[0], lineno[1]) - lines = [] + source = [] with open(self.file, 'r') as f: start = self.lineno[0] - 1 stop = self.lineno[1] - 1 for line in itertools.islice(f, start, stop): - lines.append(line.strip('\n')) - self.rst = lines + source.append(line.strip('\n')) + self.source = source + + text = [] + for node in nodes: + text.extend(node.astext().split('\n')) + self.text = text # Find exactly one ID attr in nodes self.refid = None @@ -85,40 +112,63 @@ def __init__(self, *nodes: nodes.Node) -> None: break -class Text(Snippet): - #: Text of snippet - text: str - - def __init__(self, node: nodes.Node) -> None: - super().__init__(node) - self.text = node.astext() - - -class Code(Text): +class Code(Snippet): #: Language of code block - language: str - #: Caption of code block - caption: str | None + lang: str + #: Description of code block, usually the text of preceding paragraph + desc: str def __init__(self, node: nodes.literal_block) -> None: assert isinstance(node, nodes.literal_block) - super().__init__(node) - self.language = node['language'] - self.caption = node.get('caption') - -class Title(Text): - def __init__(self, node: nodes.title) -> None: - assert isinstance(node, nodes.title) - super().__init__(node) + self.lang = node['language'] + if self.lang not in BashSessionLexer.aliases: # TODO: support more language + raise ValueError( + f'Language {self.lang} is not supported', + ) + + self.desc = '' + # Use the preceding paragraph as descritpion. We usually write some + # descritpions before a code block. For example, The ``::`` syntax is + # a common way to create code block:: + # + # | Foo:: | + # | | Foo: + # | Bar | + # | | Bar + # + # In this case, the paragraph "Foo:" is the descritpion of the code block. + # This convention also applies to the code, code-block, sourcecode directive. + if isinstance(para := node.previous_sibling(), nodes.paragraph): + # For better display, the trailing colon is removed. + # TODO: https://en.wikipedia.org/wiki/Colon_(punctuation)#Computing + self.desc += para.astext().replace('\n', ' ').rstrip(':::︁︓﹕') + if caption := node.get('caption'): + # Use caption as descritpion. + # All of code-block, sourcecode and code directives have caption option. + # https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-code-block + self.desc += caption + if not self.desc: + raise ValueError( + f'Node f{node} lacks description: a preceding paragraph or a caption' + ) + + if isinstance(para, nodes.paragraph): + # If we have a paragraph preceding code block, include it. + super().__init__(para, node) + # Fixup text field, it should be pure code. + self.text = node.astext().split('\n') + else: + super().__init__(node) class WithTitle(object): - title: Title | None + title: str - def __init__(self, node: nodes.Node) -> None: - title_node = node.next_node(nodes.title) - self.title = Title(title_node) if title_node else None + def __init__(self, node: nodes.Element) -> None: + if not (title := node.next_node(nodes.title)): + raise ValueError(f'Node f{node} lacks title') + self.title = title.astext() class Section(Snippet, WithTitle): @@ -169,7 +219,7 @@ def _line_of_start(node: nodes.Node) -> int: return node.line -def _line_of_end(node: nodes.Node) -> int | None: +def _line_of_end(node: nodes.Node) -> int: next_node = node.next_node(descend=False, siblings=True, ascend=True) while next_node: if next_node.line: @@ -184,7 +234,7 @@ def _line_of_end(node: nodes.Node) -> int | None: siblings=True, ) # No line found, return the max line of source file - if node.source: + if node.source and path.exists(node.source): with open(node.source) as f: - return sum(1 for line in f) + return sum(1 for _ in f) raise AttributeError('None source attr of node %s' % node) diff --git a/src/sphinxnotes/snippet/utils/titlepath.py b/src/sphinxnotes/snippet/utils/titlepath.py index 1e827d3..eaa6bc3 100644 --- a/src/sphinxnotes/snippet/utils/titlepath.py +++ b/src/sphinxnotes/snippet/utils/titlepath.py @@ -14,17 +14,16 @@ from docutils import nodes if TYPE_CHECKING: - from sphinx.enviornment import BuilderEnviornment + from sphinx.environment import BuildEnvironment def resolve( - env: BuilderEnviornment, docname: str, node: nodes.Node + env: BuildEnvironment, docname: str, node: nodes.Element ) -> list[nodes.title]: return resolve_section(node) + resolve_document(env, docname) -def resolve_section(node: nodes.section) -> list[nodes.title]: - # FIXME: doc is None +def resolve_section(node: nodes.Element) -> list[nodes.title]: titlenodes = [] while node: if len(node) > 0 and isinstance(node[0], nodes.title): @@ -33,10 +32,8 @@ def resolve_section(node: nodes.section) -> list[nodes.title]: return titlenodes -def resolve_document(env: BuilderEnviornment, docname: str) -> list[nodes.title]: - """ - .. note:: Title of document itself does not included in the returned list - """ +def resolve_document(env: BuildEnvironment, docname: str) -> list[nodes.title]: + """NOTE: Title of document itself does not included in the returned list""" titles = [] master_doc = env.config.master_doc v = docname.split('/') From d6db43f1c82ca22c28c58fee013e5bcd0d935b82 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sun, 27 Oct 2024 12:53:30 +0800 Subject: [PATCH 12/17] feat(integration): General vim input helper (#38) * feat(integration): General vim input helper * refactor(integration): Refactor vim plugin --- .../snippet/integration/binding.nvim | 6 +- .../snippet/integration/binding.vim | 47 +++------ .../snippet/integration/plugin.vim | 99 ++++++++++++++++--- 3 files changed, 105 insertions(+), 47 deletions(-) diff --git a/src/sphinxnotes/snippet/integration/binding.nvim b/src/sphinxnotes/snippet/integration/binding.nvim index 0414214..25f0f34 100644 --- a/src/sphinxnotes/snippet/integration/binding.nvim +++ b/src/sphinxnotes/snippet/integration/binding.nvim @@ -8,10 +8,10 @@ " TODO: Support vim? function! g:SphinxNotesSnippetListAndView() - function! s:CallView(selection) - call g:SphinxNotesSnippetView(s:SplitID(a:selection)) + function! ListAndView_CB(id) + call g:SphinxNotesSnippetView(a:id) endfunction - call g:SphinxNotesSnippetList(function('s:CallView'), '*') + call g:SphinxNotesSnippetList('"*"', function('ListAndView_CB')) endfunction " https://github.com/anhmv/vim-float-window/blob/master/plugin/float-window.vim diff --git a/src/sphinxnotes/snippet/integration/binding.vim b/src/sphinxnotes/snippet/integration/binding.vim index 5919f0b..6225080 100644 --- a/src/sphinxnotes/snippet/integration/binding.vim +++ b/src/sphinxnotes/snippet/integration/binding.vim @@ -6,9 +6,9 @@ " :Version: 20211114 " -function! g:SphinxNotesSnippetEdit(id) - let file = system(join([s:snippet, 'get', '--file', a:id, '2>/dev/null'], ' ')) - let line = system(join([s:snippet, 'get', '--line-start', a:id, '2>/dev/null'], ' ')) +function g:SphinxNotesSnippetEdit(id) + let file = g:SphinxNotesSnippetGet(a:id, 'file')[0] + let line = g:SphinxNotesSnippetGet(a:id, 'line-start')[0] if &modified execute 'vsplit ' . file else @@ -17,48 +17,29 @@ function! g:SphinxNotesSnippetEdit(id) execute line endfunction -function! g:SphinxNotesSnippetListAndEdit() - function! s:CallEdit(selection) - call g:SphinxNotesSnippetEdit(s:SplitID(a:selection)) +function g:SphinxNotesSnippetListAndEdit() + function! ListAndEdit_CB(id) + call g:SphinxNotesSnippetEdit(a:id) endfunction - call g:SphinxNotesSnippetList(function('s:CallEdit'), 'ds') + call g:SphinxNotesSnippetList('ds', function('ListAndEdit_CB')) endfunction -function! g:SphinxNotesSnippetUrl(id) - let url_list = systemlist(join([s:snippet, 'get', '--url', a:id, '2>/dev/null'], ' ')) +function g:SphinxNotesSnippetUrl(id) + let url_list = g:SphinxNotesSnippetGet(a:id, 'url') for url in url_list echo system('xdg-open ' . shellescape(url)) endfor endfunction -function! g:SphinxNotesSnippetListAndUrl() - function! s:CallUrl(selection) - call g:SphinxNotesSnippetUrl(s:SplitID(a:selection)) +function g:SphinxNotesSnippetListAndUrl() + function! ListAndUrl_CB(id) + call g:SphinxNotesSnippetUrl(a:id) endfunction - call g:SphinxNotesSnippetList(function('s:CallUrl'), 'ds') -endfunction - -function! g:SphinxNotesSnippetInput(id, item) - let content = system(join([s:snippet, 'get', '--' . a:item, a:id, '2>/dev/null'], ' ')) - let content = substitute(content, '\n\+$', '', '') " skip trailing \n - if a:item == 'docname' - " Create doc reference. - let content = ':doc:`/' . content . '`' - endif - execute 'normal! i' . content -endfunction - -function! g:SphinxNotesSnippetListAndInputDocname() - function! s:InputDocname(selection) - call g:SphinxNotesSnippetInput(s:SplitID(a:selection), 'docname') - endfunction - call g:SphinxNotesSnippetList(function('s:InputDocname'), 'd') + call g:SphinxNotesSnippetList('ds', function('ListAndUrl_CB')) endfunction nmap e :call g:SphinxNotesSnippetListAndEdit() nmap u :call g:SphinxNotesSnippetListAndUrl() -nmap d :call g:SphinxNotesSnippetListAndInputDocname() -" FIXME: can't return to insert mode even use a/startinsert!/C-o -imap d :call g:SphinxNotesSnippetListAndInputDocname() +nmap i :call g:SphinxNotesSnippetListAndInput() " vim: set shiftwidth=2: diff --git a/src/sphinxnotes/snippet/integration/plugin.vim b/src/sphinxnotes/snippet/integration/plugin.vim index da79b48..3cf01c5 100644 --- a/src/sphinxnotes/snippet/integration/plugin.vim +++ b/src/sphinxnotes/snippet/integration/plugin.vim @@ -8,25 +8,102 @@ " NOTE: junegunn/fzf.vim is required let s:snippet = 'snippet' +let s:width = 0.9 +let s:height = 0.6 -function! s:SplitID(row) - return split(a:row, ' ')[0] -endfunction - -" TODO: extra opts -function! g:SphinxNotesSnippetList(callback, tags) - let l:width = 0.9 +" Use fzf to list all snippets, callback with argument id. +function g:SphinxNotesSnippetList(tags, callback) let cmd = [s:snippet, 'list', \ '--tags', a:tags, - \ '--width', float2nr(&columns * l:width) - 2, + \ '--width', float2nr(&columns * s:width) - 2, \ ] + + " Use closure keyword so that inner function can access outer one's + " localvars (l:) and arguments (a:). + " https://vi.stackexchange.com/a/21807 + function! List_CB(selection) closure + let id = split(a:selection, ' ')[0] + call a:callback(id) + endfunction + " https://github.com/junegunn/fzf/blob/master/README-VIM.md#fzfrun call fzf#run({ \ 'source': join(cmd, ' '), - \ 'sink': a:callback, + \ 'sink': function('List_CB'), \ 'options': ['--with-nth', '2..', '--no-hscroll', '--header-lines', '1'], - \ 'window': {'width': l:width, 'height': 0.6}, + \ 'window': {'width': s:width, 'height': s:height}, \ }) endfunction -" vim: set shiftwidth=2: +" Return the attribute value of specific snippet. +function g:SphinxNotesSnippetGet(id, attr) + let cmd = [s:snippet, 'get', a:id, '--' . a:attr] + return systemlist(join(cmd, ' ')) +endfunction + +" Use fzf to list all attr of specific snippet, +" callback with arguments (attr_name, attr_value). +function g:SphinxNotesSnippetListSnippetAttrs(id, callback) + " Display attr -> Identify attr (also used as CLI option) + let attrs = { + \ 'Source': 'src', + \ 'URL': 'url', + \ 'Docname': 'docname', + \ 'Dependent files': 'deps', + \ 'Text': 'text', + \ 'Title': 'title', + \ } + let delim = ' ' + let table = ['OPTION' . delim . 'ATTRIBUTE'] + for name in keys(attrs) + call add(table, attrs[name] . delim . name) + endfor + + function! ListSnippetAttrs_CB(selection) closure + let opt = split(a:selection, ' ')[0] + let val = g:SphinxNotesSnippetGet(a:id, opt) + call a:callback(opt, val) " finally call user's cb + endfunction + + let preview_cmd = [s:snippet, 'get', a:id, '--$(echo {} | cut -d " " -f1)'] + let info_cmd = ['echo', 'Index ID:', a:id] + call fzf#run({ + \ 'source': table, + \ 'sink': function('ListSnippetAttrs_CB'), + \ 'options': [ + \ '--header-lines', '1', + \ '--with-nth', '2..', + \ '--preview', join(preview_cmd, ' '), + \ '--preview-window', ',wrap', + \ '--info-command', join(info_cmd, ' '), + \ ], + \ 'window': {'width': s:width, 'height': s:height}, + \ }) +endfunction + +function g:SphinxNotesSnippetInput(id) + function! Input_CB(attr, val) " TODO: became g:func. + if a:attr == 'docname' + " Create doc reference. + let content = ':doc:`/' . a:val[0] . '`' + elseif a:attr == 'title' + " Create local section reference. + let content = '`' . a:val[0] . '`_' + else + let content = join(a:val, '') + endif + execute 'normal! i' . content + endfunction + + call g:SphinxNotesSnippetListSnippetAttrs(a:id, function('Input_CB')) +endfunction + +function g:SphinxNotesSnippetListAndInput() + function! ListAndInput_CB(id) + call g:SphinxNotesSnippetInput(a:id) + endfunction + + call g:SphinxNotesSnippetList('"*"', function('ListAndInput_CB')) +endfunction + + " vim: set shiftwidth=2: From 633a342381c9d356a227ca434744dd68ec31afbc Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sat, 15 Feb 2025 05:29:36 +0800 Subject: [PATCH 13/17] fix(picker): Always pick document --- src/sphinxnotes/snippet/ext.py | 11 +++++++++-- src/sphinxnotes/snippet/picker.py | 14 ++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/sphinxnotes/snippet/ext.py b/src/sphinxnotes/snippet/ext.py index 7a863cc..178c98c 100644 --- a/src/sphinxnotes/snippet/ext.py +++ b/src/sphinxnotes/snippet/ext.py @@ -119,9 +119,11 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N doc = [] snippets = pick(app, doctree, docname) + tags = [] for s, n in snippets: # FIXME: Better filter logic. - if extract_tags(s) not in allowed_tags: + tags.append(extract_tags(s)) + if tags[-1] not in allowed_tags: continue tpath = [x.astext() for x in titlepath.resolve(app.env, docname, n)] if isinstance(s, Section): @@ -144,7 +146,12 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N del cache[cache_key] logger.debug( - '[snippet] picked %s/%s snippetes in %s', len(doc), len(snippets), docname + '[snippet] picked %s/%s snippets in %s, tags: %s, allowed tags: %s', + len(doc), + len(snippets), + docname, + tags, + allowed_tags, ) diff --git a/src/sphinxnotes/snippet/picker.py b/src/sphinxnotes/snippet/picker.py index 667bcc1..ea1b4cf 100644 --- a/src/sphinxnotes/snippet/picker.py +++ b/src/sphinxnotes/snippet/picker.py @@ -35,15 +35,16 @@ def pick( """ # FIXME: Why doctree.source is always None? if not doctree.attributes.get('source'): - logger.debug('Skip document without source') + logger.debug('Skip document %s: no source', docname) return [] metadata = app.env.metadata.get(docname, {}) if 'no-search' in metadata or 'nosearch' in metadata: - logger.debug('Skip document with nosearch metadata') + logger.debug('Skip document %s: have :no[-]search: metadata', docname) return [] # Walk doctree and pick snippets. + picker = SnippetPicker(doctree) doctree.walkabout(picker) @@ -83,13 +84,14 @@ def depart_section(self, node: nodes.section) -> None: section = self._sections.pop() assert section == node + # Always pick document. + if len(self._sections) == 0: + self.snippets.append((Document(self.document), node)) + return # Skip non-leaf section without content if self._is_empty_non_leaf_section(node): return - if len(self._sections) == 0: - self.snippets.append((Document(self.document), node)) - else: - self.snippets.append((Section(node), node)) + self.snippets.append((Section(node), node)) def unknown_visit(self, node: nodes.Node) -> None: pass # Ignore any unknown node From 1ca3c9f588213df2952ee19710f962ce4ff4a014 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sat, 15 Feb 2025 05:32:04 +0800 Subject: [PATCH 14/17] chore: Update project template to sphinx-notes/cookiecutter@7e07548a --- .cruft.json | 5 ++-- docs/_templates/confval.rst | 11 -------- docs/_templates/example.rst | 31 ----------------------- docs/_templates/version.rst | 8 ------ docs/conf.py | 50 +++---------------------------------- pyproject.toml | 2 +- 6 files changed, 7 insertions(+), 100 deletions(-) delete mode 100644 docs/_templates/confval.rst delete mode 100644 docs/_templates/example.rst delete mode 100644 docs/_templates/version.rst diff --git a/.cruft.json b/.cruft.json index 7919327..cea31ef 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/sphinx-notes/template", - "commit": "c4f14dab2840eeff6352647a923642b6377d1f49", + "commit": "7e07548aad55432b86d881ccdab66eb1c89986c0", "checkout": null, "context": { "cookiecutter": { @@ -14,7 +14,8 @@ "github_repo": "snippet", "pypi_name": "sphinxnotes-snippet", "pypi_owner": "SilverRainZ", - "_template": "https://github.com/sphinx-notes/template" + "_template": "https://github.com/sphinx-notes/template", + "_commit": "7e07548aad55432b86d881ccdab66eb1c89986c0" } }, "directory": null diff --git a/docs/_templates/confval.rst b/docs/_templates/confval.rst deleted file mode 100644 index fbbaa4c..0000000 --- a/docs/_templates/confval.rst +++ /dev/null @@ -1,11 +0,0 @@ - -:Type: :py:class:`{{ type }}` -:Default: ``{{ default }}`` -{% if choice %}:Choices: {% for c in choice %}``{{ c }}`` {% endfor %}{% endif %} -{% if versionadded %}:Version added: :version:`{{ versionadded }}`{% endif %} -{% if versionchanged %}:Version changed:{% for i in range(0, versionchanged|count -1, 2) %} - :version:`{{ versionchanged[i] }}` - {{ versionchanged[i+1] }}{% endfor %}{% endif %} - -{{ content }} - diff --git a/docs/_templates/example.rst b/docs/_templates/example.rst deleted file mode 100644 index 18093e6..0000000 --- a/docs/_templates/example.rst +++ /dev/null @@ -1,31 +0,0 @@ - -{% if style is not defined or style == 'tab' %} -.. tab-set:: - - .. tab-item:: Result - - {% for line in content %}{{ line }} - {% endfor %} - - .. tab-item:: reStructuredText - - .. code:: rst - - {% for line in content %}{{ line }} - {% endfor %} -{% elif style == 'grid' %} -.. grid:: 2 - - .. grid-item-card:: reStructuredText - - .. code:: rst - - {% for line in content %}{{ line }} - {% endfor %} - - .. grid-item-card:: Result - - {% for line in content %}{{ line }} - {% endfor %} -{% endif %} - diff --git a/docs/_templates/version.rst b/docs/_templates/version.rst deleted file mode 100644 index eac376e..0000000 --- a/docs/_templates/version.rst +++ /dev/null @@ -1,8 +0,0 @@ - -:Date: :ref:`📅{{ date }} ` -:Download: :tag:`{{ title }}` - -{% for line in content %} -{{ line }} -{% endfor %} - diff --git a/docs/conf.py b/docs/conf.py index d6cdf88..18d2567 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -74,49 +74,6 @@ # -- Extensions ------------------------------------------------------------- -# -extensions.append('sphinxnotes.any') -from sphinxnotes.any import Schema, Field as F -# -version_schema = Schema('version', - name=F(unique=True, referenceable=True, required=True, form=F.Form.LINES), - attrs={'date': F(referenceable=True)}, - content=F(form=F.Form.LINES), - description_template=open('_templates/version.rst', 'r').read(), - reference_template='🏷️{{ title }}', - missing_reference_template='🏷️{{ title }}', - ambiguous_reference_template='🏷️{{ title }}') -confval_schema = Schema('confval', - name=F(unique=True, referenceable=True, required=True, form=F.Form.LINES), - attrs={ - 'type': F(), - 'default': F(), - 'choice': F(form=F.Form.WORDS), - 'versionadded': F(), - 'versionchanged': F(form=F.Form.LINES), - }, - content=F(), - description_template=open('_templates/confval.rst', 'r').read(), - reference_template='⚙️{{ title }}', - missing_reference_template='⚙️{{ title }}', - ambiguous_reference_template='⚙️{{ title }}') -example_schema = Schema('example', - name=F(referenceable=True), - attrs={'style': F()}, - content=F(form=F.Form.LINES), - description_template=open('_templates/example.rst', 'r').read(), - reference_template='📝{{ title }}', - missing_reference_template='📝{{ title }}', - ambiguous_reference_template='📝{{ title }}') -# -any_schemas = [ - version_schema, - confval_schema, - example_schema, -] -primary_domain = 'any' -# - extensions.append('sphinx.ext.extlinks') extlinks = { 'issue': ('https://github.com/sphinx-notes/snippet/issues/%s', '💬%s'), @@ -138,14 +95,14 @@ 'jinja': ('https://jinja.palletsprojects.com/en/latest/', None), } -# extensions.append('sphinxnotes.comboroles') comboroles_roles = { 'parsed_literal': (['literal'], True), } -# -# +extensions.append('sphinxnotes.project') +primary_domain = 'any' + # -- Eat your own dog food -------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, @@ -163,4 +120,3 @@ extensions.append('sphinxcontrib.asciinema') # DOG FOOD CONFIGURATION END -# diff --git a/pyproject.toml b/pyproject.toml index 9642379..97c091c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ test = [ ] docs = [ "furo", - "sphinxnotes-any", + "sphinxnotes-project", "sphinx_design", "sphinx_copybutton", "sphinxcontrib-gtagjs", From fe70d415f8fb5dfc2f8467d2abc480cc6cd72264 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Sat, 26 Apr 2025 23:02:13 +0800 Subject: [PATCH 15/17] chore: Update project template to sphinx-notes/cookiecutter@0b096cd7 --- .cruft.json | 4 ++-- .github/workflows/pages.yml | 2 +- docs/index.rst | 8 ++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.cruft.json b/.cruft.json index cea31ef..1292e75 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/sphinx-notes/template", - "commit": "7e07548aad55432b86d881ccdab66eb1c89986c0", + "commit": "0b096cd7ecff1b28ee8c2887ba3321418be34b9a", "checkout": null, "context": { "cookiecutter": { @@ -15,7 +15,7 @@ "pypi_name": "sphinxnotes-snippet", "pypi_owner": "SilverRainZ", "_template": "https://github.com/sphinx-notes/template", - "_commit": "7e07548aad55432b86d881ccdab66eb1c89986c0" + "_commit": "0b096cd7ecff1b28ee8c2887ba3321418be34b9a" } }, "directory": null diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 4b60562..0cf70c8 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -13,7 +13,7 @@ concurrency: jobs: pages: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/docs/index.rst b/docs/index.rst index b2f2962..c5067de 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,7 +5,7 @@ sphinxnotes-snippet =================== -.. |docs| image:: https://img.shields.io/github/deployments/sphinx-notes/snippet/github-pages +.. |docs| image:: https://img.shields.io/github/deployments/sphinx-notes/snippet/github-pages?label=docs :target: https://sphinx.silverrainz.me/snippet :alt: Documentation Status @@ -21,7 +21,11 @@ sphinxnotes-snippet :target: https://pypi.python.org/pypi/sphinxnotes-snippet :alt: PyPI Package Downloads -|docs| |license| |pypi| |download| +.. |github| image:: https://img.shields.io/badge/GitHub-181717?style=flat&logo=github&logoColor=white/ + :target: https://github.com/sphinx-notes/snippet + :alt: GitHub Repository + +|docs| |license| |pypi| |download| |github| Introduction ============ From 6afcc0cdef4d1641f9dc7db3c9b71bd8b07ed349 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 14 Oct 2025 00:24:34 +0800 Subject: [PATCH 16/17] chore: Update project template to sphinx-notes/cookiecutter@800a79a0 --- .cruft.json | 9 ++++-- Makefile | 52 +++++++++++++++++++------------- README.rst | 3 -- docs/conf.py | 19 ++++++++---- docs/index.rst | 6 ++-- pyproject.toml | 41 ++++++++++++++++++++----- ruff.toml | 5 +++ src/sphinxnotes/snippet/meta.py | 35 +++++++++++++++++++++ src/sphinxnotes/snippet/py.typed | 0 9 files changed, 126 insertions(+), 44 deletions(-) create mode 100644 src/sphinxnotes/snippet/meta.py create mode 100644 src/sphinxnotes/snippet/py.typed diff --git a/.cruft.json b/.cruft.json index 1292e75..99ef9a6 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/sphinx-notes/template", - "commit": "0b096cd7ecff1b28ee8c2887ba3321418be34b9a", + "commit": "800a79a06f9a67494516bc14bcda85b3472dcac9", "checkout": null, "context": { "cookiecutter": { @@ -14,8 +14,13 @@ "github_repo": "snippet", "pypi_name": "sphinxnotes-snippet", "pypi_owner": "SilverRainZ", + "is_python_project": true, + "python_version": "3.12", + "is_sphinx_extension": true, + "sphinx_version": "7.0", + "development_status": "3 - Alpha", "_template": "https://github.com/sphinx-notes/template", - "_commit": "0b096cd7ecff1b28ee8c2887ba3321418be34b9a" + "_commit": "800a79a06f9a67494516bc14bcda85b3472dcac9" } }, "directory": null diff --git a/Makefile b/Makefile index c2786f4..e3d46d8 100644 --- a/Makefile +++ b/Makefile @@ -9,30 +9,30 @@ RM = rm -rf GIT = git OPEN = xdg-open -# Build sphinx documentation. .PHONY: docs docs: $(MAKE) -C docs/ -# View sphinx HTML documentation in browser. .PHONY: view view: $(OPEN) docs/_build/html/index.html .PHONY: clean clean: - $(MAKE) -C docs/ clean | true - $(RM) dist/ | true + $(MAKE) -C docs/ clean; $(RM) dist/ -.PHONY: clean +.PHONY: fmt fmt: - ruff format src/ + ruff format src/ && ruff check --fix src/ -# Run unittest. .PHONY: test test: $(PY) -m unittest discover -s tests -v +################################################################################ +# Distribution Package +################################################################################ + # Build distribution package, for "install" or "upload". .PHONY: dist dist: pyproject.toml clean @@ -54,33 +54,43 @@ install: dist upload: dist $(PY) -m twine upload --repository pypi $. -.PHONY: upload-test -upload-test: dist - $(PY) -m twine upload --repository testpypi $=3.12" dependencies = [ - "Sphinx >= 4", + "Sphinx >= 7.0", + + # CUSTOM DEPENDENCIES START "langid", "jieba", "python-pinyin", @@ -38,6 +52,7 @@ dependencies = [ "setuptools", # req by stopwordsiso, https://stackoverflow.com/a/39930983/4799273 "wcwidth", "wordsegment", + # CUSTOM DEPENDENCIES END ] dynamic = ["version"] # required by setuptools_scm, see section [build-system] @@ -47,20 +62,28 @@ dev = [ "build", "twine", "cruft", - "ruff", - "pre-commit" + "ruff>=0.11.10", # pwndbg#2716 + "pre-commit", ] test = [ "pytest", ] docs = [ "furo", - "sphinxnotes-project", "sphinx_design", "sphinx_copybutton", "sphinxcontrib-gtagjs", + "sphinx-sitemap", + "sphinxext-opengraph", + "sphinx-last-updated-by-git", + + # Dependencies of sphinxnotes projcts. + "sphinxnotes-project", "sphinxnotes-comboroles", + + # CUSTOM DOCS DEPENDENCIES START "sphinxcontrib.asciinema", + # CUSTOM DOCS DEPENDENCIES END ] [project.urls] @@ -87,6 +110,8 @@ version_scheme = "no-guess-dev" # check out https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#finding-namespace-packages where = ["src"] +# CUSTOM CONFIGURATION + [tool.setuptools.package-data] # A maps from PACKAGE NAMES to lists of glob patterns, # see also https://setuptools.pypa.io/en/latest/userguide/datafiles.html diff --git a/ruff.toml b/ruff.toml index 08e71c6..1774a67 100644 --- a/ruff.toml +++ b/ruff.toml @@ -7,3 +7,8 @@ exclude = [ [format] quote-style = "single" + +[lint] +ignore = [ + "E741", # Checks for the use of the characters 'l', 'O', or 'I' as variable names +] diff --git a/src/sphinxnotes/snippet/meta.py b/src/sphinxnotes/snippet/meta.py new file mode 100644 index 0000000..e7dbf2c --- /dev/null +++ b/src/sphinxnotes/snippet/meta.py @@ -0,0 +1,35 @@ +# This file is generated from sphinx-notes/cookiecutter. +# DO NOT EDIT!!! + +################################################################################ +# Project meta infos. +################################################################################ + +from __future__ import annotations +from importlib import metadata + +__project__ = 'sphinxnotes-snippet' +__author__ = 'Shengyu Zhang' +__desc__ = 'Sphinx documentation snippets manager' + +try: + __version__ = metadata.version('sphinxnotes-snippet') +except metadata.PackageNotFoundError: + __version__ = 'unknown' + + +################################################################################ +# Sphinx extension utils. +################################################################################ + + +def pre_setup(app): + app.require_sphinx('7.0') + + +def post_setup(app): + return { + 'version': __version__, + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/src/sphinxnotes/snippet/py.typed b/src/sphinxnotes/snippet/py.typed new file mode 100644 index 0000000..e69de29 From eb010e1e6fbd50d65cca0d8caa41db1d024c11e7 Mon Sep 17 00:00:00 2001 From: Shengyu Zhang Date: Tue, 21 Oct 2025 14:43:59 +0800 Subject: [PATCH 17/17] chore: Update project template to sphinx-notes/cookiecutter@4736b672 --- .cruft.json | 4 ++-- README.rst | 2 +- docs/index.rst | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.cruft.json b/.cruft.json index 99ef9a6..42d9051 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/sphinx-notes/template", - "commit": "800a79a06f9a67494516bc14bcda85b3472dcac9", + "commit": "4736b672023e3bf4bc80d0d51850d7bc45ca1e12", "checkout": null, "context": { "cookiecutter": { @@ -20,7 +20,7 @@ "sphinx_version": "7.0", "development_status": "3 - Alpha", "_template": "https://github.com/sphinx-notes/template", - "_commit": "800a79a06f9a67494516bc14bcda85b3472dcac9" + "_commit": "4736b672023e3bf4bc80d0d51850d7bc45ca1e12" } }, "directory": null diff --git a/README.rst b/README.rst index 5155885..2c1e79d 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,7 @@ sphinxnotes-snippet :target: https://pypi.python.org/pypi/sphinxnotes-snippet :alt: PyPI Package .. |download| image:: https://img.shields.io/pypi/dm/sphinxnotes-snippet - :target: https://pypi.python.org/pypi/sphinxnotes-snippet + :target: https://pypistats.org/packages/sphinxnotes-snippet :alt: PyPI Package Downloads |docs| |license| |pypi| |download| diff --git a/docs/index.rst b/docs/index.rst index bdd551f..2894d52 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,7 +12,7 @@ sphinxnotes-snippet :target: https://github.com/sphinx-notes/snippet/blob/master/LICENSE :alt: Open Source License .. |pypi| image:: https://img.shields.io/pypi/v/sphinxnotes-snippet.svg - :target: https://pypi.python.org/pypi/sphinxnotes-snippet + :target: https://pypistats.org/packages/sphinxnotes-snippet :alt: PyPI Package .. |download| image:: https://img.shields.io/pypi/dm/sphinxnotes-snippet :target: https://pypi.python.org/pypi/sphinxnotes-snippet