diff --git a/.cruft.json b/.cruft.json index e3324c6..42d9051 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/sphinx-notes/template", - "commit": "80a61fa9abcd9474d8cfbc36d0bf5d41f99c916c", + "commit": "4736b672023e3bf4bc80d0d51850d7bc45ca1e12", "checkout": null, "context": { "cookiecutter": { @@ -14,7 +14,13 @@ "github_repo": "snippet", "pypi_name": "sphinxnotes-snippet", "pypi_owner": "SilverRainZ", - "_template": "https://github.com/sphinx-notes/template" + "is_python_project": true, + "python_version": "3.12", + "is_sphinx_extension": true, + "sphinx_version": "7.0", + "development_status": "3 - Alpha", + "_template": "https://github.com/sphinx-notes/template", + "_commit": "4736b672023e3bf4bc80d0d51850d7bc45ca1e12" } }, "directory": null diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 4b60562..0cf70c8 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -13,7 +13,7 @@ concurrency: jobs: pages: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/Makefile b/Makefile index d1f5d08..e3d46d8 100644 --- a/Makefile +++ b/Makefile @@ -9,30 +9,30 @@ RM = rm -rf GIT = git OPEN = xdg-open -# Build sphinx documentation. .PHONY: docs docs: $(MAKE) -C docs/ -# View sphinx HTML documentation in browser. .PHONY: view view: $(OPEN) docs/_build/html/index.html .PHONY: clean clean: - $(MAKE) -C docs/ clean - $(RM) dist/ + $(MAKE) -C docs/ clean; $(RM) dist/ -.PHONY: clean +.PHONY: fmt fmt: - ruff format src/ + ruff format src/ && ruff check --fix src/ -# Run unittest. .PHONY: test test: $(PY) -m unittest discover -s tests -v +################################################################################ +# Distribution Package +################################################################################ + # Build distribution package, for "install" or "upload". .PHONY: dist dist: pyproject.toml clean @@ -54,33 +54,43 @@ install: dist upload: dist $(PY) -m twine upload --repository pypi $. -.PHONY: upload-test -upload-test: dist - $(PY) -m twine upload --repository testpypi $` -:Download: :tag:`{{ title }}` - -{% for line in content %} -{{ line }} -{% endfor %} - diff --git a/docs/conf.py b/docs/conf.py index c7d27b2..638392c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,9 +7,6 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html -import os -import sys - # -- Project information ----------------------------------------------------- project = 'sphinxnotes-snippet' @@ -28,6 +25,7 @@ 'sphinx.ext.githubpages', 'sphinx_design', 'sphinx_copybutton', + 'sphinx_last_updated_by_git', ] # Add any paths that contain templates here, relative to this directory. @@ -74,49 +72,6 @@ # -- Extensions ------------------------------------------------------------- -# -extensions.append('sphinxnotes.any') -from sphinxnotes.any import Schema, Field as F -# -version_schema = Schema('version', - name=F(unique=True, referenceable=True, required=True, form=F.Form.LINES), - attrs={'date': F(referenceable=True)}, - content=F(form=F.Form.LINES), - description_template=open('_templates/version.rst', 'r').read(), - reference_template='🏷ī¸{{ title }}', - missing_reference_template='🏷ī¸{{ title }}', - ambiguous_reference_template='🏷ī¸{{ title }}') -confval_schema = Schema('confval', - name=F(unique=True, referenceable=True, required=True, form=F.Form.LINES), - attrs={ - 'type': F(), - 'default': F(), - 'choice': F(form=F.Form.WORDS), - 'versionadded': F(), - 'versionchanged': F(form=F.Form.LINES), - }, - content=F(), - description_template=open('_templates/confval.rst', 'r').read(), - reference_template='⚙ī¸{{ title }}', - missing_reference_template='⚙ī¸{{ title }}', - ambiguous_reference_template='⚙ī¸{{ title }}') -example_schema = Schema('example', - name=F(referenceable=True), - attrs={'style': F()}, - content=F(form=F.Form.LINES), - description_template=open('_templates/example.rst', 'r').read(), - reference_template='📝{{ title }}', - missing_reference_template='📝{{ title }}', - ambiguous_reference_template='📝{{ title }}') -# -any_schemas = [ - version_schema, - confval_schema, - example_schema, -] -primary_domain = 'any' -# - extensions.append('sphinx.ext.extlinks') extlinks = { 'issue': ('https://github.com/sphinx-notes/snippet/issues/%s', 'đŸ’Ŧ%s'), @@ -138,29 +93,37 @@ 'jinja': ('https://jinja.palletsprojects.com/en/latest/', None), } -# +extensions.append('sphinx_sitemap') +sitemap_filename = "sitemap.xml" +sitemap_url_scheme = "{link}" + +extensions.append('sphinxext.opengraph') +ogp_site_url = html_baseurl +ogp_site_name = project +ogp_image = html_baseurl + '/' + html_logo + extensions.append('sphinxnotes.comboroles') comboroles_roles = { 'parsed_literal': (['literal'], True), } -# -# +extensions.append('sphinxnotes.project') +primary_domain = 'any' + # -- Eat your own dog food -------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +import os +import sys sys.path.insert(0, os.path.abspath('../src/sphinxnotes')) -extensions.append('snippet.ext') +extensions.append('snippet') -# DOG FOOD CONFIGURATION START +# CUSTOM CONFIGURATION snippet_config = { 'cache_dir': '/tmp/' + 'sphinxnotes-snippet' } extensions.append('sphinxcontrib.asciinema') - -# DOG FOOD CONFIGURATION END -# diff --git a/docs/index.rst b/docs/index.rst index b2f2962..2894d52 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,23 +5,23 @@ sphinxnotes-snippet =================== -.. |docs| image:: https://img.shields.io/github/deployments/sphinx-notes/snippet/github-pages +.. |docs| image:: https://img.shields.io/github/deployments/sphinx-notes/snippet/github-pages?label=docs :target: https://sphinx.silverrainz.me/snippet :alt: Documentation Status - .. |license| image:: https://img.shields.io/github/license/sphinx-notes/snippet :target: https://github.com/sphinx-notes/snippet/blob/master/LICENSE :alt: Open Source License - .. |pypi| image:: https://img.shields.io/pypi/v/sphinxnotes-snippet.svg - :target: https://pypi.python.org/pypi/sphinxnotes-snippet + :target: https://pypistats.org/packages/sphinxnotes-snippet :alt: PyPI Package - .. |download| image:: https://img.shields.io/pypi/dm/sphinxnotes-snippet :target: https://pypi.python.org/pypi/sphinxnotes-snippet :alt: PyPI Package Downloads +.. |github| image:: https://img.shields.io/badge/GitHub-181717?style=flat&logo=github&logoColor=white/ + :target: https://github.com/sphinx-notes/snippet + :alt: GitHub Repository -|docs| |license| |pypi| |download| +|docs| |license| |pypi| |download| |github| Introduction ============ @@ -44,12 +44,14 @@ Getting Started We assume you already have a Sphinx documentation, if not, see `Getting Started with Sphinx`_. + First, downloading extension from PyPI: .. code-block:: console $ pip install sphinxnotes-snippet + Then, add the extension name to ``extensions`` configuration item in your :parsed_literal:`conf.py_`: diff --git a/pyproject.toml b/pyproject.toml index c64582d..4e6b345 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,27 +9,41 @@ name = "sphinxnotes-snippet" description = "Sphinx documentation snippets manager" readme = "README.rst" -license = { file = "BSD-3-Clause" } +license = "BSD-3-Clause" +license-files = ["LICENSE"] authors = [ { name = "Shengyu Zhang" } ] maintainers = [ { name = "Shengyu Zhang" } ] -keywords = ["sphinx", "extension", "documentation"] # TOOD: additional_keywords +keywords = [ + "sphinx", + "extension", + "documentation", + "sphinxnotes", + + # CUSTOM KEYWORDS START + # CUSTOM KEYWORDS END +] classifiers = [ - "Development Status :: 4 - Beta", + "Development Status :: 3 - Alpha", "Environment :: Plugins", "Framework :: Sphinx", "Framework :: Sphinx :: Extension", - "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Topic :: Documentation", "Topic :: Documentation :: Sphinx", + + # CUSTOM CLASSIFIERS START "Topic :: Utilities", + # CUSTOM CLASSIFIERS END ] -requires-python = ">=3.8" +# See ``make pyver`` for more details. +requires-python = ">=3.12" dependencies = [ - "Sphinx >= 4", + "Sphinx >= 7.0", + + # CUSTOM DEPENDENCIES START "langid", "jieba", "python-pinyin", @@ -38,6 +52,7 @@ dependencies = [ "setuptools", # req by stopwordsiso, https://stackoverflow.com/a/39930983/4799273 "wcwidth", "wordsegment", + # CUSTOM DEPENDENCIES END ] dynamic = ["version"] # required by setuptools_scm, see section [build-system] @@ -47,20 +62,28 @@ dev = [ "build", "twine", "cruft", - "ruff", - "pre-commit" + "ruff>=0.11.10", # pwndbg#2716 + "pre-commit", ] test = [ "pytest", ] docs = [ "furo", - "sphinxnotes-any", "sphinx_design", "sphinx_copybutton", "sphinxcontrib-gtagjs", + "sphinx-sitemap", + "sphinxext-opengraph", + "sphinx-last-updated-by-git", + + # Dependencies of sphinxnotes projcts. + "sphinxnotes-project", "sphinxnotes-comboroles", + + # CUSTOM DOCS DEPENDENCIES START "sphinxcontrib.asciinema", + # CUSTOM DOCS DEPENDENCIES END ] [project.urls] @@ -87,6 +110,8 @@ version_scheme = "no-guess-dev" # check out https://setuptools.pypa.io/en/latest/userguide/package_discovery.html#finding-namespace-packages where = ["src"] +# CUSTOM CONFIGURATION + [tool.setuptools.package-data] # A maps from PACKAGE NAMES to lists of glob patterns, # see also https://setuptools.pypa.io/en/latest/userguide/datafiles.html diff --git a/ruff.toml b/ruff.toml index 08e71c6..1774a67 100644 --- a/ruff.toml +++ b/ruff.toml @@ -7,3 +7,8 @@ exclude = [ [format] quote-style = "single" + +[lint] +ignore = [ + "E741", # Checks for the use of the characters 'l', 'O', or 'I' as variable names +] diff --git a/src/sphinxnotes/snippet/__init__.py b/src/sphinxnotes/snippet/__init__.py index 042d354..0fd7955 100644 --- a/src/sphinxnotes/snippet/__init__.py +++ b/src/sphinxnotes/snippet/__init__.py @@ -2,185 +2,35 @@ sphinxnotes.snippet ~~~~~~~~~~~~~~~~~~~ -:copyright: Copyright 2020 Shengyu Zhang +Sphinx extension entrypoint. + +:copyright: Copyright 2024 Shengyu Zhang :license: BSD, see LICENSE for details. """ -from __future__ import annotations -from typing import List, Tuple, Optional, TYPE_CHECKING -import itertools - -from docutils import nodes - -if TYPE_CHECKING: - from sphinx.environment import BuildEnvironment - -__version__ = '1.1.1' - - -class Snippet(object): - """ - Snippet is base class of reStructuredText snippet. - - :param nodes: Document nodes that make up this snippet - """ - - #: docname where the snippet is located, can be referenced by - # :rst:role:`doc`. - docname: str - - #: Source file path of snippet - file: str - - #: Line number range of snippet, in the source file which is left closed - #: and right opened. - lineno: Tuple[int, int] - - #: The original reStructuredText of snippet - rst: List[str] - - #: The possible identifier key of snippet, which is picked from nodes' - #: (or nodes' parent's) `ids attr`_. - #: - #: .. _ids attr: https://docutils.sourceforge.io/docs/ref/doctree.html#ids - refid: Optional[str] - - def __init__(self, *nodes: nodes.Node) -> None: - assert len(nodes) != 0 - - env: BuildEnvironment = nodes[0].document.settings.env - self.file = nodes[0].source - self.docname = env.path2doc(self.file) - - lineno = [float('inf'), -float('inf')] - for node in nodes: - if not node.line: - continue # Skip node that have None line, I dont know why - lineno[0] = min(lineno[0], _line_of_start(node)) - lineno[1] = max(lineno[1], _line_of_end(node)) - self.lineno = lineno - - lines = [] - with open(self.file, 'r') as f: - start = self.lineno[0] - 1 - stop = self.lineno[1] - 1 - for line in itertools.islice(f, start, stop): - lines.append(line.strip('\n')) - self.rst = lines - - # Find exactly one ID attr in nodes - self.refid = None - for node in nodes: - if node['ids']: - self.refid = node['ids'][0] - break - - # If no ID found, try parent - if not self.refid: - for node in nodes: - if node.parent['ids']: - self.refid = node.parent['ids'][0] - break - - -class Text(Snippet): - #: Text of snippet - text: str - - def __init__(self, node: nodes.Node) -> None: - super().__init__(node) - self.text = node.astext() - - -class CodeBlock(Text): - #: Language of code block - language: str - #: Caption of code block - caption: Optional[str] - - def __init__(self, node: nodes.literal_block) -> None: - assert isinstance(node, nodes.literal_block) - super().__init__(node) - self.language = node['language'] - self.caption = node.get('caption') - - -class WithCodeBlock(object): - code_blocks: List[CodeBlock] - - def __init__(self, nodes: nodes.Nodes) -> None: - self.code_blocks = [] - for n in nodes.traverse(nodes.literal_block): - self.code_blocks.append(self.CodeBlock(n)) - - -class Title(Text): - def __init__(self, node: nodes.title) -> None: - assert isinstance(node, nodes.title) - super().__init__(node) - - -class WithTitle(object): - title: Optional[Title] - - def __init__(self, node: nodes.Node) -> None: - title_node = node.next_node(nodes.title) - self.title = Title(title_node) if title_node else None - - -class Section(Snippet, WithTitle): - def __init__(self, node: nodes.section) -> None: - assert isinstance(node, nodes.section) - Snippet.__init__(self, node) - WithTitle.__init__(self, node) - - -class Document(Section): - def __init__(self, node: nodes.document) -> None: - assert isinstance(node, nodes.document) - super().__init__(node.next_node(nodes.section)) - - -################ -# Nodes helper # -################ - - -def _line_of_start(node: nodes.Node) -> int: - assert node.line - if isinstance(node, nodes.title): - if isinstance(node.parent.parent, nodes.document): - # Spceial case for Document Title / Subtitle - return 1 - else: - # Spceial case for section title - return node.line - 1 - elif isinstance(node, nodes.section): - if isinstance(node.parent, nodes.document): - # Spceial case for top level section - return 1 - else: - # Spceial case for section - return node.line - 1 - return node.line - -def _line_of_end(node: nodes.Node) -> Optional[int]: - next_node = node.next_node(descend=False, siblings=True, ascend=True) - while next_node: - if next_node.line: - return _line_of_start(next_node) - next_node = next_node.next_node( - # Some nodes' line attr is always None, but their children has - # valid line attr - descend=True, - # If node and its children have not valid line attr, try use line - # of next node - ascend=True, - siblings=True, - ) - # No line found, return the max line of source file - if node.source: - with open(node.source) as f: - return sum(1 for line in f) - raise AttributeError('None source attr of node %s' % node) +def setup(app): + # **WARNING**: We don't import these packages globally, because the current + # package (sphinxnotes.snippet) is always resloved when importing + # sphinxnotes.snippet.*. If we import packages here, eventually we will + # load a lot of packages from the Sphinx. It will seriously **SLOW DOWN** + # the startup time of our CLI tool (sphinxnotes.snippet.cli). + # + # .. seealso:: https://github.com/sphinx-notes/snippet/pull/31 + from .ext import ( + SnippetBuilder, + on_config_inited, + on_env_get_outdated, + on_doctree_resolved, + on_builder_finished, + ) + + app.add_builder(SnippetBuilder) + + app.add_config_value('snippet_config', {}, '') + app.add_config_value('snippet_patterns', {'*': ['.*']}, '') + + app.connect('config-inited', on_config_inited) + app.connect('env-get-outdated', on_env_get_outdated) + app.connect('doctree-resolved', on_doctree_resolved) + app.connect('build-finished', on_builder_finished) diff --git a/src/sphinxnotes/snippet/cache.py b/src/sphinxnotes/snippet/cache.py index 5c5c4e3..78a2efa 100644 --- a/src/sphinxnotes/snippet/cache.py +++ b/src/sphinxnotes/snippet/cache.py @@ -1,4 +1,5 @@ -"""sphinxnotes.snippet.cache +""" +sphinxnotes.snippet.cache ~~~~~~~~~~~~~~~~~~~~~~~~~ :copyright: Copyright 2021 Shengyu Zhang @@ -6,10 +7,9 @@ """ from __future__ import annotations -from typing import List, Tuple, Dict, Optional from dataclasses import dataclass -from . import Snippet +from .snippets import Snippet from .utils.pdict import PDict @@ -18,25 +18,25 @@ class Item(object): """Item of snippet cache.""" snippet: Snippet - tags: List[str] + tags: str excerpt: str - titlepath: List[str] - keywords: List[str] + titlepath: list[str] + keywords: list[str] -DocID = Tuple[str, str] # (project, docname) +DocID = tuple[str, str] # (project, docname) IndexID = str # UUID -Index = Tuple[str, str, List[str], List[str]] # (tags, excerpt, titlepath, keywords) +Index = tuple[str, str, list[str], list[str]] # (tags, excerpt, titlepath, keywords) -class Cache(PDict): - """A DocID -> List[Item] Cache.""" +class Cache(PDict[DocID, list[Item]]): + """A DocID -> list[Item] Cache.""" - indexes: Dict[IndexID, Index] - index_id_to_doc_id: Dict[IndexID, Tuple[DocID, int]] - doc_id_to_index_ids: Dict[DocID, List[IndexID]] - num_snippets_by_project: Dict[str, int] - num_snippets_by_docid: Dict[DocID, int] + indexes: dict[IndexID, Index] + index_id_to_doc_id: dict[IndexID, tuple[DocID, int]] + doc_id_to_index_ids: dict[DocID, list[IndexID]] + num_snippets_by_project: dict[str, int] + num_snippets_by_docid: dict[DocID, int] def __init__(self, dirname: str) -> None: self.indexes = {} @@ -46,7 +46,7 @@ def __init__(self, dirname: str) -> None: self.num_snippets_by_docid = {} super().__init__(dirname) - def post_dump(self, key: DocID, items: List[Item]) -> None: + def post_dump(self, key: DocID, value: list[Item]) -> None: """Overwrite PDict.post_dump.""" # Remove old indexes and index IDs if exists @@ -55,7 +55,7 @@ def post_dump(self, key: DocID, items: List[Item]) -> None: del self.indexes[old_index_id] # Add new index to every where - for i, item in enumerate(items): + for i, item in enumerate(value): index_id = self.gen_index_id() self.indexes[index_id] = ( item.tags, @@ -69,12 +69,12 @@ def post_dump(self, key: DocID, items: List[Item]) -> None: # Update statistic if key[0] not in self.num_snippets_by_project: self.num_snippets_by_project[key[0]] = 0 - self.num_snippets_by_project[key[0]] += len(items) + self.num_snippets_by_project[key[0]] += len(value) if key not in self.num_snippets_by_docid: self.num_snippets_by_docid[key] = 0 - self.num_snippets_by_docid[key] += len(items) + self.num_snippets_by_docid[key] += len(value) - def post_purge(self, key: DocID, items: List[Item]) -> None: + def post_purge(self, key: DocID, value: list[Item]) -> None: """Overwrite PDict.post_purge.""" # Purge indexes @@ -83,17 +83,17 @@ def post_purge(self, key: DocID, items: List[Item]) -> None: del self.indexes[index_id] # Update statistic - self.num_snippets_by_project[key[0]] -= len(items) + self.num_snippets_by_project[key[0]] -= len(value) if self.num_snippets_by_project[key[0]] == 0: del self.num_snippets_by_project[key[0]] - self.num_snippets_by_docid[key] -= len(items) + self.num_snippets_by_docid[key] -= len(value) if self.num_snippets_by_docid[key] == 0: del self.num_snippets_by_docid[key] - def get_by_index_id(self, key: IndexID) -> Optional[Item]: + def get_by_index_id(self, key: IndexID) -> Item | None: """Like get(), but use IndexID as key.""" doc_id, item_index = self.index_id_to_doc_id.get(key, (None, None)) - if not doc_id: + if not doc_id or item_index is None: return None return self[doc_id][item_index] @@ -103,6 +103,6 @@ def gen_index_id(self) -> str: return uuid.uuid4().hex[:7] - def stringify(self, key: DocID, items: List[Item]) -> str: + def stringify(self, key: DocID, value: list[Item]) -> str: """Overwrite PDict.stringify.""" - return key[1] + return key[1] # docname diff --git a/src/sphinxnotes/snippet/cli.py b/src/sphinxnotes/snippet/cli.py index dcc732f..a52b17a 100644 --- a/src/sphinxnotes/snippet/cli.py +++ b/src/sphinxnotes/snippet/cli.py @@ -2,24 +2,30 @@ sphinxnotes.snippet.cli ~~~~~~~~~~~~~~~~~~~~~~~ -:copyright: Copyright 2020 Shengyu Zhang +Command line entrypoint. + +:copyright: Copyright 2024 Shengyu Zhang :license: BSD, see LICENSE for details. """ +# **NOTE**: Import new packages with caution: +# Importing complex packages (like sphinx.*) will directly slow down the +# startup of the CLI tool. from __future__ import annotations import sys -import argparse -from typing import List +import os from os import path +import argparse +from typing import Iterable from textwrap import dedent from shutil import get_terminal_size import posixpath from xdg.BaseDirectory import xdg_config_home -from . import __version__ +from .snippets import Document from .config import Config -from .cache import Cache +from .cache import Cache, IndexID, Index from .table import tablify, COLUMNS DEFAULT_CONFIG_FILE = path.join(xdg_config_home, 'sphinxnotes', 'snippet', 'conf.py') @@ -38,14 +44,14 @@ def get_integration_file(fn: str) -> str: .. seealso:: see ``[tool.setuptools.package-data]`` section of pyproject.toml to know - how files are included. + how files are included. """ # TODO: use https://docs.python.org/3/library/importlib.resources.html#importlib.resources.files prefix = path.abspath(path.dirname(__file__)) return path.join(prefix, 'integration', fn) -def main(argv: List[str] = sys.argv[1:]): +def main(argv: list[str] = sys.argv[1:]): """Command line entrypoint.""" parser = argparse.ArgumentParser( @@ -54,13 +60,17 @@ def main(argv: List[str] = sys.argv[1:]): formatter_class=HelpFormatter, epilog=dedent(""" snippet tags: - d (document) a reST document - s (section) a reST section - c (code) snippet with code blocks + d (document) a document + s (section) a section + c (code) a code block * (any) wildcard for any snippet"""), ) parser.add_argument( - '-v', '--version', action='version', version='%(prog)s ' + __version__ + '--version', + # add_argument provides action='version', but it requires a version + # literal and doesn't support lazily obtaining version. + action='store_true', + help="show program's version number and exit", ) parser.add_argument( '-c', '--config', default=DEFAULT_CONFIG_FILE, help='path to configuration file' @@ -83,7 +93,14 @@ def main(argv: List[str] = sys.argv[1:]): help='list snippet indexes, columns of indexes: %s' % COLUMNS, ) listparser.add_argument( - '--tags', '-t', type=str, default='*', help='list specified tags only' + '--tags', '-t', type=str, default='*', help='list snippets with specified tags' + ) + listparser.add_argument( + '--docname', + '-d', + type=str, + default='**', + help='list snippets whose docname matches shell-style glob pattern', ) listparser.add_argument( '--width', @@ -106,6 +123,9 @@ def main(argv: List[str] = sys.argv[1:]): getparser.add_argument( '--file', '-f', action='store_true', help='get source file path of snippet' ) + getparser.add_argument( + '--deps', action='store_true', help='get dependent files of document' + ) getparser.add_argument( '--line-start', action='store_true', @@ -120,7 +140,12 @@ def main(argv: List[str] = sys.argv[1:]): '--text', '-t', action='store_true', - help='get source reStructuredText of snippet', + help='get text representation of snippet', + ) + getparser.add_argument( + '--src', + action='store_true', + help='get source text of snippet', ) getparser.add_argument( '--url', @@ -165,6 +190,16 @@ def main(argv: List[str] = sys.argv[1:]): # Parse command line arguments args = parser.parse_args(argv) + # Print version message. + # See parser.add_argument('--version', ...) for more detais. + if args.version: + # NOTE: Importing is slow, do it on demand. + from importlib.metadata import version + + pkgname = 'sphinxnotes.snippet' + print(pkgname, version(pkgname)) + parser.exit() + # Load config from file if args.config == DEFAULT_CONFIG_FILE and not path.isfile(DEFAULT_CONFIG_FILE): print( @@ -205,24 +240,62 @@ def _on_command_stat(args: argparse.Namespace): print(f'\t {v} snippets(s)') +def _filter_list_items( + cache: Cache, tags: str, docname_glob: str +) -> Iterable[tuple[IndexID, Index]]: + # NOTE: Importing is slow, do it on demand. + from sphinx.util.matching import patmatch + + for index_id, index in cache.indexes.items(): + # Filter by tags. + if index[0] not in tags and '*' not in tags: + continue + # Filter by docname. + (_, docname), _ = cache.index_id_to_doc_id[index_id] + if not patmatch(docname, docname_glob): + continue + yield (index_id, index) + + def _on_command_list(args: argparse.Namespace): - rows = tablify(args.cache.indexes, args.tags, args.width) - for row in rows: + items = _filter_list_items(args.cache, args.tags, args.docname) + for row in tablify(items, args.width): print(row) def _on_command_get(args: argparse.Namespace): + # Wrapper for warning when nothing is printed + printed = False + + def p(*args, **opts): + nonlocal printed + printed = True + print(*args, **opts) + for index_id in args.index_id: item = args.cache.get_by_index_id(index_id) if not item: - print('no such index ID', file=sys.stderr) + p('no such index ID', file=sys.stderr) sys.exit(1) if args.text: - print('\n'.join(item.snippet.rst)) + p('\n'.join(item.snippet.text)) + if args.src: + p('\n'.join(item.snippet.source)) if args.docname: - print(item.snippet.docname) + p(item.snippet.docname) if args.file: - print(item.snippet.file) + p(item.snippet.file) + if args.deps: + if not isinstance(item.snippet, Document): + print( + f'{type(item.snippet)} ({index_id}) is not a document', + file=sys.stderr, + ) + sys.exit(1) + if len(item.snippet.deps) == 0: + p('') # prevent print nothing warning + for dep in item.snippet.deps: + p(dep) if args.url: # HACK: get doc id in better way doc_id, _ = args.cache.index_id_to_doc_id.get(index_id) @@ -236,11 +309,15 @@ def _on_command_get(args: argparse.Namespace): url = posixpath.join(base_url, doc_id[1] + '.html') if item.snippet.refid: url += '#' + item.snippet.refid - print(url) + p(url) if args.line_start: - print(item.snippet.lineno[0]) + p(item.snippet.lineno[0]) if args.line_end: - print(item.snippet.lineno[1]) + p(item.snippet.lineno[1]) + + if not printed: + print('please specify at least one argument', file=sys.stderr) + sys.exit(1) def _on_command_integration(args: argparse.Namespace): @@ -275,4 +352,13 @@ def _on_command_integration(args: argparse.Namespace): if __name__ == '__main__': - sys.exit(main()) + # Prevent "[Errno 32] Broken pipe" error. + # https://docs.python.org/3/library/signal.html#note-on-sigpipe + try: + sys.exit(main()) + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown. + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + sys.exit(1) # Python exits with error code 1 on EPIPE diff --git a/src/sphinxnotes/snippet/ext.py b/src/sphinxnotes/snippet/ext.py index e37fd23..178c98c 100644 --- a/src/sphinxnotes/snippet/ext.py +++ b/src/sphinxnotes/snippet/ext.py @@ -1,15 +1,15 @@ """ -sphinxnotes.ext.snippet +sphinxnotes.snippet.ext ~~~~~~~~~~~~~~~~~~~~~~~ -Sphinx extension for sphinxnotes.snippet. +Sphinx extension implementation, but the entrypoint is located at __init__.py. -:copyright: Copyright 2021 Shengyu Zhang +:copyright: Copyright 2024 Shengyu Zhang :license: BSD, see LICENSE for details. """ from __future__ import annotations -from typing import List, Set, TYPE_CHECKING, Dict +from typing import TYPE_CHECKING import re from os import path import time @@ -26,7 +26,7 @@ from collections.abc import Iterator from .config import Config -from . import Snippet, WithTitle, Document, Section +from .snippets import Snippet, WithTitle, Document, Section, Code from .picker import pick from .cache import Cache, Item from .keyword import Extractor @@ -45,53 +45,38 @@ def extract_tags(s: Snippet) -> str: tags += 'd' elif isinstance(s, Section): tags += 's' + elif isinstance(s, Code): + tags += 'c' return tags def extract_excerpt(s: Snippet) -> str: if isinstance(s, Document) and s.title is not None: - return '<' + s.title.text + '>' + return '<' + s.title + '>' elif isinstance(s, Section) and s.title is not None: - return '[' + s.title.text + ']' + return '[' + s.title + ']' + elif isinstance(s, Code): + return '`' + (s.lang + ':').ljust(8, ' ') + ' ' + s.desc + '`' return '' -def extract_keywords(s: Snippet) -> List[str]: - keywords = [] - # TODO: Deal with more snippet +def extract_keywords(s: Snippet) -> list[str]: + keywords = [s.docname] if isinstance(s, WithTitle) and s.title is not None: - keywords.extend(extractor.extract(s.title.text, strip_stopwords=False)) + keywords.extend(extractor.extract(s.title, strip_stopwords=False)) + if isinstance(s, Code): + keywords.extend(extractor.extract(s.desc, strip_stopwords=False)) return keywords -def is_document_matched( - pats: Dict[str, List[str]], docname: str -) -> Dict[str, List[str]]: - """Whether the docname matched by given patterns pats""" - new_pats = {} - for tag, ps in pats.items(): +def _get_document_allowed_tags(pats: dict[str, list[str]], docname: str) -> str: + """Return the tags of snippets that are allowed to be picked from the document.""" + allowed_tags = '' + for tags, ps in pats.items(): for pat in ps: if re.match(pat, docname): - new_pats.setdefault(tag, []).append(pat) - return new_pats - - -def is_snippet_matched(pats: Dict[str, List[str]], s: [Snippet], docname: str) -> bool: - """Whether the snippet's tags and docname matched by given patterns pats""" - if '*' in pats: # Wildcard - for pat in pats['*']: - if re.match(pat, docname): - return True - - not_in_pats = True - for k in extract_tags(s): - if k not in pats: - continue - not_in_pats = False - for pat in pats[k]: - if re.match(pat, docname): - return True - return not_in_pats + allowed_tags += tags + return allowed_tags def on_config_inited(app: Sphinx, appcfg: SphinxConfig) -> None: @@ -108,11 +93,12 @@ def on_config_inited(app: Sphinx, appcfg: SphinxConfig) -> None: def on_env_get_outdated( app: Sphinx, env: BuildEnvironment, - added: Set[str], - changed: Set[str], - removed: Set[str], -) -> List[str]: + added: set[str], + changed: set[str], + removed: set[str], +) -> list[str]: # Remove purged indexes and snippetes from db + assert cache is not None for docname in removed: del cache[(app.config.project, docname)] return [] @@ -126,15 +112,18 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N ) return - pats = is_document_matched(app.config.snippet_patterns, docname) - if len(pats) == 0: - logger.debug('[snippet] skip picking because %s is not matched', docname) + allowed_tags = _get_document_allowed_tags(app.config.snippet_patterns, docname) + if not allowed_tags: + logger.debug('[snippet] skip picking: no tag allowed for document %s', docname) return doc = [] snippets = pick(app, doctree, docname) + tags = [] for s, n in snippets: - if not is_snippet_matched(pats, s, docname): + # FIXME: Better filter logic. + tags.append(extract_tags(s)) + if tags[-1] not in allowed_tags: continue tpath = [x.astext() for x in titlepath.resolve(app.env, docname, n)] if isinstance(s, Section): @@ -150,17 +139,24 @@ def on_doctree_resolved(app: Sphinx, doctree: nodes.document, docname: str) -> N ) cache_key = (app.config.project, docname) + assert cache is not None if len(doc) != 0: cache[cache_key] = doc elif cache_key in cache: del cache[cache_key] logger.debug( - '[snippet] picked %s/%s snippetes in %s', len(doc), len(snippets), docname + '[snippet] picked %s/%s snippets in %s, tags: %s, allowed tags: %s', + len(doc), + len(snippets), + docname, + tags, + allowed_tags, ) def on_builder_finished(app: Sphinx, exception) -> None: + assert cache is not None cache.dump() @@ -206,15 +202,3 @@ def _format_modified_time(timestamp: float) -> str: """Return an RFC 3339 formatted string representing the given timestamp.""" seconds, fraction = divmod(timestamp, 1) return time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(seconds)) + f'.{fraction:.3f}' - - -def setup(app: Sphinx): - app.add_builder(SnippetBuilder) - - app.add_config_value('snippet_config', {}, '') - app.add_config_value('snippet_patterns', {'*': ['.*']}, '') - - app.connect('config-inited', on_config_inited) - app.connect('env-get-outdated', on_env_get_outdated) - app.connect('doctree-resolved', on_doctree_resolved) - app.connect('build-finished', on_builder_finished) diff --git a/src/sphinxnotes/snippet/integration/binding.nvim b/src/sphinxnotes/snippet/integration/binding.nvim index 6f05c5f..25f0f34 100644 --- a/src/sphinxnotes/snippet/integration/binding.nvim +++ b/src/sphinxnotes/snippet/integration/binding.nvim @@ -8,10 +8,10 @@ " TODO: Support vim? function! g:SphinxNotesSnippetListAndView() - function! s:CallView(selection) - call g:SphinxNotesSnippetView(s:SplitID(a:selection)) + function! ListAndView_CB(id) + call g:SphinxNotesSnippetView(a:id) endfunction - call g:SphinxNotesSnippetList(function('s:CallView'), 'ds') + call g:SphinxNotesSnippetList('"*"', function('ListAndView_CB')) endfunction " https://github.com/anhmv/vim-float-window/blob/master/plugin/float-window.vim @@ -40,7 +40,7 @@ function! g:SphinxNotesSnippetView(id) " Press enter to return nmap :call nvim_win_close(g:sphinx_notes_snippet_win, v:true) - let cmd = [s:snippet, 'get', '--text', a:id] + let cmd = [s:snippet, 'get', '--src', a:id] call append(line('$'), ['.. hint:: Press to return']) execute '$read !' . '..' execute '$read !' . join(cmd, ' ') diff --git a/src/sphinxnotes/snippet/integration/binding.sh b/src/sphinxnotes/snippet/integration/binding.sh index a07dd17..f831d58 100644 --- a/src/sphinxnotes/snippet/integration/binding.sh +++ b/src/sphinxnotes/snippet/integration/binding.sh @@ -3,10 +3,10 @@ # # :Author: Shengyu Zhang # :Date: 2021-08-14 -# :Version: 20211114 +# :Version: 20240828 function snippet_view() { - selection=$(snippet_list ds) + selection=$(snippet_list) [ -z "$selection" ] && return # Make sure we have $PAGER @@ -18,18 +18,18 @@ function snippet_view() { fi fi - echo "$SNIPPET get --text $selection | $PAGER" + echo "$SNIPPET get --src $selection | $PAGER" } function snippet_edit() { - selection=$(snippet_list ds) + selection=$(snippet_list --tags ds) [ -z "$selection" ] && return echo "vim +\$($SNIPPET get --line-start $selection) \$($SNIPPET get --file $selection)" } function snippet_url() { - selection=$(snippet_list ds) + selection=$(snippet_list --tags ds) [ -z "$selection" ] && return echo "xdg-open \$($SNIPPET get --url $selection)" diff --git a/src/sphinxnotes/snippet/integration/binding.vim b/src/sphinxnotes/snippet/integration/binding.vim index 3b4539d..6225080 100644 --- a/src/sphinxnotes/snippet/integration/binding.vim +++ b/src/sphinxnotes/snippet/integration/binding.vim @@ -6,39 +6,40 @@ " :Version: 20211114 " -function! g:SphinxNotesSnippetEdit(id) - let file = system(join([s:snippet, 'get', '--file', a:id, '2>/dev/null'], ' ')) - let line = system(join([s:snippet, 'get', '--line-start', a:id, '2>/dev/null'], ' ')) +function g:SphinxNotesSnippetEdit(id) + let file = g:SphinxNotesSnippetGet(a:id, 'file')[0] + let line = g:SphinxNotesSnippetGet(a:id, 'line-start')[0] if &modified - execute 'tabedit ' . file + execute 'vsplit ' . file else execute 'edit ' . file endif execute line endfunction -function! g:SphinxNotesSnippetListAndEdit() - function! s:CallEdit(selection) - call g:SphinxNotesSnippetEdit(s:SplitID(a:selection)) +function g:SphinxNotesSnippetListAndEdit() + function! ListAndEdit_CB(id) + call g:SphinxNotesSnippetEdit(a:id) endfunction - call g:SphinxNotesSnippetList(function('s:CallEdit'), 'ds') + call g:SphinxNotesSnippetList('ds', function('ListAndEdit_CB')) endfunction -function! g:SphinxNotesSnippetUrl(id) - let url_list = systemlist(join([s:snippet, 'get', '--url', a:id, '2>/dev/null'], ' ')) +function g:SphinxNotesSnippetUrl(id) + let url_list = g:SphinxNotesSnippetGet(a:id, 'url') for url in url_list echo system('xdg-open ' . shellescape(url)) endfor endfunction -function! g:SphinxNotesSnippetListAndUrl() - function! s:CallUrl(selection) - call g:SphinxNotesSnippetUrl(s:SplitID(a:selection)) +function g:SphinxNotesSnippetListAndUrl() + function! ListAndUrl_CB(id) + call g:SphinxNotesSnippetUrl(a:id) endfunction - call g:SphinxNotesSnippetList(function('s:CallUrl'), 'ds') + call g:SphinxNotesSnippetList('ds', function('ListAndUrl_CB')) endfunction nmap e :call g:SphinxNotesSnippetListAndEdit() nmap u :call g:SphinxNotesSnippetListAndUrl() +nmap i :call g:SphinxNotesSnippetListAndInput() " vim: set shiftwidth=2: diff --git a/src/sphinxnotes/snippet/integration/plugin.sh b/src/sphinxnotes/snippet/integration/plugin.sh index 5b75e0a..3d24a91 100644 --- a/src/sphinxnotes/snippet/integration/plugin.sh +++ b/src/sphinxnotes/snippet/integration/plugin.sh @@ -3,15 +3,15 @@ # # :Author: Shengyu Zhang # :Date: 2021-03-20 -# :Version: 20211114 +# :Version: 20240828 # Make sure we have $SNIPPET [ -z "$SNIPPET"] && SNIPPET='snippet' -# Arguments: $1: kinds +# Arguments: $*: Extra opts of ``snippet list`` # Returns: snippet_id function snippet_list() { - $SNIPPET list --tags $1 --width $(($(tput cols) - 2)) | \ + $SNIPPET list --width $(($(tput cols) - 2)) "$@" | \ fzf --with-nth 2.. \ --no-hscroll \ --header-lines 1 \ diff --git a/src/sphinxnotes/snippet/integration/plugin.vim b/src/sphinxnotes/snippet/integration/plugin.vim index cfe77f7..3cf01c5 100644 --- a/src/sphinxnotes/snippet/integration/plugin.vim +++ b/src/sphinxnotes/snippet/integration/plugin.vim @@ -8,24 +8,102 @@ " NOTE: junegunn/fzf.vim is required let s:snippet = 'snippet' +let s:width = 0.9 +let s:height = 0.6 -function! s:SplitID(row) - return split(a:row, ' ')[0] -endfunction - -function! g:SphinxNotesSnippetList(callback, tags) - let l:width = 0.9 +" Use fzf to list all snippets, callback with argument id. +function g:SphinxNotesSnippetList(tags, callback) let cmd = [s:snippet, 'list', \ '--tags', a:tags, - \ '--width', float2nr(&columns * l:width) - 2, + \ '--width', float2nr(&columns * s:width) - 2, \ ] + + " Use closure keyword so that inner function can access outer one's + " localvars (l:) and arguments (a:). + " https://vi.stackexchange.com/a/21807 + function! List_CB(selection) closure + let id = split(a:selection, ' ')[0] + call a:callback(id) + endfunction + " https://github.com/junegunn/fzf/blob/master/README-VIM.md#fzfrun call fzf#run({ \ 'source': join(cmd, ' '), - \ 'sink': a:callback, + \ 'sink': function('List_CB'), \ 'options': ['--with-nth', '2..', '--no-hscroll', '--header-lines', '1'], - \ 'window': {'width': l:width, 'height': 0.6}, + \ 'window': {'width': s:width, 'height': s:height}, \ }) endfunction -" vim: set shiftwidth=2: +" Return the attribute value of specific snippet. +function g:SphinxNotesSnippetGet(id, attr) + let cmd = [s:snippet, 'get', a:id, '--' . a:attr] + return systemlist(join(cmd, ' ')) +endfunction + +" Use fzf to list all attr of specific snippet, +" callback with arguments (attr_name, attr_value). +function g:SphinxNotesSnippetListSnippetAttrs(id, callback) + " Display attr -> Identify attr (also used as CLI option) + let attrs = { + \ 'Source': 'src', + \ 'URL': 'url', + \ 'Docname': 'docname', + \ 'Dependent files': 'deps', + \ 'Text': 'text', + \ 'Title': 'title', + \ } + let delim = ' ' + let table = ['OPTION' . delim . 'ATTRIBUTE'] + for name in keys(attrs) + call add(table, attrs[name] . delim . name) + endfor + + function! ListSnippetAttrs_CB(selection) closure + let opt = split(a:selection, ' ')[0] + let val = g:SphinxNotesSnippetGet(a:id, opt) + call a:callback(opt, val) " finally call user's cb + endfunction + + let preview_cmd = [s:snippet, 'get', a:id, '--$(echo {} | cut -d " " -f1)'] + let info_cmd = ['echo', 'Index ID:', a:id] + call fzf#run({ + \ 'source': table, + \ 'sink': function('ListSnippetAttrs_CB'), + \ 'options': [ + \ '--header-lines', '1', + \ '--with-nth', '2..', + \ '--preview', join(preview_cmd, ' '), + \ '--preview-window', ',wrap', + \ '--info-command', join(info_cmd, ' '), + \ ], + \ 'window': {'width': s:width, 'height': s:height}, + \ }) +endfunction + +function g:SphinxNotesSnippetInput(id) + function! Input_CB(attr, val) " TODO: became g:func. + if a:attr == 'docname' + " Create doc reference. + let content = ':doc:`/' . a:val[0] . '`' + elseif a:attr == 'title' + " Create local section reference. + let content = '`' . a:val[0] . '`_' + else + let content = join(a:val, '') + endif + execute 'normal! i' . content + endfunction + + call g:SphinxNotesSnippetListSnippetAttrs(a:id, function('Input_CB')) +endfunction + +function g:SphinxNotesSnippetListAndInput() + function! ListAndInput_CB(id) + call g:SphinxNotesSnippetInput(a:id) + endfunction + + call g:SphinxNotesSnippetList('"*"', function('ListAndInput_CB')) +endfunction + + " vim: set shiftwidth=2: diff --git a/src/sphinxnotes/snippet/keyword.py b/src/sphinxnotes/snippet/keyword.py index 6ff21d5..80d45f8 100644 --- a/src/sphinxnotes/snippet/keyword.py +++ b/src/sphinxnotes/snippet/keyword.py @@ -9,7 +9,6 @@ """ from __future__ import annotations -from typing import List, Optional import string from collections import Counter @@ -47,8 +46,8 @@ def __init__(self): ) def extract( - self, text: str, top_n: Optional[int] = None, strip_stopwords: bool = True - ) -> List[str]: + self, text: str, top_n: int | None = None, strip_stopwords: bool = True + ) -> list[str]: """Return keywords of given text.""" # TODO: zh -> en # Normalize @@ -87,7 +86,7 @@ def normalize(self, text: str) -> str: text = text.replace('\n', ' ') return text - def tokenize(self, text: str) -> List[str]: + def tokenize(self, text: str) -> list[str]: # Get top most 5 langs langs = self._detect_langs(text)[:5] tokens = [text] @@ -104,10 +103,10 @@ def tokenize(self, text: str) -> List[str]: new_tokens = [] return tokens - def trans_to_pinyin(self, word: str) -> Optional[str]: + def trans_to_pinyin(self, word: str) -> str | None: return ' '.join(self._pinyin(word, errors='ignore')) - def strip_stopwords(self, words: List[str]) -> List[str]: + def strip_stopwords(self, words: list[str]) -> list[str]: stw = self._stopwords(['en', 'zh']) new_words = [] for word in words: @@ -115,5 +114,5 @@ def strip_stopwords(self, words: List[str]) -> List[str]: new_words.append(word) return new_words - def strip_invalid_token(self, tokens: List[str]) -> List[str]: + def strip_invalid_token(self, tokens: list[str]) -> list[str]: return [token for token in tokens if token != ''] diff --git a/src/sphinxnotes/snippet/meta.py b/src/sphinxnotes/snippet/meta.py new file mode 100644 index 0000000..e7dbf2c --- /dev/null +++ b/src/sphinxnotes/snippet/meta.py @@ -0,0 +1,35 @@ +# This file is generated from sphinx-notes/cookiecutter. +# DO NOT EDIT!!! + +################################################################################ +# Project meta infos. +################################################################################ + +from __future__ import annotations +from importlib import metadata + +__project__ = 'sphinxnotes-snippet' +__author__ = 'Shengyu Zhang' +__desc__ = 'Sphinx documentation snippets manager' + +try: + __version__ = metadata.version('sphinxnotes-snippet') +except metadata.PackageNotFoundError: + __version__ = 'unknown' + + +################################################################################ +# Sphinx extension utils. +################################################################################ + + +def pre_setup(app): + app.require_sphinx('7.0') + + +def post_setup(app): + return { + 'version': __version__, + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/src/sphinxnotes/snippet/picker.py b/src/sphinxnotes/snippet/picker.py index 9b2ac2a..ea1b4cf 100644 --- a/src/sphinxnotes/snippet/picker.py +++ b/src/sphinxnotes/snippet/picker.py @@ -15,7 +15,7 @@ from sphinx.util import logging -from . import Snippet, Section, Document +from .snippets import Snippet, Section, Document, Code if TYPE_CHECKING: from sphinx.application import Sphinx @@ -25,81 +25,73 @@ def pick( app: Sphinx, doctree: nodes.document, docname: str -) -> list[tuple[Snippet, nodes.section]]: +) -> list[tuple[Snippet, nodes.Element]]: """ - Pick snippets from document, return a list of snippet and the section - it belongs to. + Pick snippets from document, return a list of snippet and the related node. + + As :class:`Snippet` can not hold any refs to doctree, we additionly returns + the related nodes here. To ensure the caller can back reference to original + document node and do more things (e.g. generate title path). """ # FIXME: Why doctree.source is always None? if not doctree.attributes.get('source'): - logger.debug('Skipped document without source') + logger.debug('Skip document %s: no source', docname) return [] metadata = app.env.metadata.get(docname, {}) if 'no-search' in metadata or 'nosearch' in metadata: - logger.debug('Skipped document with nosearch metadata') + logger.debug('Skip document %s: have :no[-]search: metadata', docname) return [] - snippets = [] - - # Pick document - toplevel_section = doctree.next_node(nodes.section) - if toplevel_section: - snippets.append((Document(doctree), toplevel_section)) - else: - logger.warning('can not pick document without child section: %s', doctree) - - # Pick sections - section_picker = SectionPicker(doctree) - doctree.walkabout(section_picker) - snippets.extend(section_picker.sections) + # Walk doctree and pick snippets. - return snippets + picker = SnippetPicker(doctree) + doctree.walkabout(picker) + return picker.snippets -class SectionPicker(nodes.SparseNodeVisitor): - """Node visitor for picking code snippet from document.""" - #: Constant list of unsupported languages (:class:`pygments.lexers.Lexer`) - UNSUPPORTED_LANGUAGES: list[str] = ['default'] +class SnippetPicker(nodes.SparseNodeVisitor): + """Node visitor for picking snippets from document.""" - #: List of picked section snippets and the section it belongs to - sections: list[tuple[Section, nodes.section]] + #: List of picked snippets and the section it belongs to + snippets: list[tuple[Snippet, nodes.Element]] - _section_has_code_block: bool - _section_level: int + #: Stack of nested sections. + _sections: list[nodes.section] - def __init__(self, document: nodes.document) -> None: - super().__init__(document) - self.sections = [] - self._section_has_code_block = False - self._section_level = 0 + def __init__(self, doctree: nodes.document) -> None: + super().__init__(doctree) + self.snippets = [] + self._sections = [] ################### # Visitor methods # ################### def visit_literal_block(self, node: nodes.literal_block) -> None: - if node['language'] in self.UNSUPPORTED_LANGUAGES: + try: + code = Code(node) + except ValueError as e: + logger.debug(f'skip {node}: {e}') raise nodes.SkipNode - self._has_code_block = True + self.snippets.append((code, node)) def visit_section(self, node: nodes.section) -> None: - self._section_level += 1 + self._sections.append(node) def depart_section(self, node: nodes.section) -> None: - self._section_level -= 1 - self._has_code_block = False + section = self._sections.pop() + assert section == node + # Always pick document. + if len(self._sections) == 0: + self.snippets.append((Document(self.document), node)) + return # Skip non-leaf section without content if self._is_empty_non_leaf_section(node): return - # Skip toplevel section, we generate :class:`Document` for it - if self._section_level == 0: - return - - # TODO: code block - self.sections.append((Section(node), node)) + self.snippets.append((Section(node), node)) def unknown_visit(self, node: nodes.Node) -> None: pass # Ignore any unknown node diff --git a/src/sphinxnotes/snippet/py.typed b/src/sphinxnotes/snippet/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/sphinxnotes/snippet/snippets.py b/src/sphinxnotes/snippet/snippets.py new file mode 100644 index 0000000..f14661a --- /dev/null +++ b/src/sphinxnotes/snippet/snippets.py @@ -0,0 +1,240 @@ +""" +sphinxnotes.snippet.snippets +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Definitions of various snippets. + +:copyright: Copyright 2024 Shengyu Zhang +:license: BSD, see LICENSE for details. +""" + +from __future__ import annotations +from typing import TYPE_CHECKING +import itertools +from os import path +import sys +from pygments.lexers.shell import BashSessionLexer + +from docutils import nodes + +if TYPE_CHECKING: + from sphinx.environment import BuildEnvironment + + +class Snippet(object): + """ + Snippet is structured fragments extracted from a single Sphinx document + (usually, also a single reStructuredText file). + + :param nodes: nodes of doctree that make up this snippet. + + .. warning:: + + Snippet will be persisted to disk via pickle, to keep it simple, + it CAN NOT holds reference to any doctree ``nodes`` + (or even any non-std module). + """ + + #: docname where the snippet is located, can be referenced by + # :rst:role:`doc`. + docname: str + + #: Absolute path to the source file. + file: str + + #: Line number range of source file (:attr:`Snippet.file`), + #: left closed and right opened. + lineno: tuple[int, int] + + #: The source text read from source file (:attr:`Snippet.file`), + # in Markdown or reStructuredText. + source: list[str] + + #: Text representation of the snippet, usually generated form + # :meth:`nodes.Element.astext`. + text: list[str] + + #: The possible identifier key of snippet, which is picked from nodes' + #: (or nodes' parent's) `ids attr`_. + #: + #: .. _ids attr: https://docutils.sourceforge.io/docs/ref/doctree.html#ids + refid: str | None + + def __init__(self, *nodes: nodes.Element) -> None: + assert len(nodes) != 0 + + env: BuildEnvironment = nodes[0].document.settings.env # type: ignore + + file, docname = None, None + for node in nodes: + if (src := nodes[0].source) and path.exists(src): + file = src + docname = env.path2doc(file) + break + if not file or not docname: + raise ValueError(f'Nodes {nodes} lacks source file or docname') + self.file = file + self.docname = docname + + lineno = [sys.maxsize, -sys.maxsize] + for node in nodes: + if not node.line: + continue # Skip node that have None line, I dont know why + lineno[0] = min(lineno[0], _line_of_start(node)) + lineno[1] = max(lineno[1], _line_of_end(node)) + self.lineno = (lineno[0], lineno[1]) + + source = [] + with open(self.file, 'r') as f: + start = self.lineno[0] - 1 + stop = self.lineno[1] - 1 + for line in itertools.islice(f, start, stop): + source.append(line.strip('\n')) + self.source = source + + text = [] + for node in nodes: + text.extend(node.astext().split('\n')) + self.text = text + + # Find exactly one ID attr in nodes + self.refid = None + for node in nodes: + if node['ids']: + self.refid = node['ids'][0] + break + + # If no ID found, try parent + if not self.refid: + for node in nodes: + if node.parent['ids']: + self.refid = node.parent['ids'][0] + break + + +class Code(Snippet): + #: Language of code block + lang: str + #: Description of code block, usually the text of preceding paragraph + desc: str + + def __init__(self, node: nodes.literal_block) -> None: + assert isinstance(node, nodes.literal_block) + + self.lang = node['language'] + if self.lang not in BashSessionLexer.aliases: # TODO: support more language + raise ValueError( + f'Language {self.lang} is not supported', + ) + + self.desc = '' + # Use the preceding paragraph as descritpion. We usually write some + # descritpions before a code block. For example, The ``::`` syntax is + # a common way to create code block:: + # + # | Foo:: | + # | | Foo: + # | Bar | + # | | Bar + # + # In this case, the paragraph "Foo:" is the descritpion of the code block. + # This convention also applies to the code, code-block, sourcecode directive. + if isinstance(para := node.previous_sibling(), nodes.paragraph): + # For better display, the trailing colon is removed. + # TODO: https://en.wikipedia.org/wiki/Colon_(punctuation)#Computing + self.desc += para.astext().replace('\n', ' ').rstrip(':īŧšīŧšī¸ī¸“īš•') + if caption := node.get('caption'): + # Use caption as descritpion. + # All of code-block, sourcecode and code directives have caption option. + # https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-code-block + self.desc += caption + if not self.desc: + raise ValueError( + f'Node f{node} lacks description: a preceding paragraph or a caption' + ) + + if isinstance(para, nodes.paragraph): + # If we have a paragraph preceding code block, include it. + super().__init__(para, node) + # Fixup text field, it should be pure code. + self.text = node.astext().split('\n') + else: + super().__init__(node) + + +class WithTitle(object): + title: str + + def __init__(self, node: nodes.Element) -> None: + if not (title := node.next_node(nodes.title)): + raise ValueError(f'Node f{node} lacks title') + self.title = title.astext() + + +class Section(Snippet, WithTitle): + def __init__(self, node: nodes.section) -> None: + assert isinstance(node, nodes.section) + Snippet.__init__(self, node) + WithTitle.__init__(self, node) + + +class Document(Section): + #: A set of absolute paths of dependent files for document. + #: Obtained from :attr:`BuildEnvironment.dependencies`. + deps: set[str] + + def __init__(self, node: nodes.document) -> None: + assert isinstance(node, nodes.document) + super().__init__(node.next_node(nodes.section)) + + # Record document's dependent files + self.deps = set() + env: BuildEnvironment = node.settings.env + for dep in env.dependencies[self.docname]: + # Relative to documentation root -> Absolute path of file system. + self.deps.add(path.join(env.srcdir, dep)) + + +################ +# Nodes helper # +################ + + +def _line_of_start(node: nodes.Node) -> int: + assert node.line + if isinstance(node, nodes.title): + if isinstance(node.parent.parent, nodes.document): + # Spceial case for Document Title / Subtitle + return 1 + else: + # Spceial case for section title + return node.line - 1 + elif isinstance(node, nodes.section): + if isinstance(node.parent, nodes.document): + # Spceial case for top level section + return 1 + else: + # Spceial case for section + return node.line - 1 + return node.line + + +def _line_of_end(node: nodes.Node) -> int: + next_node = node.next_node(descend=False, siblings=True, ascend=True) + while next_node: + if next_node.line: + return _line_of_start(next_node) + next_node = next_node.next_node( + # Some nodes' line attr is always None, but their children has + # valid line attr + descend=True, + # If node and its children have not valid line attr, try use line + # of next node + ascend=True, + siblings=True, + ) + # No line found, return the max line of source file + if node.source and path.exists(node.source): + with open(node.source) as f: + return sum(1 for _ in f) + raise AttributeError('None source attr of node %s' % node) diff --git a/src/sphinxnotes/snippet/table.py b/src/sphinxnotes/snippet/table.py index 26a8a70..8138573 100644 --- a/src/sphinxnotes/snippet/table.py +++ b/src/sphinxnotes/snippet/table.py @@ -7,7 +7,7 @@ """ from __future__ import annotations -from typing import Iterator, Dict +from typing import Iterable from .cache import Index, IndexID from .utils import ellipsis @@ -17,8 +17,8 @@ COLUMN_DELIMITER = ' ' -def tablify(indexes: Dict[IndexID, Index], tags: str, width: int) -> Iterator[str]: - """Create a table from sequence of cache.Index.""" +def tablify(indexes: Iterable[tuple[IndexID, Index]], width: int) -> Iterable[str]: + """Create a table from sequence of indices""" # Calcuate width width = width @@ -41,10 +41,8 @@ def tablify(indexes: Dict[IndexID, Index], tags: str, width: int) -> Iterator[st yield header # Write rows - for index_id, index in indexes.items(): + for index_id, index in indexes: # TODO: assert index? - if index[0] not in tags and '*' not in tags: - continue row = COLUMN_DELIMITER.join( [ index_id, # ID diff --git a/src/sphinxnotes/snippet/utils/ellipsis.py b/src/sphinxnotes/snippet/utils/ellipsis.py index 11bfcc5..d46bda8 100644 --- a/src/sphinxnotes/snippet/utils/ellipsis.py +++ b/src/sphinxnotes/snippet/utils/ellipsis.py @@ -9,12 +9,11 @@ """ from __future__ import annotations -from typing import List from wcwidth import wcswidth def ellipsis( - text: str, width: int, ellipsis_sym: str = '..', blank_sym: str = None + text: str, width: int, ellipsis_sym: str = '..', blank_sym: str | None = None ) -> str: text_width = wcswidth(text) if text_width <= width: @@ -34,7 +33,7 @@ def ellipsis( def join( - lst: List[str], + lst: list[str], total_width: int, title_width: int, separate_sym: str = '/', diff --git a/src/sphinxnotes/snippet/utils/pdict.py b/src/sphinxnotes/snippet/utils/pdict.py index 59d293f..ef93a38 100644 --- a/src/sphinxnotes/snippet/utils/pdict.py +++ b/src/sphinxnotes/snippet/utils/pdict.py @@ -11,7 +11,7 @@ from __future__ import annotations import os from os import path -from typing import Dict, Optional, Iterable, TypeVar +from typing import Iterator, TypeVar import pickle from collections.abc import MutableMapping from hashlib import sha1 @@ -21,16 +21,16 @@ # FIXME: PDict is buggy -class PDict(MutableMapping): +class PDict(MutableMapping[K, V]): """A persistent dict with event handlers.""" dirname: str # The real in memory store of values - _store: Dict[K, V] + _store: dict[K, V | None] # Items that need write back to store - _dirty_items: Dict[K, V] + _dirty_items: dict[K, V] # Items that need purge from store - _orphan_items: Dict[K, V] + _orphan_items: dict[K, V] def __init__(self, dirname: str) -> None: self.dirname = dirname @@ -38,7 +38,7 @@ def __init__(self, dirname: str) -> None: self._dirty_items = {} self._orphan_items = {} - def __getitem__(self, key: K) -> Optional[V]: + def __getitem__(self, key: K) -> V: if key not in self._store: raise KeyError value = self._store[key] @@ -65,7 +65,7 @@ def __delitem__(self, key: K) -> None: else: self._orphan_items[key] = value - def __iter__(self) -> Iterable: + def __iter__(self) -> Iterator: return iter(self._store) def __len__(self) -> int: @@ -142,4 +142,4 @@ def post_purge(self, key: K, value: V) -> None: pass def stringify(self, key: K, value: V) -> str: - return key + return str(key) diff --git a/src/sphinxnotes/snippet/utils/titlepath.py b/src/sphinxnotes/snippet/utils/titlepath.py index 9231537..eaa6bc3 100644 --- a/src/sphinxnotes/snippet/utils/titlepath.py +++ b/src/sphinxnotes/snippet/utils/titlepath.py @@ -9,22 +9,21 @@ """ from __future__ import annotations -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING from docutils import nodes if TYPE_CHECKING: - from sphinx.enviornment import BuilderEnviornment + from sphinx.environment import BuildEnvironment def resolve( - env: BuilderEnviornment, docname: str, node: nodes.Node -) -> List[nodes.title]: + env: BuildEnvironment, docname: str, node: nodes.Element +) -> list[nodes.title]: return resolve_section(node) + resolve_document(env, docname) -def resolve_section(node: nodes.section) -> List[nodes.title]: - # FIXME: doc is None +def resolve_section(node: nodes.Element) -> list[nodes.title]: titlenodes = [] while node: if len(node) > 0 and isinstance(node[0], nodes.title): @@ -33,10 +32,8 @@ def resolve_section(node: nodes.section) -> List[nodes.title]: return titlenodes -def resolve_document(env: BuilderEnviornment, docname: str) -> List[nodes.title]: - """ - .. note:: Title of document itself does not included in the returned list - """ +def resolve_document(env: BuildEnvironment, docname: str) -> list[nodes.title]: + """NOTE: Title of document itself does not included in the returned list""" titles = [] master_doc = env.config.master_doc v = docname.split('/')