ENH: Add support for variable doc-comments (starting with '#:') (#292)

* Add support for variable docstrings starting with '#:'

* Get tests passing on Windows

* Updates from the PRD

* Add some unit tests, lint

* Update pdoc/__init__.py

* PR fixes

* PR updates, add documentation

* Revert unrelated change

* Touch ups

* Reword docs

Author: csm10495

pdoc/__init__.py

@@ -244,7 +244,7 @@ def _pep224_docstrings(doc_obj: Union['Module', 'Class'], *,
                        _init_tree=None) -> Tuple[Dict[str, str],
                                                  Dict[str, str]]:
     """
-    Extracts PEP-224 docstrings for variables of `doc_obj`
+    Extracts PEP-224 docstrings and doc-comments (`#: ...`) for variables of `doc_obj`
     (either a `pdoc.Module` or `pdoc.Class`).
 
     Returns a tuple of two dicts mapping variable names to their docstrings.
@@ -284,12 +284,7 @@ def _pep224_docstrings(doc_obj: Union['Module', 'Class'], *,
                     instance_vars, _ = _pep224_docstrings(doc_obj, _init_tree=node)
                     break
 
-    for assign_node, str_node in _pairwise(ast.iter_child_nodes(tree)):
-        if not (isinstance(assign_node, (ast.Assign, ast.AnnAssign)) and
-                isinstance(str_node, ast.Expr) and
-                isinstance(str_node.value, ast.Str)):
-            continue
-
+    def get_name(assign_node):
         if isinstance(assign_node, ast.Assign) and len(assign_node.targets) == 1:
             target = assign_node.targets[0]
         elif isinstance(assign_node, ast.AnnAssign):
@@ -298,7 +293,7 @@ def _pep224_docstrings(doc_obj: Union['Module', 'Class'], *,
             # > Putting the instance variable annotations together in the class
             # > makes it easier to find them, and helps a first-time reader of the code.
         else:
-            continue
+            return None
 
         if not _init_tree and isinstance(target, ast.Name):
             name = target.id
@@ -308,9 +303,22 @@ def _pep224_docstrings(doc_obj: Union['Module', 'Class'], *,
               target.value.id == 'self'):
             name = target.attr
         else:
-            continue
+            return None
 
         if not _is_public(name) and not _is_whitelisted(name, doc_obj):
+            return None
+
+        return name
+
+    # For handling PEP-224 docstrings for variables
+    for assign_node, str_node in _pairwise(ast.iter_child_nodes(tree)):
+        if not (isinstance(assign_node, (ast.Assign, ast.AnnAssign)) and
+                isinstance(str_node, ast.Expr) and
+                isinstance(str_node.value, ast.Str)):
+            continue
+
+        name = get_name(assign_node)
+        if not name:
             continue
 
         docstring = inspect.cleandoc(str_node.value.s).strip()
@@ -319,6 +327,43 @@ def _pep224_docstrings(doc_obj: Union['Module', 'Class'], *,
 
         vars[name] = docstring
 
+    # For handling '#:' docstrings for variables
+    for assign_node in ast.iter_child_nodes(tree):
+        if not isinstance(assign_node, (ast.Assign, ast.AnnAssign)):
+            continue
+
+        name = get_name(assign_node)
+        if not name:
+            continue
+
+        # Already documented. PEP-224 method above takes precedence.
+        if name in vars:
+            continue
+
+        def get_indent(line):
+            return len(line) - len(line.lstrip())
+
+        source_lines = doc_obj.source.splitlines()  # type: ignore
+        assign_line = source_lines[assign_node.lineno - 1]
+        assign_indent = get_indent(assign_line)
+        comment_lines = []
+        MARKER = '#: '
+        for line in reversed(source_lines[:assign_node.lineno - 1]):
+            if get_indent(line) == assign_indent and line.lstrip().startswith(MARKER):
+                comment_lines.append(line.split(MARKER, maxsplit=1)[1])
+            else:
+                break
+
+        # Since we went 'up' need to reverse lines to be in correct order
+        comment_lines = comment_lines[::-1]
+
+        # Finally: check for a '#: ' comment at the end of the assignment line itself.
+        if MARKER in assign_line:
+            comment_lines.append(assign_line.rsplit(MARKER, maxsplit=1)[1])
+
+        if comment_lines:
+            vars[name] = '\n'.join(comment_lines)
+
     return vars, instance_vars
 
 

pdoc/documentation.md

@@ -107,24 +107,26 @@ In the default HTML template, such inherited docstrings are greyed out.
 [variable docstrings]: #docstrings-for-variables
 
 Python by itself [doesn't allow docstrings attached to variables][PEP-224].
-However, `pdoc` supports docstrings attached to module (or global)
-variables, class variables, and object instance variables; all in
-the same way as proposed in [PEP-224], with a docstring following the
-variable assignment.
+However, `pdoc` supports documenting module (or global)
+variables, class variables, and object instance variables via
+two different mechanisms: [PEP-224] and `#:` doc-comments.
+
 For example:
 
 [PEP-224]: http://www.python.org/dev/peps/pep-0224
 
     module_variable = 1
-    """Docstring for module_variable."""
+    """PEP 224 docstring for module_variable."""
 
     class C:
-        class_variable = 2
-        """Docstring for class_variable."""
+        #: Documentation comment for class_variable
+        #: spanning over three lines.
+        class_variable = 2  #: Assignment line is included.
 
         def __init__(self):
+            #: Instance variable's doc-comment
             self.variable = 3
-            """Docstring for instance variable."""
+            """But note, PEP 224 docstrings take precedence."""
 
 While the resulting variables have no `__doc__` attribute,
 `pdoc` compensates by reading the source code (when available)

pdoc/html_helpers.py

@@ -560,6 +560,11 @@ def format_git_link(template: str, dobj: pdoc.Doc):
             commit = _git_head_commit()
         abs_path = inspect.getfile(inspect.unwrap(dobj.obj))
         path = _project_relative_path(abs_path)
+
+        # Urls should always use / instead of \\
+        if os.name == 'nt':
+            path = path.replace('\\', '/')
+
         lines, start_line = inspect.getsourcelines(dobj.obj)
         end_line = start_line + len(lines) - 1
         url = template.format(**locals())

pdoc/test/__init__.py

@@ -106,25 +106,21 @@ class CliTest(unittest.TestCase):
     """
     ALL_FILES = [
         'example_pkg',
-        'example_pkg/index.html',
-        'example_pkg/index.m.html',
-        'example_pkg/module.html',
-        'example_pkg/_private',
-        'example_pkg/_private/index.html',
-        'example_pkg/_private/module.html',
-        'example_pkg/subpkg',
-        'example_pkg/subpkg/_private.html',
-        'example_pkg/subpkg/index.html',
-        'example_pkg/subpkg2',
-        'example_pkg/subpkg2/_private.html',
-        'example_pkg/subpkg2/module.html',
-        'example_pkg/subpkg2/index.html',
+        os.path.join('example_pkg', 'index.html'),
+        os.path.join('example_pkg', 'index.m.html'),
+        os.path.join('example_pkg', 'module.html'),
+        os.path.join('example_pkg', '_private'),
+        os.path.join('example_pkg', '_private', 'index.html'),
+        os.path.join('example_pkg', '_private', 'module.html'),
+        os.path.join('example_pkg', 'subpkg'),
+        os.path.join('example_pkg', 'subpkg', '_private.html'),
+        os.path.join('example_pkg', 'subpkg', 'index.html'),
+        os.path.join('example_pkg', 'subpkg2'),
+        os.path.join('example_pkg', 'subpkg2', '_private.html'),
+        os.path.join('example_pkg', 'subpkg2', 'module.html'),
+        os.path.join('example_pkg', 'subpkg2', 'index.html'),
     ]
-    PUBLIC_FILES = [f for f in ALL_FILES if '/_' not in f]
-
-    if os.name == 'nt':
-        ALL_FILES = [i.replace('/', '\\') for i in ALL_FILES]
-        PUBLIC_FILES = [i.replace('/', '\\') for i in PUBLIC_FILES]
+    PUBLIC_FILES = [f for f in ALL_FILES if (os.path.sep + '_') not in f]
 
     def setUp(self):
         pdoc.reset()
@@ -190,7 +186,7 @@ def test_html(self):
             '.subpkg2': [f for f in self.PUBLIC_FILES
                          if 'subpkg2' in f or f == EXAMPLE_MODULE],
             '._private': [f for f in self.ALL_FILES
-                          if EXAMPLE_MODULE + '/_private' in f or f == EXAMPLE_MODULE],
+                          if EXAMPLE_MODULE + os.path.sep + '_private' in f or f == EXAMPLE_MODULE],
         }
         for package, expected_files in package_files.items():
             with self.subTest(package=package):
@@ -203,7 +199,8 @@ def test_html(self):
         filenames_files = {
             ('module.py',): ['module.html'],
             ('module.py', 'subpkg2'): ['module.html', 'subpkg2',
-                                       'subpkg2/index.html', 'subpkg2/module.html'],
+                                       os.path.join('subpkg2', 'index.html'),
+                                       os.path.join('subpkg2', 'module.html')],
         }
         with chdir(TESTS_BASEDIR):
             for filenames, expected_files in filenames_files.items():
@@ -216,9 +213,11 @@ def test_html(self):
 
     def test_html_multiple_files(self):
         with chdir(TESTS_BASEDIR):
-            with run_html(EXAMPLE_MODULE + '/module.py', EXAMPLE_MODULE + '/subpkg2'):
-                self._basic_html_assertions(
-                    ['module.html', 'subpkg2', 'subpkg2/index.html', 'subpkg2/module.html'])
+            with run_html(os.path.join(EXAMPLE_MODULE, 'module.py'),
+                          os.path.join(EXAMPLE_MODULE, 'subpkg2')):
+                self._basic_html_assertions(['module.html', 'subpkg2',
+                                             os.path.join('subpkg2', 'index.html'),
+                                             os.path.join('subpkg2', 'module.html')])
 
     def test_html_identifier(self):
         for package in ('', '._private'):
@@ -232,7 +231,7 @@ def test_html_identifier(self):
     def test_html_ref_links(self):
         with run_html(EXAMPLE_MODULE, config='show_source_code=False'):
             self._check_files(
-                file_pattern=EXAMPLE_MODULE + '/index.html',
+                file_pattern=os.path.join(EXAMPLE_MODULE, 'index.html'),
                 include_patterns=[
                     'href="#example_pkg.B">',
                     'href="#example_pkg.A">',
@@ -1099,6 +1098,39 @@ def func(self):
         self.assertIsInstance(cls.doc['name'], pdoc.Variable)
         self.assertEqual(cls.doc['name'].type_annotation(), 'str')
 
+    def test_doc_comment_docstrings(self):
+        with temp_dir() as path:
+            filename = os.path.join(path, 'doc_comment_docstrs.py')
+            with open(filename, 'w') as f:
+                f.write('''#: Not included
+
+#: Line 1
+#: Line 2
+var1 = 1  #: Line 3
+
+#: Not included
+var2 = 1
+"""PEP-224 takes precedence"""
+
+#: This should not appear
+class C:
+    #: class var
+    class_var = 1
+
+    #: This also should not show
+    def __init__(self):
+       #: instance var
+       self.instance_var = 1
+''')
+
+            mod = pdoc.Module(pdoc.import_module(filename))
+
+            self.assertEqual(mod.doc['var1'].docstring, 'Line 1\nLine 2\nLine 3')
+            self.assertEqual(mod.doc['var2'].docstring, 'PEP-224 takes precedence')
+            self.assertEqual(mod.doc['C'].docstring, '')
+            self.assertEqual(mod.doc['C'].doc['class_var'].docstring, 'class var')
+            self.assertEqual(mod.doc['C'].doc['instance_var'].docstring, 'instance var')
+
 
 class HtmlHelpersTest(unittest.TestCase):
     """