Various fixes for own page output

Also added tests for own page output.
Fix some inherited members always being rendered.
Own page members of an entity are linked to after the docstring
of the parent entity.
Fix entities below the "class" level that have their own page
from rendering incorrectly.
Rename "single page output" to "own page output". An entity does
not have a "single page" when its members are spread across
their own pages.
Properties are linked to on their parent classes page.
Children not present in `__all__` are not rendered.
Fixed emitting ignore event twice for methods.
Corrected documentation around `imported-members` to reflect that it
applies only to objects imported into a package, not modules.
Fixed path error on Windows.

Author: AWhetter

autoapi/extension.py

@@ -35,6 +35,13 @@
     "special-members",
     "imported-members",
 ]
+_VALID_PAGE_LEVELS = [
+    "module",
+    "class",
+    "function",
+    "method",
+    "attribute",
+]
 _VIEWCODE_CACHE: Dict[str, Tuple[str, Dict]] = {}
 """Caches a module's parse results for use in viewcode."""
 
@@ -75,6 +82,10 @@ def run_autoapi(app):
         if app.config.autoapi_include_summaries:
             app.config.autoapi_options.append("show-module-summary")
 
+    own_page_level = app.config.autoapi_own_page_level
+    if own_page_level not in _VALID_PAGE_LEVELS:
+        raise ValueError(f"Invalid autoapi_own_page_level '{own_page_level}")
+
     # Make sure the paths are full
     normalised_dirs = _normalise_autoapi_dirs(app.config.autoapi_dirs, app.srcdir)
     for _dir in normalised_dirs:
@@ -101,7 +112,7 @@ def run_autoapi(app):
                 RemovedInAutoAPI3Warning,
             )
     sphinx_mapper_obj = PythonSphinxMapper(
-        app, template_dir=template_dir, url_root=url_root
+        app, template_dir=template_dir, dir_root=normalized_root, url_root=url_root
     )
 
     if app.config.autoapi_file_patterns:
@@ -128,7 +139,7 @@ def run_autoapi(app):
         sphinx_mapper_obj.map(options=app.config.autoapi_options)
 
         if app.config.autoapi_generate_api_docs:
-            sphinx_mapper_obj.output_rst(root=normalized_root, source_suffix=out_suffix)
+            sphinx_mapper_obj.output_rst(source_suffix=out_suffix)
 
 
 def build_finished(app, exception):

autoapi/mappers/base.py

@@ -1,10 +1,9 @@
-import os
-import fnmatch
 from collections import OrderedDict, namedtuple
+import fnmatch
+import os
+import pathlib
 import re
 
-import anyascii
-from docutils.parsers.rst import convert_directive_function
 from jinja2 import Environment, FileSystemLoader, TemplateNotFound
 import sphinx
 import sphinx.util
@@ -13,7 +12,7 @@
 from sphinx.util.osutil import ensuredir
 import sphinx.util.logging
 
-from ..settings import API_ROOT, TEMPLATE_DIR
+from ..settings import TEMPLATE_DIR
 
 LOGGER = sphinx.util.logging.getLogger(__name__)
 _OWN_PAGE_LEVELS = [
@@ -24,8 +23,8 @@
     "function",
     "method",
     "property",
-    "attribute",
     "data",
+    "attribute",
 ]
 
 Path = namedtuple("Path", ["absolute", "relative"])
@@ -38,14 +37,10 @@ class PythonMapperBase:
     and map that onto this standard Python object.
     Subclasses may also include language-specific attributes on this object.
 
-    Arguments:
-
     Args:
         obj: JSON object representing this object
         jinja_env: A template environment for rendering this object
 
-    Required attributes:
-
     Attributes:
         id (str): A globally unique identifier for this object.
             Generally a fully qualified name, including namespace.
@@ -55,25 +50,21 @@ class PythonMapperBase:
         children (list): Children of this object
         parameters (list): Parameters to this object
         methods (list): Methods on this object
-
-    Optional attributes:
-
     """
 
     language = "base"
     type = "base"
-    # Create a page in the output for this object.
-    top_level_object = False
     _RENDER_LOG_LEVEL = "VERBOSE"
 
-    def __init__(self, obj, jinja_env, app, options=None):
+    def __init__(self, obj, jinja_env, app, url_root, options=None):
         self.app = app
         self.obj = obj
         self.options = options
         self.jinja_env = jinja_env
-        self.url_root = os.path.join("/", API_ROOT)
+        self.url_root = url_root
 
         self.name = None
+        self.qual_name = None
         self.id = None
 
     def __getstate__(self):
@@ -103,7 +94,7 @@ def rendered(self):
     def get_context_data(self):
         own_page_level = self.app.config.autoapi_own_page_level
         desired_page_level = _OWN_PAGE_LEVELS.index(own_page_level)
-        own_page_types = set(_OWN_PAGE_LEVELS[:desired_page_level+1])
+        own_page_types = set(_OWN_PAGE_LEVELS[: desired_page_level + 1])
 
         return {
             "autoapi_options": self.app.config.autoapi_options,
@@ -127,28 +118,19 @@ def short_name(self):
         """Shorten name property"""
         return self.name.split(".")[-1]
 
-    @property
-    def pathname(self):
-        """Sluggified path for filenames
+    def output_dir(self, root):
+        """The directory to render this object."""
+        module = self.id[: -(len("." + self.qual_name))]
+        parts = [root] + module.split(".")
+        return pathlib.PurePosixPath(*parts)
 
-        Slugs to a filename using the follow steps
+    def output_filename(self):
+        """The name of the file to render into, without a file suffix."""
+        filename = self.qual_name
+        if filename == "index":
+            filename = ".index"
 
-        * Decode unicode to approximate ascii
-        * Remove existing hyphens
-        * Substitute hyphens for non-word characters
-        * Break up the string as paths
-        """
-        slug = self.name
-        slug = anyascii.anyascii(slug)
-        slug = slug.replace("-", "")
-        slug = re.sub(r"[^\w\.]+", "-", slug).strip("-")
-        return os.path.join(*slug.split("."))
-
-    def include_dir(self, root):
-        """Return directory of file"""
-        parts = [root]
-        parts.extend(self.pathname.split(os.path.sep))
-        return "/".join(parts)
+        return filename
 
     @property
     def include_path(self):
@@ -157,9 +139,7 @@ def include_path(self):
         This is used in ``toctree`` directives, as Sphinx always expects Unix
         path separators
         """
-        parts = [self.include_dir(root=self.url_root)]
-        parts.append("index")
-        return "/".join(parts)
+        return str(self.output_dir(self.url_root) / self.output_filename())
 
     @property
     def display(self):
@@ -185,7 +165,7 @@ class SphinxMapperBase:
         app: Sphinx application instance
     """
 
-    def __init__(self, app, template_dir=None, url_root=None):
+    def __init__(self, app, template_dir=None, dir_root=None, url_root=None):
         self.app = app
 
         template_paths = [TEMPLATE_DIR]
@@ -209,8 +189,9 @@ def _wrapped_prepare(value):
 
         own_page_level = self.app.config.autoapi_own_page_level
         desired_page_level = _OWN_PAGE_LEVELS.index(own_page_level)
-        self.own_page_types = set(_OWN_PAGE_LEVELS[:desired_page_level+1])
+        self.own_page_types = set(_OWN_PAGE_LEVELS[: desired_page_level + 1])
 
+        self.dir_root = dir_root
         self.url_root = url_root
 
         # Mapping of {filepath -> raw data}
@@ -298,14 +279,17 @@ def add_object(self, obj):
         Args:
             obj: Instance of a AutoAPI object
         """
-        if obj.type in self.own_page_types:
+        display = obj.display
+        if display and obj.type in self.own_page_types:
             self.objects_to_render[obj.id] = obj
 
         self.all_objects[obj.id] = obj
         child_stack = list(obj.children)
         while child_stack:
             child = child_stack.pop()
             self.all_objects[child.id] = child
+            if display and child.type in self.own_page_types:
+                self.objects_to_render[child.id] = child
             child_stack.extend(getattr(child, "children", ()))
 
     def map(self, options=None):
@@ -327,81 +311,32 @@ def create_class(self, data, options=None, **kwargs):
         """
         raise NotImplementedError
 
-    def output_child_rst(self, obj, obj_parent, detail_dir, source_suffix):
-
-        if not obj.display:
-            return
-
-        # Skip nested cases like functions in functions or clases in clases
-        if obj.type == obj_parent.type:
-            return
-
-        obj_child_page_level = _OWN_PAGE_LEVELS.index(obj.type)
-        desired_page_level = _OWN_PAGE_LEVELS.index(self.app.config.autoapi_own_page_level)
-        is_own_page = obj_child_page_level <= desired_page_level
-        if not is_own_page:
-            return
-
-        obj_child_rst = obj.render(
-            is_own_page=is_own_page,
-        )
-        if not obj_child_rst:
-            return
-
-        function_page_level = _OWN_PAGE_LEVELS.index("function")
-        is_level_beyond_function = function_page_level < desired_page_level
-        if obj.type in ["exception", "class"]:
-            if not is_level_beyond_function:
-                outfile = f"{obj.short_name}{source_suffix}"
-                path = os.path.join(detail_dir, outfile)
-            else:
-                outdir = os.path.join(detail_dir, obj.short_name)
-                ensuredir(outdir)
-                path = os.path.join(outdir, f"index{source_suffix}")
-        else:
-            is_parent_in_detail_dir = detail_dir.endswith(obj_parent.short_name)
-            outdir = detail_dir if is_parent_in_detail_dir else os.path.join(detail_dir, obj_parent.short_name)
-            ensuredir(outdir)
-            path = os.path.join(outdir, f"{obj.short_name}{source_suffix}")
-
-        with open(path, "wb+") as obj_child_detail_file:
-            obj_child_detail_file.write(obj_child_rst.encode("utf-8"))
-
-        for obj_child in obj.children:
-            child_detail_dir = os.path.join(detail_dir, obj.name)
-            self.output_child_rst(obj_child, obj, child_detail_dir, source_suffix)
-
-    def output_rst(self, root, source_suffix):
+    def output_rst(self, source_suffix):
         for _, obj in status_iterator(
             self.objects_to_render.items(),
             colorize("bold", "[AutoAPI] ") + "Rendering Data... ",
             length=len(self.objects_to_render),
             verbosity=1,
             stringify_func=(lambda x: x[0]),
         ):
-            if not obj.display:
-                continue
-
             rst = obj.render(is_own_page=True)
             if not rst:
                 continue
 
-            detail_dir = obj.include_dir(root=root)
-            ensuredir(detail_dir)
-            path = os.path.join(detail_dir, f"index{source_suffix}")
+            output_dir = obj.output_dir(self.dir_root)
+            ensuredir(output_dir)
+            output_path = output_dir / obj.output_filename()
+            path = f"{output_path}{source_suffix}"
             with open(path, "wb+") as detail_file:
                 detail_file.write(rst.encode("utf-8"))
-            
-            for child in obj.children:
-                self.output_child_rst(child, obj, detail_dir, source_suffix)
 
         if self.app.config.autoapi_add_toctree_entry:
-            self._output_top_rst(root)
+            self._output_top_rst()
 
-    def _output_top_rst(self, root):
+    def _output_top_rst(self):
         # Render Top Index
-        top_level_index = os.path.join(root, "index.rst")
-        pages = self.objects_to_render.values()
+        top_level_index = os.path.join(self.dir_root, "index.rst")
+        pages = [obj for obj in self.objects_to_render.values() if obj.display]
         with open(top_level_index, "wb") as top_level_file:
             content = self.jinja_env.get_template("index.rst")
             top_level_file.write(content.render(pages=pages).encode("utf-8"))

autoapi/mappers/python/mapper.py

@@ -62,12 +62,14 @@ def _expand_wildcard_placeholder(original_module, originals_map, placeholder):
     placeholders = []
     for original in originals:
         new_full_name = placeholder["full_name"].replace("*", original["name"])
+        new_qual_name = placeholder["qual_name"].replace("*", original["name"])
         new_original_path = placeholder["original_path"].replace("*", original["name"])
         if "original_path" in original:
             new_original_path = original["original_path"]
         new_placeholder = dict(
             placeholder,
             name=original["name"],
+            qual_name=new_qual_name,
             full_name=new_full_name,
             original_path=new_original_path,
         )
@@ -167,6 +169,7 @@ def _resolve_placeholder(placeholder, original):
     assert original["type"] != "placeholder"
     # The name remains the same.
     new["name"] = placeholder["name"]
+    new["qual_name"] = placeholder["qual_name"]
     new["full_name"] = placeholder["full_name"]
     # Record where the placeholder originally came from.
     new["original_path"] = original["full_name"]
@@ -217,7 +220,7 @@ def _link_objs(value):
 
 
 class PythonSphinxMapper(SphinxMapperBase):
-    """Auto API domain handler for Python
+    """AutoAPI domain handler for Python
 
     Parses directly from Python files.
 
@@ -240,8 +243,8 @@ class PythonSphinxMapper(SphinxMapperBase):
         )
     }
 
-    def __init__(self, app, template_dir=None, url_root=None):
-        super().__init__(app, template_dir, url_root)
+    def __init__(self, app, template_dir=None, dir_root=None, url_root=None):
+        super().__init__(app, template_dir, dir_root, url_root)
 
         self.jinja_env.filters["link_objs"] = _link_objs
         self._use_implicit_namespace = (
@@ -340,15 +343,33 @@ def _resolve_placeholders(self):
             visit_path = collections.OrderedDict()
             _resolve_module_placeholders(modules, module_name, visit_path, resolved)
 
+    def _hide_yo_kids(self):
+        """For all direct children of a module/package, hide them if needed."""
+        for module in self.paths.values():
+            if module["all"] is not None:
+                all_names = set(module["all"])
+                for child in module["children"]:
+                    if child["qual_name"] not in all_names:
+                        child["hide"] = True
+            elif module["type"] == "module":
+                for child in module["children"]:
+                    if child.get("imported"):
+                        child["hide"] = True
+
     def map(self, options=None):
         self._resolve_placeholders()
+        self._hide_yo_kids()
         self.app.env.autoapi_annotations = {}
 
         super().map(options)
 
-        top_level_objects = {obj.id: obj for obj in self.all_objects.values() if isinstance(obj, TopLevelPythonPythonMapper)}
+        top_level_objects = {
+            obj.id: obj
+            for obj in self.all_objects.values()
+            if isinstance(obj, TopLevelPythonPythonMapper)
+        }
         parents = {obj.name: obj for obj in top_level_objects.values()}
-        for obj in self.objects_to_render.values():
+        for obj in top_level_objects.values():
             parent_name = obj.name.rsplit(".", 1)[0]
             if parent_name in parents and parent_name != obj.name:
                 parent = parents[parent_name]
@@ -380,9 +401,9 @@ def create_class(self, data, options=None, **kwargs):
                 options=self.app.config.autoapi_options,
                 jinja_env=self.jinja_env,
                 app=self.app,
+                url_root=self.url_root,
                 **kwargs,
             )
-            obj.url_root = self.url_root
 
             for child_data in data.get("children", []):
                 for child_obj in self.create_class(