Merge pull request #2524 from strictdoc-project/stanislaw/line_marker

feat(backend/sdoc_source_code): add support for multiline scope=line markers in Python code

Author: stanislaw

strictdoc/backend/sdoc_source_code/marker_parser.py

@@ -175,8 +175,12 @@ def _parse_relation_marker(
             range_marker.ng_source_column_begin = (
                 element_.meta.column + col_offset
             )
-            range_marker.ng_range_line_begin = line_start
-            range_marker.ng_range_line_end = line_end
+            range_marker.ng_range_line_begin = (
+                comment_line_start + element_.meta.line - 1
+            )
+            range_marker.ng_range_line_end = (
+                comment_line_start + element_.meta.end_line - 1
+            )
             markers.append(range_marker)
         elif relation_scope == "line":
             line_marker = LineMarker(None, requirements, role=relation_role)
@@ -186,8 +190,12 @@ def _parse_relation_marker(
             line_marker.ng_source_column_begin = (
                 element_.meta.column + col_offset
             )
-            line_marker.ng_range_line_begin = line_start
-            line_marker.ng_range_line_end = line_end + 1
+            line_marker.ng_range_line_begin = (
+                comment_line_start + element_.meta.line - 1
+            )
+            line_marker.ng_range_line_end = (
+                comment_line_start + element_.meta.end_line
+            )
             markers.append(line_marker)
         else:
             raise NotImplementedError

strictdoc/backend/sdoc_source_code/reader_python.py

@@ -3,7 +3,7 @@
 """
 
 from itertools import islice
-from typing import List, Optional, Sequence
+from typing import Any, List, Optional, Sequence, Tuple
 
 import tree_sitter_python
 from tree_sitter import Language, Node, Parser
@@ -62,6 +62,8 @@ def read(
 
         nodes = traverse_tree(tree)
         map_function_to_node = {}
+
+        visited_comments = set()
         for node_ in nodes:
             if node_.type == "module":
                 function = Function(
@@ -214,16 +216,34 @@ def read(
                     function_markers
                 )
             elif node_.type == "comment":
+                if node_ in visited_comments:
+                    continue
+
+                assert node_.parent is not None
                 assert node_.text is not None, (
                     f"Comment without a text: {node_}"
                 )
 
-                node_text_string = node_.text.decode("utf8")
+                if not SourceFileTraceabilityReader_Python.is_comment_alone_on_line(
+                    node_
+                ):
+                    continue
+
+                merged_comments, last_idx = (
+                    SourceFileTraceabilityReader_Python.collect_consecutive_comments(
+                        node_
+                    )
+                )
+
+                for j in range(node_.parent.children.index(node_), last_idx):
+                    visited_comments.add(node_.parent.children[j])
+
+                last_comment = node_.parent.children[last_idx - 1]
 
                 source_node = MarkerParser.parse(
-                    node_text_string,
+                    merged_comments,
                     node_.start_point[0] + 1,
-                    node_.end_point[0] + 1,
+                    last_comment.end_point[0] + 1,
                     node_.start_point[0] + 1,
                     None,
                 )
@@ -286,3 +306,62 @@ def get_node_ns(node: Node) -> Sequence[str]:
         # The array now contains the "fully qualified" node name,
         # we want to return the namespace, so don't return the last part.
         return parent_scopes[:-1]
+
+    @staticmethod
+    def collect_consecutive_comments(comment_node: Any) -> Tuple[str, int]:
+        parent = comment_node.parent
+
+        siblings = parent.children
+        idx = siblings.index(comment_node)
+
+        merged_texts = []
+
+        last_node = None
+
+        while idx < len(siblings) and siblings[idx].type == "comment":
+            n = siblings[idx]
+            assert n.text is not None
+            text = n.text.decode("utf8")
+
+            if last_node is not None:
+                # Tree-sitter line numbers are 0-based
+                last_end_line = last_node.end_point[0]
+                curr_start_line = n.start_point[0]
+
+                # Stop merging if there is an empty line between comments
+                if curr_start_line > last_end_line + 1:
+                    break
+
+            merged_texts.append(text)
+            last_node = n
+            idx += 1
+
+        return "\n".join(merged_texts), idx
+
+    @staticmethod
+    def is_comment_alone_on_line(node: Any) -> bool:
+        """
+        Return True if the comment node is the only thing on its line (ignoring whitespace).
+        """
+
+        if node.type != "comment":
+            return False
+
+        parent = node.parent
+        assert parent is not None
+
+        comment_line = node.start_point[0]
+
+        for sibling in parent.children:
+            if sibling is node:
+                continue
+            start_line = sibling.start_point[0]
+            end_line = sibling.end_point[0]
+
+            # If sibling shares the same line as comment
+            if start_line <= comment_line <= end_line:
+                # If it's not a comment (code, punctuation, etc.)
+                if sibling.type != "comment":
+                    return False
+
+        return True

strictdoc/export/html/templates/screens/source_file_view/main.jinja

@@ -82,7 +82,7 @@
 
       {#-- decide closer candidate: explicit end vs implicit close --#}
       {%- set marker_is_end = (is_marker and line.is_range_marker() and line.is_end()) -%}
-      {%- set implicit_close = (is_markup and ns.prev_line != none) -%}
+      {%- set implicit_close = (is_markup and ns.prev_line != none and ns.prev_line.ng_range_line_end == loop.index) -%}
 
       {%- if marker_is_end -%}
         {%- set range_closer_line = line -%}

tests/integration/features/source_code_traceability/_language_parsers/python/30_py_line_marker_singline/file.py

@@ -0,0 +1,8 @@
+def hello_world():
+    # @relation(REQ-001, scope=line)
+    print("Line marker")  # noqa: T201
+
+    # @relation(REQ-001, scope=range_start)
+    print("ignored hello world")  # noqa: T201
+    print("ignored hello world")  # noqa: T201
+    # @relation(REQ-001, scope=range_end)

tests/integration/features/source_code_traceability/_language_parsers/python/30_py_line_marker_singline/input.sdoc

@@ -0,0 +1,10 @@
+[DOCUMENT]
+TITLE: Hello world doc
+
+[REQUIREMENT]
+UID: REQ-001
+TITLE: Requirement Title
+STATEMENT: Requirement Statement
+RELATIONS:
+- TYPE: File
+  VALUE: file.py

tests/integration/features/source_code_traceability/_language_parsers/python/30_py_line_marker_singline/strictdoc.toml

@@ -0,0 +1,10 @@
+[project]
+
+features = [
+  "REQUIREMENT_TO_SOURCE_TRACEABILITY",
+  "SOURCE_FILE_LANGUAGE_PARSERS",
+]
+
+exclude_source_paths = [
+  "test.itest",
+]

tests/integration/features/source_code_traceability/_language_parsers/python/30_py_line_marker_singline/test.itest

@@ -0,0 +1,10 @@
+# @relation(SDOC-SRS-124, scope=file)
+
+RUN: %strictdoc export %S --output-dir %T | filecheck %s --dump-input=fail
+CHECK: Published: Hello world doc
+
+RUN: %check_exists --file "%T/html/_source_files/file.py.html"
+
+RUN: %cat %T/html/_source_files/file.py.html | filecheck %s --dump-input=fail --check-prefix CHECK-SOURCE-FILE
+CHECK-SOURCE-FILE: [ 2-3 ]
+CHECK-SOURCE-FILE: [ 5-8 ]

tests/integration/features/source_code_traceability/_language_parsers/python/31_py_line_marker_multiline/file.py

@@ -0,0 +1,15 @@
+# ruff: noqa
+
+def hello_world():
+    # @relation(
+    #     REQ-001,
+    #     REQ-002,
+    #     REQ-003,
+    #     scope=line
+    # )
+    print("Line marker")  # noqa: T201
+
+    # @relation(REQ-001, scope=range_start)
+    print("ignored hello world")  # noqa: T201
+    print("ignored hello world")  # noqa: T201
+    # @relation(REQ-001, scope=range_end)

tests/integration/features/source_code_traceability/_language_parsers/python/31_py_line_marker_multiline/input.sdoc

@@ -0,0 +1,20 @@
+[DOCUMENT]
+TITLE: Hello world doc
+
+[REQUIREMENT]
+UID: REQ-001
+TITLE: Requirement Title #1
+STATEMENT: Requirement Statement #1
+RELATIONS:
+- TYPE: File
+  VALUE: file.py
+
+[REQUIREMENT]
+UID: REQ-002
+TITLE: Requirement Title #2
+STATEMENT: Requirement Statement #2
+
+[REQUIREMENT]
+UID: REQ-003
+TITLE: Requirement Title #3
+STATEMENT: Requirement Statement #3

tests/integration/features/source_code_traceability/_language_parsers/python/31_py_line_marker_multiline/strictdoc.toml

@@ -0,0 +1,10 @@
+[project]
+
+features = [
+  "REQUIREMENT_TO_SOURCE_TRACEABILITY",
+  "SOURCE_FILE_LANGUAGE_PARSERS",
+]
+
+exclude_source_paths = [
+  "test.itest",
+]