Merge pull request #2543 from haxtibal/tdmg/source_node_remap

feat(backend/sdoc_source_code): Support remapping custom tags to sdoc fields

Author: stanislaw

strictdoc/backend/sdoc_source_code/caching_reader.py

@@ -4,7 +4,6 @@
 
 from typing import Optional, Union
 
-from strictdoc.backend.sdoc.models.grammar_element import GrammarElement
 from strictdoc.backend.sdoc.pickle_cache import PickleCache
 from strictdoc.backend.sdoc_source_code.models.source_file_info import (
     SourceFileTraceabilityInfo,
@@ -29,7 +28,7 @@ class SourceFileTraceabilityCachingReader:
     def read_from_file(
         path_to_file: str,
         project_config: ProjectConfig,
-        source_node_grammar_element: Optional[GrammarElement],
+        source_node_tags: Optional[set[str]] = None,
     ) -> Optional[SourceFileTraceabilityInfo]:
         unpickled_content = PickleCache.read_from_cache(
             path_to_file, project_config, "source_file"
@@ -42,7 +41,9 @@ def read_from_file(
             return unpickled_content
 
         reader = SourceFileTraceabilityCachingReader._get_reader(
-            path_to_file, project_config, source_node_grammar_element
+            path_to_file,
+            project_config,
+            source_node_tags,
         )
         try:
             traceability_info = reader.read_from_file(path_to_file)
@@ -64,7 +65,7 @@ def read_from_file(
     def _get_reader(
         path_to_file: str,
         project_config: ProjectConfig,
-        source_node_grammar_element: Optional[GrammarElement],
+        source_node_tags: Optional[set[str]] = None,
     ) -> Union[
         SourceFileTraceabilityReader,
         SourceFileTraceabilityReader_Python,
@@ -82,12 +83,9 @@ def _get_reader(
                 or path_to_file.endswith(".hpp")
                 or path_to_file.endswith(".cpp")
             ):
-                custom_tags = (
-                    source_node_grammar_element.get_field_titles()
-                    if source_node_grammar_element is not None
-                    else None
+                return SourceFileTraceabilityReader_C(
+                    custom_tags=source_node_tags
                 )
-                return SourceFileTraceabilityReader_C(custom_tags=custom_tags)
             if path_to_file.endswith(".robot"):
                 return SourceFileTraceabilityReader_Robot()
         return SourceFileTraceabilityReader()

strictdoc/backend/sdoc_source_code/comment_parser/marker_lexer.py

@@ -67,7 +67,7 @@ class GrammarTemplate(Template):
 class MarkerLexer:
     @staticmethod
     def parse(
-        source_input: str, custom_tags: Optional[list[str]] = None
+        source_input: str, custom_tags: Optional[set[str]] = None
     ) -> ParseTree:
         if custom_tags is not None:
             grammar_extension = NODE_GRAMMAR_EXTENSION.substitute(

strictdoc/backend/sdoc_source_code/marker_parser.py

@@ -32,7 +32,7 @@ def parse(
         comment_line_start: int,
         entity_name: Optional[str] = None,
         col_offset: int = 0,
-        custom_tags: Optional[list[str]] = None,
+        custom_tags: Optional[set[str]] = None,
     ) -> SourceNode:
         """
         Parse relation markers from source file comments.

strictdoc/backend/sdoc_source_code/models/source_node.py

@@ -3,7 +3,7 @@
 """
 
 from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Union
+from typing import List, Optional, Union
 
 from strictdoc.backend.sdoc_source_code.models.function import Function
 from strictdoc.backend.sdoc_source_code.models.function_range_marker import (
@@ -13,6 +13,7 @@
 from strictdoc.backend.sdoc_source_code.models.range_marker import (
     RangeMarker,
 )
+from strictdoc.core.project_config import SourceNodesEntry
 
 
 @dataclass
@@ -21,5 +22,33 @@ class SourceNode:
     markers: List[Union[FunctionRangeMarker, RangeMarker, LineMarker]] = field(
         default_factory=list
     )
-    fields: Dict[str, str] = field(default_factory=dict)
+    fields: dict[str, str] = field(default_factory=dict)
     function: Optional[Function] = None
+
+    def get_sdoc_field(
+        self,
+        field_name: str,
+        cfg_entry: SourceNodesEntry,
+    ) -> Optional[str]:
+        if field_name in cfg_entry.sdoc_to_source_map:
+            field_name = cfg_entry.sdoc_to_source_map[field_name]
+        return self.fields.get(field_name)
+
+    def get_sdoc_fields(self, cfg_entry: SourceNodesEntry) -> dict[str, str]:
+        """
+        Get SDoc representation of all available fields.
+
+        Returns a dict equivalent to self.fields unless config option 'sdoc_to_source_map' is set.
+        """
+        fields = {
+            field_name: self.fields[field_name]
+            for field_name in self.fields
+            if field_name not in cfg_entry.sdoc_to_source_map.values()
+        }
+        for (
+            field_name,
+            mapped_field_name,
+        ) in cfg_entry.sdoc_to_source_map.items():
+            if mapped_field_name in self.fields:
+                fields[field_name] = self.fields[mapped_field_name]
+        return fields

strictdoc/backend/sdoc_source_code/reader_c.py

@@ -40,8 +40,8 @@
 
 
 class SourceFileTraceabilityReader_C:
-    def __init__(self, custom_tags: Optional[list[str]] = None) -> None:
-        self.custom_tags: Optional[list[str]] = custom_tags
+    def __init__(self, custom_tags: Optional[set[str]] = None) -> None:
+        self.custom_tags: Optional[set[str]] = custom_tags
 
     def read(
         self,

strictdoc/core/file_traceability_index.py

@@ -648,41 +648,46 @@ def create_folder_section(
 
             for source_node_ in traceability_info_.source_nodes:
                 assert source_node_.entity_name is not None
-                source_sdoc_node_uid = f"{document_uid}/{path_to_source_file_}/{source_node_.entity_name}"
-
-                source_sdoc_node = traceability_index.get_node_by_uid_weak(
-                    source_sdoc_node_uid
+                sdoc_node_uid = source_node_.get_sdoc_field(
+                    "UID", relevant_source_node_entry
+                )
+                if sdoc_node_uid is None:
+                    sdoc_node_uid = f"{document_uid}/{path_to_source_file_}/{source_node_.entity_name}"
+                sdoc_node = traceability_index.get_node_by_uid_weak(
+                    sdoc_node_uid
                 )
-                if source_sdoc_node is not None:
-                    source_sdoc_node = assert_cast(source_sdoc_node, SDocNode)
+
+                if sdoc_node is not None:
+                    sdoc_node = assert_cast(sdoc_node, SDocNode)
                     self.merge_sdoc_node_with_source_node(
-                        source_sdoc_node,
+                        relevant_source_node_entry,
                         source_node_,
+                        sdoc_node,
                         document,
-                        relevant_source_node_entry,
                     )
                 else:
-                    source_sdoc_node = self.create_sdoc_node_from_source_node(
+                    sdoc_node = self.create_sdoc_node_from_source_node(
                         source_node_,
-                        source_sdoc_node_uid,
-                        document,
                         relevant_source_node_entry,
+                        sdoc_node_uid,
+                        document,
                     )
-                    current_top_node.section_contents.append(source_sdoc_node)
+                    sdoc_node_uid = assert_cast(sdoc_node.reserved_uid, str)
+                    current_top_node.section_contents.append(sdoc_node)
                     traceability_index.graph_database.create_link(
                         link_type=GraphLinkType.UID_TO_NODE,
-                        lhs_node=source_sdoc_node_uid,
-                        rhs_node=source_sdoc_node,
+                        lhs_node=sdoc_node_uid,
+                        rhs_node=sdoc_node,
                     )
 
                 self.connect_source_node_function(
-                    source_node_, source_sdoc_node_uid, traceability_info_
+                    source_node_, sdoc_node_uid, traceability_info_
                 )
                 self.connect_sdoc_node_with_file_path(
-                    source_sdoc_node, path_to_source_file_
+                    sdoc_node, path_to_source_file_
                 )
                 self.connect_source_node_requirements(
-                    source_node_, source_sdoc_node, traceability_index
+                    source_node_, sdoc_node, traceability_index
                 )
 
         # Warn if source_node was not matched by any include_source_paths, it indicates misconfiguration
@@ -975,45 +980,37 @@ def connect_source_node_function(
     @staticmethod
     def create_sdoc_node_from_source_node(
         source_node: SourceNode,
-        uid: str,
+        source_node_config_entry: SourceNodesEntry,
+        sdoc_node_uid: str,
         parent_document: SDocDocumentIF,
-        relevant_source_node_entry: SourceNodesEntry,
     ) -> SDocNode:
-        source_sdoc_node = SDocNode(
+        sdoc_node = SDocNode(
             parent=parent_document,
-            node_type=relevant_source_node_entry.node_type,
+            node_type=source_node_config_entry.node_type,
             fields=[],
             relations=[],
             # It is important that this autogenerated node is marked as such.
             autogen=True,
         )
-        source_sdoc_node.ng_document_reference = DocumentReference()
-        source_sdoc_node.ng_document_reference.set_document(parent_document)
-        source_sdoc_node.ng_including_document_reference = DocumentReference()
-        source_sdoc_node.set_field_value(
-            field_name="UID",
-            form_field_index=0,
-            value=uid,
-        )
-        source_sdoc_node.set_field_value(
-            field_name="TITLE",
-            form_field_index=0,
-            value=source_node.entity_name,
-        )
-        for node_name_, node_value_ in source_node.fields.items():
-            source_sdoc_node.set_field_value(
-                field_name=node_name_,
-                form_field_index=0,
-                value=node_value_,
-            )
-        return source_sdoc_node
+        sdoc_node.ng_document_reference = DocumentReference()
+        sdoc_node.ng_document_reference.set_document(parent_document)
+        sdoc_node.ng_including_document_reference = DocumentReference()
+        sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
+        sdoc_node_fields["UID"] = sdoc_node_uid
+        if (
+            "TITLE" not in sdoc_node_fields
+            and source_node.entity_name is not None
+        ):
+            sdoc_node_fields["TITLE"] = source_node.entity_name
+        FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
+        return sdoc_node
 
     @staticmethod
     def merge_sdoc_node_with_source_node(
-        sdoc_node: SDocNode,
+        source_node_config_entry: SourceNodesEntry,
         source_node: SourceNode,
+        sdoc_node: SDocNode,
         parent_document: SDocDocumentIF,
-        source_node_config_entry: SourceNodesEntry,
     ) -> None:
         # First check if grammar element definitions are compatible.
         source_node_type = source_node_config_entry.node_type
@@ -1038,19 +1035,19 @@ def merge_sdoc_node_with_source_node(
                 f"Grammar element {sdoc_node_document.reserved_uid}::{source_node_type} "
                 f"incompatible with {parent_document.reserved_uid}::{source_node_type}"
             )
-        # Merge strategy: overwrite title if there's a TITLE from custom tags.
-        if "TITLE" in source_node.fields:
-            sdoc_node.set_field_value(
-                field_name="TITLE",
-                form_field_index=0,
-                value=source_node.fields["TITLE"],
-            )
         # Merge strategy: overwrite any field if there's a field with same name from custom tags.
-        for node_name_, node_value_ in source_node.fields.items():
+        sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
+        FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
+
+    @staticmethod
+    def set_sdoc_node_fields(
+        sdoc_node: SDocNode, sdoc_node_fields: dict[str, str]
+    ) -> None:
+        for field_name, field_value in sdoc_node_fields.items():
             sdoc_node.set_field_value(
-                field_name=node_name_,
+                field_name=field_name,
                 form_field_index=0,
-                value=node_value_,
+                value=field_value,
             )
 
     def connect_sdoc_node_with_file_path(

strictdoc/core/project_config.py

@@ -8,7 +8,7 @@
 import sys
 import tempfile
 import types
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
@@ -47,12 +47,8 @@ class SourceNodesEntry:
     path: str
     uid: str
     node_type: str
-    full_path: Optional[Path]
-
-    @classmethod
-    def from_cfg_data(cls, data: dict[str, str]) -> "SourceNodesEntry":
-        full_path = Path(data["full_path"]) if "full_path" in data else None
-        return cls(data["path"], data["uid"], data["node_type"], full_path)
+    sdoc_to_source_map: Dict[str, str] = field(default_factory=dict)
+    full_path: Optional[Path] = None
 
 
 class ProjectFeature(str, Enum):
@@ -124,7 +120,7 @@ def __init__(
         include_source_paths: Optional[List[str]] = None,
         exclude_source_paths: Optional[List[str]] = None,
         test_report_root_dict: Optional[Dict[str, str]] = None,
-        source_nodes: Optional[List[Dict[str, str]]] = None,
+        source_nodes: Optional[List[SourceNodesEntry]] = None,
         html2pdf_strict: bool = False,
         html2pdf_template: Optional[str] = None,
         bundle_document_version: Optional[
@@ -209,12 +205,7 @@ def __init__(
             test_report_root_dict if test_report_root_dict is not None else {}
         )
         self.source_nodes: List[SourceNodesEntry] = (
-            [
-                SourceNodesEntry.from_cfg_data(source_node)
-                for source_node in source_nodes
-            ]
-            if source_nodes is not None
-            else []
+            source_nodes if source_nodes is not None else []
         )
 
         # Settings derived from the command-line parameters.
@@ -585,7 +576,7 @@ def _load_from_dictionary(
         include_source_paths: List[str] = []
         exclude_source_paths: List[str] = []
         test_report_root_dict: Dict[str, str] = {}
-        source_nodes: List[Dict[str, str]] = []
+        source_nodes: List[SourceNodesEntry] = []
         html2pdf_strict: bool = False
         html2pdf_template: Optional[str] = None
         bundle_document_version = (
@@ -801,14 +792,16 @@ def _load_from_dictionary(
                 assert isinstance(source_nodes_config, list)
                 for item_ in source_nodes_config:
                     source_node_path = next(iter(item_))
-                    source_node_uid = item_[source_node_path]["uid"]
-                    source_node_node_type = item_[source_node_path]["node_type"]
+                    source_node_item = item_[source_node_path]
                     source_nodes.append(
-                        {
-                            "path": source_node_path,
-                            "uid": source_node_uid,
-                            "node_type": source_node_node_type,
-                        }
+                        SourceNodesEntry(
+                            path=source_node_path,
+                            uid=source_node_item["uid"],
+                            node_type=source_node_item["node_type"],
+                            sdoc_to_source_map=source_node_item["map"]
+                            if "map" in source_node_item
+                            else {},
+                        )
                     )
 
         if "server" in config_dict:

strictdoc/core/traceability_index.py

@@ -10,6 +10,7 @@
 from strictdoc.backend.sdoc.models.anchor import Anchor
 from strictdoc.backend.sdoc.models.document import SDocDocument
 from strictdoc.backend.sdoc.models.document_config import DocumentConfig
+from strictdoc.backend.sdoc.models.grammar_element import GrammarElement
 from strictdoc.backend.sdoc.models.inline_link import InlineLink
 from strictdoc.backend.sdoc.models.node import SDocNode
 from strictdoc.backend.sdoc_source_code.models.source_file_info import (
@@ -353,6 +354,14 @@ def get_incoming_links(
         # FIXME: Should the graph database return OrderedSet or a copied list()?
         return list(incoming_links)
 
+    def get_grammar_element(
+        self, document_uid: str, node_type: str
+    ) -> Optional[GrammarElement]:
+        document = self.get_node_by_uid_weak2(document_uid)
+        if isinstance(document, SDocDocument) and document.grammar is not None:
+            return document.grammar.elements_by_type.get(node_type)
+        return None
+
     def create_traceability_info(
         self,
         source_file: SourceFile,