feat(backend/sdoc_source_code): Support remapping custom tags to sdoc fields

This introduces a config option for source_nodes to describe how tags
from an annotated source node map to *different*  SDocNode fields.

It's most useful in cases where one can't control the custom tag scheme
at source code side, but still wants to reuse some of the custom fields
for reserved StrictDoc fields with special meaning like UID or TITLE.
For example, one can now use the value from "THEIR-PROPRIETARY-ID" as
UID within StrictDoc, with all the special meanings of UID.

Author: haxtibal

strictdoc/backend/sdoc_source_code/caching_reader.py

@@ -4,7 +4,6 @@
 
 from typing import Optional, Union
 
-from strictdoc.backend.sdoc.models.grammar_element import GrammarElement
 from strictdoc.backend.sdoc.pickle_cache import PickleCache
 from strictdoc.backend.sdoc_source_code.models.source_file_info import (
     SourceFileTraceabilityInfo,
@@ -29,7 +28,7 @@ class SourceFileTraceabilityCachingReader:
     def read_from_file(
         path_to_file: str,
         project_config: ProjectConfig,
-        source_node_grammar_element: Optional[GrammarElement],
+        source_node_tags: Optional[set[str]] = None,
     ) -> Optional[SourceFileTraceabilityInfo]:
         unpickled_content = PickleCache.read_from_cache(
             path_to_file, project_config, "source_file"
@@ -42,7 +41,9 @@ def read_from_file(
             return unpickled_content
 
         reader = SourceFileTraceabilityCachingReader._get_reader(
-            path_to_file, project_config, source_node_grammar_element
+            path_to_file,
+            project_config,
+            source_node_tags,
         )
         try:
             traceability_info = reader.read_from_file(path_to_file)
@@ -64,7 +65,7 @@ def read_from_file(
     def _get_reader(
         path_to_file: str,
         project_config: ProjectConfig,
-        source_node_grammar_element: Optional[GrammarElement],
+        source_node_tags: Optional[set[str]] = None,
     ) -> Union[
         SourceFileTraceabilityReader,
         SourceFileTraceabilityReader_Python,
@@ -82,12 +83,9 @@ def _get_reader(
                 or path_to_file.endswith(".hpp")
                 or path_to_file.endswith(".cpp")
             ):
-                custom_tags = (
-                    source_node_grammar_element.get_field_titles()
-                    if source_node_grammar_element is not None
-                    else None
+                return SourceFileTraceabilityReader_C(
+                    custom_tags=source_node_tags
                 )
-                return SourceFileTraceabilityReader_C(custom_tags=custom_tags)
             if path_to_file.endswith(".robot"):
                 return SourceFileTraceabilityReader_Robot()
         return SourceFileTraceabilityReader()

strictdoc/backend/sdoc_source_code/comment_parser/marker_lexer.py

@@ -67,7 +67,7 @@ class GrammarTemplate(Template):
 class MarkerLexer:
     @staticmethod
     def parse(
-        source_input: str, custom_tags: Optional[list[str]] = None
+        source_input: str, custom_tags: Optional[set[str]] = None
     ) -> ParseTree:
         if custom_tags is not None:
             grammar_extension = NODE_GRAMMAR_EXTENSION.substitute(

strictdoc/backend/sdoc_source_code/marker_parser.py

@@ -32,7 +32,7 @@ def parse(
         comment_line_start: int,
         entity_name: Optional[str] = None,
         col_offset: int = 0,
-        custom_tags: Optional[list[str]] = None,
+        custom_tags: Optional[set[str]] = None,
     ) -> SourceNode:
         """
         Parse relation markers from source file comments.

strictdoc/backend/sdoc_source_code/models/source_node.py

@@ -3,7 +3,7 @@
 """
 
 from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Union
+from typing import List, Optional, Union
 
 from strictdoc.backend.sdoc_source_code.models.function import Function
 from strictdoc.backend.sdoc_source_code.models.function_range_marker import (
@@ -13,6 +13,7 @@
 from strictdoc.backend.sdoc_source_code.models.range_marker import (
     RangeMarker,
 )
+from strictdoc.core.project_config import SourceNodesEntry
 
 
 @dataclass
@@ -21,5 +22,33 @@ class SourceNode:
     markers: List[Union[FunctionRangeMarker, RangeMarker, LineMarker]] = field(
         default_factory=list
     )
-    fields: Dict[str, str] = field(default_factory=dict)
+    fields: dict[str, str] = field(default_factory=dict)
     function: Optional[Function] = None
+
+    def get_sdoc_field(
+        self,
+        field_name: str,
+        cfg_entry: SourceNodesEntry,
+    ) -> Optional[str]:
+        if field_name in cfg_entry.sdoc_to_source_map:
+            field_name = cfg_entry.sdoc_to_source_map[field_name]
+        return self.fields.get(field_name)
+
+    def get_sdoc_fields(self, cfg_entry: SourceNodesEntry) -> dict[str, str]:
+        """
+        Get SDoc representation of all available fields.
+
+        Returns a dict equivalent to self.fields unless config option 'sdoc_to_source_map' is set.
+        """
+        fields = {
+            field_name: self.fields[field_name]
+            for field_name in self.fields
+            if field_name not in cfg_entry.sdoc_to_source_map.values()
+        }
+        for (
+            field_name,
+            mapped_field_name,
+        ) in cfg_entry.sdoc_to_source_map.items():
+            if mapped_field_name in self.fields:
+                fields[field_name] = self.fields[mapped_field_name]
+        return fields

strictdoc/backend/sdoc_source_code/reader_c.py

@@ -40,8 +40,8 @@
 
 
 class SourceFileTraceabilityReader_C:
-    def __init__(self, custom_tags: Optional[list[str]] = None) -> None:
-        self.custom_tags: Optional[list[str]] = custom_tags
+    def __init__(self, custom_tags: Optional[set[str]] = None) -> None:
+        self.custom_tags: Optional[set[str]] = custom_tags
 
     def read(
         self,

strictdoc/core/file_traceability_index.py

@@ -648,41 +648,46 @@ def create_folder_section(
 
             for source_node_ in traceability_info_.source_nodes:
                 assert source_node_.entity_name is not None
-                source_sdoc_node_uid = f"{document_uid}/{path_to_source_file_}/{source_node_.entity_name}"
-
-                source_sdoc_node = traceability_index.get_node_by_uid_weak(
-                    source_sdoc_node_uid
+                sdoc_node_uid = source_node_.get_sdoc_field(
+                    "UID", relevant_source_node_entry
+                )
+                if sdoc_node_uid is None:
+                    sdoc_node_uid = f"{document_uid}/{path_to_source_file_}/{source_node_.entity_name}"
+                sdoc_node = traceability_index.get_node_by_uid_weak(
+                    sdoc_node_uid
                 )
-                if source_sdoc_node is not None:
-                    source_sdoc_node = assert_cast(source_sdoc_node, SDocNode)
+
+                if sdoc_node is not None:
+                    sdoc_node = assert_cast(sdoc_node, SDocNode)
                     self.merge_sdoc_node_with_source_node(
-                        source_sdoc_node,
+                        relevant_source_node_entry,
                         source_node_,
+                        sdoc_node,
                         document,
-                        relevant_source_node_entry,
                     )
                 else:
-                    source_sdoc_node = self.create_sdoc_node_from_source_node(
+                    sdoc_node = self.create_sdoc_node_from_source_node(
                         source_node_,
-                        source_sdoc_node_uid,
-                        document,
                         relevant_source_node_entry,
+                        sdoc_node_uid,
+                        document,
                     )
-                    current_top_node.section_contents.append(source_sdoc_node)
+                    sdoc_node_uid = assert_cast(sdoc_node.reserved_uid, str)
+                    current_top_node.section_contents.append(sdoc_node)
                     traceability_index.graph_database.create_link(
                         link_type=GraphLinkType.UID_TO_NODE,
-                        lhs_node=source_sdoc_node_uid,
-                        rhs_node=source_sdoc_node,
+                        lhs_node=sdoc_node_uid,
+                        rhs_node=sdoc_node,
                     )
 
                 self.connect_source_node_function(
-                    source_node_, source_sdoc_node_uid, traceability_info_
+                    source_node_, sdoc_node_uid, traceability_info_
                 )
                 self.connect_sdoc_node_with_file_path(
-                    source_sdoc_node, path_to_source_file_
+                    sdoc_node, path_to_source_file_
                 )
                 self.connect_source_node_requirements(
-                    source_node_, source_sdoc_node, traceability_index
+                    source_node_, sdoc_node, traceability_index
                 )
 
         # Warn if source_node was not matched by any include_source_paths, it indicates misconfiguration
@@ -975,45 +980,37 @@ def connect_source_node_function(
     @staticmethod
     def create_sdoc_node_from_source_node(
         source_node: SourceNode,
-        uid: str,
+        source_node_config_entry: SourceNodesEntry,
+        sdoc_node_uid: str,
         parent_document: SDocDocumentIF,
-        relevant_source_node_entry: SourceNodesEntry,
     ) -> SDocNode:
-        source_sdoc_node = SDocNode(
+        sdoc_node = SDocNode(
             parent=parent_document,
-            node_type=relevant_source_node_entry.node_type,
+            node_type=source_node_config_entry.node_type,
             fields=[],
             relations=[],
             # It is important that this autogenerated node is marked as such.
             autogen=True,
         )
-        source_sdoc_node.ng_document_reference = DocumentReference()
-        source_sdoc_node.ng_document_reference.set_document(parent_document)
-        source_sdoc_node.ng_including_document_reference = DocumentReference()
-        source_sdoc_node.set_field_value(
-            field_name="UID",
-            form_field_index=0,
-            value=uid,
-        )
-        source_sdoc_node.set_field_value(
-            field_name="TITLE",
-            form_field_index=0,
-            value=source_node.entity_name,
-        )
-        for node_name_, node_value_ in source_node.fields.items():
-            source_sdoc_node.set_field_value(
-                field_name=node_name_,
-                form_field_index=0,
-                value=node_value_,
-            )
-        return source_sdoc_node
+        sdoc_node.ng_document_reference = DocumentReference()
+        sdoc_node.ng_document_reference.set_document(parent_document)
+        sdoc_node.ng_including_document_reference = DocumentReference()
+        sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
+        sdoc_node_fields["UID"] = sdoc_node_uid
+        if (
+            "TITLE" not in sdoc_node_fields
+            and source_node.entity_name is not None
+        ):
+            sdoc_node_fields["TITLE"] = source_node.entity_name
+        FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
+        return sdoc_node
 
     @staticmethod
     def merge_sdoc_node_with_source_node(
-        sdoc_node: SDocNode,
+        source_node_config_entry: SourceNodesEntry,
         source_node: SourceNode,
+        sdoc_node: SDocNode,
         parent_document: SDocDocumentIF,
-        source_node_config_entry: SourceNodesEntry,
     ) -> None:
         # First check if grammar element definitions are compatible.
         source_node_type = source_node_config_entry.node_type
@@ -1038,19 +1035,19 @@ def merge_sdoc_node_with_source_node(
                 f"Grammar element {sdoc_node_document.reserved_uid}::{source_node_type} "
                 f"incompatible with {parent_document.reserved_uid}::{source_node_type}"
             )
-        # Merge strategy: overwrite title if there's a TITLE from custom tags.
-        if "TITLE" in source_node.fields:
-            sdoc_node.set_field_value(
-                field_name="TITLE",
-                form_field_index=0,
-                value=source_node.fields["TITLE"],
-            )
         # Merge strategy: overwrite any field if there's a field with same name from custom tags.
-        for node_name_, node_value_ in source_node.fields.items():
+        sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
+        FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
+
+    @staticmethod
+    def set_sdoc_node_fields(
+        sdoc_node: SDocNode, sdoc_node_fields: dict[str, str]
+    ) -> None:
+        for field_name, field_value in sdoc_node_fields.items():
             sdoc_node.set_field_value(
-                field_name=node_name_,
+                field_name=field_name,
                 form_field_index=0,
-                value=node_value_,
+                value=field_value,
             )
 
     def connect_sdoc_node_with_file_path(

strictdoc/core/project_config.py

@@ -8,7 +8,7 @@
 import sys
 import tempfile
 import types
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
@@ -47,12 +47,8 @@ class SourceNodesEntry:
     path: str
     uid: str
     node_type: str
-    full_path: Optional[Path]
-
-    @classmethod
-    def from_cfg_data(cls, data: dict[str, str]) -> "SourceNodesEntry":
-        full_path = Path(data["full_path"]) if "full_path" in data else None
-        return cls(data["path"], data["uid"], data["node_type"], full_path)
+    sdoc_to_source_map: Dict[str, str] = field(default_factory=dict)
+    full_path: Optional[Path] = None
 
 
 class ProjectFeature(str, Enum):
@@ -124,7 +120,7 @@ def __init__(
         include_source_paths: Optional[List[str]] = None,
         exclude_source_paths: Optional[List[str]] = None,
         test_report_root_dict: Optional[Dict[str, str]] = None,
-        source_nodes: Optional[List[Dict[str, str]]] = None,
+        source_nodes: Optional[List[SourceNodesEntry]] = None,
         html2pdf_strict: bool = False,
         html2pdf_template: Optional[str] = None,
         bundle_document_version: Optional[
@@ -209,12 +205,7 @@ def __init__(
             test_report_root_dict if test_report_root_dict is not None else {}
         )
         self.source_nodes: List[SourceNodesEntry] = (
-            [
-                SourceNodesEntry.from_cfg_data(source_node)
-                for source_node in source_nodes
-            ]
-            if source_nodes is not None
-            else []
+            source_nodes if source_nodes is not None else []
         )
 
         # Settings derived from the command-line parameters.
@@ -585,7 +576,7 @@ def _load_from_dictionary(
         include_source_paths: List[str] = []
         exclude_source_paths: List[str] = []
         test_report_root_dict: Dict[str, str] = {}
-        source_nodes: List[Dict[str, str]] = []
+        source_nodes: List[SourceNodesEntry] = []
         html2pdf_strict: bool = False
         html2pdf_template: Optional[str] = None
         bundle_document_version = (
@@ -801,14 +792,16 @@ def _load_from_dictionary(
                 assert isinstance(source_nodes_config, list)
                 for item_ in source_nodes_config:
                     source_node_path = next(iter(item_))
-                    source_node_uid = item_[source_node_path]["uid"]
-                    source_node_node_type = item_[source_node_path]["node_type"]
+                    source_node_item = item_[source_node_path]
                     source_nodes.append(
-                        {
-                            "path": source_node_path,
-                            "uid": source_node_uid,
-                            "node_type": source_node_node_type,
-                        }
+                        SourceNodesEntry(
+                            path=source_node_path,
+                            uid=source_node_item["uid"],
+                            node_type=source_node_item["node_type"],
+                            sdoc_to_source_map=source_node_item["map"]
+                            if "map" in source_node_item
+                            else {},
+                        )
                     )
 
         if "server" in config_dict:

strictdoc/core/traceability_index.py

@@ -10,6 +10,7 @@
 from strictdoc.backend.sdoc.models.anchor import Anchor
 from strictdoc.backend.sdoc.models.document import SDocDocument
 from strictdoc.backend.sdoc.models.document_config import DocumentConfig
+from strictdoc.backend.sdoc.models.grammar_element import GrammarElement
 from strictdoc.backend.sdoc.models.inline_link import InlineLink
 from strictdoc.backend.sdoc.models.node import SDocNode
 from strictdoc.backend.sdoc_source_code.models.source_file_info import (
@@ -353,6 +354,14 @@ def get_incoming_links(
         # FIXME: Should the graph database return OrderedSet or a copied list()?
         return list(incoming_links)
 
+    def get_grammar_element(
+        self, document_uid: str, node_type: str
+    ) -> Optional[GrammarElement]:
+        document = self.get_node_by_uid_weak2(document_uid)
+        if isinstance(document, SDocDocument) and document.grammar is not None:
+            return document.grammar.elements_by_type.get(node_type)
+        return None
+
     def create_traceability_info(
         self,
         source_file: SourceFile,