draft: feat(backend/sdoc_source_code): Allow empty lines in source node fields

strictdoc/backend/sdoc_source_code/comment_parser/marker_lexer.py

@@ -22,12 +22,12 @@ class GrammarTemplate(Template):
 NODE_GRAMMAR_EXTENSION = GrammarTemplate("""
 node_field: node_name ":" node_multiline_value
 node_name: /##CUSTOM_TAGS/
-node_multiline_value: (_WS_INLINE | _NL) (NODE_FIRST_STRING_VALUE _NL) (NODE_STRING_VALUE _NL)*
+node_multiline_value: (_WS_INLINE | _NL) (NODE_FIRST_STRING_VALUE NEWLINE) (NODE_STRING_VALUE NEWLINE)*
 
 NODE_FIRST_STRING_VALUE.2: /\\s*[^\n\r]+/x
-NODE_STRING_VALUE.2: /(?![ ]*##RELATION_MARKER_START)(?!\\s*[A-Z_]+: )[^\n\r]+/x
+NODE_STRING_VALUE.2: /(?![ ]*##RELATION_MARKER_START)(?!\\s*(##CUSTOM_TAGS):\\s)(?!\\s*##NODE_FIELD_END_MARKER)[^\n\r]+/x
 
-_NORMAL_STRING_NO_MARKER_NO_NODE: /(?!\\s*##RELATION_MARKER_START)((?!\\s*(##CUSTOM_TAGS): )|(##RESERVED_KEYWORDS)).+/
+_NORMAL_STRING_NO_MARKER_NO_NODE: /(?!\\s*##RELATION_MARKER_START)((?!\\s*(##CUSTOM_TAGS):\\s)|(##RESERVED_KEYWORDS)).+/
 """)
 
 GRAMMAR = GrammarTemplate("""
@@ -74,6 +74,7 @@ def parse(
                 CUSTOM_TAGS="|".join(f"{tag}(?=:)" for tag in custom_tags),
                 RESERVED_KEYWORDS=RESERVED_KEYWORDS,
                 RELATION_MARKER_START=RELATION_MARKER_START,
+                NODE_FIELD_END_MARKER="SPDX-Req-End",
             )
             start = "(relation_marker | node_field | _NORMAL_STRING_NO_MARKER_NO_NODE | _WS)*"
         else:

strictdoc/backend/sdoc_source_code/marker_parser.py

@@ -216,17 +216,11 @@ def _parse_node_field(
         assert isinstance(node_value_node, Tree)
         assert node_value_node.data == "node_multiline_value"
 
-        processed_node_values = []
+        node_value = ""
         for node_value_component_ in node_value_node.children:
             assert isinstance(node_value_component_, Token)
-            processed_node_value = node_value_component_.value.strip()
-            if "\\n\\n" in processed_node_value:
-                processed_node_value = processed_node_value.replace(
-                    "\\n\\n", ""
-                )
-
-            processed_node_values.append(processed_node_value)
+            node_value += node_value_component_.value
 
-        node_value = "\n".join(processed_node_values)
+        node_value = node_value.rstrip()
 
         return node_name, node_value

strictdoc/core/file_traceability_index.py

@@ -47,6 +47,7 @@
 from strictdoc.helpers.cast import assert_cast
 from strictdoc.helpers.exception import StrictDocException
 from strictdoc.helpers.google_test import convert_function_name_to_gtest_macro
+from strictdoc.helpers.mid import MID
 from strictdoc.helpers.ordered_set import OrderedSet
 
 if TYPE_CHECKING:
@@ -601,15 +602,36 @@ def validate_and_resolve(
                     continue
 
                 assert source_node_.entity_name is not None
+                sdoc_node = None
                 sdoc_node_uid = source_node_.get_sdoc_field(
                     "UID", relevant_source_node_entry
                 )
-                if sdoc_node_uid is None:
-                    sdoc_node_uid = f"{document_uid}/{path_to_source_file_}/{source_node_.entity_name}"
-                sdoc_node = traceability_index.get_node_by_uid_weak(
-                    sdoc_node_uid
+                mid = source_node_.get_sdoc_field(
+                    "MID", relevant_source_node_entry
                 )
 
+                # First merge criterion: Merge if SDoc node with same MID exists.
+                if mid is not None:
+                    sdoc_node_mid = MID(mid)
+                    merge_candidate_sdoc_node = (
+                        traceability_index.get_node_by_mid_weak(sdoc_node_mid)
+                    )
+                    if isinstance(merge_candidate_sdoc_node, SDocNode):
+                        sdoc_node = merge_candidate_sdoc_node
+                        sdoc_node_uid = sdoc_node.reserved_uid
+
+                if sdoc_node is None:
+                    # If no UID from source code field or merge-by-MID, create UID by conventional scheme.
+                    if sdoc_node_uid is None:
+                        sdoc_node_uid = f"{document_uid}/{path_to_source_file_}/{source_node_.entity_name}"
+                    # Second merge criterion: Merge if SDoc node with same UID exists.
+                    tmp_sdoc_node = traceability_index.get_node_by_uid_weak(
+                        sdoc_node_uid
+                    )
+                    if isinstance(tmp_sdoc_node, SDocNode):
+                        sdoc_node = tmp_sdoc_node
+
+                assert sdoc_node_uid is not None
                 if sdoc_node is not None:
                     sdoc_node = assert_cast(sdoc_node, SDocNode)
                     self.merge_sdoc_node_with_source_node(
@@ -626,11 +648,6 @@ def validate_and_resolve(
                         document,
                     )
                     sdoc_node_uid = assert_cast(sdoc_node.reserved_uid, str)
-                    traceability_index.graph_database.create_link(
-                        link_type=GraphLinkType.UID_TO_NODE,
-                        lhs_node=sdoc_node_uid,
-                        rhs_node=sdoc_node,
-                    )
                     if current_top_node is None:
                         current_top_node = (
                             FileTraceabilityIndex.create_source_node_section(
@@ -998,6 +1015,25 @@ def merge_sdoc_node_with_source_node(
             )
         # Merge strategy: overwrite any field if there's a field with same name from custom tags.
         sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
+
+        # Sanity check: Nor UID neither MID must conflict (early auto-MID is allowed to be overwritten)
+        if (
+            "MID" in sdoc_node.ordered_fields_lookup
+            and "MID" in sdoc_node_fields
+        ):
+            sdoc_mid_field = sdoc_node.get_field_by_name("MID").get_text_value()
+            if sdoc_mid_field != sdoc_node_fields["MID"]:
+                raise StrictDocException(
+                    f"Can't merge node by UID {sdoc_node.reserved_uid}: "
+                    f"Conflicting MID: {sdoc_mid_field} != {sdoc_node_fields['MID']}"
+                )
+        if sdoc_node.reserved_uid is not None and "UID" in sdoc_node_fields:
+            if sdoc_node.reserved_uid != sdoc_node_fields["UID"]:
+                raise StrictDocException(
+                    f"Can't merge node by MID {sdoc_node.reserved_mid}: "
+                    f"Conflicting UID: {sdoc_node.reserved_uid} != {sdoc_node_fields['UID']}"
+                )
+
         FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
 
     @staticmethod
@@ -1081,6 +1117,45 @@ def connect_source_node_requirements(
 
         Here we link REQ and sdoc_node bidirectional.
         """
+        if (
+            sdoc_node.reserved_uid is not None
+            and not traceability_index.graph_database.has_link(
+                link_type=GraphLinkType.UID_TO_NODE,
+                lhs_node=sdoc_node.reserved_uid,
+                rhs_node=sdoc_node,
+            )
+        ):
+            traceability_index.graph_database.create_link(
+                link_type=GraphLinkType.UID_TO_NODE,
+                lhs_node=sdoc_node.reserved_uid,
+                rhs_node=sdoc_node,
+            )
+
+        # A merge procedure may have overwritten the MID,
+        # in which case the graph database and search index needs an update.
+        if "MID" in sdoc_node.ordered_fields_lookup != sdoc_node.reserved_mid:
+            sdoc_mid_field = sdoc_node.get_field_by_name("MID").get_text_value()
+            if sdoc_mid_field != sdoc_node.reserved_mid:
+                # TODO:
+                # If we really want to support changing the auto-assigned MID,
+                # at least the graph database and the document search index need an update (remove old MID, add new MID).
+                # I currently struggle to update the search index.
+                parent_document = sdoc_node.get_parent_or_including_document()
+                sdoc_node.reserved_mid = MID(sdoc_mid_field)
+                if parent_document.config.enable_mid:
+                    sdoc_node.mid_permanent = True
+
+        if not traceability_index.graph_database.has_link(
+            link_type=GraphLinkType.MID_TO_NODE,
+            lhs_node=sdoc_node.reserved_mid,
+            rhs_node=sdoc_node,
+        ):
+            traceability_index.graph_database.create_link(
+                link_type=GraphLinkType.MID_TO_NODE,
+                lhs_node=sdoc_node.reserved_mid,
+                rhs_node=sdoc_node,
+            )
+
         for marker_ in source_node.markers:
             if not isinstance(marker_, FunctionRangeMarker):
                 continue

tests/integration/features/source_code_traceability/_source_nodes/30_merge_with_static_node/source_node_base.sdoc → tests/integration/features/source_code_traceability/_source_nodes/30_merge_with_sdoc_by_default_uid/source_node_base.sdoc

@@ -39,7 +39,7 @@ ELEMENTS:
 TITLE: Merge example.c into static nodes
 
 [REQUIREMENT]
-UID: SRC-NODES-BASE/src/example/example.c/example_1
+UID: SRC-NODES-BASE/src/example.c/example_1
 TITLE: TITLE from sdoc
 FOO: FOO text from sdoc
 BAR: BAR text from sdoc

tests/integration/features/source_code_traceability/_source_nodes/30_merge_with_sdoc_by_default_uid/test.itest

@@ -0,0 +1,30 @@
+#
+# This test verifies that a source nodes is merged with a static SDoc node if
+# - the source nodes is not marked up with a UID field (i.e., default UID is effective), and
+# - static SDoc node was explicitly given the default UID.
+#
+# @relation(SDOC-SRS-141, scope=file)
+#
+
+RUN: %strictdoc --debug export %S --output-dir %T | filecheck %s
+
+CHECK: Published: Hello world doc
+
+RUN: %check_exists --file "%T/html/_source_files/src/example.c.html"
+
+RUN: %cat %T/html/%THIS_TEST_FOLDER/source_node_base.html | filecheck %s --check-prefix CHECK-HTML
+CHECK-HTML: Requirements from Source Nodes
+CHECK-HTML: SRC-NODES-BASE/src/example.c/example_1
+CHECK-HTML: TITLE from sdoc
+CHECK-HTML: class="requirement__link-parent" href="../30_merge_with_sdoc_by_default_uid/parent.html#REQ-1"
+CHECK-HTML: src/example.c, <i>lines: 3-14</i>, function example_1()
+CHECK-HTML-NOT: FOO text from sdoc
+CHECK-HTML: FOO text from example.c
+CHECK-HTML-NOT: BAR text from sdoc
+CHECK-HTML: BAR text from example.c
+
+RUN: %cat %T/html/_source_files/src/example.c.html | filecheck %s --check-prefix CHECK-SOURCE-FILE
+CHECK-SOURCE-FILE: SRC-NODES-BASE/src/example.c/example_1
+
+RUN: %cat %T/html/source_coverage.html | filecheck %s --check-prefix CHECK-SOURCE-COVERAGE
+CHECK-SOURCE-COVERAGE: 100.0

tests/integration/features/source_code_traceability/_source_nodes/31_merge_with_sdoc_by_uid/parent.sdoc

@@ -0,0 +1,17 @@
+[DOCUMENT]
+TITLE: Hello world doc
+
+[REQUIREMENT]
+UID: REQ-1
+TITLE: Requirement Title
+STATEMENT: Requirement Statement
+
+[REQUIREMENT]
+UID: REQ-2
+TITLE: Requirement Title #2
+STATEMENT: Requirement Statement #2
+
+[REQUIREMENT]
+UID: REQ-3
+TITLE: Requirement Title #3
+STATEMENT: Requirement Statement #3

tests/integration/features/source_code_traceability/_source_nodes/31_merge_with_sdoc_by_uid/source_node_base.sdoc

@@ -0,0 +1,72 @@
+[DOCUMENT]
+MID: c2d4542d5f1741c88dfcb4f68ad7dcbd
+TITLE: Requirements from Source Nodes
+UID: SRC-NODES-BASE
+
+[GRAMMAR]
+ELEMENTS:
+- TAG: SECTION
+  PROPERTIES:
+    IS_COMPOSITE: True
+  FIELDS:
+  - TITLE: UID
+    TYPE: String
+    REQUIRED: False
+  - TITLE: TITLE
+    TYPE: String
+    REQUIRED: True
+- TAG: REQUIREMENT
+  PROPERTIES:
+    VIEW_STYLE: Narrative
+  FIELDS:
+  - TITLE: UID
+    TYPE: String
+    REQUIRED: False
+  - TITLE: MID
+    TYPE: String
+    REQUIRED: False
+  - TITLE: TITLE
+    TYPE: String
+    REQUIRED: False
+  - TITLE: FOO
+    TYPE: String
+    REQUIRED: False
+  - TITLE: BAR
+    TYPE: String
+    REQUIRED: False
+  RELATIONS:
+  - TYPE: Parent
+  - TYPE: File
+
+[[SECTION]]
+TITLE: Merge example.c into static nodes
+
+[REQUIREMENT]
+UID: REQ-SOURCE-1
+TITLE: TITLE1 from sdoc
+FOO: FOO1 text from sdoc
+BAR: BAR1 text from sdoc
+RELATIONS:
+- TYPE: Parent
+  VALUE: REQ-1
+
+[REQUIREMENT]
+UID: REQ-SOURCE-2
+MID: 80cd685d-0e18-44b8-9842-c1863a2eb9ec
+TITLE: TITLE2 from sdoc
+FOO: FOO2 text from sdoc
+BAR: BAR2 text from sdoc
+RELATIONS:
+- TYPE: Parent
+  VALUE: REQ-2
+
+[REQUIREMENT]
+UID: REQ-SOURCE-3
+TITLE: TITLE3 from sdoc
+FOO: FOO3 text from sdoc
+BAR: BAR3 text from sdoc
+RELATIONS:
+- TYPE: Parent
+  VALUE: REQ-3
+
+[[/SECTION]]

tests/integration/features/source_code_traceability/_source_nodes/31_merge_with_sdoc_by_uid/src/example.c

@@ -0,0 +1,48 @@
+#include <stdio.h>
+
+/**
+ * Some text.
+ *
+ * @relation(REQ-1, scope=function)
+ *
+ * UID: REQ-SOURCE-1
+ *
+ * FOO: FOO1 text from example.c
+ *
+ * BAR: BAR1 text from example.c
+ */
+void example_1(void) {
+    print("hello world\n");
+}
+
+/**
+ * Some text.
+ *
+ * @relation(REQ-2, scope=function)
+ *
+ * UID: REQ-SOURCE-2
+ *
+ * FOO: FOO2 text from example.c
+ *
+ * BAR: BAR2 text from example.c
+ */
+void example_2(void) {
+    print("hello world\n");
+}
+
+/**
+ * Some text.
+ *
+ * @relation(REQ-3, scope=function)
+ *
+ * UID: REQ-SOURCE-3
+ *
+ * MID: 1973a567-a109-491d-b7f0-6bb22eafa6ab
+ *
+ * FOO: FOO3 text from example.c
+ *
+ * BAR: BAR3 text from example.c
+ */
+void example_3(void) {
+    print("hello world\n");
+}

tests/integration/features/source_code_traceability/_source_nodes/31_merge_with_sdoc_by_uid/strictdoc.toml

@@ -0,0 +1,14 @@
+[project]
+
+features = [
+  "REQUIREMENT_TO_SOURCE_TRACEABILITY",
+  "SOURCE_FILE_LANGUAGE_PARSERS",
+]
+
+source_nodes = [
+  { "src/" = { uid = "SRC-NODES-BASE", node_type = "REQUIREMENT" } }
+]
+
+exclude_source_paths = [
+    "test.itest"
+]