StrictDoc Documentation
strictdoc/core/file_traceability_index.py
Source file coverage
Path:
strictdoc/core/file_traceability_index.py
Lines:
1237
Non-empty lines:
1110
Non-empty lines covered with requirements:
1110 / 1110 (100.0%)
Functions:
29
Functions covered by requirements:
29 / 29 (100.0%)
1
"""
2
@relation(SDOC-SRS-28, SDOC-SRS-33, scope=file)
3
"""
4
 
5
import re
6
from copy import copy
7
from typing import (
8
    TYPE_CHECKING,
9
    Dict,
10
    Iterator,
11
    List,
12
    Optional,
13
    Set,
14
    Tuple,
15
    Union,
16
)
17
 
18
from strictdoc.backend.sdoc.document_reference import DocumentReference
19
from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
20
from strictdoc.backend.sdoc.models.document_grammar import (
21
    DocumentGrammar,
22
)
23
from strictdoc.backend.sdoc.models.model import SDocDocumentIF
24
from strictdoc.backend.sdoc.models.node import SDocNode, SDocNodeField
25
from strictdoc.backend.sdoc.models.reference import FileEntry, FileReference
26
from strictdoc.backend.sdoc_source_code.models.language import LanguageItem
27
from strictdoc.backend.sdoc_source_code.models.language_item_marker import (
28
    ForwardLanguageItemMarker,
29
    LanguageItemMarker,
30
    RangeMarkerType,
31
)
32
from strictdoc.backend.sdoc_source_code.models.line_marker import LineMarker
33
from strictdoc.backend.sdoc_source_code.models.range_marker import (
34
    ForwardFileMarker,
35
    ForwardRangeMarker,
36
    RangeMarker,
37
)
38
from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req
39
from strictdoc.backend.sdoc_source_code.models.source_file_info import (
40
    RelationMarkerType,
41
    SourceFileTraceabilityInfo,
42
)
43
from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
44
from strictdoc.core.constants import GraphLinkType
45
from strictdoc.core.document_iterator import SDocDocumentIterator
46
from strictdoc.core.file_system.source_tree import SourceFile
47
from strictdoc.core.project_config import ProjectConfig, SourceNodesEntry
48
from strictdoc.helpers.cast import assert_cast
49
from strictdoc.helpers.exception import StrictDocException
50
from strictdoc.helpers.google_test import convert_function_name_to_gtest_macro
51
from strictdoc.helpers.mid import MID
52
from strictdoc.helpers.ordered_set import OrderedSet
53
 
54
if TYPE_CHECKING:
55
    from strictdoc.core.traceability_index import (
56
        TraceabilityIndex,
57
    )
58
 
59
 
60
class FileTraceabilityIndex:
61
    def __init__(self) -> None:
62
        # "file.py" -> List[SDocNode]
63
        self.map_paths_to_reqs: Dict[str, OrderedSet[SDocNode]] = {}
64
 
65
        # "REQ-001" -> {"file.py", ...}
66
        self.map_reqs_uids_to_paths: Dict[str, OrderedSet[str]] = {}
67
 
68
        # "file.py" -> SourceFileTraceabilityInfo.
69
        self.map_paths_to_source_file_traceability_info: Dict[
70
            str, SourceFileTraceabilityInfo
71
        ] = {}
72
 
73
        # "file.py" -> { { "foo" -> [("REQ-1", "Impl"), ("REQ-2", "Test")] }, ... }
74
        self.map_file_function_names_to_reqs_uids: Dict[
75
            str, Dict[str, List[Tuple[str, Optional[str]]]]
76
        ] = {}
77
        self.map_file_class_names_to_reqs_uids: Dict[
78
            str, Dict[str, List[Tuple[str, Optional[str]]]]
79
        ] = {}
80
 
81
        # This is only public non-static functions from languages like C.
82
        self.map_all_function_names_to_definition_functions: Dict[
83
            str, List[LanguageItem]
84
        ] = {}
85
 
86
        # "file.py" -> [SDocNode]  # noqa: ERA001
87
        self.source_file_reqs_cache: Dict[str, Optional[List[SDocNode]]] = {}
88
 
89
        self.requirements_with_forward_links: OrderedSet[SDocNode] = (
90
            OrderedSet()
91
        )
92
        self.trace_infos: List[SourceFileTraceabilityInfo] = []
93
 
94
    def has_source_file_reqs(self, source_file_rel_path: str) -> bool:
95
        path_reqs = self.map_paths_to_reqs.get(source_file_rel_path)
96
        if path_reqs is not None and len(path_reqs) > 0:
97
            return True
98
        file_trace_info = self.map_paths_to_source_file_traceability_info[
99
            source_file_rel_path
100
        ]
101
        return len(file_trace_info.markers) > 0
102
 
103
    def get_requirement_file_links(
104
        self, requirement: SDocNode
105
    ) -> List[Tuple[str, List[RelationMarkerType]]]:
106
        if requirement.reserved_uid not in self.map_reqs_uids_to_paths:
107
            return []
108
 
109
        matching_links_with_markers: List[
110
            Tuple[str, List[RelationMarkerType]]
111
        ] = []
112
        requirement_source_paths: OrderedSet[str] = self.map_reqs_uids_to_paths[
113
            requirement.reserved_uid
114
        ]
115
 
116
        # Now that one requirement can have multiple File-relations to the same file.
117
        # This can be multiple FUNCTION: or RANGE: forward-relations.
118
        # To avoid duplication of results, visit each unique file link path only once.
119
        visited_file_links: Set[str] = set()
120
        for requirement_source_path_ in requirement_source_paths:
121
            if requirement_source_path_ in visited_file_links:
122
                continue
123
            visited_file_links.add(requirement_source_path_)
124
 
125
            source_file_traceability_info: Optional[
126
                SourceFileTraceabilityInfo
127
            ] = self.map_paths_to_source_file_traceability_info.get(
128
                requirement_source_path_
129
            )
130
            assert source_file_traceability_info is not None, (
131
                f"Requirement {requirement.reserved_uid} references a file"
132
                f" that does not exist: {requirement_source_path_}."
133
            )
134
            markers = source_file_traceability_info.ng_map_reqs_to_markers.get(
135
                requirement.reserved_uid
136
            )
137
            if markers is None or len(markers) == 0:
138
                matching_links_with_markers.append(
139
                    (requirement_source_path_, [])
140
                )
141
                continue
142
            matching_links_with_markers.append(
143
                (requirement_source_path_, markers)
144
            )
145
 
146
        return matching_links_with_markers
147
 
148
    def indexed_source_files(self) -> Iterator[SourceFile]:
149
        for _, sfti in self.map_paths_to_source_file_traceability_info.items():
150
            if sfti.source_file is not None:
151
                yield sfti.source_file
152
 
153
    def get_source_file_reqs(
154
        self, source_file_rel_path: str
155
    ) -> Optional[List[SDocNode]]:
156
        assert (
157
            source_file_rel_path
158
            in self.map_paths_to_source_file_traceability_info
159
        )
160
        if source_file_rel_path in self.source_file_reqs_cache:
161
            return self.source_file_reqs_cache[source_file_rel_path]
162
 
163
        source_file_traceability_info: SourceFileTraceabilityInfo = (
164
            self.map_paths_to_source_file_traceability_info[
165
                source_file_rel_path
166
            ]
167
        )
168
 
169
        if source_file_rel_path not in self.map_paths_to_reqs:
170
            self.source_file_reqs_cache[source_file_rel_path] = None
171
            return None
172
 
173
        requirements = self.map_paths_to_reqs[source_file_rel_path]
174
        assert len(requirements) > 0
175
        range_requirements = []
176
 
177
        for requirement in requirements:
178
            if (
179
                requirement.reserved_uid
180
                in source_file_traceability_info.ng_map_reqs_to_markers
181
            ):
182
                range_requirements.append(requirement)
183
 
184
        self.source_file_reqs_cache[source_file_rel_path] = range_requirements
185
        return range_requirements
186
 
187
    def get_coverage_info(
188
        self, source_file_rel_path: str
189
    ) -> SourceFileTraceabilityInfo:
190
        assert (
191
            source_file_rel_path
192
            in self.map_paths_to_source_file_traceability_info
193
        ), source_file_rel_path
194
        source_file_tr_info: SourceFileTraceabilityInfo = (
195
            self.map_paths_to_source_file_traceability_info[
196
                source_file_rel_path
197
            ]
198
        )
199
        return source_file_tr_info
200
 
201
    def get_coverage_info_weak(
202
        self, source_file_rel_path: str
203
    ) -> Optional[SourceFileTraceabilityInfo]:
204
        source_file_tr_info: Optional[SourceFileTraceabilityInfo] = (
205
            self.map_paths_to_source_file_traceability_info.get(
206
                source_file_rel_path
207
            )
208
        )
209
        return source_file_tr_info
210
 
211
    def validate_and_resolve(
212
        self,
213
        traceability_index: "TraceabilityIndex",
214
        project_config: ProjectConfig,
215
    ) -> None:
216
        """
217
        Resolve all source code traceability after the index is fully built.
218
        """
219
 
220
        #
221
        # STEP: Collect minimal information that will help to resolve the
222
        #       forward-declared paths/function names at the step 2.
223
        #
224
        for trace_info_ in self.trace_infos:
225
            source_file: SourceFile = assert_cast(
226
                trace_info_.source_file, SourceFile
227
            )
228
 
229
            self.map_paths_to_source_file_traceability_info[
230
                source_file.in_doctree_source_file_rel_path_posix
231
            ] = trace_info_
232
 
233
            for function_ in trace_info_.functions:
234
                if function_.is_definition() and function_.is_public():
235
                    self.map_all_function_names_to_definition_functions.setdefault(
236
                        function_.name, []
237
                    ).append(function_)
238
 
239
        #
240
        # STEP: Auto-generated SDocNodes from source file comments and register
241
        #       their UIDs. This must happen before marker validation so that
242
        #       source files can reference these UIDs via @relation.
243
        #
244
        documents_with_generated_content = set()
245
 
246
        section_cache: Dict[str, Union[SDocDocumentIF, SDocNode]] = {}
247
        source_nodes_config: List[SourceNodesEntry] = (
248
            project_config.source_nodes
249
        )
250
        unused_source_node_paths = {
251
            config_entry_.path for config_entry_ in source_nodes_config
252
        }
253
        for (
254
            path_to_source_file_,
255
            traceability_info_,
256
        ) in self.map_paths_to_source_file_traceability_info.items():
257
            if len(traceability_info_.source_nodes) == 0:
258
                continue
259
 
260
            if len(source_nodes_config) == 0:
261
                continue
262
 
263
            relevant_source_node_entry = (
264
                project_config.get_relevant_source_nodes_entry(
265
                    path_to_source_file_
266
                )
267
            )
268
            if relevant_source_node_entry is not None:
269
                unused_source_node_paths.discard(
270
                    relevant_source_node_entry.path
271
                )
272
            else:
273
                continue
274
 
275
            document_uid = relevant_source_node_entry.uid
276
            document = traceability_index.get_node_by_uid(document_uid)
277
            documents_with_generated_content.add(document)
278
            current_top_node = None
279
 
280
            for source_node_ in traceability_info_.source_nodes:
281
                if len(source_node_.fields) == 0:
282
                    continue
283
 
284
                assert source_node_.entity_name is not None
285
                sdoc_node = None
286
                sdoc_node_uid = source_node_.get_sdoc_field(
287
                    "UID", relevant_source_node_entry
288
                )
289
                mid = source_node_.get_sdoc_field(
290
                    "MID", relevant_source_node_entry
291
                )
292
 
293
                # First merge criterion: Merge if SDoc node with same MID exists.
294
                if mid is not None:
295
                    sdoc_node_mid = MID(mid)
296
                    merge_candidate_sdoc_node = (
297
                        traceability_index.get_node_by_mid_weak(sdoc_node_mid)
298
                    )
299
                    if isinstance(merge_candidate_sdoc_node, SDocNode):
300
                        sdoc_node = merge_candidate_sdoc_node
301
                        sdoc_node_uid = sdoc_node.reserved_uid
302
 
303
                if sdoc_node is None:
304
                    # If no UID from source code field or merge-by-MID, create UID by conventional scheme.
305
                    if sdoc_node_uid is None:
306
                        sdoc_node_uid = f"{document_uid}/{path_to_source_file_}/{source_node_.entity_name}"
307
                    # Second merge criterion: Merge if SDoc node with same UID exists.
308
                    tmp_sdoc_node = traceability_index.get_node_by_uid_weak(
309
                        sdoc_node_uid
310
                    )
311
                    if isinstance(tmp_sdoc_node, SDocNode):
312
                        sdoc_node = tmp_sdoc_node
313
 
314
                assert sdoc_node_uid is not None
315
                if sdoc_node is not None:
316
                    sdoc_node = assert_cast(sdoc_node, SDocNode)
317
                    self.merge_sdoc_node_with_source_node(
318
                        relevant_source_node_entry,
319
                        source_node_,
320
                        sdoc_node,
321
                        document,
322
                    )
323
                else:
324
                    sdoc_node = self.create_sdoc_node_from_source_node(
325
                        source_node_,
326
                        relevant_source_node_entry,
327
                        sdoc_node_uid,
328
                        document,
329
                    )
330
                    sdoc_node_uid = assert_cast(sdoc_node.reserved_uid, str)
331
                    if current_top_node is None:
332
                        current_top_node, created_sections = (
333
                            FileTraceabilityIndex.create_source_node_section(
334
                                document,
335
                                path_to_source_file_,
336
                                section_cache,
337
                            )
338
                        )
339
                        for created_section in created_sections:
340
                            traceability_index.graph_database.create_link(
341
                                link_type=GraphLinkType.MID_TO_NODE,
342
                                lhs_node=created_section.reserved_mid,
343
                                rhs_node=created_section,
344
                            )
345
                    current_top_node.section_contents.append(sdoc_node)
346
 
347
                self.connect_source_node_function(
348
                    source_node_, sdoc_node_uid, traceability_info_
349
                )
350
                self.connect_sdoc_node_with_file_path(
351
                    sdoc_node, path_to_source_file_
352
                )
353
                self.connect_source_node_requirements(
354
                    source_node_, sdoc_node, traceability_index
355
                )
356
 
357
        # Warn if source_node was not matched by any include_source_paths, it indicates misconfiguration
358
        for unused_source_node_path in unused_source_node_paths:
359
            print(  # noqa: T201
360
                f"warning: source_node path {unused_source_node_path} doesn't match any source file. "
361
                "Hint: Check include_source_paths."
362
            )
363
 
364
        # Iterate over all generated documents to calculate all node levels.
365
        for document_ in documents_with_generated_content:
366
            document_iterator = SDocDocumentIterator(document_)
367
            for _, _ in document_iterator.all_content(
368
                print_fragments=False,
369
            ):
370
                pass
371
 
372
        #
373
        # STEP: Resolve requirements that have forward links.
374
        #       Some requirements can come from the SDoc documents generated
375
        #       on the fly from JUnit XML documents.
376
        #
377
        for forward_requirement_ in self.requirements_with_forward_links:
378
            assert forward_requirement_.reserved_uid is not None
379
 
380
            for relation_ in forward_requirement_.relations:
381
                if not isinstance(relation_, FileReference):
382
                    continue
383
 
384
                file_reference: FileReference = assert_cast(
385
                    relation_, FileReference
386
                )
387
                file_posix_path = file_reference.get_posix_path()
388
 
389
                if file_posix_path == "#FORWARD#":
390
                    test_function = (
391
                        forward_requirement_.get_meta_field_value_by_title(
392
                            "TEST_FUNCTION"
393
                        )
394
                    )
395
                    assert test_function is not None
396
 
397
                    functions: List[LanguageItem]
398
                    if test_function.startswith("#GTEST#"):
399
                        test_function = test_function.removeprefix("#GTEST#")
400
                        possible_gtest_functions = (
401
                            convert_function_name_to_gtest_macro(test_function)
402
                        )
403
                        for (
404
                            possible_gtest_function_
405
                        ) in possible_gtest_functions:
406
                            if (
407
                                possible_gtest_function_
408
                                in self.map_all_function_names_to_definition_functions
409
                            ):
410
                                test_function = possible_gtest_function_
411
                                break
412
                        else:
413
                            raise RuntimeError(
414
                                "Could not find a matching Google Test function: "
415
                                f"{possible_gtest_functions}"
416
                            )  # pragma: no cover
417
                        forward_requirement_.set_field_value(
418
                            field_name="TEST_FUNCTION",
419
                            form_field_index=0,
420
                            value=test_function,
421
                        )
422
                    functions = (
423
                        self.map_all_function_names_to_definition_functions[
424
                            test_function
425
                        ]
426
                    )
427
                    assert len(functions) == 1
428
 
429
                    function: LanguageItem = functions[0]
430
                    resolved_path_to_function_file = function.parent.source_file.in_doctree_source_file_rel_path_posix
431
                    file_posix_path = resolved_path_to_function_file
432
 
433
                    file_reference.g_file_entry = FileEntry(
434
                        relation_,
435
                        g_file_format=relation_.g_file_entry.g_file_format,
436
                        g_file_path=resolved_path_to_function_file,
437
                        g_line_range=None,
438
                        element="function",
439
                        id=test_function,
440
                    )
441
 
442
                    forward_requirement_.set_field_value(
443
                        field_name="TEST_PATH",
444
                        form_field_index=0,
445
                        value=resolved_path_to_function_file,
446
                    )
447
 
448
                    #
449
                    # This transitively connects requirements and test results
450
                    # through the test source files.
451
                    #
452
                    for language_item_marker_ in function.markers:
453
                        for req_ in language_item_marker_.reqs:
454
                            node = traceability_index.get_node_by_uid_weak2(
455
                                req_
456
                            )
457
                            traceability_index.graph_database.create_link(
458
                                link_type=GraphLinkType.NODE_TO_PARENT_NODES,
459
                                lhs_node=forward_requirement_,
460
                                rhs_node=node,
461
                                edge="Satisfies",
462
                            )
463
                            traceability_index.graph_database.create_link(
464
                                link_type=GraphLinkType.NODE_TO_CHILD_NODES,
465
                                lhs_node=node,
466
                                rhs_node=forward_requirement_,
467
                                edge="IsSatisfiedBy",
468
                            )
469
                #
470
                # Validate that all requirements reference existing files.
471
                #
472
                source_file_traceability_info: Optional[
473
                    SourceFileTraceabilityInfo
474
                ] = self.map_paths_to_source_file_traceability_info.get(
475
                    file_posix_path
476
                )
477
                if source_file_traceability_info is None:
478
                    raise StrictDocException(
479
                        f"Requirement {forward_requirement_.reserved_uid} "
480
                        "references a file that does not exist: "
481
                        f"{file_posix_path}."
482
                    )
483
 
484
                #
485
                # Now that the test reports related fixups are done, the
486
                # following code registers the requirements with forward links.
487
                #
488
                self.map_paths_to_reqs.setdefault(
489
                    file_posix_path, OrderedSet()
490
                ).add(forward_requirement_)
491
 
492
                assert forward_requirement_.reserved_uid is not None
493
                self.map_reqs_uids_to_paths.setdefault(
494
                    forward_requirement_.reserved_uid, OrderedSet()
495
                ).add(file_posix_path)
496
 
497
                if (
498
                    file_reference.g_file_entry.element == "function"
499
                    and file_reference.g_file_entry.id is not None
500
                ):
501
                    one_file_function_name_to_reqs_uids = (
502
                        self.map_file_function_names_to_reqs_uids.setdefault(
503
                            file_posix_path, {}
504
                        )
505
                    )
506
                    one_file_function_name_to_reqs_uids.setdefault(
507
                        file_reference.g_file_entry.id, []
508
                    ).append(
509
                        (forward_requirement_.reserved_uid, relation_.role)
510
                    )
511
                elif (
512
                    file_reference.g_file_entry.element == "class"
513
                    and file_reference.g_file_entry.id is not None
514
                ):
515
                    one_file_class_name_to_reqs_uids = (
516
                        self.map_file_class_names_to_reqs_uids.setdefault(
517
                            file_posix_path, {}
518
                        )
519
                    )
520
                    one_file_class_name_to_reqs_uids.setdefault(
521
                        file_reference.g_file_entry.id, []
522
                    ).append(
523
                        (forward_requirement_.reserved_uid, relation_.role)
524
                    )
525
                elif file_reference.g_file_entry.line_range is not None:
526
                    line_range = file_reference.g_file_entry.line_range
527
                    uid = forward_requirement_.reserved_uid
528
                    source_file_info = (
529
                        self.map_paths_to_source_file_traceability_info[
530
                            file_posix_path
531
                        ]
532
                    )
533
                    start_marker, end_marker = (
534
                        self.forward_range_markers_from_range(
535
                            line_range, uid, relation_.role
536
                        )
537
                    )
538
                    source_file_info.ng_map_reqs_to_markers.setdefault(
539
                        uid, []
540
                    ).append(start_marker)
541
                    source_file_info.markers.append(start_marker)
542
                    source_file_info.markers.append(end_marker)
543
                else:
544
                    uid = forward_requirement_.reserved_uid
545
                    source_file_info = (
546
                        self.map_paths_to_source_file_traceability_info[
547
                            file_posix_path
548
                        ]
549
                    )
550
                    forward_file_marker = (
551
                        self.forward_file_marker_from_file_info(
552
                            source_file_info,
553
                            uid,
554
                            relation_.role,
555
                        )
556
                    )
557
                    source_file_info.ng_map_reqs_to_markers.setdefault(
558
                        forward_requirement_.reserved_uid, []
559
                    ).append(forward_file_marker)
560
                    source_file_info.markers.append(forward_file_marker)
561
 
562
        #
563
        # STEP: Add markers for forward relations to functions and classes
564
        #
565
        for trace_info_ in self.trace_infos:
566
            source_file = assert_cast(trace_info_.source_file, SourceFile)
567
 
568
            self.map_paths_to_source_file_traceability_info[
569
                source_file.in_doctree_source_file_rel_path_posix
570
            ] = trace_info_
571
 
572
            for function_ in trace_info_.functions:
573
                # FIXME: Using display_name, not name. A separate exercise is
574
                #        to disambiguate forward links to C++ overloaded functions.
575
                if (
576
                    reqs_uids := self.get_req_uids_by_function_name(
577
                        source_file.in_doctree_source_file_rel_path_posix,
578
                        function_.display_name,
579
                    )
580
                ) is not None:
581
                    self.create_traceability_info_shared_markers_for_function(
582
                        trace_info_,
583
                        function_,
584
                        RangeMarkerType.FUNCTION,
585
                        reqs_uids,
586
                    )
587
                if (
588
                    reqs_uids := self.get_req_uids_by_class_name(
589
                        source_file.in_doctree_source_file_rel_path_posix,
590
                        function_.display_name,
591
                    )
592
                ) is not None:
593
                    self.create_traceability_info_shared_markers_for_function(
594
                        trace_info_,
595
                        function_,
596
                        RangeMarkerType.CLASS,
597
                        reqs_uids,
598
                    )
599
 
600
            marker_: Union[
601
                LanguageItemMarker, LineMarker, RangeMarker, ForwardRangeMarker
602
            ]
603
            for marker_ in copy(trace_info_.markers):
604
                # FIXME: Is this 'continue' needed here?
605
                if isinstance(marker_, ForwardRangeMarker):
606
                    continue
607
                for requirement_uid_ in marker_.reqs:
608
                    node = traceability_index.get_node_by_uid_weak2(
609
                        requirement_uid_
610
                    )
611
                    if node is None:
612
                        raise StrictDocException(
613
                            f"Source file {source_file.in_doctree_source_file_rel_path_posix} references "
614
                            f"a requirement that does not exist: {requirement_uid_}."
615
                        )
616
 
617
                    self.map_reqs_uids_to_paths.setdefault(
618
                        requirement_uid_, OrderedSet()
619
                    ).add(source_file.in_doctree_source_file_rel_path_posix)
620
 
621
                    self.map_paths_to_reqs.setdefault(
622
                        source_file.in_doctree_source_file_rel_path_posix,
623
                        OrderedSet(),
624
                    ).add(node)
625
 
626
                if isinstance(marker_, LanguageItemMarker):
627
                    marker_copy = marker_.create_end_marker()
628
                    trace_info_.markers.append(marker_copy)
629
 
630
        #
631
        # Resolve definitions to declarations (only applicable for C and C++).
632
        #
633
 
634
        reversed_trace_info = {
635
            value: key
636
            for key, value in self.map_paths_to_source_file_traceability_info.items()
637
        }
638
 
639
        for (
640
            traceability_info_
641
        ) in self.map_paths_to_source_file_traceability_info.values():
642
            for function_ in traceability_info_.functions:
643
                if (
644
                    function_.is_declaration()
645
                    and function_.name
646
                    in self.map_all_function_names_to_definition_functions
647
                ):
648
                    definition_functions: List[LanguageItem] = []
649
                    if not function_.is_public():
650
                        definition_function = traceability_info_.ng_map_names_to_definition_functions.get(
651
                            function_.name, None
652
                        )
653
                        if definition_function is not None:
654
                            definition_functions.append(definition_function)
655
                    else:
656
                        mapped_definition_functions = (
657
                            self.map_all_function_names_to_definition_functions[
658
                                function_.name
659
                            ]
660
                        )
661
                        definition_functions.extend(mapped_definition_functions)
662
                    if len(definition_functions) == 0:
663
                        continue
664
 
665
                    for definition_function_ in definition_functions:
666
                        definition_function_trace_info: SourceFileTraceabilityInfo = definition_function_.parent
667
 
668
                        for marker_ in function_.markers:
669
                            language_item_marker = self.forward_marker_from_language_item(
670
                                function=definition_function_,
671
                                marker_type=RangeMarkerType.FUNCTION,
672
                                reqs=marker_.reqs_objs,
673
                                role=marker_.role,
674
                                description=f"function {function_.display_name}()",
675
                            )
676
 
677
                            for req_uid_ in marker_.reqs:
678
                                definition_function_trace_info.ng_map_reqs_to_markers.setdefault(
679
                                    req_uid_, []
680
                                ).append(language_item_marker)
681
 
682
                                path_to_info = reversed_trace_info[
683
                                    definition_function_trace_info
684
                                ]
685
                                self.map_reqs_uids_to_paths.setdefault(
686
                                    req_uid_, OrderedSet()
687
                                ).add(path_to_info)
688
 
689
                                node = traceability_index.get_node_by_uid(
690
                                    req_uid_
691
                                )
692
                                self.map_paths_to_reqs.setdefault(
693
                                    path_to_info, OrderedSet()
694
                                ).add(node)
695
 
696
                            definition_function_trace_info.markers.append(
697
                                language_item_marker
698
                            )
699
 
700
        #
701
        # STEP: Calculate requirements coverage by code. Sort nodes.
702
        #
703
        self.calculate_code_coverage_and_sort_nodes(traceability_index)
704
 
705
    def create_requirement_with_forward_source_links(
706
        self, requirement: SDocNode
707
    ) -> None:
708
        self.requirements_with_forward_links.add(requirement)
709
 
710
    def create_traceability_info(
711
        self,
712
        source_file: SourceFile,
713
        traceability_info: SourceFileTraceabilityInfo,
714
    ) -> None:
715
        assert isinstance(traceability_info, SourceFileTraceabilityInfo)
716
        traceability_info.source_file = source_file
717
 
718
        self.trace_infos.append(traceability_info)
719
 
720
    def get_req_uids_by_function_name(
721
        self, rel_path_posix: str, name: str
722
    ) -> Optional[List[Tuple[str, Optional[str]]]]:
723
        """
724
        @relation(SDOC-LLR-207, scope=function)
725
        """
726
 
727
        if rel_path_posix not in self.map_file_function_names_to_reqs_uids:
728
            return None
729
 
730
        function_names_to_reqs_uids = self.map_file_function_names_to_reqs_uids[
731
            rel_path_posix
732
        ]
733
 
734
        matching_req_uids: List[Tuple[str, Optional[str]]] = []
735
        exact_matching_req_uids = function_names_to_reqs_uids.get(name, None)
736
        if exact_matching_req_uids is not None:
737
            matching_req_uids.extend(exact_matching_req_uids)
738
 
739
        for function_name_, req_uids_ in function_names_to_reqs_uids.items():
740
            if not FileTraceabilityIndex.is_regex_function_name(function_name_):
741
                continue
742
 
743
            regex_pattern = function_name_[1:-1]
744
            try:
745
                if re.search(regex_pattern, name) is not None:
746
                    matching_req_uids.extend(req_uids_)
747
            except re.error as exception:
748
                raise StrictDocException(
749
                    "Invalid regular expression in FUNCTION relation "
750
                    f"{function_name_}: {exception}."
751
                ) from exception
752
 
753
        return matching_req_uids if len(matching_req_uids) > 0 else None
720 - 753 | function FileTraceabilityIndex.get_req_uids_by_function_name()
754
 
755
    @staticmethod
756
    def is_regex_function_name(name: str) -> bool:
757
        return len(name) >= 2 and name[0] == "/" and name[-1] == "/"
758
 
759
    def get_req_uids_by_class_name(
760
        self, rel_path_posix: str, name: str
761
    ) -> Optional[List[Tuple[str, Optional[str]]]]:
762
        if rel_path_posix in self.map_file_class_names_to_reqs_uids:
763
            return self.map_file_class_names_to_reqs_uids[rel_path_posix].get(
764
                name, None
765
            )
766
        return None
767
 
768
    @staticmethod
769
    def create_traceability_info_shared_markers_for_function(
770
        traceability_info: SourceFileTraceabilityInfo,
771
        function: LanguageItem,
772
        marker_type: RangeMarkerType,
773
        reqs_uids: List[Tuple[str, Optional[str]]],
774
    ) -> None:
775
        markers_by_role = {}
776
        for req_uid_, role in reqs_uids:
777
            req = Req(None, req_uid_)
778
            if role not in markers_by_role:
779
                markers_by_role[role] = (
780
                    FileTraceabilityIndex.forward_marker_from_language_item(
781
                        function, marker_type, [req], role
782
                    )
783
                )
784
            else:
785
                markers_by_role[role].reqs_objs.append(req)
786
 
787
        for req_uid_, role in reqs_uids:
788
            markers = traceability_info.ng_map_reqs_to_markers.setdefault(
789
                req_uid_, []
790
            )
791
            markers.append(markers_by_role[role])
792
 
793
        traceability_info.markers.extend(markers_by_role.values())
794
 
795
    @staticmethod
796
    def forward_marker_from_language_item(
797
        function: LanguageItem,
798
        marker_type: RangeMarkerType,
799
        reqs: List[Req],
800
        role: Optional[str],
801
        description: Optional[str] = None,
802
    ) -> ForwardLanguageItemMarker:
803
        language_item_marker = ForwardLanguageItemMarker(
804
            parent=None, reqs_objs=reqs, scope=marker_type.value
805
        )
806
        language_item_marker.ng_source_line_begin = function.line_begin
807
        language_item_marker.ng_range_line_begin = function.line_begin
808
        language_item_marker.ng_range_line_end = function.line_end
809
        language_item_marker.role = role
810
        if description is not None:
811
            language_item_marker.set_description(description)
812
        elif marker_type == RangeMarkerType.FUNCTION:
813
            language_item_marker.set_description(
814
                f"function {function.display_name}()"
815
            )
816
        elif marker_type == RangeMarkerType.CLASS:
817
            language_item_marker.set_description(f"class {function.name}")
818
        return language_item_marker
819
 
820
    @staticmethod
821
    def forward_range_markers_from_range(
822
        file_range: Tuple[int, int], requirement_uid_: str, role: Optional[str]
823
    ) -> Tuple[ForwardRangeMarker, ForwardRangeMarker]:
824
        start_marker = ForwardRangeMarker(
825
            start_or_end=True,
826
            reqs_objs=[Req(parent=None, uid=requirement_uid_)],
827
            role=role,
828
        )
829
        start_marker.ng_range_line_begin = file_range[0]
830
        start_marker.ng_source_line_begin = file_range[0]
831
        start_marker.ng_range_line_end = file_range[1]
832
 
833
        end_marker = ForwardRangeMarker(
834
            start_or_end=False,
835
            reqs_objs=[Req(parent=None, uid=requirement_uid_)],
836
            role=role,
837
        )
838
        end_marker.ng_source_line_begin = file_range[1]
839
        end_marker.ng_range_line_begin = file_range[0]
840
        end_marker.ng_range_line_end = file_range[1]
841
 
842
        return start_marker, end_marker
843
 
844
    @staticmethod
845
    def forward_file_marker_from_file_info(
846
        file_info: SourceFileTraceabilityInfo,
847
        requirement_uid_: str,
848
        role: Optional[str],
849
    ) -> ForwardFileMarker:
850
        marker = ForwardFileMarker(
851
            reqs_objs=[Req(parent=None, uid=requirement_uid_)],
852
            role=role,
853
        )
854
        marker.ng_range_line_begin = 1
855
        marker.ng_source_line_begin = 1
856
        marker.ng_range_line_end = file_info.file_stats.lines_total
857
        return marker
858
 
859
    def calculate_code_coverage_and_sort_nodes(
860
        self, traceability_index: "TraceabilityIndex"
861
    ) -> None:
862
        """
863
        Finalize code coverage and sort all nodes.
864
 
865
        For each trace info object:
866
        - Sort the markers according to their source location.
867
        - Calculate coverage information.
868
        """
869
 
870
        for (
871
            path,
872
            traceability_info_,
873
        ) in self.map_paths_to_source_file_traceability_info.items():
874
 
875
            def marker_comparator_start(
876
                marker: RelationMarkerType,
877
            ) -> int:
878
                assert marker.ng_range_line_begin is not None
879
                return marker.ng_range_line_begin
880
 
881
            sorted_markers = sorted(
882
                traceability_info_.markers, key=marker_comparator_start
883
            )
884
 
885
            traceability_info_.markers = sorted_markers
886
            # Finding how many lines are covered by the requirements in the file.
887
            # Quick and dirty: https://stackoverflow.com/a/15273749/598057
888
            merged_ranges: List[List[int]] = []
889
            for marker_ in traceability_info_.markers:
890
                assert isinstance(
891
                    marker_,
892
                    (
893
                        LanguageItemMarker,
894
                        ForwardRangeMarker,
895
                        RangeMarker,
896
                        LineMarker,
897
                    ),
898
                ), marker_
899
                if marker_.ng_is_nodoc:
900
                    continue
901
                if not marker_.is_begin():
902
                    continue
903
                begin, end = (
904
                    assert_cast(marker_.ng_range_line_begin, int),
905
                    assert_cast(marker_.ng_range_line_end, int),
906
                )
907
                if merged_ranges and merged_ranges[-1][1] >= (begin - 1):
908
                    merged_ranges[-1][1] = max(merged_ranges[-1][1], end)
909
                else:
910
                    merged_ranges.append([begin, end])
911
            coverage = 0
912
            for merged_range in merged_ranges:
913
                for line_ in range(merged_range[0], merged_range[1] + 1):
914
                    if traceability_info_.file_stats.lines_info[line_]:
915
                        coverage += 1
916
 
917
            for function_ in traceability_info_.functions:
918
                for merged_range in merged_ranges:
919
                    if (
920
                        function_.line_begin >= merged_range[0]
921
                        and function_.line_end <= merged_range[1]
922
                    ):
923
                        traceability_info_.covered_functions += 1
924
                        break
925
 
926
            traceability_info_.set_coverage_stats(merged_ranges, coverage)
927
 
928
            for (
929
                req_uid_,
930
                markers_,
931
            ) in traceability_info_.ng_map_reqs_to_markers.items():
932
 
933
                def marker_comparator_range(
934
                    marker: RelationMarkerType,
935
                ) -> Tuple[int, int]:
936
                    assert marker.ng_range_line_begin is not None
937
                    assert marker.ng_range_line_end is not None
938
                    return marker.ng_range_line_begin, marker.ng_range_line_end
939
 
940
                markers_.sort(key=marker_comparator_range)
941
 
942
                # Validate here, SDocNode.relations doesn't track marker roles.
943
                node = traceability_index.get_node_by_uid(req_uid_)
944
                document = node.get_document()
945
                assert document is not None
946
                assert document.grammar is not None
947
                grammar_element = document.grammar.elements_by_type[
948
                    node.node_type
949
                ]
950
                for marker in markers_:
951
                    # Backwards markers do not require referenced node grammar
952
                    # to have the relation/role registered in the grammar.
953
                    if isinstance(marker, (LanguageItemMarker, RangeMarker)):
954
                        continue
955
 
956
                    if not grammar_element.has_relation_type_role(
957
                        relation_type="File",
958
                        relation_role=marker.role,
959
                    ):
960
                        raise StrictDocSemanticError.invalid_marker_role(
961
                            node=node,
962
                            marker=marker,
963
                            path_to_src_file=path,
964
                        )
965
 
966
        # Sort by paths alphabetically.
967
        for paths_with_role in self.map_reqs_uids_to_paths.values():
968
            paths_with_role.sort()
969
 
970
        # Sort by node UID alphabetically.
971
        for path_requirements_ in self.map_paths_to_reqs.values():
972
 
973
            def compare_sdocnode_by_uid(node_: SDocNode) -> str:
974
                return assert_cast(node_.reserved_uid, str)
975
 
976
            path_requirements_.sort(key=compare_sdocnode_by_uid)
977
 
978
    def connect_source_node_function(
979
        self,
980
        source_node: SourceNode,
981
        source_sdoc_node_uid: str,
982
        traceability_info: SourceFileTraceabilityInfo,
983
    ) -> None:
984
        source_node_function = source_node.function
985
        assert source_node_function is not None
986
 
987
        language_item_marker = self.forward_marker_from_language_item(
988
            function=source_node_function,
989
            marker_type=RangeMarkerType.FUNCTION,
990
            reqs=[Req(None, source_sdoc_node_uid)],
991
            role=None,
992
            description=f"function {source_node_function.display_name}()",
993
        )
994
 
995
        traceability_info.ng_map_reqs_to_markers.setdefault(
996
            source_sdoc_node_uid, []
997
        ).append(language_item_marker)
998
        language_item_marker_copy = language_item_marker.create_end_marker()
999
        traceability_info.markers.append(language_item_marker)
1000
        traceability_info.markers.append(language_item_marker_copy)
1001
 
1002
    @staticmethod
1003
    def create_sdoc_node_from_source_node(
1004
        source_node: SourceNode,
1005
        source_node_config_entry: SourceNodesEntry,
1006
        sdoc_node_uid: str,
1007
        parent_document: SDocDocumentIF,
1008
    ) -> SDocNode:
1009
        sdoc_node = SDocNode(
1010
            parent=parent_document,
1011
            node_type=source_node_config_entry.node_type,
1012
            fields=[],
1013
            relations=[],
1014
            # It is important that this autogenerated node is marked as such.
1015
            autogen=True,
1016
        )
1017
        sdoc_node.ng_document_reference = DocumentReference()
1018
        sdoc_node.ng_document_reference.set_document(parent_document)
1019
        sdoc_node.ng_including_document_reference = DocumentReference()
1020
        sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
1021
        sdoc_node_fields["UID"] = sdoc_node_uid
1022
        if (
1023
            "TITLE" not in sdoc_node_fields
1024
            and source_node.entity_name is not None
1025
        ):
1026
            sdoc_node_fields["TITLE"] = source_node.entity_name
1027
        FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
1028
        return sdoc_node
1029
 
1030
    @staticmethod
1031
    def merge_sdoc_node_with_source_node(
1032
        source_node_config_entry: SourceNodesEntry,
1033
        source_node: SourceNode,
1034
        sdoc_node: SDocNode,
1035
        parent_document: SDocDocumentIF,
1036
    ) -> None:
1037
        # First check if grammar element definitions are compatible.
1038
        source_node_type = source_node_config_entry.node_type
1039
        source_node_grammar = assert_cast(
1040
            parent_document.grammar, DocumentGrammar
1041
        )
1042
        source_node_grammar_element = source_node_grammar.elements_by_type[
1043
            source_node_type
1044
        ]
1045
        sdoc_node_document = assert_cast(
1046
            sdoc_node.get_document(), SDocDocumentIF
1047
        )
1048
        sdoc_node_grammar = assert_cast(
1049
            sdoc_node_document.grammar, DocumentGrammar
1050
        )
1051
        sdoc_node_grammar_element = sdoc_node_grammar.elements_by_type[
1052
            source_node_type
1053
        ]
1054
        if source_node_grammar_element != sdoc_node_grammar_element:
1055
            raise StrictDocException(
1056
                f"Can't merge node {sdoc_node.reserved_uid} with source portion: "
1057
                f"Grammar element {sdoc_node_document.reserved_uid}::{source_node_type} "
1058
                f"incompatible with {parent_document.reserved_uid}::{source_node_type}"
1059
            )
1060
        # Merge strategy: overwrite any field if there's a field with same name from custom tags.
1061
        sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
1062
 
1063
        # Sanity check: Nor UID neither MID must conflict (early auto-MID is allowed to be overwritten)
1064
        if (
1065
            "MID" in sdoc_node.ordered_fields_lookup
1066
            and "MID" in sdoc_node_fields
1067
        ):
1068
            sdoc_mid_field = sdoc_node.get_field_by_name("MID").get_text_value()
1069
            if sdoc_mid_field != sdoc_node_fields["MID"]:
1070
                raise StrictDocException(
1071
                    f"Can't merge node by UID {sdoc_node.reserved_uid}: "
1072
                    f"Conflicting MID: {sdoc_mid_field} != {sdoc_node_fields['MID']}"
1073
                )
1074
        if sdoc_node.reserved_uid is not None and "UID" in sdoc_node_fields:
1075
            if sdoc_node.reserved_uid != sdoc_node_fields["UID"]:
1076
                raise StrictDocException(
1077
                    f"Can't merge node by MID {sdoc_node.reserved_mid}: "
1078
                    f"Conflicting UID: {sdoc_node.reserved_uid} != {sdoc_node_fields['UID']}"
1079
                )
1080
 
1081
        FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
1082
        source_node.sdoc_node = sdoc_node
1083
 
1084
    @staticmethod
1085
    def set_sdoc_node_fields(
1086
        sdoc_node: SDocNode, sdoc_node_fields: dict[str, str]
1087
    ) -> None:
1088
        for field_name, field_value in sdoc_node_fields.items():
1089
            sdoc_node.set_field_value(
1090
                field_name=field_name,
1091
                form_field_index=0,
1092
                value=field_value,
1093
            )
1094
 
1095
            # As we overwrite the field's content from the source code,
1096
            # we mark the field as source_origin here.
1097
            new_field: SDocNodeField = sdoc_node.ordered_fields_lookup[
1098
                field_name
1099
            ][0]
1100
            new_field.mark_as_source_origin()
1101
 
1102
    @staticmethod
1103
    def create_source_node_section(
1104
        document: SDocDocumentIF,
1105
        path_to_source_file: str,
1106
        section_cache: Dict[str, Union[SDocDocumentIF, SDocNode]],
1107
    ) -> Tuple[Union[SDocDocumentIF, SDocNode], List[SDocNode]]:
1108
        """
1109
        Add a subsection for each path components in a given file path.
1110
        """
1111
        current_top_node: Union[SDocDocumentIF, SDocNode] = document
1112
        created_sections: List[SDocNode] = []
1113
        path_components = path_to_source_file.split("/")
1114
        for path_component_idx_, path_component_ in enumerate(path_components):
1115
            if path_component_ not in section_cache:
1116
                path_component_title = (
1117
                    path_component_ + "/"
1118
                    if path_component_idx_ < (len(path_components) - 1)
1119
                    else path_component_
1120
                )
1121
                current_section = SDocNode(
1122
                    parent=current_top_node,
1123
                    node_type="SECTION",
1124
                    fields=[],
1125
                    relations=[],
1126
                    is_composite=True,
1127
                    node_type_close="SECTION",
1128
                    # It is important that this autogenerated node is marked as such.
1129
                    autogen=True,
1130
                )
1131
                current_section.ng_document_reference = DocumentReference()
1132
                current_section.ng_document_reference.set_document(document)
1133
                current_section.ng_including_document_reference = (
1134
                    DocumentReference()
1135
                )
1136
                current_section.set_field_value(
1137
                    field_name="TITLE",
1138
                    form_field_index=0,
1139
                    value=path_component_title,
1140
                )
1141
 
1142
                current_top_node.section_contents.append(current_section)
1143
                section_cache[path_component_] = current_section
1144
                created_sections.append(current_section)
1145
            current_top_node = section_cache[path_component_]
1146
        return current_top_node, created_sections
1147
 
1148
    def connect_sdoc_node_with_file_path(
1149
        self, sdoc_node: SDocNode, path_to_source_file_: str
1150
    ) -> None:
1151
        uid = sdoc_node.reserved_uid
1152
        assert uid is not None
1153
        self.map_reqs_uids_to_paths.setdefault(uid, OrderedSet()).add(
1154
            path_to_source_file_
1155
        )
1156
        self.map_paths_to_reqs.setdefault(
1157
            path_to_source_file_, OrderedSet()
1158
        ).add(sdoc_node)
1159
 
1160
    @staticmethod
1161
    def connect_source_node_requirements(
1162
        source_node: SourceNode,
1163
        sdoc_node: SDocNode,
1164
        traceability_index: "TraceabilityIndex",
1165
    ) -> None:
1166
        """
1167
        Connect auto-generated requirement with function marker and with marker target requirement.
1168
 
1169
        If function comment has @relation(REQ, scope=function), connections shall become
1170
        [REQ] <-parent- [auto-generated/merged sdoc_node] -file-> [function marker]
1171
 
1172
        Here we link REQ and sdoc_node bidirectional.
1173
        """
1174
        if (
1175
            sdoc_node.reserved_uid is not None
1176
            and not traceability_index.graph_database.has_link(
1177
                link_type=GraphLinkType.UID_TO_NODE,
1178
                lhs_node=sdoc_node.reserved_uid,
1179
                rhs_node=sdoc_node,
1180
            )
1181
        ):
1182
            traceability_index.graph_database.create_link(
1183
                link_type=GraphLinkType.UID_TO_NODE,
1184
                lhs_node=sdoc_node.reserved_uid,
1185
                rhs_node=sdoc_node,
1186
            )
1187
 
1188
        # A merge procedure may have overwritten the MID,
1189
        # in which case the graph database and search index needs an update.
1190
        if "MID" in sdoc_node.ordered_fields_lookup != sdoc_node.reserved_mid:
1191
            sdoc_mid_field = sdoc_node.get_field_by_name("MID").get_text_value()
1192
            if sdoc_mid_field != sdoc_node.reserved_mid:
1193
                # TODO:
1194
                # If we really want to support changing the auto-assigned MID,
1195
                # at least the graph database and the document search index need an update (remove old MID, add new MID).
1196
                # I currently struggle to update the search index.
1197
                parent_document = sdoc_node.get_parent_or_including_document()
1198
                sdoc_node.reserved_mid = MID(sdoc_mid_field)
1199
                if parent_document.config.enable_mid:
1200
                    sdoc_node.mid_permanent = True
1201
 
1202
        if not traceability_index.graph_database.has_link(
1203
            link_type=GraphLinkType.MID_TO_NODE,
1204
            lhs_node=sdoc_node.reserved_mid,
1205
            rhs_node=sdoc_node,
1206
        ):
1207
            traceability_index.graph_database.create_link(
1208
                link_type=GraphLinkType.MID_TO_NODE,
1209
                lhs_node=sdoc_node.reserved_mid,
1210
                rhs_node=sdoc_node,
1211
            )
1212
 
1213
        for marker_ in source_node.markers:
1214
            if not isinstance(marker_, LanguageItemMarker):
1215
                continue
1216
            for req_ in marker_.reqs:
1217
                node = traceability_index.get_node_by_uid_weak2(req_)
1218
                if not traceability_index.graph_database.has_link(
1219
                    link_type=GraphLinkType.NODE_TO_PARENT_NODES,
1220
                    lhs_node=sdoc_node,
1221
                    rhs_node=node,
1222
                ):
1223
                    traceability_index.graph_database.create_link(
1224
                        link_type=GraphLinkType.NODE_TO_PARENT_NODES,
1225
                        lhs_node=sdoc_node,
1226
                        rhs_node=node,
1227
                    )
1228
                if not traceability_index.graph_database.has_link(
1229
                    link_type=GraphLinkType.NODE_TO_CHILD_NODES,
1230
                    lhs_node=node,
1231
                    rhs_node=sdoc_node,
1232
                ):
1233
                    traceability_index.graph_database.create_link(
1234
                        link_type=GraphLinkType.NODE_TO_CHILD_NODES,
1235
                        lhs_node=node,
1236
                        rhs_node=sdoc_node,
1237
                    )
1 - 1237 | entire file