Path:
strictdoc/core/file_traceability_index.py
Lines:
1274
Non-empty lines:
1147
Non-empty lines covered with requirements:
1147 / 1147 (100.0%)
Functions:
29
Functions covered by requirements:
29 / 29 (100.0%)
- "4.1. Traceability index" (REQUIREMENT)
- "7.1. Link requirements with source files" (REQUIREMENT)
1
"""2
@relation(SDOC-SRS-28, SDOC-SRS-33, scope=file)3
"""4
5
import re
6
from copy import copy
7
from typing import (
8
TYPE_CHECKING,
9
Dict,
10
Iterator,
11
List,
12
Optional,
13
Set,
14
Tuple,
15
Union,
16
)17
18
from strictdoc.backend.sdoc.document_reference import DocumentReference
19
from strictdoc.backend.sdoc.error_handling import StrictDocSemanticError
20
from strictdoc.backend.sdoc.models.document_grammar import (
21
DocumentGrammar,
22
)23
from strictdoc.backend.sdoc.models.model import SDocDocumentIF
24
from strictdoc.backend.sdoc.models.node import SDocNode, SDocNodeField
25
from strictdoc.backend.sdoc.models.reference import FileEntry, FileReference
26
from strictdoc.backend.sdoc_source_code.models.language import LanguageItem
27
from strictdoc.backend.sdoc_source_code.models.language_item_marker import (
28
ForwardLanguageItemMarker,
29
LanguageItemMarker,
30
RangeMarkerType,
31
)32
from strictdoc.backend.sdoc_source_code.models.line_marker import LineMarker
33
from strictdoc.backend.sdoc_source_code.models.range_marker import (
34
ForwardFileMarker,
35
ForwardRangeMarker,
36
RangeMarker,
37
)38
from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req
39
from strictdoc.backend.sdoc_source_code.models.source_file_info import (
40
RelationMarkerType,
41
SourceFileTraceabilityInfo,
42
)43
from strictdoc.backend.sdoc_source_code.models.source_node import SourceNode
44
from strictdoc.core.constants import GraphLinkType
45
from strictdoc.core.document_iterator import SDocDocumentIterator
46
from strictdoc.core.file_system.source_tree import SourceFile
47
from strictdoc.core.project_config import ProjectConfig, SourceNodesEntry
48
from strictdoc.helpers.cargo_nextest import (
49
convert_nextest_test_to_rust_canonical_paths,
50
)51
from strictdoc.helpers.cast import assert_cast
52
from strictdoc.helpers.exception import StrictDocException
53
from strictdoc.helpers.google_test import convert_function_name_to_gtest_macro
54
from strictdoc.helpers.mid import MID
55
from strictdoc.helpers.ordered_set import OrderedSet
56
57
if TYPE_CHECKING:
58
from strictdoc.core.traceability_index import (
59
TraceabilityIndex,
60
)61
62
63
class FileTraceabilityIndex:
64
def __init__(self) -> None:
65
# "file.py" -> List[SDocNode]66
self.map_paths_to_reqs: Dict[str, OrderedSet[SDocNode]] = {}
67
68
# "REQ-001" -> {"file.py", ...}69
self.map_reqs_uids_to_paths: Dict[str, OrderedSet[str]] = {}
70
71
# "file.py" -> SourceFileTraceabilityInfo.72
self.map_paths_to_source_file_traceability_info: Dict[
73
str, SourceFileTraceabilityInfo
74
] = {}
75
76
# "file.py" -> { { "foo" -> [("REQ-1", "Impl"), ("REQ-2", "Test")] }, ... }77
self.map_file_function_names_to_reqs_uids: Dict[
78
str, Dict[str, List[Tuple[str, Optional[str]]]]
79
] = {}
80
self.map_file_class_names_to_reqs_uids: Dict[
81
str, Dict[str, List[Tuple[str, Optional[str]]]]
82
] = {}
83
84
# This is only public non-static functions from languages like C.85
self.map_all_function_names_to_definition_functions: Dict[
86
str, List[LanguageItem]
87
] = {}
88
89
# "file.py" -> [SDocNode] # noqa: ERA00190
self.source_file_reqs_cache: Dict[str, Optional[List[SDocNode]]] = {}
91
92
self.requirements_with_forward_links: OrderedSet[SDocNode] = (
93
OrderedSet()
94
)95
self.trace_infos: List[SourceFileTraceabilityInfo] = []
96
97
def has_source_file_reqs(self, source_file_rel_path: str) -> bool:
98
path_reqs = self.map_paths_to_reqs.get(source_file_rel_path)
99
if path_reqs is not None and len(path_reqs) > 0:
100
return True
101
file_trace_info = self.map_paths_to_source_file_traceability_info[
102
source_file_rel_path103
]104
return len(file_trace_info.markers) > 0
105
106
def get_requirement_file_links(
107
self, requirement: SDocNode
108
) -> List[Tuple[str, List[RelationMarkerType]]]:
109
if requirement.reserved_uid not in self.map_reqs_uids_to_paths:
110
return []
111
112
matching_links_with_markers: List[
113
Tuple[str, List[RelationMarkerType]]
114
] = []
115
requirement_source_paths: OrderedSet[str] = self.map_reqs_uids_to_paths[
116
requirement.reserved_uid
117
]118
119
# Now that one requirement can have multiple File-relations to the same file.120
# This can be multiple FUNCTION: or RANGE: forward-relations.121
# To avoid duplication of results, visit each unique file link path only once.122
visited_file_links: Set[str] = set()
123
for requirement_source_path_ in requirement_source_paths:
124
if requirement_source_path_ in visited_file_links:
125
continue126
visited_file_links.add(requirement_source_path_)
127
128
source_file_traceability_info: Optional[
129
SourceFileTraceabilityInfo130
] = self.map_paths_to_source_file_traceability_info.get(
131
requirement_source_path_132
)133
assert source_file_traceability_info is not None, (
134
f"Requirement {requirement.reserved_uid} references a file"
135
f" that does not exist: {requirement_source_path_}."
136
)137
markers = source_file_traceability_info.ng_map_reqs_to_markers.get(
138
requirement.reserved_uid
139
)140
if markers is None or len(markers) == 0:
141
matching_links_with_markers.append(
142
(requirement_source_path_, [])
143
)144
continue145
matching_links_with_markers.append(
146
(requirement_source_path_, markers)
147
)148
149
return matching_links_with_markers
150
151
def indexed_source_files(self) -> Iterator[SourceFile]:
152
for _, sfti in self.map_paths_to_source_file_traceability_info.items():
153
if sfti.source_file is not None:
154
yield sfti.source_file
155
156
def get_source_file_reqs(
157
self, source_file_rel_path: str
158
) -> Optional[List[SDocNode]]:
159
assert (
160
source_file_rel_path161
in self.map_paths_to_source_file_traceability_info
162
)163
if source_file_rel_path in self.source_file_reqs_cache:
164
return self.source_file_reqs_cache[source_file_rel_path]
165
166
source_file_traceability_info: SourceFileTraceabilityInfo = (
167
self.map_paths_to_source_file_traceability_info[
168
source_file_rel_path169
]170
)171
172
if source_file_rel_path not in self.map_paths_to_reqs:
173
self.source_file_reqs_cache[source_file_rel_path] = None
174
return None
175
176
requirements = self.map_paths_to_reqs[source_file_rel_path]
177
assert len(requirements) > 0
178
range_requirements = []
179
180
for requirement in requirements:
181
if (
182
requirement.reserved_uid
183
in source_file_traceability_info.ng_map_reqs_to_markers
184
):185
range_requirements.append(requirement)
186
187
self.source_file_reqs_cache[source_file_rel_path] = range_requirements
188
return range_requirements
189
190
def get_coverage_info(
191
self, source_file_rel_path: str
192
) -> SourceFileTraceabilityInfo:
193
assert (
194
source_file_rel_path195
in self.map_paths_to_source_file_traceability_info
196
), source_file_rel_path
197
source_file_tr_info: SourceFileTraceabilityInfo = (
198
self.map_paths_to_source_file_traceability_info[
199
source_file_rel_path200
]201
)202
return source_file_tr_info
203
204
def get_coverage_info_weak(
205
self, source_file_rel_path: str
206
) -> Optional[SourceFileTraceabilityInfo]:
207
source_file_tr_info: Optional[SourceFileTraceabilityInfo] = (
208
self.map_paths_to_source_file_traceability_info.get(
209
source_file_rel_path210
)211
)212
return source_file_tr_info
213
214
def validate_and_resolve(
215
self,
216
traceability_index: "TraceabilityIndex",
217
project_config: ProjectConfig,
218
) -> None:
219
"""
220
Resolve all source code traceability after the index is fully built.221
"""222
223
#224
# STEP: Collect minimal information that will help to resolve the225
# forward-declared paths/function names at the step 2.226
#227
for trace_info_ in self.trace_infos:
228
source_file: SourceFile = assert_cast(
229
trace_info_.source_file, SourceFile
230
)231
232
self.map_paths_to_source_file_traceability_info[
233
source_file.in_doctree_source_file_rel_path_posix
234
] = trace_info_
235
236
for function_ in trace_info_.functions:
237
if function_.is_definition() and function_.is_public():
238
self.map_all_function_names_to_definition_functions.setdefault(
239
function_.name, []
240
).append(function_)
241
242
#243
# STEP: Auto-generated SDocNodes from source file comments and register244
# their UIDs. This must happen before marker validation so that245
# source files can reference these UIDs via @relation.246
#247
documents_with_generated_content = set()
248
249
section_cache: Dict[str, Union[SDocDocumentIF, SDocNode]] = {}
250
source_nodes_config: List[SourceNodesEntry] = (
251
project_config.source_nodes
252
)253
unused_source_node_paths = {
254
config_entry_.path for config_entry_ in source_nodes_config
255
}256
for (
257
path_to_source_file_,
258
traceability_info_,
259
) in self.map_paths_to_source_file_traceability_info.items():
260
if len(traceability_info_.source_nodes) == 0:
261
continue262
263
if len(source_nodes_config) == 0:
264
continue265
266
relevant_source_node_entry = (
267
project_config.get_relevant_source_nodes_entry(
268
path_to_source_file_269
)270
)271
if relevant_source_node_entry is not None:
272
unused_source_node_paths.discard(
273
relevant_source_node_entry.path
274
)275
else:
276
continue277
278
document_uid = relevant_source_node_entry.uid
279
document = traceability_index.get_node_by_uid(document_uid)
280
documents_with_generated_content.add(document)
281
current_top_node = None
282
283
for source_node_ in traceability_info_.source_nodes:
284
if len(source_node_.fields) == 0:
285
continue286
287
assert source_node_.entity_name is not None
288
sdoc_node = None
289
sdoc_node_uid = source_node_.get_sdoc_field(
290
"UID", relevant_source_node_entry
291
)292
mid = source_node_.get_sdoc_field(
293
"MID", relevant_source_node_entry
294
)295
296
# First merge criterion: Merge if SDoc node with same MID exists.297
if mid is not None:
298
sdoc_node_mid = MID(mid)
299
merge_candidate_sdoc_node = (
300
traceability_index.get_node_by_mid_weak(sdoc_node_mid)
301
)302
if isinstance(merge_candidate_sdoc_node, SDocNode):
303
sdoc_node = merge_candidate_sdoc_node
304
sdoc_node_uid = sdoc_node.reserved_uid
305
306
if sdoc_node is None:
307
# If no UID from source code field or merge-by-MID, create UID by conventional scheme.308
if sdoc_node_uid is None:
309
sdoc_node_uid = f"{document_uid}/{path_to_source_file_}/{source_node_.entity_name}"
310
# Second merge criterion: Merge if SDoc node with same UID exists.311
tmp_sdoc_node = traceability_index.get_node_by_uid_weak(
312
sdoc_node_uid313
)314
if isinstance(tmp_sdoc_node, SDocNode):
315
sdoc_node = tmp_sdoc_node
316
317
assert sdoc_node_uid is not None
318
if sdoc_node is not None:
319
sdoc_node = assert_cast(sdoc_node, SDocNode)
320
self.merge_sdoc_node_with_source_node(
321
relevant_source_node_entry,
322
source_node_,
323
sdoc_node,
324
document,
325
)326
else:
327
sdoc_node = self.create_sdoc_node_from_source_node(
328
source_node_,
329
relevant_source_node_entry,
330
sdoc_node_uid,
331
document,
332
)333
sdoc_node_uid = assert_cast(sdoc_node.reserved_uid, str)
334
if current_top_node is None:
335
current_top_node, created_sections = (
336
FileTraceabilityIndex.create_source_node_section(
337
document,
338
path_to_source_file_,
339
section_cache,
340
)341
)342
for created_section in created_sections:
343
traceability_index.graph_database.create_link(
344
link_type=GraphLinkType.MID_TO_NODE,
345
lhs_node=created_section.reserved_mid,
346
rhs_node=created_section,
347
)348
current_top_node.section_contents.append(sdoc_node)
349
350
self.connect_source_node_function(
351
source_node_, sdoc_node_uid, traceability_info_
352
)353
self.connect_sdoc_node_with_file_path(
354
sdoc_node, path_to_source_file_
355
)356
self.connect_source_node_requirements(
357
source_node_, sdoc_node, traceability_index
358
)359
360
# Warn if source_node was not matched by any include_source_paths, it indicates misconfiguration361
for unused_source_node_path in unused_source_node_paths:
362
print( # noqa: T201
363
f"warning: source_node path {unused_source_node_path} doesn't match any source file. "
364
"Hint: Check include_source_paths."365
)366
367
# Iterate over all generated documents to calculate all node levels.368
for document_ in documents_with_generated_content:
369
document_iterator = SDocDocumentIterator(document_)
370
for _, _ in document_iterator.all_content(
371
print_fragments=False,
372
):373
pass374
375
#376
# STEP: Resolve requirements that have forward links.377
# Some requirements can come from the SDoc documents generated378
# on the fly from JUnit XML documents.379
#380
for forward_requirement_ in self.requirements_with_forward_links:
381
assert forward_requirement_.reserved_uid is not None
382
383
for relation_ in forward_requirement_.relations:
384
if not isinstance(relation_, FileReference):
385
continue386
387
file_reference: FileReference = assert_cast(
388
relation_, FileReference
389
)390
file_posix_path = file_reference.get_posix_path()
391
392
if file_posix_path == "#FORWARD#":
393
test_function = (
394
forward_requirement_.get_meta_field_value_by_title(
395
"TEST_FUNCTION"396
)397
)398
assert test_function is not None
399
400
functions: List[LanguageItem]
401
if test_function.startswith("#GTEST#"):
402
test_function = test_function.removeprefix("#GTEST#")
403
possible_gtest_functions = (
404
convert_function_name_to_gtest_macro(test_function)
405
)406
for (
407
possible_gtest_function_408
) in possible_gtest_functions:
409
if (
410
possible_gtest_function_411
in self.map_all_function_names_to_definition_functions
412
):413
test_function = possible_gtest_function_
414
break415
else:
416
raise RuntimeError(
417
"Could not find a matching Google Test function: "418
f"{possible_gtest_functions}"
419
) # pragma: no cover
420
forward_requirement_.set_field_value(
421
field_name="TEST_FUNCTION",
422
form_field_index=0,
423
value=test_function,
424
)425
elif test_function.startswith("#NEXTEST#"):
426
# The payload after #NEXTEST# is "<classname>|<name>"427
# as taken straight from the cargo-nextest JUnit XML.428
nextest_payload = test_function.removeprefix(
429
"#NEXTEST#"430
)431
nextest_classname, _, nextest_name = (
432
nextest_payload.partition("|")
433
)434
possible_nextest_functions = (
435
convert_nextest_test_to_rust_canonical_paths(
436
nextest_classname, nextest_name
437
)438
)439
for (
440
possible_nextest_function_441
) in possible_nextest_functions:
442
if (
443
possible_nextest_function_444
in self.map_all_function_names_to_definition_functions
445
):446
test_function = possible_nextest_function_
447
break448
else:
449
raise RuntimeError(
450
"Could not find a matching Rust function for "451
"cargo-nextest test: "452
f"{possible_nextest_functions}"
453
) # pragma: no cover
454
forward_requirement_.set_field_value(
455
field_name="TEST_FUNCTION",
456
form_field_index=0,
457
value=test_function,
458
)459
functions = (
460
self.map_all_function_names_to_definition_functions[
461
test_function462
]463
)464
assert len(functions) == 1
465
466
function: LanguageItem = functions[0]
467
resolved_path_to_function_file = function.parent.source_file.in_doctree_source_file_rel_path_posix
468
file_posix_path = resolved_path_to_function_file
469
470
file_reference.g_file_entry = FileEntry(
471
relation_,
472
g_file_format=relation_.g_file_entry.g_file_format,
473
g_file_path=resolved_path_to_function_file,
474
g_line_range=None,
475
element="function",
476
id=test_function,
477
)478
479
forward_requirement_.set_field_value(
480
field_name="TEST_PATH",
481
form_field_index=0,
482
value=resolved_path_to_function_file,
483
)484
485
#486
# This transitively connects requirements and test results487
# through the test source files.488
#489
for language_item_marker_ in function.markers:
490
for req_ in language_item_marker_.reqs:
491
node = traceability_index.get_node_by_uid_weak2(
492
req_493
)494
traceability_index.graph_database.create_link(
495
link_type=GraphLinkType.NODE_TO_PARENT_NODES,
496
lhs_node=forward_requirement_,
497
rhs_node=node,
498
edge="Satisfies",
499
)500
traceability_index.graph_database.create_link(
501
link_type=GraphLinkType.NODE_TO_CHILD_NODES,
502
lhs_node=node,
503
rhs_node=forward_requirement_,
504
edge="IsSatisfiedBy",
505
)506
#507
# Validate that all requirements reference existing files.508
#509
source_file_traceability_info: Optional[
510
SourceFileTraceabilityInfo511
] = self.map_paths_to_source_file_traceability_info.get(
512
file_posix_path513
)514
if source_file_traceability_info is None:
515
raise StrictDocException(
516
f"Requirement {forward_requirement_.reserved_uid} "
517
"references a file that does not exist: "518
f"{file_posix_path}."
519
)520
521
#522
# Now that the test reports related fixups are done, the523
# following code registers the requirements with forward links.524
#525
self.map_paths_to_reqs.setdefault(
526
file_posix_path, OrderedSet()
527
).add(forward_requirement_)
528
529
assert forward_requirement_.reserved_uid is not None
530
self.map_reqs_uids_to_paths.setdefault(
531
forward_requirement_.reserved_uid, OrderedSet()
532
).add(file_posix_path)
533
534
if (
535
file_reference.g_file_entry.element == "function"
536
and file_reference.g_file_entry.id is not None
537
):538
one_file_function_name_to_reqs_uids = (
539
self.map_file_function_names_to_reqs_uids.setdefault(
540
file_posix_path, {}
541
)542
)543
one_file_function_name_to_reqs_uids.setdefault(
544
file_reference.g_file_entry.id, []
545
).append(
546
(forward_requirement_.reserved_uid, relation_.role)
547
)548
elif (
549
file_reference.g_file_entry.element == "class"
550
and file_reference.g_file_entry.id is not None
551
):552
one_file_class_name_to_reqs_uids = (
553
self.map_file_class_names_to_reqs_uids.setdefault(
554
file_posix_path, {}
555
)556
)557
one_file_class_name_to_reqs_uids.setdefault(
558
file_reference.g_file_entry.id, []
559
).append(
560
(forward_requirement_.reserved_uid, relation_.role)
561
)562
elif file_reference.g_file_entry.line_range is not None:
563
line_range = file_reference.g_file_entry.line_range
564
uid = forward_requirement_.reserved_uid
565
source_file_info = (
566
self.map_paths_to_source_file_traceability_info[
567
file_posix_path568
]569
)570
start_marker, end_marker = (
571
self.forward_range_markers_from_range(
572
line_range, uid, relation_.role
573
)574
)575
source_file_info.ng_map_reqs_to_markers.setdefault(
576
uid, []
577
).append(start_marker)
578
source_file_info.markers.append(start_marker)
579
source_file_info.markers.append(end_marker)
580
else:
581
uid = forward_requirement_.reserved_uid
582
source_file_info = (
583
self.map_paths_to_source_file_traceability_info[
584
file_posix_path585
]586
)587
forward_file_marker = (
588
self.forward_file_marker_from_file_info(
589
source_file_info,
590
uid,
591
relation_.role,
592
)593
)594
source_file_info.ng_map_reqs_to_markers.setdefault(
595
forward_requirement_.reserved_uid, []
596
).append(forward_file_marker)
597
source_file_info.markers.append(forward_file_marker)
598
599
#600
# STEP: Add markers for forward relations to functions and classes601
#602
for trace_info_ in self.trace_infos:
603
source_file = assert_cast(trace_info_.source_file, SourceFile)
604
605
self.map_paths_to_source_file_traceability_info[
606
source_file.in_doctree_source_file_rel_path_posix
607
] = trace_info_
608
609
for function_ in trace_info_.functions:
610
# FIXME: Using display_name, not name. A separate exercise is611
# to disambiguate forward links to C++ overloaded functions.612
if (
613
reqs_uids := self.get_req_uids_by_function_name(
614
source_file.in_doctree_source_file_rel_path_posix,
615
function_.display_name,
616
)617
) is not None:
618
self.create_traceability_info_shared_markers_for_function(
619
trace_info_,
620
function_,
621
RangeMarkerType.FUNCTION,
622
reqs_uids,
623
)624
if (
625
reqs_uids := self.get_req_uids_by_class_name(
626
source_file.in_doctree_source_file_rel_path_posix,
627
function_.display_name,
628
)629
) is not None:
630
self.create_traceability_info_shared_markers_for_function(
631
trace_info_,
632
function_,
633
RangeMarkerType.CLASS,
634
reqs_uids,
635
)636
637
marker_: Union[
638
LanguageItemMarker, LineMarker, RangeMarker, ForwardRangeMarker
639
]640
for marker_ in copy(trace_info_.markers):
641
# FIXME: Is this 'continue' needed here?642
if isinstance(marker_, ForwardRangeMarker):
643
continue644
for requirement_uid_ in marker_.reqs:
645
node = traceability_index.get_node_by_uid_weak2(
646
requirement_uid_647
)648
if node is None:
649
raise StrictDocException(
650
f"Source file {source_file.in_doctree_source_file_rel_path_posix} references "
651
f"a requirement that does not exist: {requirement_uid_}."
652
)653
654
self.map_reqs_uids_to_paths.setdefault(
655
requirement_uid_, OrderedSet()
656
).add(source_file.in_doctree_source_file_rel_path_posix)
657
658
self.map_paths_to_reqs.setdefault(
659
source_file.in_doctree_source_file_rel_path_posix,
660
OrderedSet(),
661
).add(node)
662
663
if isinstance(marker_, LanguageItemMarker):
664
marker_copy = marker_.create_end_marker()
665
trace_info_.markers.append(marker_copy)
666
667
#668
# Resolve definitions to declarations (only applicable for C and C++).669
#670
671
reversed_trace_info = {
672
value: key
673
for key, value in self.map_paths_to_source_file_traceability_info.items()
674
}675
676
for (
677
traceability_info_678
) in self.map_paths_to_source_file_traceability_info.values():
679
for function_ in traceability_info_.functions:
680
if (
681
function_.is_declaration()
682
and function_.name
683
in self.map_all_function_names_to_definition_functions
684
):685
definition_functions: List[LanguageItem] = []
686
if not function_.is_public():
687
definition_function = traceability_info_.ng_map_names_to_definition_functions.get(
688
function_.name, None
689
)690
if definition_function is not None:
691
definition_functions.append(definition_function)
692
else:
693
mapped_definition_functions = (
694
self.map_all_function_names_to_definition_functions[
695
function_.name
696
]697
)698
definition_functions.extend(mapped_definition_functions)
699
if len(definition_functions) == 0:
700
continue701
702
for definition_function_ in definition_functions:
703
definition_function_trace_info: SourceFileTraceabilityInfo = definition_function_.parent
704
705
for marker_ in function_.markers:
706
language_item_marker = self.forward_marker_from_language_item(
707
function=definition_function_,
708
marker_type=RangeMarkerType.FUNCTION,
709
reqs=marker_.reqs_objs,
710
role=marker_.role,
711
description=f"function {function_.display_name}()",
712
)713
714
for req_uid_ in marker_.reqs:
715
definition_function_trace_info.ng_map_reqs_to_markers.setdefault(
716
req_uid_, []
717
).append(language_item_marker)
718
719
path_to_info = reversed_trace_info[
720
definition_function_trace_info721
]722
self.map_reqs_uids_to_paths.setdefault(
723
req_uid_, OrderedSet()
724
).add(path_to_info)
725
726
node = traceability_index.get_node_by_uid(
727
req_uid_728
)729
self.map_paths_to_reqs.setdefault(
730
path_to_info, OrderedSet()
731
).add(node)
732
733
definition_function_trace_info.markers.append(
734
language_item_marker735
)736
737
#738
# STEP: Calculate requirements coverage by code. Sort nodes.739
#740
self.calculate_code_coverage_and_sort_nodes(traceability_index)
741
742
def create_requirement_with_forward_source_links(
743
self, requirement: SDocNode
744
) -> None:
745
self.requirements_with_forward_links.add(requirement)
746
747
def create_traceability_info(
748
self,
749
source_file: SourceFile,
750
traceability_info: SourceFileTraceabilityInfo,
751
) -> None:
752
assert isinstance(traceability_info, SourceFileTraceabilityInfo)
753
traceability_info.source_file = source_file
754
755
self.trace_infos.append(traceability_info)
756
- "3.2.2. Forward links to macro definitions using regexes" (REQUIREMENT)
757
def get_req_uids_by_function_name(
758
self, rel_path_posix: str, name: str
759
) -> Optional[List[Tuple[str, Optional[str]]]]:
760
"""
761
@relation(SDOC-LLR-207, scope=function)762
"""763
764
if rel_path_posix not in self.map_file_function_names_to_reqs_uids:
765
return None
766
767
function_names_to_reqs_uids = self.map_file_function_names_to_reqs_uids[
768
rel_path_posix769
]770
771
matching_req_uids: List[Tuple[str, Optional[str]]] = []
772
exact_matching_req_uids = function_names_to_reqs_uids.get(name, None)
773
if exact_matching_req_uids is not None:
774
matching_req_uids.extend(exact_matching_req_uids)
775
776
for function_name_, req_uids_ in function_names_to_reqs_uids.items():
777
if not FileTraceabilityIndex.is_regex_function_name(function_name_):
778
continue779
780
regex_pattern = function_name_[1:-1]
781
try:
782
if re.search(regex_pattern, name) is not None:
783
matching_req_uids.extend(req_uids_)
784
except re.error as exception:
785
raise StrictDocException(
786
"Invalid regular expression in FUNCTION relation "787
f"{function_name_}: {exception}."
788
) from exception
789
790
return matching_req_uids if len(matching_req_uids) > 0 else None
791
792
@staticmethod793
def is_regex_function_name(name: str) -> bool:
794
return len(name) >= 2 and name[0] == "/" and name[-1] == "/"
795
796
def get_req_uids_by_class_name(
797
self, rel_path_posix: str, name: str
798
) -> Optional[List[Tuple[str, Optional[str]]]]:
799
if rel_path_posix in self.map_file_class_names_to_reqs_uids:
800
return self.map_file_class_names_to_reqs_uids[rel_path_posix].get(
801
name, None
802
)803
return None
804
805
@staticmethod806
def create_traceability_info_shared_markers_for_function(
807
traceability_info: SourceFileTraceabilityInfo,
808
function: LanguageItem,
809
marker_type: RangeMarkerType,
810
reqs_uids: List[Tuple[str, Optional[str]]],
811
) -> None:
812
markers_by_role = {}
813
for req_uid_, role in reqs_uids:
814
req = Req(None, req_uid_)
815
if role not in markers_by_role:
816
markers_by_role[role] = (
817
FileTraceabilityIndex.forward_marker_from_language_item(
818
function, marker_type, [req], role
819
)820
)821
else:
822
markers_by_role[role].reqs_objs.append(req)
823
824
for req_uid_, role in reqs_uids:
825
markers = traceability_info.ng_map_reqs_to_markers.setdefault(
826
req_uid_, []
827
)828
markers.append(markers_by_role[role])
829
830
traceability_info.markers.extend(markers_by_role.values())
831
832
@staticmethod833
def forward_marker_from_language_item(
834
function: LanguageItem,
835
marker_type: RangeMarkerType,
836
reqs: List[Req],
837
role: Optional[str],
838
description: Optional[str] = None,
839
) -> ForwardLanguageItemMarker:
840
language_item_marker = ForwardLanguageItemMarker(
841
parent=None, reqs_objs=reqs, scope=marker_type.value
842
)843
language_item_marker.ng_source_line_begin = function.line_begin
844
language_item_marker.ng_range_line_begin = function.line_begin
845
language_item_marker.ng_range_line_end = function.line_end
846
language_item_marker.role = role
847
if description is not None:
848
language_item_marker.set_description(description)
849
elif marker_type == RangeMarkerType.FUNCTION:
850
language_item_marker.set_description(
851
f"function {function.display_name}()"
852
)853
elif marker_type == RangeMarkerType.CLASS:
854
language_item_marker.set_description(f"class {function.name}")
855
return language_item_marker
856
857
@staticmethod858
def forward_range_markers_from_range(
859
file_range: Tuple[int, int], requirement_uid_: str, role: Optional[str]
860
) -> Tuple[ForwardRangeMarker, ForwardRangeMarker]:
861
start_marker = ForwardRangeMarker(
862
start_or_end=True,
863
reqs_objs=[Req(parent=None, uid=requirement_uid_)],
864
role=role,
865
)866
start_marker.ng_range_line_begin = file_range[0]
867
start_marker.ng_source_line_begin = file_range[0]
868
start_marker.ng_range_line_end = file_range[1]
869
870
end_marker = ForwardRangeMarker(
871
start_or_end=False,
872
reqs_objs=[Req(parent=None, uid=requirement_uid_)],
873
role=role,
874
)875
end_marker.ng_source_line_begin = file_range[1]
876
end_marker.ng_range_line_begin = file_range[0]
877
end_marker.ng_range_line_end = file_range[1]
878
879
return start_marker, end_marker
880
881
@staticmethod882
def forward_file_marker_from_file_info(
883
file_info: SourceFileTraceabilityInfo,
884
requirement_uid_: str,
885
role: Optional[str],
886
) -> ForwardFileMarker:
887
marker = ForwardFileMarker(
888
reqs_objs=[Req(parent=None, uid=requirement_uid_)],
889
role=role,
890
)891
marker.ng_range_line_begin = 1
892
marker.ng_source_line_begin = 1
893
marker.ng_range_line_end = file_info.file_stats.lines_total
894
return marker
895
896
def calculate_code_coverage_and_sort_nodes(
897
self, traceability_index: "TraceabilityIndex"
898
) -> None:
899
"""
900
Finalize code coverage and sort all nodes.901
902
For each trace info object:903
- Sort the markers according to their source location.904
- Calculate coverage information.905
"""906
907
for (
908
path,
909
traceability_info_,
910
) in self.map_paths_to_source_file_traceability_info.items():
911
912
def marker_comparator_start(
913
marker: RelationMarkerType,
914
) -> int:
915
assert marker.ng_range_line_begin is not None
916
return marker.ng_range_line_begin
917
918
sorted_markers = sorted(
919
traceability_info_.markers, key=marker_comparator_start
920
)921
922
traceability_info_.markers = sorted_markers
923
# Finding how many lines are covered by the requirements in the file.924
# Quick and dirty: https://stackoverflow.com/a/15273749/598057925
merged_ranges: List[List[int]] = []
926
for marker_ in traceability_info_.markers:
927
assert isinstance(
928
marker_,
929
(930
LanguageItemMarker,
931
ForwardRangeMarker,
932
RangeMarker,
933
LineMarker,
934
),935
), marker_
936
if marker_.ng_is_nodoc:
937
continue938
if not marker_.is_begin():
939
continue940
begin, end = (
941
assert_cast(marker_.ng_range_line_begin, int),
942
assert_cast(marker_.ng_range_line_end, int),
943
)944
if merged_ranges and merged_ranges[-1][1] >= (begin - 1):
945
merged_ranges[-1][1] = max(merged_ranges[-1][1], end)
946
else:
947
merged_ranges.append([begin, end])
948
coverage = 0
949
for merged_range in merged_ranges:
950
for line_ in range(merged_range[0], merged_range[1] + 1):
951
if traceability_info_.file_stats.lines_info[line_]:
952
coverage += 1
953
954
for function_ in traceability_info_.functions:
955
for merged_range in merged_ranges:
956
if (
957
function_.line_begin >= merged_range[0]
958
and function_.line_end <= merged_range[1]
959
):960
traceability_info_.covered_functions += 1
961
break962
963
traceability_info_.set_coverage_stats(merged_ranges, coverage)
964
965
for (
966
req_uid_,
967
markers_,
968
) in traceability_info_.ng_map_reqs_to_markers.items():
969
970
def marker_comparator_range(
971
marker: RelationMarkerType,
972
) -> Tuple[int, int]:
973
assert marker.ng_range_line_begin is not None
974
assert marker.ng_range_line_end is not None
975
return marker.ng_range_line_begin, marker.ng_range_line_end
976
977
markers_.sort(key=marker_comparator_range)
978
979
# Validate here, SDocNode.relations doesn't track marker roles.980
node = traceability_index.get_node_by_uid(req_uid_)
981
document = node.get_document()
982
assert document is not None
983
assert document.grammar is not None
984
grammar_element = document.grammar.elements_by_type[
985
node.node_type
986
]987
for marker in markers_:
988
# Backwards markers do not require referenced node grammar989
# to have the relation/role registered in the grammar.990
if isinstance(marker, (LanguageItemMarker, RangeMarker)):
991
continue992
993
if not grammar_element.has_relation_type_role(
994
relation_type="File",
995
relation_role=marker.role,
996
):997
raise StrictDocSemanticError.invalid_marker_role(
998
node=node,
999
marker=marker,
1000
path_to_src_file=path,
1001
)1002
1003
# Sort by paths alphabetically.1004
for paths_with_role in self.map_reqs_uids_to_paths.values():
1005
paths_with_role.sort()
1006
1007
# Sort by node UID alphabetically.1008
for path_requirements_ in self.map_paths_to_reqs.values():
1009
1010
def compare_sdocnode_by_uid(node_: SDocNode) -> str:
1011
return assert_cast(node_.reserved_uid, str)
1012
1013
path_requirements_.sort(key=compare_sdocnode_by_uid)
1014
1015
def connect_source_node_function(
1016
self,
1017
source_node: SourceNode,
1018
source_sdoc_node_uid: str,
1019
traceability_info: SourceFileTraceabilityInfo,
1020
) -> None:
1021
source_node_function = source_node.function
1022
assert source_node_function is not None
1023
1024
language_item_marker = self.forward_marker_from_language_item(
1025
function=source_node_function,
1026
marker_type=RangeMarkerType.FUNCTION,
1027
reqs=[Req(None, source_sdoc_node_uid)],
1028
role=None,
1029
description=f"function {source_node_function.display_name}()",
1030
)1031
1032
traceability_info.ng_map_reqs_to_markers.setdefault(
1033
source_sdoc_node_uid, []
1034
).append(language_item_marker)
1035
language_item_marker_copy = language_item_marker.create_end_marker()
1036
traceability_info.markers.append(language_item_marker)
1037
traceability_info.markers.append(language_item_marker_copy)
1038
1039
@staticmethod1040
def create_sdoc_node_from_source_node(
1041
source_node: SourceNode,
1042
source_node_config_entry: SourceNodesEntry,
1043
sdoc_node_uid: str,
1044
parent_document: SDocDocumentIF,
1045
) -> SDocNode:
1046
sdoc_node = SDocNode(
1047
parent=parent_document,
1048
node_type=source_node_config_entry.node_type,
1049
fields=[],
1050
relations=[],
1051
# It is important that this autogenerated node is marked as such.1052
autogen=True,
1053
)1054
sdoc_node.ng_document_reference = DocumentReference()
1055
sdoc_node.ng_document_reference.set_document(parent_document)
1056
sdoc_node.ng_including_document_reference = DocumentReference()
1057
sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
1058
sdoc_node_fields["UID"] = sdoc_node_uid
1059
if (
1060
"TITLE" not in sdoc_node_fields
1061
and source_node.entity_name is not None
1062
):1063
sdoc_node_fields["TITLE"] = source_node.entity_name
1064
FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
1065
return sdoc_node
1066
1067
@staticmethod1068
def merge_sdoc_node_with_source_node(
1069
source_node_config_entry: SourceNodesEntry,
1070
source_node: SourceNode,
1071
sdoc_node: SDocNode,
1072
parent_document: SDocDocumentIF,
1073
) -> None:
1074
# First check if grammar element definitions are compatible.1075
source_node_type = source_node_config_entry.node_type
1076
source_node_grammar = assert_cast(
1077
parent_document.grammar, DocumentGrammar
1078
)1079
source_node_grammar_element = source_node_grammar.elements_by_type[
1080
source_node_type1081
]1082
sdoc_node_document = assert_cast(
1083
sdoc_node.get_document(), SDocDocumentIF
1084
)1085
sdoc_node_grammar = assert_cast(
1086
sdoc_node_document.grammar, DocumentGrammar
1087
)1088
sdoc_node_grammar_element = sdoc_node_grammar.elements_by_type[
1089
source_node_type1090
]1091
if source_node_grammar_element != sdoc_node_grammar_element:
1092
raise StrictDocException(
1093
f"Can't merge node {sdoc_node.reserved_uid} with source portion: "
1094
f"Grammar element {sdoc_node_document.reserved_uid}::{source_node_type} "
1095
f"incompatible with {parent_document.reserved_uid}::{source_node_type}"
1096
)1097
# Merge strategy: overwrite any field if there's a field with same name from custom tags.1098
sdoc_node_fields = source_node.get_sdoc_fields(source_node_config_entry)
1099
1100
# Sanity check: Nor UID neither MID must conflict (early auto-MID is allowed to be overwritten)1101
if (
1102
"MID" in sdoc_node.ordered_fields_lookup
1103
and "MID" in sdoc_node_fields
1104
):1105
sdoc_mid_field = sdoc_node.get_field_by_name("MID").get_text_value()
1106
if sdoc_mid_field != sdoc_node_fields["MID"]:
1107
raise StrictDocException(
1108
f"Can't merge node by UID {sdoc_node.reserved_uid}: "
1109
f"Conflicting MID: {sdoc_mid_field} != {sdoc_node_fields['MID']}"
1110
)1111
if sdoc_node.reserved_uid is not None and "UID" in sdoc_node_fields:
1112
if sdoc_node.reserved_uid != sdoc_node_fields["UID"]:
1113
raise StrictDocException(
1114
f"Can't merge node by MID {sdoc_node.reserved_mid}: "
1115
f"Conflicting UID: {sdoc_node.reserved_uid} != {sdoc_node_fields['UID']}"
1116
)1117
1118
FileTraceabilityIndex.set_sdoc_node_fields(sdoc_node, sdoc_node_fields)
1119
source_node.sdoc_node = sdoc_node
1120
1121
@staticmethod1122
def set_sdoc_node_fields(
1123
sdoc_node: SDocNode, sdoc_node_fields: dict[str, str]
1124
) -> None:
1125
for field_name, field_value in sdoc_node_fields.items():
1126
sdoc_node.set_field_value(
1127
field_name=field_name,
1128
form_field_index=0,
1129
value=field_value,
1130
)1131
1132
# As we overwrite the field's content from the source code,1133
# we mark the field as source_origin here.1134
new_field: SDocNodeField = sdoc_node.ordered_fields_lookup[
1135
field_name1136
][0]
1137
new_field.mark_as_source_origin()
1138
1139
@staticmethod1140
def create_source_node_section(
1141
document: SDocDocumentIF,
1142
path_to_source_file: str,
1143
section_cache: Dict[str, Union[SDocDocumentIF, SDocNode]],
1144
) -> Tuple[Union[SDocDocumentIF, SDocNode], List[SDocNode]]:
1145
"""
1146
Add a subsection for each path components in a given file path.1147
"""1148
current_top_node: Union[SDocDocumentIF, SDocNode] = document
1149
created_sections: List[SDocNode] = []
1150
path_components = path_to_source_file.split("/")
1151
for path_component_idx_, path_component_ in enumerate(path_components):
1152
if path_component_ not in section_cache:
1153
path_component_title = (
1154
path_component_ + "/"
1155
if path_component_idx_ < (len(path_components) - 1)
1156
else path_component_
1157
)1158
current_section = SDocNode(
1159
parent=current_top_node,
1160
node_type="SECTION",
1161
fields=[],
1162
relations=[],
1163
is_composite=True,
1164
node_type_close="SECTION",
1165
# It is important that this autogenerated node is marked as such.1166
autogen=True,
1167
)1168
current_section.ng_document_reference = DocumentReference()
1169
current_section.ng_document_reference.set_document(document)
1170
current_section.ng_including_document_reference = (
1171
DocumentReference()
1172
)1173
current_section.set_field_value(
1174
field_name="TITLE",
1175
form_field_index=0,
1176
value=path_component_title,
1177
)1178
1179
current_top_node.section_contents.append(current_section)
1180
section_cache[path_component_] = current_section
1181
created_sections.append(current_section)
1182
current_top_node = section_cache[path_component_]
1183
return current_top_node, created_sections
1184
1185
def connect_sdoc_node_with_file_path(
1186
self, sdoc_node: SDocNode, path_to_source_file_: str
1187
) -> None:
1188
uid = sdoc_node.reserved_uid
1189
assert uid is not None
1190
self.map_reqs_uids_to_paths.setdefault(uid, OrderedSet()).add(
1191
path_to_source_file_1192
)1193
self.map_paths_to_reqs.setdefault(
1194
path_to_source_file_, OrderedSet()
1195
).add(sdoc_node)
1196
1197
@staticmethod1198
def connect_source_node_requirements(
1199
source_node: SourceNode,
1200
sdoc_node: SDocNode,
1201
traceability_index: "TraceabilityIndex",
1202
) -> None:
1203
"""
1204
Connect auto-generated requirement with function marker and with marker target requirement.1205
1206
If function comment has @relation(REQ, scope=function), connections shall become1207
[REQ] <-parent- [auto-generated/merged sdoc_node] -file-> [function marker]1208
1209
Here we link REQ and sdoc_node bidirectional.1210
"""1211
if (
1212
sdoc_node.reserved_uid is not None
1213
and not traceability_index.graph_database.has_link(
1214
link_type=GraphLinkType.UID_TO_NODE,
1215
lhs_node=sdoc_node.reserved_uid,
1216
rhs_node=sdoc_node,
1217
)1218
):1219
traceability_index.graph_database.create_link(
1220
link_type=GraphLinkType.UID_TO_NODE,
1221
lhs_node=sdoc_node.reserved_uid,
1222
rhs_node=sdoc_node,
1223
)1224
1225
# A merge procedure may have overwritten the MID,1226
# in which case the graph database and search index needs an update.1227
if "MID" in sdoc_node.ordered_fields_lookup != sdoc_node.reserved_mid:
1228
sdoc_mid_field = sdoc_node.get_field_by_name("MID").get_text_value()
1229
if sdoc_mid_field != sdoc_node.reserved_mid:
1230
# TODO:1231
# If we really want to support changing the auto-assigned MID,1232
# at least the graph database and the document search index need an update (remove old MID, add new MID).1233
# I currently struggle to update the search index.1234
parent_document = sdoc_node.get_parent_or_including_document()
1235
sdoc_node.reserved_mid = MID(sdoc_mid_field)
1236
if parent_document.config.enable_mid:
1237
sdoc_node.mid_permanent = True
1238
1239
if not traceability_index.graph_database.has_link(
1240
link_type=GraphLinkType.MID_TO_NODE,
1241
lhs_node=sdoc_node.reserved_mid,
1242
rhs_node=sdoc_node,
1243
):1244
traceability_index.graph_database.create_link(
1245
link_type=GraphLinkType.MID_TO_NODE,
1246
lhs_node=sdoc_node.reserved_mid,
1247
rhs_node=sdoc_node,
1248
)1249
1250
for marker_ in source_node.markers:
1251
if not isinstance(marker_, LanguageItemMarker):
1252
continue1253
for req_ in marker_.reqs:
1254
node = traceability_index.get_node_by_uid_weak2(req_)
1255
if not traceability_index.graph_database.has_link(
1256
link_type=GraphLinkType.NODE_TO_PARENT_NODES,
1257
lhs_node=sdoc_node,
1258
rhs_node=node,
1259
):1260
traceability_index.graph_database.create_link(
1261
link_type=GraphLinkType.NODE_TO_PARENT_NODES,
1262
lhs_node=sdoc_node,
1263
rhs_node=node,
1264
)1265
if not traceability_index.graph_database.has_link(
1266
link_type=GraphLinkType.NODE_TO_CHILD_NODES,
1267
lhs_node=node,
1268
rhs_node=sdoc_node,
1269
):1270
traceability_index.graph_database.create_link(
1271
link_type=GraphLinkType.NODE_TO_CHILD_NODES,
1272
lhs_node=node,
1273
rhs_node=sdoc_node,
1274
)